CI / test (pull_request) Successful in 40s
- collection/etl/review_reddit.py (`python -m collection review-reddit`) and .gitea/workflows/reddit-review.yml: on every scraper pull request, one comment (refreshed on each push) listing the runs added, changed and gone, the runs that break a game rule (left out of the model and stats until corrected), other flags, runs that may also be in the form, and result pages already counted; plus how to reject / fix / add. forge.upsert_comment keeps it to one comment. collection/README.md: "Reviewing Reddit runs". - analysis/overlap/overlap.py -> data/public/overlap.json: a Reddit run with at least 4 route answers "matches" a form row giving the same answers within a day; chance is the same test a few weeks off. 9 of 36 comparable runs match, ~7.3 by chance: about 2 of 82 look reported twice, too few to matter. Nothing is removed. Shown on the page's Data & methods tab; in the dataset metadata; checked in test:py; part of `rebuild`. - Neither the form nor the register records a result-page id, so the only proven copy (comment <-> result page) stays the one #18 applies. - data/public/stats.json: the statistics of #42 (impossible runs left out), which did not make it into that merge. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
187 lines
7.6 KiB
Python
187 lines
7.6 KiB
Python
"""The repository host (forge.yaml): links, and the few API calls the pipeline makes, for Gitea and GitHub.
|
|
|
|
python -m collection forge open-pr --head BRANCH --title T --body-file F [--automerge] [--repo OWNER/NAME --base B]
|
|
python -m collection forge open-issue --title T --body-file F (skipped if an open issue has that title)
|
|
python -m collection forge ensure-label --name N --color HEX
|
|
|
|
The token comes from ATLAS_BOT_TOKEN, then GITEA_TOKEN / GITHUB_TOKEN / GH_TOKEN.
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
from .common import read_yaml
|
|
|
|
USER_AGENT = "threshing-atlas-collector/1.0"
|
|
|
|
|
|
def config():
|
|
return read_yaml("forge.yaml")
|
|
|
|
|
|
def repo_url(f=None):
|
|
f = f or config()
|
|
return f"{f['url'].rstrip('/')}/{f['repo']}"
|
|
|
|
|
|
def file_url(path, f=None, raw=True):
|
|
"""A file in the repository on its main branch (raw content, or the page that shows it)."""
|
|
f = f or config()
|
|
if f["kind"] == "github":
|
|
if raw:
|
|
return f"https://raw.githubusercontent.com/{f['repo']}/{f['branch']}/{path}"
|
|
return f"{repo_url(f)}/blob/{f['branch']}/{path}"
|
|
return f"{repo_url(f)}/{'raw' if raw else 'src'}/branch/{f['branch']}/{path}"
|
|
|
|
|
|
def new_issue_url(template, f=None):
|
|
"""The "new issue" page for an issue form in .github/ISSUE_TEMPLATE/."""
|
|
f = f or config()
|
|
if f["kind"] == "github":
|
|
return f"{repo_url(f)}/issues/new?template={os.path.basename(template)}"
|
|
return f"{repo_url(f)}/issues/new?template={urllib.parse.quote(template)}"
|
|
|
|
|
|
def api_base(f=None):
|
|
f = f or config()
|
|
if f["kind"] == "github":
|
|
return f"https://api.github.com/repos/{f['repo']}"
|
|
return f"{f['url'].rstrip('/')}/api/v1/repos/{f['repo']}"
|
|
|
|
|
|
def token(env=None):
|
|
env = os.environ if env is None else env
|
|
for k in ("ATLAS_BOT_TOKEN", "GITEA_TOKEN", "GITHUB_TOKEN", "GH_TOKEN"):
|
|
if env.get(k):
|
|
return env[k]
|
|
return None
|
|
|
|
|
|
def _call(method, url, body=None, tok=None, f=None):
|
|
f = f or config()
|
|
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
|
|
if tok:
|
|
headers["Authorization"] = f"{'Bearer' if f['kind'] == 'github' else 'token'} {tok}"
|
|
data = None
|
|
if body is not None:
|
|
data = json.dumps(body).encode("utf-8")
|
|
headers["Content-Type"] = "application/json"
|
|
req = urllib.request.Request(url, data=data, method=method, headers=headers)
|
|
with urllib.request.urlopen(req, timeout=60) as r:
|
|
text = r.read().decode("utf-8")
|
|
return json.loads(text) if text.strip() else None
|
|
|
|
|
|
def list_issues(label, env=None, f=None):
|
|
"""Every issue (open and closed, no pull requests) with the label, oldest first, in GitHub's field names
|
|
(number, body, labels, state, state_reason, created_at), which both hosts share."""
|
|
f = f or config()
|
|
tok = token(env)
|
|
lab = urllib.parse.quote(label)
|
|
out, page = [], 1
|
|
while True:
|
|
if f["kind"] == "github":
|
|
url = f"{api_base(f)}/issues?labels={lab}&state=all&per_page=100&page={page}"
|
|
else:
|
|
url = f"{api_base(f)}/issues?labels={lab}&state=all&type=issues&limit=50&page={page}"
|
|
batch = _call("GET", url, tok=tok, f=f) or []
|
|
out += [i for i in batch if "pull_request" not in i or not i["pull_request"]]
|
|
if len(batch) < (100 if f["kind"] == "github" else 50):
|
|
break
|
|
page += 1
|
|
for i in out:
|
|
i.setdefault("state_reason", None) # Gitea has no close reason: reports are rejected with labels
|
|
return sorted(out, key=lambda i: i["number"])
|
|
|
|
|
|
def open_pr(head, title, body, automerge=False, env=None, f=None, repo=None, base=None):
|
|
"""Open a pull request (on this repository, or `repo` on the same host). If one is open for the branch already,
|
|
it is reused: pushing to its branch has updated it."""
|
|
f = dict(f or config())
|
|
if repo:
|
|
f["repo"] = repo
|
|
base = base or f["branch"]
|
|
tok = token(env)
|
|
open_prs = _call("GET", f"{api_base(f)}/pulls?state=open&limit=50&per_page=100", tok=tok, f=f) or []
|
|
same = [p for p in open_prs if (p.get("head") or {}).get("ref") == head]
|
|
if same:
|
|
return same[0]["number"]
|
|
pr = _call("POST", f"{api_base(f)}/pulls", {"head": head, "base": base, "title": title, "body": body}, tok, f)
|
|
n = pr["number"]
|
|
if automerge:
|
|
if f["kind"] == "github":
|
|
# GitHub's auto-merge needs GraphQL; the repo's auto-merge setting merges it when checks pass
|
|
print(f"opened #{n}; turn on auto-merge in the pull request page if it is not on by default")
|
|
else:
|
|
_call("POST", f"{api_base(f)}/pulls/{n}/merge",
|
|
{"Do": "squash", "merge_when_checks_succeed": True, "delete_branch_after_merge": True}, tok, f)
|
|
return n
|
|
|
|
|
|
def open_issue(title, body, env=None, f=None):
|
|
f = f or config()
|
|
tok = token(env)
|
|
q = urllib.parse.quote(title)
|
|
found = _call("GET", f"{api_base(f)}/issues?state=open&type=issues&q={q}" if f["kind"] == "gitea"
|
|
else f"{api_base(f)}/issues?state=open&per_page=100", tok=tok, f=f) or []
|
|
if any(i.get("title") == title for i in found):
|
|
return None
|
|
return _call("POST", f"{api_base(f)}/issues", {"title": title, "body": body}, tok, f)["number"]
|
|
|
|
|
|
def upsert_comment(number, marker, body, env=None, f=None):
|
|
"""Post `body` on issue or pull request `number`, replacing an earlier comment that starts with `marker`
|
|
(so a summary refreshed on every push stays one comment)."""
|
|
f = f or config()
|
|
tok = token(env)
|
|
comments = _call("GET", f"{api_base(f)}/issues/{number}/comments?limit=50&per_page=100", tok=tok, f=f) or []
|
|
mine = [c for c in comments if (c.get("body") or "").startswith(marker)]
|
|
if mine:
|
|
return _call("PATCH", f"{api_base(f)}/issues/comments/{mine[-1]['id']}", {"body": body}, tok, f)["id"]
|
|
return _call("POST", f"{api_base(f)}/issues/{number}/comments", {"body": body}, tok, f)["id"]
|
|
|
|
|
|
def ensure_label(name, color, env=None, f=None):
|
|
f = f or config()
|
|
tok = token(env)
|
|
labels = _call("GET", f"{api_base(f)}/labels?limit=50&per_page=100", tok=tok, f=f) or []
|
|
if any(lb["name"] == name for lb in labels):
|
|
return False
|
|
_call("POST", f"{api_base(f)}/labels", {"name": name, "color": ("#" if f["kind"] == "gitea" else "") + color.lstrip("#")}, tok, f)
|
|
return True
|
|
|
|
|
|
def main(argv):
|
|
import argparse
|
|
ap = argparse.ArgumentParser(prog="python -m collection forge")
|
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
p = sub.add_parser("open-pr")
|
|
p.add_argument("--head", required=True)
|
|
p.add_argument("--title", required=True)
|
|
p.add_argument("--body-file", required=True)
|
|
p.add_argument("--automerge", action="store_true")
|
|
p.add_argument("--repo", help="another repository on the same host (OWNER/NAME)")
|
|
p.add_argument("--base", help="the branch to merge into (default: forge.yaml branch)")
|
|
i = sub.add_parser("open-issue")
|
|
i.add_argument("--title", required=True)
|
|
i.add_argument("--body-file", required=True)
|
|
lb = sub.add_parser("ensure-label")
|
|
lb.add_argument("--name", required=True)
|
|
lb.add_argument("--color", default="e8b07e")
|
|
a = ap.parse_args(argv)
|
|
if a.cmd == "open-pr":
|
|
body = open(a.body_file, encoding="utf-8").read()
|
|
print(f"pull request #{open_pr(a.head, a.title, body, a.automerge, repo=a.repo, base=a.base)}")
|
|
elif a.cmd == "open-issue":
|
|
n = open_issue(a.title, open(a.body_file, encoding="utf-8").read())
|
|
print(f"issue #{n}" if n else "an open issue with that title exists already")
|
|
elif a.cmd == "ensure-label":
|
|
print("created" if ensure_label(a.name, a.color) else "exists")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|