Files
threshing-atlas/collection/etl/forge.py
kathleenandClaude Opus 5.5 78e069f03f
CI / test (pull_request) Successful in 39s
Bot handoff: the daily check pulls Ken's bot repo; route model updates go to the bot by pull request
- update-sources: download DragonKind/DragonKind (read with ATLAS_BOT_TOKEN, or DRAGONKIND_TOKEN if set) and
  import it as the bot source. Needs DK_ACCOUNT_SALT (the same salt as earlier imports, so accounts keep their keys);
  without it the bot source is skipped as before.
- The bot load keeps one file, data/bot/trajectories-bot.jsonl: the bot's repo holds every account's full history,
  so each import is merged by attempt id (attempts already published stay; older per-import files are folded in).
  One copy of each attempt also keeps build-graph.js from counting an attempt twice.
- New workflow push-policy: when policy.json, solver.js or graph.js change on main, open (or update) a pull request
  on DragonKind/DragonKind that replaces vendor/threshing-atlas/ with them. Ken merges it.
- forge open-pr: --repo/--base for another repository, and reuse the open pull request for the same branch.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
2026-11-19 02:20:35 +00:00

175 lines
7.0 KiB
Python

"""The repository host (forge.yaml): links, and the few API calls the pipeline makes, for Gitea and GitHub.
python -m collection forge open-pr --head BRANCH --title T --body-file F [--automerge] [--repo OWNER/NAME --base B]
python -m collection forge open-issue --title T --body-file F (skipped if an open issue has that title)
python -m collection forge ensure-label --name N --color HEX
The token comes from ATLAS_BOT_TOKEN, then GITEA_TOKEN / GITHUB_TOKEN / GH_TOKEN.
"""
import json
import os
import sys
import urllib.parse
import urllib.request
from .common import read_yaml
USER_AGENT = "threshing-atlas-collector/1.0"
def config():
return read_yaml("forge.yaml")
def repo_url(f=None):
f = f or config()
return f"{f['url'].rstrip('/')}/{f['repo']}"
def file_url(path, f=None, raw=True):
"""A file in the repository on its main branch (raw content, or the page that shows it)."""
f = f or config()
if f["kind"] == "github":
if raw:
return f"https://raw.githubusercontent.com/{f['repo']}/{f['branch']}/{path}"
return f"{repo_url(f)}/blob/{f['branch']}/{path}"
return f"{repo_url(f)}/{'raw' if raw else 'src'}/branch/{f['branch']}/{path}"
def new_issue_url(template, f=None):
"""The "new issue" page for an issue form in .github/ISSUE_TEMPLATE/."""
f = f or config()
if f["kind"] == "github":
return f"{repo_url(f)}/issues/new?template={os.path.basename(template)}"
return f"{repo_url(f)}/issues/new?template={urllib.parse.quote(template)}"
def api_base(f=None):
f = f or config()
if f["kind"] == "github":
return f"https://api.github.com/repos/{f['repo']}"
return f"{f['url'].rstrip('/')}/api/v1/repos/{f['repo']}"
def token(env=None):
env = os.environ if env is None else env
for k in ("ATLAS_BOT_TOKEN", "GITEA_TOKEN", "GITHUB_TOKEN", "GH_TOKEN"):
if env.get(k):
return env[k]
return None
def _call(method, url, body=None, tok=None, f=None):
f = f or config()
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
if tok:
headers["Authorization"] = f"{'Bearer' if f['kind'] == 'github' else 'token'} {tok}"
data = None
if body is not None:
data = json.dumps(body).encode("utf-8")
headers["Content-Type"] = "application/json"
req = urllib.request.Request(url, data=data, method=method, headers=headers)
with urllib.request.urlopen(req, timeout=60) as r:
text = r.read().decode("utf-8")
return json.loads(text) if text.strip() else None
def list_issues(label, env=None, f=None):
"""Every issue (open and closed, no pull requests) with the label, oldest first, in GitHub's field names
(number, body, labels, state, state_reason, created_at), which both hosts share."""
f = f or config()
tok = token(env)
lab = urllib.parse.quote(label)
out, page = [], 1
while True:
if f["kind"] == "github":
url = f"{api_base(f)}/issues?labels={lab}&state=all&per_page=100&page={page}"
else:
url = f"{api_base(f)}/issues?labels={lab}&state=all&type=issues&limit=50&page={page}"
batch = _call("GET", url, tok=tok, f=f) or []
out += [i for i in batch if "pull_request" not in i or not i["pull_request"]]
if len(batch) < (100 if f["kind"] == "github" else 50):
break
page += 1
for i in out:
i.setdefault("state_reason", None) # Gitea has no close reason: reports are rejected with labels
return sorted(out, key=lambda i: i["number"])
def open_pr(head, title, body, automerge=False, env=None, f=None, repo=None, base=None):
"""Open a pull request (on this repository, or `repo` on the same host). If one is open for the branch already,
it is reused: pushing to its branch has updated it."""
f = dict(f or config())
if repo:
f["repo"] = repo
base = base or f["branch"]
tok = token(env)
open_prs = _call("GET", f"{api_base(f)}/pulls?state=open&limit=50&per_page=100", tok=tok, f=f) or []
same = [p for p in open_prs if (p.get("head") or {}).get("ref") == head]
if same:
return same[0]["number"]
pr = _call("POST", f"{api_base(f)}/pulls", {"head": head, "base": base, "title": title, "body": body}, tok, f)
n = pr["number"]
if automerge:
if f["kind"] == "github":
# GitHub's auto-merge needs GraphQL; the repo's auto-merge setting merges it when checks pass
print(f"opened #{n}; turn on auto-merge in the pull request page if it is not on by default")
else:
_call("POST", f"{api_base(f)}/pulls/{n}/merge",
{"Do": "squash", "merge_when_checks_succeed": True, "delete_branch_after_merge": True}, tok, f)
return n
def open_issue(title, body, env=None, f=None):
f = f or config()
tok = token(env)
q = urllib.parse.quote(title)
found = _call("GET", f"{api_base(f)}/issues?state=open&type=issues&q={q}" if f["kind"] == "gitea"
else f"{api_base(f)}/issues?state=open&per_page=100", tok=tok, f=f) or []
if any(i.get("title") == title for i in found):
return None
return _call("POST", f"{api_base(f)}/issues", {"title": title, "body": body}, tok, f)["number"]
def ensure_label(name, color, env=None, f=None):
f = f or config()
tok = token(env)
labels = _call("GET", f"{api_base(f)}/labels?limit=50&per_page=100", tok=tok, f=f) or []
if any(lb["name"] == name for lb in labels):
return False
_call("POST", f"{api_base(f)}/labels", {"name": name, "color": ("#" if f["kind"] == "gitea" else "") + color.lstrip("#")}, tok, f)
return True
def main(argv):
import argparse
ap = argparse.ArgumentParser(prog="python -m collection forge")
sub = ap.add_subparsers(dest="cmd", required=True)
p = sub.add_parser("open-pr")
p.add_argument("--head", required=True)
p.add_argument("--title", required=True)
p.add_argument("--body-file", required=True)
p.add_argument("--automerge", action="store_true")
p.add_argument("--repo", help="another repository on the same host (OWNER/NAME)")
p.add_argument("--base", help="the branch to merge into (default: forge.yaml branch)")
i = sub.add_parser("open-issue")
i.add_argument("--title", required=True)
i.add_argument("--body-file", required=True)
lb = sub.add_parser("ensure-label")
lb.add_argument("--name", required=True)
lb.add_argument("--color", default="e8b07e")
a = ap.parse_args(argv)
if a.cmd == "open-pr":
body = open(a.body_file, encoding="utf-8").read()
print(f"pull request #{open_pr(a.head, a.title, body, a.automerge, repo=a.repo, base=a.base)}")
elif a.cmd == "open-issue":
n = open_issue(a.title, open(a.body_file, encoding="utf-8").read())
print(f"issue #{n}" if n else "an open issue with that title exists already")
elif a.cmd == "ensure-label":
print("created" if ensure_label(a.name, a.color) else "exists")
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))