Host on Gitea: forge.yaml, Gitea workflows, host-neutral links
- forge.yaml: the repository's host (Gitea at git.kenschiano.net, limited visibility); the "Report a run" collector, the page's links (data/public/links.json, filled in by site/build.js) and the dataset metadata read it - collection/etl/forge.py: issues, pull requests (merge when checks succeed) and labels for Gitea and GitHub; `python -m collection forge ...` - .gitea/workflows: CI and the daily update (pushes a branch, opens a pull request that merges itself, opens an issue when a source breaks); .github/workflows kept for GitHub - docs: Gitea setup (REPO_SETTINGS), the bot reading atlas files from the repository Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
8713ccff1a
commit
614bfcda7c
@@ -4,8 +4,24 @@
|
||||
"""
|
||||
import json
|
||||
|
||||
from . import forge
|
||||
from .common import path, read_yaml
|
||||
|
||||
LINKS = "data/public/links.json"
|
||||
|
||||
|
||||
def links():
|
||||
"""The links the page shows, from forge.yaml and pipeline.yaml (site/build.js fills them into the page)."""
|
||||
f = forge.config()
|
||||
src = read_yaml("collection/pipeline.yaml")["sources"]
|
||||
host = f["url"].split("://", 1)[-1].rstrip("/")
|
||||
return {
|
||||
"repo": forge.repo_url(f), "repo_label": f"{host}/{f['repo']}", "host": host, "kind": f["kind"],
|
||||
"visibility": f.get("visibility", "public"),
|
||||
"report": forge.new_issue_url(".github/ISSUE_TEMPLATE/report-a-run.yml", f),
|
||||
"form": src["reddit_spreadsheet"]["links"]["form"], "sheet": src["reddit_spreadsheet"]["links"]["sheet"],
|
||||
}
|
||||
|
||||
GRAPH_JS = "core/graph.js"
|
||||
BEGIN = " // BEGIN CROSSWALK"
|
||||
END = " // END CROSSWALK\n"
|
||||
|
||||
@@ -64,22 +64,10 @@ def reddit(spec, env):
|
||||
return json.dumps(threads, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
|
||||
|
||||
def github_issues(spec, env):
|
||||
"""Every issue with the label (open and closed), through the GitHub REST API."""
|
||||
headers = {"Accept": "application/vnd.github+json"}
|
||||
token = env.get("GITHUB_TOKEN") or env.get("GH_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
label = urllib.parse.quote(spec["label"])
|
||||
issues, page = [], 1
|
||||
while True:
|
||||
url = f"https://api.github.com/repos/{spec['repo']}/issues?labels={label}&state=all&per_page=100&page={page}"
|
||||
batch = json.loads(_get(url, headers=headers))
|
||||
issues += [i for i in batch if "pull_request" not in i]
|
||||
if len(batch) < 100:
|
||||
break
|
||||
page += 1
|
||||
issues.sort(key=lambda i: i["number"])
|
||||
def forge_issues(spec, env):
|
||||
"""Every issue with the label (open and closed), from the repository's host (forge.yaml: Gitea or GitHub)."""
|
||||
from . import forge
|
||||
issues = forge.list_issues(spec["label"], env)
|
||||
return json.dumps(issues, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
|
||||
|
||||
@@ -119,7 +107,7 @@ def pack_local(spec, src):
|
||||
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
|
||||
|
||||
EXTRACTORS = {"http": http, "reddit": reddit, "github_issues": github_issues, "local_dir": local_dir}
|
||||
EXTRACTORS = {"http": http, "reddit": reddit, "forge_issues": forge_issues, "local_dir": local_dir}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the raw store
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
"""The repository host (forge.yaml): links, and the few API calls the pipeline makes, for Gitea and GitHub.
|
||||
|
||||
python -m collection forge open-pr --head BRANCH --title T --body-file F [--automerge]
|
||||
python -m collection forge open-issue --title T --body-file F (skipped if an open issue has that title)
|
||||
python -m collection forge ensure-label --name N --color HEX
|
||||
|
||||
The token comes from ATLAS_BOT_TOKEN, then GITEA_TOKEN / GITHUB_TOKEN / GH_TOKEN.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
from .common import read_yaml
|
||||
|
||||
USER_AGENT = "threshing-atlas-collector/1.0"
|
||||
|
||||
|
||||
def config():
|
||||
return read_yaml("forge.yaml")
|
||||
|
||||
|
||||
def repo_url(f=None):
|
||||
f = f or config()
|
||||
return f"{f['url'].rstrip('/')}/{f['repo']}"
|
||||
|
||||
|
||||
def file_url(path, f=None, raw=True):
|
||||
"""A file in the repository on its main branch (raw content, or the page that shows it)."""
|
||||
f = f or config()
|
||||
if f["kind"] == "github":
|
||||
if raw:
|
||||
return f"https://raw.githubusercontent.com/{f['repo']}/{f['branch']}/{path}"
|
||||
return f"{repo_url(f)}/blob/{f['branch']}/{path}"
|
||||
return f"{repo_url(f)}/{'raw' if raw else 'src'}/branch/{f['branch']}/{path}"
|
||||
|
||||
|
||||
def new_issue_url(template, f=None):
|
||||
"""The "new issue" page for an issue form in .github/ISSUE_TEMPLATE/."""
|
||||
f = f or config()
|
||||
if f["kind"] == "github":
|
||||
return f"{repo_url(f)}/issues/new?template={os.path.basename(template)}"
|
||||
return f"{repo_url(f)}/issues/new?template={urllib.parse.quote(template)}"
|
||||
|
||||
|
||||
def api_base(f=None):
|
||||
f = f or config()
|
||||
if f["kind"] == "github":
|
||||
return f"https://api.github.com/repos/{f['repo']}"
|
||||
return f"{f['url'].rstrip('/')}/api/v1/repos/{f['repo']}"
|
||||
|
||||
|
||||
def token(env=None):
|
||||
env = os.environ if env is None else env
|
||||
for k in ("ATLAS_BOT_TOKEN", "GITEA_TOKEN", "GITHUB_TOKEN", "GH_TOKEN"):
|
||||
if env.get(k):
|
||||
return env[k]
|
||||
return None
|
||||
|
||||
|
||||
def _call(method, url, body=None, tok=None, f=None):
|
||||
f = f or config()
|
||||
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
|
||||
if tok:
|
||||
headers["Authorization"] = f"{'Bearer' if f['kind'] == 'github' else 'token'} {tok}"
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(url, data=data, method=method, headers=headers)
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
text = r.read().decode("utf-8")
|
||||
return json.loads(text) if text.strip() else None
|
||||
|
||||
|
||||
def list_issues(label, env=None, f=None):
|
||||
"""Every issue (open and closed, no pull requests) with the label, oldest first, in GitHub's field names
|
||||
(number, body, labels, state, state_reason, created_at), which both hosts share."""
|
||||
f = f or config()
|
||||
tok = token(env)
|
||||
lab = urllib.parse.quote(label)
|
||||
out, page = [], 1
|
||||
while True:
|
||||
if f["kind"] == "github":
|
||||
url = f"{api_base(f)}/issues?labels={lab}&state=all&per_page=100&page={page}"
|
||||
else:
|
||||
url = f"{api_base(f)}/issues?labels={lab}&state=all&type=issues&limit=50&page={page}"
|
||||
batch = _call("GET", url, tok=tok, f=f) or []
|
||||
out += [i for i in batch if "pull_request" not in i or not i["pull_request"]]
|
||||
if len(batch) < (100 if f["kind"] == "github" else 50):
|
||||
break
|
||||
page += 1
|
||||
for i in out:
|
||||
i.setdefault("state_reason", None) # Gitea has no close reason: reports are rejected with labels
|
||||
return sorted(out, key=lambda i: i["number"])
|
||||
|
||||
|
||||
def open_pr(head, title, body, automerge=False, env=None, f=None):
|
||||
f = f or config()
|
||||
tok = token(env)
|
||||
pr = _call("POST", f"{api_base(f)}/pulls", {"head": head, "base": f["branch"], "title": title, "body": body}, tok, f)
|
||||
n = pr["number"]
|
||||
if automerge:
|
||||
if f["kind"] == "github":
|
||||
# GitHub's auto-merge needs GraphQL; the repo's auto-merge setting merges it when checks pass
|
||||
print(f"opened #{n}; turn on auto-merge in the pull request page if it is not on by default")
|
||||
else:
|
||||
_call("POST", f"{api_base(f)}/pulls/{n}/merge",
|
||||
{"Do": "squash", "merge_when_checks_succeed": True, "delete_branch_after_merge": True}, tok, f)
|
||||
return n
|
||||
|
||||
|
||||
def open_issue(title, body, env=None, f=None):
|
||||
f = f or config()
|
||||
tok = token(env)
|
||||
q = urllib.parse.quote(title)
|
||||
found = _call("GET", f"{api_base(f)}/issues?state=open&type=issues&q={q}" if f["kind"] == "gitea"
|
||||
else f"{api_base(f)}/issues?state=open&per_page=100", tok=tok, f=f) or []
|
||||
if any(i.get("title") == title for i in found):
|
||||
return None
|
||||
return _call("POST", f"{api_base(f)}/issues", {"title": title, "body": body}, tok, f)["number"]
|
||||
|
||||
|
||||
def ensure_label(name, color, env=None, f=None):
|
||||
f = f or config()
|
||||
tok = token(env)
|
||||
labels = _call("GET", f"{api_base(f)}/labels?limit=50&per_page=100", tok=tok, f=f) or []
|
||||
if any(lb["name"] == name for lb in labels):
|
||||
return False
|
||||
_call("POST", f"{api_base(f)}/labels", {"name": name, "color": ("#" if f["kind"] == "gitea" else "") + color.lstrip("#")}, tok, f)
|
||||
return True
|
||||
|
||||
|
||||
def main(argv):
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser(prog="python -m collection forge")
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
p = sub.add_parser("open-pr")
|
||||
p.add_argument("--head", required=True)
|
||||
p.add_argument("--title", required=True)
|
||||
p.add_argument("--body-file", required=True)
|
||||
p.add_argument("--automerge", action="store_true")
|
||||
i = sub.add_parser("open-issue")
|
||||
i.add_argument("--title", required=True)
|
||||
i.add_argument("--body-file", required=True)
|
||||
lb = sub.add_parser("ensure-label")
|
||||
lb.add_argument("--name", required=True)
|
||||
lb.add_argument("--color", default="e8b07e")
|
||||
a = ap.parse_args(argv)
|
||||
if a.cmd == "open-pr":
|
||||
body = open(a.body_file, encoding="utf-8").read()
|
||||
print(f"pull request #{open_pr(a.head, a.title, body, a.automerge)}")
|
||||
elif a.cmd == "open-issue":
|
||||
n = open_issue(a.title, open(a.body_file, encoding="utf-8").read())
|
||||
print(f"issue #{n}" if n else "an open issue with that title exists already")
|
||||
elif a.cmd == "ensure-label":
|
||||
print("created" if ensure_label(a.name, a.color) else "exists")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -7,6 +7,7 @@ Written by `python -m collection build` (a test checks it is up to date).
|
||||
"""
|
||||
import json
|
||||
|
||||
from . import forge
|
||||
from .common import ROOT, pipeline, read_yaml, sha256_file
|
||||
|
||||
OUT = "data/croissant.json"
|
||||
@@ -67,6 +68,7 @@ def field_description(spec):
|
||||
def build(dictionary=None):
|
||||
d = dictionary or load_dictionary()
|
||||
ds = d["dataset"]
|
||||
fg = forge.config()
|
||||
sources = pipeline()["sources"]
|
||||
|
||||
distribution = []
|
||||
@@ -77,7 +79,7 @@ def build(dictionary=None):
|
||||
"@id": key,
|
||||
"name": key,
|
||||
"description": " ".join(spec["description"].split()),
|
||||
"contentUrl": spec.get("url") or f"{ds['repository']}/blob/main/{spec['path']}",
|
||||
"contentUrl": forge.file_url(spec["path"], fg),
|
||||
"encodingFormat": spec["format"],
|
||||
"sha256": sha256(path),
|
||||
"contentSize": f"{path.stat().st_size} B",
|
||||
@@ -102,12 +104,12 @@ def build(dictionary=None):
|
||||
"conformsTo": "http://mlcommons.org/croissant/1.0",
|
||||
"name": ds["name"],
|
||||
"description": " ".join(ds["description"].split()),
|
||||
"url": ds["url"],
|
||||
"sameAs": ds["repository"],
|
||||
"url": fg.get("site") or forge.repo_url(fg),
|
||||
"sameAs": forge.repo_url(fg),
|
||||
"license": ds["license"],
|
||||
"creator": {"@type": "sc:Person", "name": ds["creator"], "url": f"https://github.com/{ds['creator']}"},
|
||||
"creator": {"@type": "sc:Person", "name": ds["creator"]},
|
||||
"keywords": ds["keywords"],
|
||||
"citeAs": ds["citation"],
|
||||
"citeAs": ds["citation"].format(repository=forge.repo_url(fg)),
|
||||
"isLiveDataset": True,
|
||||
"isBasedOn": [
|
||||
{"@type": "sc:CreativeWork", "name": s["label"], "url": next(iter(s["links"].values()))}
|
||||
|
||||
@@ -66,11 +66,11 @@ sources:
|
||||
runs: reddit_results
|
||||
|
||||
reported:
|
||||
label: Runs reported on the Atlas ("Report a run", a GitHub issue form)
|
||||
label: Runs reported on the Atlas ("Report a run", an issue form on the Atlas's repository)
|
||||
run_source: reported
|
||||
links: { report: "https://github.com/ekcSchiano/threshing-atlas/issues/new?template=report-a-run.yml" }
|
||||
extract: { kind: github_issues, repo: ekcSchiano/threshing-atlas, label: run report, ext: json }
|
||||
# GitHub usernames are dropped; issues closed as "not planned" or labelled "invalid" are left out
|
||||
# the repository's host is forge.yaml (Gitea or GitHub); its "new issue" link is made from it
|
||||
extract: { kind: forge_issues, label: run report, ext: json }
|
||||
# usernames are dropped; issues closed as "not planned" (GitHub) or labelled invalid/duplicate are left out
|
||||
transform: { name: reports, form: collection/specs/report_form.yaml, reject_labels: [invalid, duplicate] }
|
||||
load: data/public/reports.json
|
||||
runs: reports
|
||||
|
||||
@@ -78,6 +78,8 @@ def build(check=False, report=None):
|
||||
out.append(report_form.TEMPLATE)
|
||||
if L.write_bytes(codegen.GRAPH_JS, codegen.graph_js().encode("utf-8"), check):
|
||||
out.append(codegen.GRAPH_JS)
|
||||
if L.json_file(codegen.LINKS, codegen.links(), check, compact=False):
|
||||
out.append(codegen.LINKS)
|
||||
# the metadata hashes the files above, so it is built from what is (or would be) on disk
|
||||
if not check or not out:
|
||||
if L.write_bytes(metadata.OUT, metadata.render(metadata.build()).encode("utf-8"), check):
|
||||
@@ -100,12 +102,17 @@ def main(argv=None):
|
||||
up.add_argument("--report")
|
||||
b = sub.add_parser("build")
|
||||
b.add_argument("--check", action="store_true")
|
||||
fg = sub.add_parser("forge", help="pull requests, issues and labels on the repository's host (forge.yaml)")
|
||||
fg.add_argument("args", nargs=argparse.REMAINDER)
|
||||
k = sub.add_parser("export-bot-kit", help="zip policy.json, solver.js and graph.js for the bot")
|
||||
k.add_argument("--out", default="exports")
|
||||
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
|
||||
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
|
||||
a = ap.parse_args(argv)
|
||||
|
||||
if a.cmd == "forge":
|
||||
from .etl import forge
|
||||
return forge.main(a.args)
|
||||
if a.cmd == "export-bot-kit":
|
||||
from .etl.handoff import export_bot_kit
|
||||
print(f"wrote {export_bot_kit(a.out)}")
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# "Report a run": the questions of the GitHub issue form, and how each answer becomes a column of runs_clean.csv.
|
||||
# "Report a run": the questions of the issue form (on the Atlas's repository: forge.yaml), and how each answer becomes a column of runs_clean.csv.
|
||||
#
|
||||
# collection/etl/report_form.py writes .github/ISSUE_TEMPLATE/report-a-run.yml from this file (a test checks it is
|
||||
# up to date), and collection/etl/runs.py codes every report with it. Each option is "label shown: column value";
|
||||
@@ -11,7 +11,8 @@ title: "Run report"
|
||||
label: run report
|
||||
intro: |
|
||||
Thanks for reporting a run! Answer what you remember; leave anything you're unsure of on "Not sure".
|
||||
Every answer here is **public** (this is a GitHub issue) and goes into the Atlas's open data as source "reported".
|
||||
Every answer here is visible to everyone who can see this repository (it is an issue) and goes into the Atlas's
|
||||
open data as source "reported".
|
||||
Please don't include your DragonKind account name or anything personal.
|
||||
You can also use the community [Threshing Routes form](https://forms.gle/DRpj9tG8gb5E4Hcu7) linked from the r/fourthwing megathread; please report each run only once.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user