Host on Gitea: forge.yaml, Gitea workflows, host-neutral links

- forge.yaml: the repository's host (Gitea at git.kenschiano.net, limited visibility);
  the "Report a run" collector, the page's links (data/public/links.json, filled in by
  site/build.js) and the dataset metadata read it
- collection/etl/forge.py: issues, pull requests (merge when checks succeed) and labels
  for Gitea and GitHub; `python -m collection forge ...`
- .gitea/workflows: CI and the daily update (pushes a branch, opens a pull request that
  merges itself, opens an issue when a source breaks); .github/workflows kept for GitHub
- docs: Gitea setup (REPO_SETTINGS), the bot reading atlas files from the repository

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
ekcSchiano
2026-11-12 02:46:17 +00:00
co-authored by Claude Opus 5.5
parent 8713ccff1a
commit 614bfcda7c
20 changed files with 479 additions and 47 deletions
+16
View File
@@ -4,8 +4,24 @@
"""
import json
from . import forge
from .common import path, read_yaml
LINKS = "data/public/links.json"
def links():
"""The links the page shows, from forge.yaml and pipeline.yaml (site/build.js fills them into the page)."""
f = forge.config()
src = read_yaml("collection/pipeline.yaml")["sources"]
host = f["url"].split("://", 1)[-1].rstrip("/")
return {
"repo": forge.repo_url(f), "repo_label": f"{host}/{f['repo']}", "host": host, "kind": f["kind"],
"visibility": f.get("visibility", "public"),
"report": forge.new_issue_url(".github/ISSUE_TEMPLATE/report-a-run.yml", f),
"form": src["reddit_spreadsheet"]["links"]["form"], "sheet": src["reddit_spreadsheet"]["links"]["sheet"],
}
GRAPH_JS = "core/graph.js"
BEGIN = " // BEGIN CROSSWALK"
END = " // END CROSSWALK\n"
+5 -17
View File
@@ -64,22 +64,10 @@ def reddit(spec, env):
return json.dumps(threads, ensure_ascii=False, sort_keys=True).encode("utf-8")
def github_issues(spec, env):
"""Every issue with the label (open and closed), through the GitHub REST API."""
headers = {"Accept": "application/vnd.github+json"}
token = env.get("GITHUB_TOKEN") or env.get("GH_TOKEN")
if token:
headers["Authorization"] = f"Bearer {token}"
label = urllib.parse.quote(spec["label"])
issues, page = [], 1
while True:
url = f"https://api.github.com/repos/{spec['repo']}/issues?labels={label}&state=all&per_page=100&page={page}"
batch = json.loads(_get(url, headers=headers))
issues += [i for i in batch if "pull_request" not in i]
if len(batch) < 100:
break
page += 1
issues.sort(key=lambda i: i["number"])
def forge_issues(spec, env):
"""Every issue with the label (open and closed), from the repository's host (forge.yaml: Gitea or GitHub)."""
from . import forge
issues = forge.list_issues(spec["label"], env)
return json.dumps(issues, ensure_ascii=False, sort_keys=True).encode("utf-8")
@@ -119,7 +107,7 @@ def pack_local(spec, src):
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
EXTRACTORS = {"http": http, "reddit": reddit, "github_issues": github_issues, "local_dir": local_dir}
EXTRACTORS = {"http": http, "reddit": reddit, "forge_issues": forge_issues, "local_dir": local_dir}
# ---------------------------------------------------------------- the raw store
+163
View File
@@ -0,0 +1,163 @@
"""The repository host (forge.yaml): links, and the few API calls the pipeline makes, for Gitea and GitHub.
python -m collection forge open-pr --head BRANCH --title T --body-file F [--automerge]
python -m collection forge open-issue --title T --body-file F (skipped if an open issue has that title)
python -m collection forge ensure-label --name N --color HEX
The token comes from ATLAS_BOT_TOKEN, then GITEA_TOKEN / GITHUB_TOKEN / GH_TOKEN.
"""
import json
import os
import sys
import urllib.parse
import urllib.request
from .common import read_yaml
USER_AGENT = "threshing-atlas-collector/1.0"
def config():
return read_yaml("forge.yaml")
def repo_url(f=None):
f = f or config()
return f"{f['url'].rstrip('/')}/{f['repo']}"
def file_url(path, f=None, raw=True):
"""A file in the repository on its main branch (raw content, or the page that shows it)."""
f = f or config()
if f["kind"] == "github":
if raw:
return f"https://raw.githubusercontent.com/{f['repo']}/{f['branch']}/{path}"
return f"{repo_url(f)}/blob/{f['branch']}/{path}"
return f"{repo_url(f)}/{'raw' if raw else 'src'}/branch/{f['branch']}/{path}"
def new_issue_url(template, f=None):
"""The "new issue" page for an issue form in .github/ISSUE_TEMPLATE/."""
f = f or config()
if f["kind"] == "github":
return f"{repo_url(f)}/issues/new?template={os.path.basename(template)}"
return f"{repo_url(f)}/issues/new?template={urllib.parse.quote(template)}"
def api_base(f=None):
f = f or config()
if f["kind"] == "github":
return f"https://api.github.com/repos/{f['repo']}"
return f"{f['url'].rstrip('/')}/api/v1/repos/{f['repo']}"
def token(env=None):
env = os.environ if env is None else env
for k in ("ATLAS_BOT_TOKEN", "GITEA_TOKEN", "GITHUB_TOKEN", "GH_TOKEN"):
if env.get(k):
return env[k]
return None
def _call(method, url, body=None, tok=None, f=None):
f = f or config()
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
if tok:
headers["Authorization"] = f"{'Bearer' if f['kind'] == 'github' else 'token'} {tok}"
data = None
if body is not None:
data = json.dumps(body).encode("utf-8")
headers["Content-Type"] = "application/json"
req = urllib.request.Request(url, data=data, method=method, headers=headers)
with urllib.request.urlopen(req, timeout=60) as r:
text = r.read().decode("utf-8")
return json.loads(text) if text.strip() else None
def list_issues(label, env=None, f=None):
"""Every issue (open and closed, no pull requests) with the label, oldest first, in GitHub's field names
(number, body, labels, state, state_reason, created_at), which both hosts share."""
f = f or config()
tok = token(env)
lab = urllib.parse.quote(label)
out, page = [], 1
while True:
if f["kind"] == "github":
url = f"{api_base(f)}/issues?labels={lab}&state=all&per_page=100&page={page}"
else:
url = f"{api_base(f)}/issues?labels={lab}&state=all&type=issues&limit=50&page={page}"
batch = _call("GET", url, tok=tok, f=f) or []
out += [i for i in batch if "pull_request" not in i or not i["pull_request"]]
if len(batch) < (100 if f["kind"] == "github" else 50):
break
page += 1
for i in out:
i.setdefault("state_reason", None) # Gitea has no close reason: reports are rejected with labels
return sorted(out, key=lambda i: i["number"])
def open_pr(head, title, body, automerge=False, env=None, f=None):
f = f or config()
tok = token(env)
pr = _call("POST", f"{api_base(f)}/pulls", {"head": head, "base": f["branch"], "title": title, "body": body}, tok, f)
n = pr["number"]
if automerge:
if f["kind"] == "github":
# GitHub's auto-merge needs GraphQL; the repo's auto-merge setting merges it when checks pass
print(f"opened #{n}; turn on auto-merge in the pull request page if it is not on by default")
else:
_call("POST", f"{api_base(f)}/pulls/{n}/merge",
{"Do": "squash", "merge_when_checks_succeed": True, "delete_branch_after_merge": True}, tok, f)
return n
def open_issue(title, body, env=None, f=None):
f = f or config()
tok = token(env)
q = urllib.parse.quote(title)
found = _call("GET", f"{api_base(f)}/issues?state=open&type=issues&q={q}" if f["kind"] == "gitea"
else f"{api_base(f)}/issues?state=open&per_page=100", tok=tok, f=f) or []
if any(i.get("title") == title for i in found):
return None
return _call("POST", f"{api_base(f)}/issues", {"title": title, "body": body}, tok, f)["number"]
def ensure_label(name, color, env=None, f=None):
f = f or config()
tok = token(env)
labels = _call("GET", f"{api_base(f)}/labels?limit=50&per_page=100", tok=tok, f=f) or []
if any(lb["name"] == name for lb in labels):
return False
_call("POST", f"{api_base(f)}/labels", {"name": name, "color": ("#" if f["kind"] == "gitea" else "") + color.lstrip("#")}, tok, f)
return True
def main(argv):
import argparse
ap = argparse.ArgumentParser(prog="python -m collection forge")
sub = ap.add_subparsers(dest="cmd", required=True)
p = sub.add_parser("open-pr")
p.add_argument("--head", required=True)
p.add_argument("--title", required=True)
p.add_argument("--body-file", required=True)
p.add_argument("--automerge", action="store_true")
i = sub.add_parser("open-issue")
i.add_argument("--title", required=True)
i.add_argument("--body-file", required=True)
lb = sub.add_parser("ensure-label")
lb.add_argument("--name", required=True)
lb.add_argument("--color", default="e8b07e")
a = ap.parse_args(argv)
if a.cmd == "open-pr":
body = open(a.body_file, encoding="utf-8").read()
print(f"pull request #{open_pr(a.head, a.title, body, a.automerge)}")
elif a.cmd == "open-issue":
n = open_issue(a.title, open(a.body_file, encoding="utf-8").read())
print(f"issue #{n}" if n else "an open issue with that title exists already")
elif a.cmd == "ensure-label":
print("created" if ensure_label(a.name, a.color) else "exists")
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
+7 -5
View File
@@ -7,6 +7,7 @@ Written by `python -m collection build` (a test checks it is up to date).
"""
import json
from . import forge
from .common import ROOT, pipeline, read_yaml, sha256_file
OUT = "data/croissant.json"
@@ -67,6 +68,7 @@ def field_description(spec):
def build(dictionary=None):
d = dictionary or load_dictionary()
ds = d["dataset"]
fg = forge.config()
sources = pipeline()["sources"]
distribution = []
@@ -77,7 +79,7 @@ def build(dictionary=None):
"@id": key,
"name": key,
"description": " ".join(spec["description"].split()),
"contentUrl": spec.get("url") or f"{ds['repository']}/blob/main/{spec['path']}",
"contentUrl": forge.file_url(spec["path"], fg),
"encodingFormat": spec["format"],
"sha256": sha256(path),
"contentSize": f"{path.stat().st_size} B",
@@ -102,12 +104,12 @@ def build(dictionary=None):
"conformsTo": "http://mlcommons.org/croissant/1.0",
"name": ds["name"],
"description": " ".join(ds["description"].split()),
"url": ds["url"],
"sameAs": ds["repository"],
"url": fg.get("site") or forge.repo_url(fg),
"sameAs": forge.repo_url(fg),
"license": ds["license"],
"creator": {"@type": "sc:Person", "name": ds["creator"], "url": f"https://github.com/{ds['creator']}"},
"creator": {"@type": "sc:Person", "name": ds["creator"]},
"keywords": ds["keywords"],
"citeAs": ds["citation"],
"citeAs": ds["citation"].format(repository=forge.repo_url(fg)),
"isLiveDataset": True,
"isBasedOn": [
{"@type": "sc:CreativeWork", "name": s["label"], "url": next(iter(s["links"].values()))}
+4 -4
View File
@@ -66,11 +66,11 @@ sources:
runs: reddit_results
reported:
label: Runs reported on the Atlas ("Report a run", a GitHub issue form)
label: Runs reported on the Atlas ("Report a run", an issue form on the Atlas's repository)
run_source: reported
links: { report: "https://github.com/ekcSchiano/threshing-atlas/issues/new?template=report-a-run.yml" }
extract: { kind: github_issues, repo: ekcSchiano/threshing-atlas, label: run report, ext: json }
# GitHub usernames are dropped; issues closed as "not planned" or labelled "invalid" are left out
# the repository's host is forge.yaml (Gitea or GitHub); its "new issue" link is made from it
extract: { kind: forge_issues, label: run report, ext: json }
# usernames are dropped; issues closed as "not planned" (GitHub) or labelled invalid/duplicate are left out
transform: { name: reports, form: collection/specs/report_form.yaml, reject_labels: [invalid, duplicate] }
load: data/public/reports.json
runs: reports
+7
View File
@@ -78,6 +78,8 @@ def build(check=False, report=None):
out.append(report_form.TEMPLATE)
if L.write_bytes(codegen.GRAPH_JS, codegen.graph_js().encode("utf-8"), check):
out.append(codegen.GRAPH_JS)
if L.json_file(codegen.LINKS, codegen.links(), check, compact=False):
out.append(codegen.LINKS)
# the metadata hashes the files above, so it is built from what is (or would be) on disk
if not check or not out:
if L.write_bytes(metadata.OUT, metadata.render(metadata.build()).encode("utf-8"), check):
@@ -100,12 +102,17 @@ def main(argv=None):
up.add_argument("--report")
b = sub.add_parser("build")
b.add_argument("--check", action="store_true")
fg = sub.add_parser("forge", help="pull requests, issues and labels on the repository's host (forge.yaml)")
fg.add_argument("args", nargs=argparse.REMAINDER)
k = sub.add_parser("export-bot-kit", help="zip policy.json, solver.js and graph.js for the bot")
k.add_argument("--out", default="exports")
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
a = ap.parse_args(argv)
if a.cmd == "forge":
from .etl import forge
return forge.main(a.args)
if a.cmd == "export-bot-kit":
from .etl.handoff import export_bot_kit
print(f"wrote {export_bot_kit(a.out)}")
+3 -2
View File
@@ -1,4 +1,4 @@
# "Report a run": the questions of the GitHub issue form, and how each answer becomes a column of runs_clean.csv.
# "Report a run": the questions of the issue form (on the Atlas's repository: forge.yaml), and how each answer becomes a column of runs_clean.csv.
#
# collection/etl/report_form.py writes .github/ISSUE_TEMPLATE/report-a-run.yml from this file (a test checks it is
# up to date), and collection/etl/runs.py codes every report with it. Each option is "label shown: column value";
@@ -11,7 +11,8 @@ title: "Run report"
label: run report
intro: |
Thanks for reporting a run! Answer what you remember; leave anything you're unsure of on "Not sure".
Every answer here is **public** (this is a GitHub issue) and goes into the Atlas's open data as source "reported".
Every answer here is visible to everyone who can see this repository (it is an issue) and goes into the Atlas's
open data as source "reported".
Please don't include your DragonKind account name or anything personal.
You can also use the community [Threshing Routes form](https://forms.gle/DRpj9tG8gb5E4Hcu7) linked from the r/fourthwing megathread; please report each run only once.