CI / test (pull_request) Successful in 39s
- update-sources: download DragonKind/DragonKind (read with ATLAS_BOT_TOKEN, or DRAGONKIND_TOKEN if set) and import it as the bot source. Needs DK_ACCOUNT_SALT (the same salt as earlier imports, so accounts keep their keys); without it the bot source is skipped as before. - The bot load keeps one file, data/bot/trajectories-bot.jsonl: the bot's repo holds every account's full history, so each import is merged by attempt id (attempts already published stay; older per-import files are folded in). One copy of each attempt also keeps build-graph.js from counting an attempt twice. - New workflow push-policy: when policy.json, solver.js or graph.js change on main, open (or update) a pull request on DragonKind/DragonKind that replaces vendor/threshing-atlas/ with them. Ken merges it. - forge open-pr: --repo/--base for another repository, and reuse the open pull request for the same branch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
191 lines
9.5 KiB
Python
191 lines
9.5 KiB
Python
"""Data collection: extract -> transform -> load for every source, then build the combined runs.
|
|
|
|
python -m collection update [--source NAME ...] [--file NAME=PATH ...] [--report FILE]
|
|
For each source (all by default): extract it (or take a local file), and when the raw file changed,
|
|
transform it and load the public source file. Then build. Exit code 0 even when a source could not be
|
|
reached: the report says what happened (and the daily workflow opens an issue).
|
|
python -m collection refresh [--bot PATH ...]
|
|
Locally, what the daily workflow does: update every source (plus bot folders or zips someone sent), then
|
|
rebuild the graph, the route model, the statistics and the build.
|
|
python -m collection export-bot-kit
|
|
exports/atlas-for-bot-<date>-<policy id>.zip: policy.json, experiments.json, solver.js and graph.js.
|
|
python -m collection cycle --bot PATH [--deliver DIR] [--community] [--force]
|
|
The automated loop with the bot: import its new runs, refit the route model, rebuild the experiment
|
|
plan (planner/experiments.yaml), and copy the bot's files into DIR. Safe to run on a timer.
|
|
python -m collection build [--check]
|
|
Code every public source file into data/public/runs_clean.csv (validated against data/dictionary.yaml),
|
|
and write the files generated from the specs: the "Report a run" issue form, the bot crosswalk in
|
|
core/graph.js, and data/croissant.json. With --check nothing is written; exit 1
|
|
if any of them is out of date (CI runs this).
|
|
|
|
Sources and their rules: collection/pipeline.yaml. What each column means: data/dictionary.yaml.
|
|
"""
|
|
import argparse
|
|
import os
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
|
|
from .etl import extract as E
|
|
from .etl import load as L
|
|
from .etl import codegen, metadata, report_form, runs, validate
|
|
from .etl.common import Report, path, pipeline, read_jsonl
|
|
from .etl.transform import TRANSFORMS
|
|
|
|
|
|
BOT_FILE = "trajectories-bot.jsonl"
|
|
|
|
|
|
def merge_trajectories(target, new):
|
|
"""Every bot attempt ever imported, in one file (data/bot/trajectories-bot.jsonl), sorted by id. The bot's repo
|
|
holds each account's full history, so an import is a superset: a re-imported attempt replaces its old copy, and
|
|
attempts no longer in the bot's repo (an archived account) stay. Older per-import files are folded in."""
|
|
d = path(target)
|
|
olds = sorted(d.glob("trajectories-*.jsonl"))
|
|
by_id = {}
|
|
for f in olds:
|
|
for t in read_jsonl(f):
|
|
by_id.setdefault(t["id"], t)
|
|
for t in new:
|
|
by_id[t["id"]] = t
|
|
wrote = L.jsonl_file(f"{target}/{BOT_FILE}", [by_id[k] for k in sorted(by_id)])
|
|
for f in olds:
|
|
if f.name != BOT_FILE:
|
|
f.unlink()
|
|
wrote = True
|
|
return wrote
|
|
|
|
|
|
def update_source(name, spec, manifest, report, local_file=None, env=None):
|
|
env = os.environ if env is None else env
|
|
try:
|
|
raw, changed = E.extract(name, spec, manifest, env, local_file)
|
|
except E.Skip as e:
|
|
report.add(f"- **{spec['label']}:** skipped ({e}).")
|
|
return False
|
|
except Exception as e: # one broken source must not block the others
|
|
report.failed.append(name)
|
|
report.add(f"- **{spec['label']}:** could not extract ({type(e).__name__}: {e}).")
|
|
return False
|
|
if not changed:
|
|
report.add(f"- **{spec['label']}:** no change.")
|
|
return False
|
|
mod = TRANSFORMS[spec["transform"]["name"]]
|
|
ctx = {"changed": manifest[name]["changed"], "env": env}
|
|
new = mod.transform(raw, spec, ctx)
|
|
target = spec.get("load")
|
|
if name == "bot":
|
|
old = {"scenes": L.read_json(f"{target}/scenes.json") or {}}
|
|
lines = mod.describe(old, new)
|
|
wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False)
|
|
wrote |= merge_trajectories(target, new["trajectories"])
|
|
elif target:
|
|
old = L.read_json(target)
|
|
lines = mod.describe(old, new)
|
|
wrote = L.json_file(target, new, compact=not target.startswith("data/derived"))
|
|
else: # nothing to load: the report is the output (e.g. Reddit result links to add by hand)
|
|
lines, wrote = mod.describe(None, new), False
|
|
report.add(f"- **{spec['label']}:** {lines[0]}")
|
|
for line in lines[1:]:
|
|
report.add(line)
|
|
return wrote
|
|
|
|
|
|
def build(check=False, report=None):
|
|
"""Combined runs, issue form and metadata. Returns the list of files that changed (or would change)."""
|
|
pipe = pipeline()
|
|
rows = runs.build_runs(pipe)
|
|
validate.check(rows, runs.DICT, "runs_clean.csv")
|
|
out = []
|
|
if L.csv_file(pipe["runs"]["path"], rows, runs.COLUMNS, check):
|
|
out.append(pipe["runs"]["path"])
|
|
if L.write_bytes(report_form.TEMPLATE, report_form.render().encode("utf-8"), check):
|
|
out.append(report_form.TEMPLATE)
|
|
if L.write_bytes(codegen.GRAPH_JS, codegen.graph_js().encode("utf-8"), check):
|
|
out.append(codegen.GRAPH_JS)
|
|
if L.json_file(codegen.LINKS, codegen.links(), check, compact=False):
|
|
out.append(codegen.LINKS)
|
|
# the metadata hashes the files above, so it is built from what is (or would be) on disk
|
|
if not check or not out:
|
|
if L.write_bytes(metadata.OUT, metadata.render(metadata.build()).encode("utf-8"), check):
|
|
out.append(metadata.OUT)
|
|
counts = {}
|
|
for r in rows:
|
|
counts[r["source"]] = counts.get(r["source"], 0) + 1
|
|
msg = f"runs_clean.csv: {len(rows)} runs ({', '.join(f'{k} {v}' for k, v in counts.items())})"
|
|
(report.add if report else print)(f"- **Combined runs:** {msg}")
|
|
return out
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(prog="python -m collection", description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
up = sub.add_parser("update")
|
|
up.add_argument("--source", action="append", help="only these sources (repeatable)")
|
|
up.add_argument("--file", action="append", default=[], help="NAME=PATH: use a local file instead of extracting")
|
|
up.add_argument("--report")
|
|
b = sub.add_parser("build")
|
|
b.add_argument("--check", action="store_true")
|
|
fg = sub.add_parser("forge", help="pull requests, issues and labels on the repository's host (forge.yaml)")
|
|
fg.add_argument("args", nargs=argparse.REMAINDER)
|
|
k = sub.add_parser("export-bot-kit", help="zip policy.json, experiments.json, solver.js and graph.js for the bot")
|
|
k.add_argument("--out", default="exports")
|
|
sub.add_parser("rebuild", help="rebuild everything made from the runs: graph, route model, plan, reports, statistics, build")
|
|
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
|
|
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
|
|
c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)")
|
|
c.add_argument("--bot", action="append", default=[], help="the bot's folder, or a .zip of its data (repeatable); default: $DRAGONKIND_BOT_DIR")
|
|
c.add_argument("--deliver", help="a folder to put the bot's files in (its vendor/threshing-atlas/)")
|
|
c.add_argument("--community", action="store_true", help="also check the community sources first")
|
|
c.add_argument("--force", action="store_true", help="rebuild even when there are no new runs")
|
|
a = ap.parse_args(argv)
|
|
|
|
if a.cmd == "cycle":
|
|
from .etl.handoff import cycle
|
|
bots = a.bot or ([os.environ["DRAGONKIND_BOT_DIR"]] if os.environ.get("DRAGONKIND_BOT_DIR") else [])
|
|
return cycle(main, bots, a.deliver, a.community, a.force)
|
|
if a.cmd == "forge":
|
|
from .etl import forge
|
|
return forge.main(a.args)
|
|
if a.cmd == "export-bot-kit":
|
|
from .etl.handoff import export_bot_kit
|
|
print(f"wrote {export_bot_kit(a.out)}")
|
|
return 0
|
|
if a.cmd == "rebuild":
|
|
from .etl.handoff import rebuild
|
|
return rebuild()
|
|
if a.cmd == "refresh":
|
|
from .etl.handoff import refresh
|
|
return refresh(main, a.bot)
|
|
|
|
if a.cmd == "build":
|
|
changed = build(check=a.check)
|
|
if a.check and changed:
|
|
print("out of date (run `python -m collection build`): " + ", ".join(changed), file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
pipe = pipeline()
|
|
manifest = L.read_json(pipe["manifest"]) or {}
|
|
files = dict(f.split("=", 1) for f in a.file)
|
|
report = Report()
|
|
for name, spec in pipe["sources"].items():
|
|
if a.source and name not in a.source:
|
|
continue
|
|
report.changed |= update_source(name, spec, manifest, report, files.get(name))
|
|
L.json_file(pipe["manifest"], dict(sorted(manifest.items())), compact=False)
|
|
try:
|
|
report.changed |= bool(build(report=report))
|
|
except validate.Invalid as e: # new answers the coding rules do not know yet: report them, load nothing
|
|
report.failed.append("build")
|
|
report.add(f"- **Combined runs not rebuilt:** {e}")
|
|
report.add(" - Add the new answers to `core/spec/form.yaml` (or `data/dictionary.yaml`), then run `python -m collection build`.")
|
|
text = report.text(f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
|
|
if a.report:
|
|
with open(a.report, "w", encoding="utf-8") as f:
|
|
f.write(text)
|
|
if os.environ.get("GITHUB_OUTPUT"):
|
|
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
|
|
f.write(f"changed={'true' if report.changed else 'false'}\nfailed={','.join(report.failed)}\n")
|
|
return 0
|