CI / test (pull_request) Successful in 38s
The daily check keeps the community route PDF only in its private raw store (never published here). When that copy differs from "Threshing Options.pdf" in the bot's private repo, `python -m collection send-pdf` opens (or updates) a pull request there with the new PDF and the lines its text gained and lost, so Ken can update the hand-written routes in src/pdfRoutes.js (the bot does not read the PDF itself). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
195 lines
9.7 KiB
Python
195 lines
9.7 KiB
Python
"""Data collection: extract -> transform -> load for every source, then build the combined runs.
|
|
|
|
python -m collection update [--source NAME ...] [--file NAME=PATH ...] [--report FILE]
|
|
For each source (all by default): extract it (or take a local file), and when the raw file changed,
|
|
transform it and load the public source file. Then build. Exit code 0 even when a source could not be
|
|
reached: the report says what happened (and the daily workflow opens an issue).
|
|
python -m collection refresh [--bot PATH ...]
|
|
Locally, what the daily workflow does: update every source (plus bot folders or zips someone sent), then
|
|
rebuild the graph, the route model, the statistics and the build.
|
|
python -m collection export-bot-kit
|
|
exports/atlas-for-bot-<date>-<policy id>.zip: policy.json, experiments.json, solver.js and graph.js.
|
|
python -m collection cycle --bot PATH [--deliver DIR] [--community] [--force]
|
|
The automated loop with the bot: import its new runs, refit the route model, rebuild the experiment
|
|
plan (planner/experiments.yaml), and copy the bot's files into DIR. Safe to run on a timer.
|
|
python -m collection build [--check]
|
|
Code every public source file into data/public/runs_clean.csv (validated against data/dictionary.yaml),
|
|
and write the files generated from the specs: the "Report a run" issue form, the bot crosswalk in
|
|
core/graph.js, and data/croissant.json. With --check nothing is written; exit 1
|
|
if any of them is out of date (CI runs this).
|
|
|
|
Sources and their rules: collection/pipeline.yaml. What each column means: data/dictionary.yaml.
|
|
"""
|
|
import argparse
|
|
import os
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
|
|
from .etl import extract as E
|
|
from .etl import load as L
|
|
from .etl import codegen, metadata, report_form, runs, validate
|
|
from .etl.common import Report, path, pipeline, read_jsonl
|
|
from .etl.transform import TRANSFORMS
|
|
|
|
|
|
BOT_FILE = "trajectories-bot.jsonl"
|
|
|
|
|
|
def merge_trajectories(target, new):
|
|
"""Every bot attempt ever imported, in one file (data/bot/trajectories-bot.jsonl), sorted by id. The bot's repo
|
|
holds each account's full history, so an import is a superset: a re-imported attempt replaces its old copy, and
|
|
attempts no longer in the bot's repo (an archived account) stay. Older per-import files are folded in."""
|
|
d = path(target)
|
|
olds = sorted(d.glob("trajectories-*.jsonl"))
|
|
by_id = {}
|
|
for f in olds:
|
|
for t in read_jsonl(f):
|
|
by_id.setdefault(t["id"], t)
|
|
for t in new:
|
|
by_id[t["id"]] = t
|
|
wrote = L.jsonl_file(f"{target}/{BOT_FILE}", [by_id[k] for k in sorted(by_id)])
|
|
for f in olds:
|
|
if f.name != BOT_FILE:
|
|
f.unlink()
|
|
wrote = True
|
|
return wrote
|
|
|
|
|
|
def update_source(name, spec, manifest, report, local_file=None, env=None):
|
|
env = os.environ if env is None else env
|
|
try:
|
|
raw, changed = E.extract(name, spec, manifest, env, local_file)
|
|
except E.Skip as e:
|
|
report.add(f"- **{spec['label']}:** skipped ({e}).")
|
|
return False
|
|
except Exception as e: # one broken source must not block the others
|
|
report.failed.append(name)
|
|
report.add(f"- **{spec['label']}:** could not extract ({type(e).__name__}: {e}).")
|
|
return False
|
|
if not changed:
|
|
report.add(f"- **{spec['label']}:** no change.")
|
|
return False
|
|
mod = TRANSFORMS[spec["transform"]["name"]]
|
|
ctx = {"changed": manifest[name]["changed"], "env": env}
|
|
new = mod.transform(raw, spec, ctx)
|
|
target = spec.get("load")
|
|
if name == "bot":
|
|
old = {"scenes": L.read_json(f"{target}/scenes.json") or {}}
|
|
lines = mod.describe(old, new)
|
|
wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False)
|
|
wrote |= merge_trajectories(target, new["trajectories"])
|
|
elif target:
|
|
old = L.read_json(target)
|
|
lines = mod.describe(old, new)
|
|
wrote = L.json_file(target, new, compact=not target.startswith("data/derived"))
|
|
else: # nothing to load: the report is the output (e.g. Reddit result links to add by hand)
|
|
lines, wrote = mod.describe(None, new), False
|
|
report.add(f"- **{spec['label']}:** {lines[0]}")
|
|
for line in lines[1:]:
|
|
report.add(line)
|
|
return wrote
|
|
|
|
|
|
def build(check=False, report=None):
|
|
"""Combined runs, issue form and metadata. Returns the list of files that changed (or would change)."""
|
|
pipe = pipeline()
|
|
rows = runs.build_runs(pipe)
|
|
validate.check(rows, runs.DICT, "runs_clean.csv")
|
|
out = []
|
|
if L.csv_file(pipe["runs"]["path"], rows, runs.COLUMNS, check):
|
|
out.append(pipe["runs"]["path"])
|
|
if L.write_bytes(report_form.TEMPLATE, report_form.render().encode("utf-8"), check):
|
|
out.append(report_form.TEMPLATE)
|
|
if L.write_bytes(codegen.GRAPH_JS, codegen.graph_js().encode("utf-8"), check):
|
|
out.append(codegen.GRAPH_JS)
|
|
if L.json_file(codegen.LINKS, codegen.links(), check, compact=False):
|
|
out.append(codegen.LINKS)
|
|
# the metadata hashes the files above, so it is built from what is (or would be) on disk
|
|
if not check or not out:
|
|
if L.write_bytes(metadata.OUT, metadata.render(metadata.build()).encode("utf-8"), check):
|
|
out.append(metadata.OUT)
|
|
counts = {}
|
|
for r in rows:
|
|
counts[r["source"]] = counts.get(r["source"], 0) + 1
|
|
msg = f"runs_clean.csv: {len(rows)} runs ({', '.join(f'{k} {v}' for k, v in counts.items())})"
|
|
(report.add if report else print)(f"- **Combined runs:** {msg}")
|
|
return out
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(prog="python -m collection", description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
up = sub.add_parser("update")
|
|
up.add_argument("--source", action="append", help="only these sources (repeatable)")
|
|
up.add_argument("--file", action="append", default=[], help="NAME=PATH: use a local file instead of extracting")
|
|
up.add_argument("--report")
|
|
b = sub.add_parser("build")
|
|
b.add_argument("--check", action="store_true")
|
|
fg = sub.add_parser("forge", help="pull requests, issues and labels on the repository's host (forge.yaml)")
|
|
fg.add_argument("args", nargs=argparse.REMAINDER)
|
|
k = sub.add_parser("export-bot-kit", help="zip policy.json, experiments.json, solver.js and graph.js for the bot")
|
|
k.add_argument("--out", default="exports")
|
|
sub.add_parser("rebuild", help="rebuild everything made from the runs: graph, route model, plan, reports, statistics, build")
|
|
sub.add_parser("send-pdf", help="open a pull request on the bot's repo when the route PDF changed (private copy only)")
|
|
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
|
|
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
|
|
c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)")
|
|
c.add_argument("--bot", action="append", default=[], help="the bot's folder, or a .zip of its data (repeatable); default: $DRAGONKIND_BOT_DIR")
|
|
c.add_argument("--deliver", help="a folder to put the bot's files in (its vendor/threshing-atlas/)")
|
|
c.add_argument("--community", action="store_true", help="also check the community sources first")
|
|
c.add_argument("--force", action="store_true", help="rebuild even when there are no new runs")
|
|
a = ap.parse_args(argv)
|
|
|
|
if a.cmd == "cycle":
|
|
from .etl.handoff import cycle
|
|
bots = a.bot or ([os.environ["DRAGONKIND_BOT_DIR"]] if os.environ.get("DRAGONKIND_BOT_DIR") else [])
|
|
return cycle(main, bots, a.deliver, a.community, a.force)
|
|
if a.cmd == "forge":
|
|
from .etl import forge
|
|
return forge.main(a.args)
|
|
if a.cmd == "export-bot-kit":
|
|
from .etl.handoff import export_bot_kit
|
|
print(f"wrote {export_bot_kit(a.out)}")
|
|
return 0
|
|
if a.cmd == "rebuild":
|
|
from .etl.handoff import rebuild
|
|
return rebuild()
|
|
if a.cmd == "send-pdf":
|
|
from .etl.send_pdf import send
|
|
return send()
|
|
if a.cmd == "refresh":
|
|
from .etl.handoff import refresh
|
|
return refresh(main, a.bot)
|
|
|
|
if a.cmd == "build":
|
|
changed = build(check=a.check)
|
|
if a.check and changed:
|
|
print("out of date (run `python -m collection build`): " + ", ".join(changed), file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
pipe = pipeline()
|
|
manifest = L.read_json(pipe["manifest"]) or {}
|
|
files = dict(f.split("=", 1) for f in a.file)
|
|
report = Report()
|
|
for name, spec in pipe["sources"].items():
|
|
if a.source and name not in a.source:
|
|
continue
|
|
report.changed |= update_source(name, spec, manifest, report, files.get(name))
|
|
L.json_file(pipe["manifest"], dict(sorted(manifest.items())), compact=False)
|
|
try:
|
|
report.changed |= bool(build(report=report))
|
|
except validate.Invalid as e: # new answers the coding rules do not know yet: report them, load nothing
|
|
report.failed.append("build")
|
|
report.add(f"- **Combined runs not rebuilt:** {e}")
|
|
report.add(" - Add the new answers to `core/spec/form.yaml` (or `data/dictionary.yaml`), then run `python -m collection build`.")
|
|
text = report.text(f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
|
|
if a.report:
|
|
with open(a.report, "w", encoding="utf-8") as f:
|
|
f.write(text)
|
|
if os.environ.get("GITHUB_OUTPUT"):
|
|
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
|
|
f.write(f"changed={'true' if report.changed else 'false'}\nfailed={','.join(report.failed)}\n")
|
|
return 0
|