"""Data collection: extract -> transform -> load for every source, then build the combined runs. python -m collection update [--source NAME ...] [--file NAME=PATH ...] [--report FILE] For each source (all by default): extract it (or take a local file), and when the raw file changed, transform it and load the public source file. Then build. Exit code 0 even when a source could not be reached: the report says what happened (and the daily workflow opens an issue). python -m collection refresh [--bot PATH ...] Locally, what the daily workflow does: update every source (plus bot folders or zips someone sent), then rebuild the graph, the route model, the statistics and the build. python -m collection export-bot-kit exports/atlas-for-bot--.zip: policy.json, solver.js and graph.js for the bot. python -m collection build [--check] Code every public source file into data/public/runs_clean.csv (validated against data/dictionary.yaml), and write the files generated from the specs: the "Report a run" issue form, the bot crosswalk in core/graph.js, and data/croissant.json. With --check nothing is written; exit 1 if any of them is out of date (CI runs this). Sources and their rules: collection/pipeline.yaml. What each column means: data/dictionary.yaml. """ import argparse import os import sys from datetime import datetime, timezone from .etl import extract as E from .etl import load as L from .etl import codegen, metadata, report_form, runs, validate from .etl.common import Report, path, pipeline from .etl.transform import TRANSFORMS def update_source(name, spec, manifest, report, local_file=None, env=None): env = os.environ if env is None else env try: raw, changed = E.extract(name, spec, manifest, env, local_file) except E.Skip as e: report.add(f"- **{spec['label']}:** skipped ({e}).") return False except Exception as e: # one broken source must not block the others report.failed.append(name) report.add(f"- **{spec['label']}:** could not extract ({type(e).__name__}: {e}).") return False if not changed: report.add(f"- **{spec['label']}:** no change.") return False mod = TRANSFORMS[spec["transform"]["name"]] ctx = {"changed": manifest[name]["changed"], "env": env} new = mod.transform(raw, spec, ctx) target = spec.get("load") if name == "bot": old = {"scenes": L.read_json(f"{target}/scenes.json") or {}} lines = mod.describe(old, new) wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False) # one file per import (date + raw checksum), so two imports on the same day (yours, Ken's) never overwrite tag = f"{ctx['changed']}-{manifest[name]['sha256'][:8]}" wrote |= L.jsonl_file(f"{target}/trajectories-{tag}.jsonl", new["trajectories"]) elif target: old = L.read_json(target) lines = mod.describe(old, new) wrote = L.json_file(target, new, compact=not target.startswith("data/derived")) else: # nothing to load: the report is the output (e.g. Reddit result links to add by hand) lines, wrote = mod.describe(None, new), False report.add(f"- **{spec['label']}:** {lines[0]}") for line in lines[1:]: report.add(line) return wrote def build(check=False, report=None): """Combined runs, issue form and metadata. Returns the list of files that changed (or would change).""" pipe = pipeline() rows = runs.build_runs(pipe) validate.check(rows, runs.DICT, "runs_clean.csv") out = [] if L.csv_file(pipe["runs"]["path"], rows, runs.COLUMNS, check): out.append(pipe["runs"]["path"]) if L.write_bytes(report_form.TEMPLATE, report_form.render().encode("utf-8"), check): out.append(report_form.TEMPLATE) if L.write_bytes(codegen.GRAPH_JS, codegen.graph_js().encode("utf-8"), check): out.append(codegen.GRAPH_JS) if L.json_file(codegen.LINKS, codegen.links(), check, compact=False): out.append(codegen.LINKS) # the metadata hashes the files above, so it is built from what is (or would be) on disk if not check or not out: if L.write_bytes(metadata.OUT, metadata.render(metadata.build()).encode("utf-8"), check): out.append(metadata.OUT) counts = {} for r in rows: counts[r["source"]] = counts.get(r["source"], 0) + 1 msg = f"runs_clean.csv: {len(rows)} runs ({', '.join(f'{k} {v}' for k, v in counts.items())})" (report.add if report else print)(f"- **Combined runs:** {msg}") return out def main(argv=None): ap = argparse.ArgumentParser(prog="python -m collection", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = ap.add_subparsers(dest="cmd", required=True) up = sub.add_parser("update") up.add_argument("--source", action="append", help="only these sources (repeatable)") up.add_argument("--file", action="append", default=[], help="NAME=PATH: use a local file instead of extracting") up.add_argument("--report") b = sub.add_parser("build") b.add_argument("--check", action="store_true") fg = sub.add_parser("forge", help="pull requests, issues and labels on the repository's host (forge.yaml)") fg.add_argument("args", nargs=argparse.REMAINDER) k = sub.add_parser("export-bot-kit", help="zip policy.json, solver.js and graph.js for the bot") k.add_argument("--out", default="exports") r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)") r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)") a = ap.parse_args(argv) if a.cmd == "forge": from .etl import forge return forge.main(a.args) if a.cmd == "export-bot-kit": from .etl.handoff import export_bot_kit print(f"wrote {export_bot_kit(a.out)}") return 0 if a.cmd == "refresh": from .etl.handoff import refresh return refresh(main, a.bot) if a.cmd == "build": changed = build(check=a.check) if a.check and changed: print("out of date (run `python -m collection build`): " + ", ".join(changed), file=sys.stderr) return 1 return 0 pipe = pipeline() manifest = L.read_json(pipe["manifest"]) or {} files = dict(f.split("=", 1) for f in a.file) report = Report() for name, spec in pipe["sources"].items(): if a.source and name not in a.source: continue report.changed |= update_source(name, spec, manifest, report, files.get(name)) L.json_file(pipe["manifest"], dict(sorted(manifest.items())), compact=False) try: report.changed |= bool(build(report=report)) except validate.Invalid as e: # new answers the coding rules do not know yet: report them, load nothing report.failed.append("build") report.add(f"- **Combined runs not rebuilt:** {e}") report.add(" - Add the new answers to `core/spec/form.yaml` (or `data/dictionary.yaml`), then run `python -m collection build`.") text = report.text(f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}") if a.report: with open(a.report, "w", encoding="utf-8") as f: f.write(text) if os.environ.get("GITHUB_OUTPUT"): with open(os.environ["GITHUB_OUTPUT"], "a") as f: f.write(f"changed={'true' if report.changed else 'false'}\nfailed={','.join(report.failed)}\n") return 0