Files
threshing-atlas/collection/run.py
T
ekcSchianoandClaude Opus 5.5 3b24a11f5a Collection as an ETL pipeline; runs_clean.csv from every source; Report a run
- collection/pipeline.yaml declares every source (extract kind, transform,
  privacy rules, load target, run source); `python -m collection update|build`
- extract keeps data/raw/<source>/current + previous only (git-ignored);
  data/manifest.json records the current raw checksum and change date
- transforms are pure; validate checks runs against data/dictionary.yaml
  before load; load writes byte-stable files only when they change
- data/public/runs_clean.csv: every run from every source with a new
  `source` column (reddit spreadsheet, reddit thread, reported, bot,
  reddit pdf); the sheet's own row label moves to `entry`; run_id is text
- bot importer ported to Python (same output as bot.js on the fixture);
  labels and crosswalk in collection/specs/bot.yaml, written into graph.js
- Reddit through the official API (result links to add by hand); the 12
  result pages move to collection/manual/reddit_results.yaml
- "Report a run": a GitHub issue form generated from
  collection/specs/report_form.yaml, collected as source "reported", plus a
  site tab pointing to it, the community spreadsheet and the megathread
- analysis and the route model read runs_clean.csv (sources in
  measures.yaml); stats.json and policy.json unchanged apart from the
  factor's new name
- daily workflow runs the pipeline; the raw store is kept between runs in
  the Actions cache, encrypted with RAW_ARCHIVE_KEY

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
2026-11-04 02:46:48 +00:00

128 lines
6.0 KiB
Python

"""Data collection: extract -> transform -> load for every source, then build the combined runs.
python -m collection update [--source NAME ...] [--file NAME=PATH ...] [--report FILE]
For each source (all by default): extract it (or take a local file), and when the raw file changed,
transform it and load the public source file. Then build. Exit code 0 even when a source could not be
reached: the report says what happened (and the daily workflow opens an issue).
python -m collection build [--check]
Code every public source file into data/public/runs_clean.csv (validated against data/dictionary.yaml),
and write the files generated from the specs: the "Report a run" issue form, the bot crosswalk in
core/graph.js, and data/croissant.json. With --check nothing is written; exit 1
if any of them is out of date (CI runs this).
Sources and their rules: collection/pipeline.yaml. What each column means: data/dictionary.yaml.
"""
import argparse
import os
import sys
from datetime import datetime, timezone
from .etl import extract as E
from .etl import load as L
from .etl import codegen, metadata, report_form, runs, validate
from .etl.common import Report, path, pipeline
from .etl.transform import TRANSFORMS
def update_source(name, spec, manifest, report, local_file=None, env=None):
env = os.environ if env is None else env
try:
raw, changed = E.extract(name, spec, manifest, env, local_file)
except E.Skip as e:
report.add(f"- **{spec['label']}:** skipped ({e}).")
return False
except Exception as e: # one broken source must not block the others
report.failed.append(name)
report.add(f"- **{spec['label']}:** could not extract ({type(e).__name__}: {e}).")
return False
if not changed:
report.add(f"- **{spec['label']}:** no change.")
return False
mod = TRANSFORMS[spec["transform"]["name"]]
ctx = {"changed": manifest[name]["changed"], "env": env}
new = mod.transform(raw, spec, ctx)
target = spec.get("load")
if name == "bot":
old = {"scenes": L.read_json(f"{target}/scenes.json") or {}}
lines = mod.describe(old, new)
wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False)
wrote |= L.jsonl_file(f"{target}/trajectories-{ctx['changed']}.jsonl", new["trajectories"])
elif target:
old = L.read_json(target)
lines = mod.describe(old, new)
wrote = L.json_file(target, new, compact=not target.startswith("data/derived"))
else: # nothing to load: the report is the output (e.g. Reddit result links to add by hand)
lines, wrote = mod.describe(None, new), False
report.add(f"- **{spec['label']}:** {lines[0]}")
for line in lines[1:]:
report.add(line)
return wrote
def build(check=False, report=None):
"""Combined runs, issue form and metadata. Returns the list of files that changed (or would change)."""
pipe = pipeline()
rows = runs.build_runs(pipe)
validate.check(rows, runs.DICT, "runs_clean.csv")
out = []
if L.csv_file(pipe["runs"]["path"], rows, runs.COLUMNS, check):
out.append(pipe["runs"]["path"])
if L.write_bytes(report_form.TEMPLATE, report_form.render().encode("utf-8"), check):
out.append(report_form.TEMPLATE)
if L.write_bytes(codegen.GRAPH_JS, codegen.graph_js().encode("utf-8"), check):
out.append(codegen.GRAPH_JS)
# the metadata hashes the files above, so it is built from what is (or would be) on disk
if not check or not out:
if L.write_bytes(metadata.OUT, metadata.render(metadata.build()).encode("utf-8"), check):
out.append(metadata.OUT)
counts = {}
for r in rows:
counts[r["source"]] = counts.get(r["source"], 0) + 1
msg = f"runs_clean.csv: {len(rows)} runs ({', '.join(f'{k} {v}' for k, v in counts.items())})"
(report.add if report else print)(f"- **Combined runs:** {msg}")
return out
def main(argv=None):
ap = argparse.ArgumentParser(prog="python -m collection", description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
sub = ap.add_subparsers(dest="cmd", required=True)
up = sub.add_parser("update")
up.add_argument("--source", action="append", help="only these sources (repeatable)")
up.add_argument("--file", action="append", default=[], help="NAME=PATH: use a local file instead of extracting")
up.add_argument("--report")
b = sub.add_parser("build")
b.add_argument("--check", action="store_true")
a = ap.parse_args(argv)
if a.cmd == "build":
changed = build(check=a.check)
if a.check and changed:
print("out of date (run `python -m collection build`): " + ", ".join(changed), file=sys.stderr)
return 1
return 0
pipe = pipeline()
manifest = L.read_json(pipe["manifest"]) or {}
files = dict(f.split("=", 1) for f in a.file)
report = Report()
for name, spec in pipe["sources"].items():
if a.source and name not in a.source:
continue
report.changed |= update_source(name, spec, manifest, report, files.get(name))
L.json_file(pipe["manifest"], dict(sorted(manifest.items())), compact=False)
try:
report.changed |= bool(build(report=report))
except validate.Invalid as e: # new answers the coding rules do not know yet: report them, load nothing
report.failed.append("build")
report.add(f"- **Combined runs not rebuilt:** {e}")
report.add(" - Add the new answers to `core/spec/form.yaml` (or `data/dictionary.yaml`), then run `python -m collection build`.")
text = report.text(f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
if a.report:
with open(a.report, "w", encoding="utf-8") as f:
f.write(text)
if os.environ.get("GITHUB_OUTPUT"):
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
f.write(f"changed={'true' if report.changed else 'false'}\nfailed={','.join(report.failed)}\n")
return 0