- collection/pipeline.yaml declares every source (extract kind, transform, privacy rules, load target, run source); `python -m collection update|build` - extract keeps data/raw/<source>/current + previous only (git-ignored); data/manifest.json records the current raw checksum and change date - transforms are pure; validate checks runs against data/dictionary.yaml before load; load writes byte-stable files only when they change - data/public/runs_clean.csv: every run from every source with a new `source` column (reddit spreadsheet, reddit thread, reported, bot, reddit pdf); the sheet's own row label moves to `entry`; run_id is text - bot importer ported to Python (same output as bot.js on the fixture); labels and crosswalk in collection/specs/bot.yaml, written into graph.js - Reddit through the official API (result links to add by hand); the 12 result pages move to collection/manual/reddit_results.yaml - "Report a run": a GitHub issue form generated from collection/specs/report_form.yaml, collected as source "reported", plus a site tab pointing to it, the community spreadsheet and the megathread - analysis and the route model read runs_clean.csv (sources in measures.yaml); stats.json and policy.json unchanged apart from the factor's new name - daily workflow runs the pipeline; the raw store is kept between runs in the Actions cache, encrypted with RAW_ARCHIVE_KEY Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
128 lines
6.0 KiB
Python
128 lines
6.0 KiB
Python
"""Data collection: extract -> transform -> load for every source, then build the combined runs.
|
|
|
|
python -m collection update [--source NAME ...] [--file NAME=PATH ...] [--report FILE]
|
|
For each source (all by default): extract it (or take a local file), and when the raw file changed,
|
|
transform it and load the public source file. Then build. Exit code 0 even when a source could not be
|
|
reached: the report says what happened (and the daily workflow opens an issue).
|
|
python -m collection build [--check]
|
|
Code every public source file into data/public/runs_clean.csv (validated against data/dictionary.yaml),
|
|
and write the files generated from the specs: the "Report a run" issue form, the bot crosswalk in
|
|
core/graph.js, and data/croissant.json. With --check nothing is written; exit 1
|
|
if any of them is out of date (CI runs this).
|
|
|
|
Sources and their rules: collection/pipeline.yaml. What each column means: data/dictionary.yaml.
|
|
"""
|
|
import argparse
|
|
import os
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
|
|
from .etl import extract as E
|
|
from .etl import load as L
|
|
from .etl import codegen, metadata, report_form, runs, validate
|
|
from .etl.common import Report, path, pipeline
|
|
from .etl.transform import TRANSFORMS
|
|
|
|
|
|
def update_source(name, spec, manifest, report, local_file=None, env=None):
|
|
env = os.environ if env is None else env
|
|
try:
|
|
raw, changed = E.extract(name, spec, manifest, env, local_file)
|
|
except E.Skip as e:
|
|
report.add(f"- **{spec['label']}:** skipped ({e}).")
|
|
return False
|
|
except Exception as e: # one broken source must not block the others
|
|
report.failed.append(name)
|
|
report.add(f"- **{spec['label']}:** could not extract ({type(e).__name__}: {e}).")
|
|
return False
|
|
if not changed:
|
|
report.add(f"- **{spec['label']}:** no change.")
|
|
return False
|
|
mod = TRANSFORMS[spec["transform"]["name"]]
|
|
ctx = {"changed": manifest[name]["changed"], "env": env}
|
|
new = mod.transform(raw, spec, ctx)
|
|
target = spec.get("load")
|
|
if name == "bot":
|
|
old = {"scenes": L.read_json(f"{target}/scenes.json") or {}}
|
|
lines = mod.describe(old, new)
|
|
wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False)
|
|
wrote |= L.jsonl_file(f"{target}/trajectories-{ctx['changed']}.jsonl", new["trajectories"])
|
|
elif target:
|
|
old = L.read_json(target)
|
|
lines = mod.describe(old, new)
|
|
wrote = L.json_file(target, new, compact=not target.startswith("data/derived"))
|
|
else: # nothing to load: the report is the output (e.g. Reddit result links to add by hand)
|
|
lines, wrote = mod.describe(None, new), False
|
|
report.add(f"- **{spec['label']}:** {lines[0]}")
|
|
for line in lines[1:]:
|
|
report.add(line)
|
|
return wrote
|
|
|
|
|
|
def build(check=False, report=None):
|
|
"""Combined runs, issue form and metadata. Returns the list of files that changed (or would change)."""
|
|
pipe = pipeline()
|
|
rows = runs.build_runs(pipe)
|
|
validate.check(rows, runs.DICT, "runs_clean.csv")
|
|
out = []
|
|
if L.csv_file(pipe["runs"]["path"], rows, runs.COLUMNS, check):
|
|
out.append(pipe["runs"]["path"])
|
|
if L.write_bytes(report_form.TEMPLATE, report_form.render().encode("utf-8"), check):
|
|
out.append(report_form.TEMPLATE)
|
|
if L.write_bytes(codegen.GRAPH_JS, codegen.graph_js().encode("utf-8"), check):
|
|
out.append(codegen.GRAPH_JS)
|
|
# the metadata hashes the files above, so it is built from what is (or would be) on disk
|
|
if not check or not out:
|
|
if L.write_bytes(metadata.OUT, metadata.render(metadata.build()).encode("utf-8"), check):
|
|
out.append(metadata.OUT)
|
|
counts = {}
|
|
for r in rows:
|
|
counts[r["source"]] = counts.get(r["source"], 0) + 1
|
|
msg = f"runs_clean.csv: {len(rows)} runs ({', '.join(f'{k} {v}' for k, v in counts.items())})"
|
|
(report.add if report else print)(f"- **Combined runs:** {msg}")
|
|
return out
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(prog="python -m collection", description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
up = sub.add_parser("update")
|
|
up.add_argument("--source", action="append", help="only these sources (repeatable)")
|
|
up.add_argument("--file", action="append", default=[], help="NAME=PATH: use a local file instead of extracting")
|
|
up.add_argument("--report")
|
|
b = sub.add_parser("build")
|
|
b.add_argument("--check", action="store_true")
|
|
a = ap.parse_args(argv)
|
|
|
|
if a.cmd == "build":
|
|
changed = build(check=a.check)
|
|
if a.check and changed:
|
|
print("out of date (run `python -m collection build`): " + ", ".join(changed), file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
pipe = pipeline()
|
|
manifest = L.read_json(pipe["manifest"]) or {}
|
|
files = dict(f.split("=", 1) for f in a.file)
|
|
report = Report()
|
|
for name, spec in pipe["sources"].items():
|
|
if a.source and name not in a.source:
|
|
continue
|
|
report.changed |= update_source(name, spec, manifest, report, files.get(name))
|
|
L.json_file(pipe["manifest"], dict(sorted(manifest.items())), compact=False)
|
|
try:
|
|
report.changed |= bool(build(report=report))
|
|
except validate.Invalid as e: # new answers the coding rules do not know yet: report them, load nothing
|
|
report.failed.append("build")
|
|
report.add(f"- **Combined runs not rebuilt:** {e}")
|
|
report.add(" - Add the new answers to `core/spec/form.yaml` (or `data/dictionary.yaml`), then run `python -m collection build`.")
|
|
text = report.text(f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
|
|
if a.report:
|
|
with open(a.report, "w", encoding="utf-8") as f:
|
|
f.write(text)
|
|
if os.environ.get("GITHUB_OUTPUT"):
|
|
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
|
|
f.write(f"changed={'true' if report.changed else 'false'}\nfailed={','.join(report.failed)}\n")
|
|
return 0
|