Files
ekcSchianoandClaude Opus 5.5 98be27fc1f
CI / test (pull_request) Successful in 40s
Reddit scraper PRs get a review summary; the form/Reddit overlap is measured (#23, #24)
- collection/etl/review_reddit.py (`python -m collection review-reddit`) and .gitea/workflows/reddit-review.yml:
  on every scraper pull request, one comment (refreshed on each push) listing the runs added, changed and gone, the
  runs that break a game rule (left out of the model and stats until corrected), other flags, runs that may also
  be in the form, and result pages already counted; plus how to reject / fix / add. forge.upsert_comment keeps it
  to one comment. collection/README.md: "Reviewing Reddit runs".
- analysis/overlap/overlap.py -> data/public/overlap.json: a Reddit run with at least 4 route answers "matches" a
  form row giving the same answers within a day; chance is the same test a few weeks off. 9 of 36 comparable runs
  match, ~7.3 by chance: about 2 of 82 look reported twice, too few to matter. Nothing is removed. Shown on the
  page's Data & methods tab; in the dataset metadata; checked in test:py; part of `rebuild`.
- Neither the form nor the register records a result-page id, so the only proven copy (comment <-> result page)
  stays the one #18 applies.
- data/public/stats.json: the statistics of #42 (impossible runs left out), which did not make it into that merge.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
2026-11-19 16:44:52 +00:00

202 lines
10 KiB
Python

"""Data collection: extract -> transform -> load for every source, then build the combined runs.
python -m collection update [--source NAME ...] [--file NAME=PATH ...] [--report FILE]
For each source (all by default): extract it (or take a local file), and when the raw file changed,
transform it and load the public source file. Then build. Exit code 0 even when a source could not be
reached: the report says what happened (and the daily workflow opens an issue).
python -m collection refresh [--bot PATH ...]
Locally, what the daily workflow does: update every source (plus bot folders or zips someone sent), then
rebuild the graph, the route model, the statistics and the build.
python -m collection export-bot-kit
exports/atlas-for-bot-<date>-<policy id>.zip: policy.json, experiments.json, solver.js and graph.js.
python -m collection cycle --bot PATH [--deliver DIR] [--community] [--force]
The automated loop with the bot: import its new runs, refit the route model, rebuild the experiment
plan (planner/experiments.yaml), and copy the bot's files into DIR. Safe to run on a timer.
python -m collection build [--check]
Code every public source file into data/public/runs_clean.csv (validated against data/dictionary.yaml),
and write the files generated from the specs: the "Report a run" issue form, the bot crosswalk in
core/graph.js, and data/croissant.json. With --check nothing is written; exit 1
if any of them is out of date (CI runs this).
Sources and their rules: collection/pipeline.yaml. What each column means: data/dictionary.yaml.
"""
import argparse
import os
import sys
from datetime import datetime, timezone
from .etl import extract as E
from .etl import load as L
from .etl import codegen, metadata, report_form, runs, validate
from .etl.common import Report, path, pipeline, read_jsonl
from .etl.transform import TRANSFORMS
BOT_FILE = "trajectories-bot.jsonl"
def merge_trajectories(target, new):
"""Every bot attempt ever imported, in one file (data/bot/trajectories-bot.jsonl), sorted by id. The bot's repo
holds each account's full history, so an import is a superset: a re-imported attempt replaces its old copy, and
attempts no longer in the bot's repo (an archived account) stay. Older per-import files are folded in."""
d = path(target)
olds = sorted(d.glob("trajectories-*.jsonl"))
by_id = {}
for f in olds:
for t in read_jsonl(f):
by_id.setdefault(t["id"], t)
for t in new:
by_id[t["id"]] = t
wrote = L.jsonl_file(f"{target}/{BOT_FILE}", [by_id[k] for k in sorted(by_id)])
for f in olds:
if f.name != BOT_FILE:
f.unlink()
wrote = True
return wrote
def update_source(name, spec, manifest, report, local_file=None, env=None):
env = os.environ if env is None else env
try:
raw, changed = E.extract(name, spec, manifest, env, local_file)
except E.Skip as e:
report.add(f"- **{spec['label']}:** skipped ({e}).")
return False
except Exception as e: # one broken source must not block the others
report.failed.append(name)
report.add(f"- **{spec['label']}:** could not extract ({type(e).__name__}: {e}).")
return False
if not changed:
report.add(f"- **{spec['label']}:** no change.")
return False
mod = TRANSFORMS[spec["transform"]["name"]]
ctx = {"changed": manifest[name]["changed"], "env": env}
new = mod.transform(raw, spec, ctx)
target = spec.get("load")
if name == "bot":
old = {"scenes": L.read_json(f"{target}/scenes.json") or {}}
lines = mod.describe(old, new)
wrote = L.json_file(f"{target}/scenes.json", mod.merge_scenes(old["scenes"], new["scenes"]), compact=False)
wrote |= merge_trajectories(target, new["trajectories"])
elif target:
old = L.read_json(target)
lines = mod.describe(old, new)
wrote = L.json_file(target, new, compact=not target.startswith("data/derived"))
else: # nothing to load: the report is the output (e.g. Reddit result links to add by hand)
lines, wrote = mod.describe(None, new), False
report.add(f"- **{spec['label']}:** {lines[0]}")
for line in lines[1:]:
report.add(line)
return wrote
def build(check=False, report=None):
"""Combined runs, issue form and metadata. Returns the list of files that changed (or would change)."""
pipe = pipeline()
rows = runs.build_runs(pipe)
validate.check(rows, runs.DICT, "runs_clean.csv")
out = []
if L.csv_file(pipe["runs"]["path"], rows, runs.COLUMNS, check):
out.append(pipe["runs"]["path"])
if L.write_bytes(report_form.TEMPLATE, report_form.render().encode("utf-8"), check):
out.append(report_form.TEMPLATE)
if L.write_bytes(codegen.GRAPH_JS, codegen.graph_js().encode("utf-8"), check):
out.append(codegen.GRAPH_JS)
if L.json_file(codegen.LINKS, codegen.links(), check, compact=False):
out.append(codegen.LINKS)
# the metadata hashes the files above, so it is built from what is (or would be) on disk
if not check or not out:
if L.write_bytes(metadata.OUT, metadata.render(metadata.build()).encode("utf-8"), check):
out.append(metadata.OUT)
counts = {}
for r in rows:
counts[r["source"]] = counts.get(r["source"], 0) + 1
msg = f"runs_clean.csv: {len(rows)} runs ({', '.join(f'{k} {v}' for k, v in counts.items())})"
(report.add if report else print)(f"- **Combined runs:** {msg}")
return out
def main(argv=None):
ap = argparse.ArgumentParser(prog="python -m collection", description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
sub = ap.add_subparsers(dest="cmd", required=True)
up = sub.add_parser("update")
up.add_argument("--source", action="append", help="only these sources (repeatable)")
up.add_argument("--file", action="append", default=[], help="NAME=PATH: use a local file instead of extracting")
up.add_argument("--report")
b = sub.add_parser("build")
b.add_argument("--check", action="store_true")
fg = sub.add_parser("forge", help="pull requests, issues and labels on the repository's host (forge.yaml)")
fg.add_argument("args", nargs=argparse.REMAINDER)
k = sub.add_parser("export-bot-kit", help="zip policy.json, experiments.json, solver.js and graph.js for the bot")
k.add_argument("--out", default="exports")
sub.add_parser("rebuild", help="rebuild everything made from the runs: graph, route model, plan, reports, statistics, build")
sub.add_parser("send-pdf", help="open a pull request on the bot's repo when the route PDF changed (private copy only)")
rv = sub.add_parser("review-reddit", help="a review summary for a Reddit scraper pull request (run `build` first)")
rv.add_argument("--base", help="data/public/reddit_comments.json as it is on main")
rv.add_argument("--out", help="write the summary here (default: print it)")
rv.add_argument("--pr", help="also post it on this pull request, replacing the previous summary")
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)")
c.add_argument("--bot", action="append", default=[], help="the bot's folder, or a .zip of its data (repeatable); default: $DRAGONKIND_BOT_DIR")
c.add_argument("--deliver", help="a folder to put the bot's files in (its vendor/threshing-atlas/)")
c.add_argument("--community", action="store_true", help="also check the community sources first")
c.add_argument("--force", action="store_true", help="rebuild even when there are no new runs")
a = ap.parse_args(argv)
if a.cmd == "cycle":
from .etl.handoff import cycle
bots = a.bot or ([os.environ["DRAGONKIND_BOT_DIR"]] if os.environ.get("DRAGONKIND_BOT_DIR") else [])
return cycle(main, bots, a.deliver, a.community, a.force)
if a.cmd == "forge":
from .etl import forge
return forge.main(a.args)
if a.cmd == "export-bot-kit":
from .etl.handoff import export_bot_kit
print(f"wrote {export_bot_kit(a.out)}")
return 0
if a.cmd == "rebuild":
from .etl.handoff import rebuild
return rebuild()
if a.cmd == "review-reddit":
from .etl.review_reddit import main as review
return review(a.base, a.out, a.pr)
if a.cmd == "send-pdf":
from .etl.send_pdf import send
return send()
if a.cmd == "refresh":
from .etl.handoff import refresh
return refresh(main, a.bot)
if a.cmd == "build":
changed = build(check=a.check)
if a.check and changed:
print("out of date (run `python -m collection build`): " + ", ".join(changed), file=sys.stderr)
return 1
return 0
pipe = pipeline()
manifest = L.read_json(pipe["manifest"]) or {}
files = dict(f.split("=", 1) for f in a.file)
report = Report()
for name, spec in pipe["sources"].items():
if a.source and name not in a.source:
continue
report.changed |= update_source(name, spec, manifest, report, files.get(name))
L.json_file(pipe["manifest"], dict(sorted(manifest.items())), compact=False)
try:
report.changed |= bool(build(report=report))
except validate.Invalid as e: # new answers the coding rules do not know yet: report them, load nothing
report.failed.append("build")
report.add(f"- **Combined runs not rebuilt:** {e}")
report.add(" - Add the new answers to `core/spec/form.yaml` (or `data/dictionary.yaml`), then run `python -m collection build`.")
text = report.text(f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}")
if a.report:
with open(a.report, "w", encoding="utf-8") as f:
f.write(text)
if os.environ.get("GITHUB_OUTPUT"):
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
f.write(f"changed={'true' if report.changed else 'false'}\nfailed={','.join(report.failed)}\n")
return 0