Files
threshing-atlas/collection/sources/update_sources.py
T
3536007ade Core: coding rules as declarative form.yaml + Python package (atlas.coding) (#9)
Step 1 of moving the repo to Python with declarative rules.

- core/spec/form.yaml: every question (how it is found in an export), every answer and what it
  is coded as, missing answers, email redaction, burn order, first-dragon pools.
- core/atlas/coding.py reads it and replaces core/python/clean.py (python -m atlas.coding).
- tests/core/test_coding.py: 16 rule tests, one hand-written row per rule, checking the rules
  are right rather than only that JS and Python agree.
- analysis/stats/analyze.py takes the colours and missing answers from the spec instead of its
  own copy; the daily update job codes rows through the package.
- pyproject.toml for the package; pyyaml and pytest added to requirements.

The coded runs are byte-identical to before (5,988 runs); all parity checks still match.


Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 15:27:13 -05:00

186 lines
8.4 KiB
Python

"""update_sources.py: check the community spreadsheets and the route PDF for new information.
python collection/sources/update_sources.py [--report report.md] [--form FILE] [--register FILE] [--pdf FILE]
Run from the repo root. Without file arguments it downloads the current versions from the URLs in
data/sources.json (the GitHub Action does this every day). With file arguments it uses local copies,
which is handy for testing or when a download link changes.
What it does:
* Threshing Routes form: re-exports data/public/form_responses.json and data/derived/runs_clean.csv
when the responses changed, and reports how many rows were added or edited.
* Dragon Register: re-exports data/public/register.json (rider names and photos dropped) when it changed.
The raw register export is never written into the repo.
* Route PDF: the PDF is not redistributed. Only a fingerprint is kept (data/derived/pdf_fingerprint.json):
the file hash, per-page text hashes and per-line hashes. When it changes, the report lists the pages that
changed and the lines that are new, so the coding rules and the scene crosswalk can be updated by hand.
Exit code is 0 either way; "changed" files are visible to git, and the report says what happened.
"""
import argparse
import hashlib
import json
import os
import shutil
import subprocess
import sys
import tempfile
import urllib.request
from datetime import datetime, timezone
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
sys.path.insert(0, HERE)
sys.path.insert(0, os.path.join(ROOT, "core"))
from export_raw import cell, load_raw # noqa: E402
P = lambda *a: os.path.join(ROOT, *a) # noqa: E731
def sha(data, n=None):
h = hashlib.sha256(data if isinstance(data, bytes) else data.encode("utf-8")).hexdigest()
return h[:n] if n else h
def download(url, dest):
req = urllib.request.Request(url, headers={"User-Agent": "threshing-atlas-updater"})
with urllib.request.urlopen(req, timeout=60) as r, open(dest, "wb") as f:
data = r.read()
f.write(data)
head = data[:15].lower()
if head.startswith(b"<!doctype html") or head.startswith(b"<html"):
raise RuntimeError(f"{url} returned a web page, not a file (is the link still shared publicly?)")
return dest
def sheet_rows(path, drop=()):
raw = load_raw(path)
raw = raw[[c for c in raw.columns if not any(d in str(c).lower() for d in drop)]]
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
return [str(c) for c in raw.columns], rows
def diff_rows(old, new):
key = lambda r: json.dumps(r, sort_keys=True, ensure_ascii=False) # noqa: E731
o, n = {key(r) for r in old}, {key(r) for r in new}
return len(n - o), len(o - n)
def update_form(src, report):
out = P("data", "public", "form_responses.json")
current = json.load(open(out, encoding="utf-8"))
headers, rows = sheet_rows(src)
added, removed = diff_rows(current["rows"], rows)
if headers == current["headers"] and not added and not removed:
report.append(f"- **Threshing Routes form:** no change ({len(rows)} responses).")
return False
subprocess.run([sys.executable, P("collection", "sources", "export_raw.py"), src, out], check=True, cwd=HERE)
from atlas import coding # noqa: E402
coding.main([out, P("data", "derived", "runs_clean.csv")])
raw_copy = P("data", "raw", "form", "Threshing_Routes_Responses.xlsx")
if os.path.abspath(src) != raw_copy: # local copy only; data/raw/form/*.xlsx is git-ignored
os.makedirs(os.path.dirname(raw_copy), exist_ok=True)
shutil.copyfile(src, raw_copy)
new_q = [h for h in headers if h not in current["headers"]]
gone_q = [h for h in current["headers"] if h not in headers]
report.append(f"- **Threshing Routes form:** {len(current['rows'])} → {len(rows)} responses "
f"({added} new or edited, {removed} removed or edited).")
if new_q:
report.append(f" - New questions (add them to `core/spec/form.yaml`): {', '.join(new_q)}")
if gone_q:
report.append(f" - Questions no longer in the sheet: {', '.join(gone_q)}")
return True
def update_register(src, report):
out = P("data", "public", "register.json")
current = json.load(open(out, encoding="utf-8"))
headers, rows = sheet_rows(src, drop=("photo", "cadet name"))
added, removed = diff_rows(current["rows"], rows)
if headers == current["headers"] and not added and not removed:
report.append(f"- **Dragon Register:** no change ({len(rows)} dragons).")
return False
subprocess.run([sys.executable, P("collection", "sources", "export_register.py"), src, out], check=True, cwd=HERE)
report.append(f"- **Dragon Register:** {len(current['rows'])} → {len(rows)} entries ({added} new or edited, {removed} removed or edited).")
return True
def pdf_lines(path):
try:
from pypdf import PdfReader
except ImportError:
sys.exit("pypdf is required: pip install -r requirements.txt")
pages = []
for page in PdfReader(path).pages:
text = page.extract_text() or ""
pages.append([" ".join(l.split()) for l in text.splitlines() if l.strip()])
return pages
def update_pdf(src, report):
out = P("data", "derived", "pdf_fingerprint.json")
old = json.load(open(out)) if os.path.exists(out) else None
data = open(src, "rb").read()
pages = pdf_lines(src)
fp = {
"sha256": sha(data),
"bytes": len(data),
"pages": len(pages),
"page_hashes": [sha("\n".join(p), 16) for p in pages],
"line_hashes": sorted({sha(l.lower(), 12) for p in pages for l in p}),
"checked": datetime.now(timezone.utc).strftime("%Y-%m-%d"),
}
if old and old["sha256"] == fp["sha256"]:
report.append(f"- **Route PDF:** no change ({fp['pages']} pages).")
return False
if old and old.get("page_hashes") == fp["page_hashes"]:
report.append("- **Route PDF:** the file changed but its text did not (layout or images only). Worth a look.")
elif old:
changed = [i + 1 for i, h in enumerate(fp["page_hashes"]) if i >= len(old["page_hashes"]) or old["page_hashes"][i] != h]
old_lines = set(old["line_hashes"])
new_lines = [l for p in pages for l in p if sha(l.lower(), 12) not in old_lines]
removed = len(old_lines - set(fp["line_hashes"]))
report.append(f"- **Route PDF:** changed. {old['pages']} → {fp['pages']} pages; pages with new text: "
f"{', '.join(map(str, changed)) or 'none'}; {len(new_lines)} new lines, {removed} lines gone.")
if new_lines:
report.append(" - New lines (update the coding rules or the scene crosswalk if they add options):")
report.extend(f" - {l}" for l in new_lines[:60])
if len(new_lines) > 60:
report.append(f" - … and {len(new_lines) - 60} more")
else:
report.append(f"- **Route PDF:** fingerprint recorded for the first time ({fp['pages']} pages).")
json.dump(fp, open(out, "w"), indent=1)
return True
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--report", default=None)
ap.add_argument("--form")
ap.add_argument("--register")
ap.add_argument("--pdf")
args = ap.parse_args()
sources = json.load(open(P("data", "sources.json")))
report, changed, failed = [], False, []
tmp = tempfile.mkdtemp(prefix="sources-")
steps = [("form", args.form, update_form), ("register", args.register, update_register), ("pdf", args.pdf, update_pdf)]
for key, local, fn in steps:
try:
path = local or download(sources[key]["download"], os.path.join(tmp, sources[key]["filename"]))
changed |= fn(os.path.abspath(path), report)
except Exception as e: # keep going: one broken link should not block the others
failed.append(key)
report.append(f"- **{sources[key]['name']}:** could not check ({e}).")
head = f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}"
text = "\n".join([f"## {head}", ""] + report) + "\n"
print(text)
if args.report:
open(args.report, "w", encoding="utf-8").write(text)
if os.environ.get("GITHUB_OUTPUT"):
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
f.write(f"changed={'true' if changed else 'false'}\nfailed={','.join(failed)}\n")
if __name__ == "__main__":
main()