"""update_sources.py: check the community spreadsheets and the route PDF for new information. python collection/sources/update_sources.py [--report report.md] [--form FILE] [--register FILE] [--pdf FILE] Run from the repo root. Without file arguments it downloads the current versions from the URLs in data/sources.json (the GitHub Action does this every day). With file arguments it uses local copies, which is handy for testing or when a download link changes. What it does: * Threshing Routes form: re-exports data/public/form_responses.json and data/derived/runs_clean.csv when the responses changed, and reports how many rows were added or edited. * Dragon Register: re-exports data/public/register.json (rider names and photos dropped) when it changed. The raw register export is never written into the repo. * Route PDF: the PDF is not redistributed. Only a fingerprint is kept (data/derived/pdf_fingerprint.json): the file hash, per-page text hashes and per-line hashes. When it changes, the report lists the pages that changed and the lines that are new, so the coding rules and the scene crosswalk can be updated by hand. Exit code is 0 either way; "changed" files are visible to git, and the report says what happened. """ import argparse import hashlib import json import os import shutil import subprocess import sys import tempfile import urllib.request from datetime import datetime, timezone HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.abspath(os.path.join(HERE, "..", "..")) sys.path.insert(0, HERE) from export_raw import cell, load_raw # noqa: E402 P = lambda *a: os.path.join(ROOT, *a) # noqa: E731 def sha(data, n=None): h = hashlib.sha256(data if isinstance(data, bytes) else data.encode("utf-8")).hexdigest() return h[:n] if n else h def download(url, dest): req = urllib.request.Request(url, headers={"User-Agent": "threshing-atlas-updater"}) with urllib.request.urlopen(req, timeout=60) as r, open(dest, "wb") as f: data = r.read() f.write(data) head = data[:15].lower() if head.startswith(b"= len(old["page_hashes"]) or old["page_hashes"][i] != h] old_lines = set(old["line_hashes"]) new_lines = [l for p in pages for l in p if sha(l.lower(), 12) not in old_lines] removed = len(old_lines - set(fp["line_hashes"])) report.append(f"- **Route PDF:** changed. {old['pages']} → {fp['pages']} pages; pages with new text: " f"{', '.join(map(str, changed)) or 'none'}; {len(new_lines)} new lines, {removed} lines gone.") if new_lines: report.append(" - New lines (update the coding rules or the scene crosswalk if they add options):") report.extend(f" - {l}" for l in new_lines[:60]) if len(new_lines) > 60: report.append(f" - … and {len(new_lines) - 60} more") else: report.append(f"- **Route PDF:** fingerprint recorded for the first time ({fp['pages']} pages).") json.dump(fp, open(out, "w"), indent=1) return True def main(): ap = argparse.ArgumentParser() ap.add_argument("--report", default=None) ap.add_argument("--form") ap.add_argument("--register") ap.add_argument("--pdf") args = ap.parse_args() sources = json.load(open(P("data", "sources.json"))) report, changed, failed = [], False, [] tmp = tempfile.mkdtemp(prefix="sources-") steps = [("form", args.form, update_form), ("register", args.register, update_register), ("pdf", args.pdf, update_pdf)] for key, local, fn in steps: try: path = local or download(sources[key]["download"], os.path.join(tmp, sources[key]["filename"])) changed |= fn(os.path.abspath(path), report) except Exception as e: # keep going: one broken link should not block the others failed.append(key) report.append(f"- **{sources[key]['name']}:** could not check ({e}).") head = f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}" text = "\n".join([f"## {head}", ""] + report) + "\n" print(text) if args.report: open(args.report, "w", encoding="utf-8").write(text) if os.environ.get("GITHUB_OUTPUT"): with open(os.environ["GITHUB_OUTPUT"], "a") as f: f.write(f"changed={'true' if changed else 'false'}\nfailed={','.join(failed)}\n") if __name__ == "__main__": main()