Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
184 lines
8.4 KiB
Python
184 lines
8.4 KiB
Python
"""update_sources.py: check the community spreadsheets and the route PDF for new information.
|
|
|
|
python research/python/update_sources.py [--report report.md] [--form FILE] [--register FILE] [--pdf FILE]
|
|
|
|
Run from the repo root. Without file arguments it downloads the current versions from the URLs in
|
|
data/sources.json (the GitHub Action does this every day). With file arguments it uses local copies,
|
|
which is handy for testing or when a download link changes.
|
|
|
|
What it does:
|
|
* Threshing Routes form: re-exports data/public/form_responses.json and data/derived/runs_clean.csv
|
|
when the responses changed, and reports how many rows were added or edited.
|
|
* Dragon Register: re-exports data/public/register.json (rider names and photos dropped) when it changed.
|
|
The raw register export is never written into the repo.
|
|
* Route PDF: the PDF is not redistributed. Only a fingerprint is kept (data/derived/pdf_fingerprint.json):
|
|
the file hash, per-page text hashes and per-line hashes. When it changes, the report lists the pages that
|
|
changed and the lines that are new, so the coding rules and the scene crosswalk can be updated by hand.
|
|
|
|
Exit code is 0 either way; "changed" files are visible to git, and the report says what happened.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import urllib.request
|
|
from datetime import datetime, timezone
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
|
sys.path.insert(0, HERE)
|
|
from export_raw import cell, load_raw # noqa: E402
|
|
|
|
P = lambda *a: os.path.join(ROOT, *a) # noqa: E731
|
|
|
|
|
|
def sha(data, n=None):
|
|
h = hashlib.sha256(data if isinstance(data, bytes) else data.encode("utf-8")).hexdigest()
|
|
return h[:n] if n else h
|
|
|
|
|
|
def download(url, dest):
|
|
req = urllib.request.Request(url, headers={"User-Agent": "threshing-atlas-updater"})
|
|
with urllib.request.urlopen(req, timeout=60) as r, open(dest, "wb") as f:
|
|
data = r.read()
|
|
f.write(data)
|
|
head = data[:15].lower()
|
|
if head.startswith(b"<!doctype html") or head.startswith(b"<html"):
|
|
raise RuntimeError(f"{url} returned a web page, not a file (is the link still shared publicly?)")
|
|
return dest
|
|
|
|
|
|
def sheet_rows(path, drop=()):
|
|
raw = load_raw(path)
|
|
raw = raw[[c for c in raw.columns if not any(d in str(c).lower() for d in drop)]]
|
|
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
|
|
return [str(c) for c in raw.columns], rows
|
|
|
|
|
|
def diff_rows(old, new):
|
|
key = lambda r: json.dumps(r, sort_keys=True, ensure_ascii=False) # noqa: E731
|
|
o, n = {key(r) for r in old}, {key(r) for r in new}
|
|
return len(n - o), len(o - n)
|
|
|
|
|
|
def update_form(src, report):
|
|
out = P("data", "public", "form_responses.json")
|
|
current = json.load(open(out, encoding="utf-8"))
|
|
headers, rows = sheet_rows(src)
|
|
added, removed = diff_rows(current["rows"], rows)
|
|
if headers == current["headers"] and not added and not removed:
|
|
report.append(f"- **Threshing Routes form:** no change ({len(rows)} responses).")
|
|
return False
|
|
subprocess.run([sys.executable, P("research", "python", "export_raw.py"), src, out], check=True, cwd=HERE)
|
|
subprocess.run([sys.executable, P("research", "python", "clean.py"), out, P("data", "derived", "runs_clean.csv")], check=True, cwd=HERE)
|
|
raw_copy = P("data", "raw", "form", "Threshing_Routes_Responses.xlsx")
|
|
if os.path.abspath(src) != raw_copy: # local copy only; data/raw/form/*.xlsx is git-ignored
|
|
os.makedirs(os.path.dirname(raw_copy), exist_ok=True)
|
|
shutil.copyfile(src, raw_copy)
|
|
new_q = [h for h in headers if h not in current["headers"]]
|
|
gone_q = [h for h in current["headers"] if h not in headers]
|
|
report.append(f"- **Threshing Routes form:** {len(current['rows'])} → {len(rows)} responses "
|
|
f"({added} new or edited, {removed} removed or edited).")
|
|
if new_q:
|
|
report.append(f" - New questions (check `COLUMN_KEYS` in engine.js and clean.py): {', '.join(new_q)}")
|
|
if gone_q:
|
|
report.append(f" - Questions no longer in the sheet: {', '.join(gone_q)}")
|
|
return True
|
|
|
|
|
|
def update_register(src, report):
|
|
out = P("data", "public", "register.json")
|
|
current = json.load(open(out, encoding="utf-8"))
|
|
headers, rows = sheet_rows(src, drop=("photo", "cadet name"))
|
|
added, removed = diff_rows(current["rows"], rows)
|
|
if headers == current["headers"] and not added and not removed:
|
|
report.append(f"- **Dragon Register:** no change ({len(rows)} dragons).")
|
|
return False
|
|
subprocess.run([sys.executable, P("research", "python", "export_register.py"), src, out], check=True, cwd=HERE)
|
|
report.append(f"- **Dragon Register:** {len(current['rows'])} → {len(rows)} entries ({added} new or edited, {removed} removed or edited).")
|
|
return True
|
|
|
|
|
|
def pdf_lines(path):
|
|
try:
|
|
from pypdf import PdfReader
|
|
except ImportError:
|
|
sys.exit("pypdf is required: pip install -r research/python/requirements.txt")
|
|
pages = []
|
|
for page in PdfReader(path).pages:
|
|
text = page.extract_text() or ""
|
|
pages.append([" ".join(l.split()) for l in text.splitlines() if l.strip()])
|
|
return pages
|
|
|
|
|
|
def update_pdf(src, report):
|
|
out = P("data", "derived", "pdf_fingerprint.json")
|
|
old = json.load(open(out)) if os.path.exists(out) else None
|
|
data = open(src, "rb").read()
|
|
pages = pdf_lines(src)
|
|
fp = {
|
|
"sha256": sha(data),
|
|
"bytes": len(data),
|
|
"pages": len(pages),
|
|
"page_hashes": [sha("\n".join(p), 16) for p in pages],
|
|
"line_hashes": sorted({sha(l.lower(), 12) for p in pages for l in p}),
|
|
"checked": datetime.now(timezone.utc).strftime("%Y-%m-%d"),
|
|
}
|
|
if old and old["sha256"] == fp["sha256"]:
|
|
report.append(f"- **Route PDF:** no change ({fp['pages']} pages).")
|
|
return False
|
|
if old and old.get("page_hashes") == fp["page_hashes"]:
|
|
report.append("- **Route PDF:** the file changed but its text did not (layout or images only). Worth a look.")
|
|
elif old:
|
|
changed = [i + 1 for i, h in enumerate(fp["page_hashes"]) if i >= len(old["page_hashes"]) or old["page_hashes"][i] != h]
|
|
old_lines = set(old["line_hashes"])
|
|
new_lines = [l for p in pages for l in p if sha(l.lower(), 12) not in old_lines]
|
|
removed = len(old_lines - set(fp["line_hashes"]))
|
|
report.append(f"- **Route PDF:** changed. {old['pages']} → {fp['pages']} pages; pages with new text: "
|
|
f"{', '.join(map(str, changed)) or 'none'}; {len(new_lines)} new lines, {removed} lines gone.")
|
|
if new_lines:
|
|
report.append(" - New lines (update the coding rules or the scene crosswalk if they add options):")
|
|
report.extend(f" - {l}" for l in new_lines[:60])
|
|
if len(new_lines) > 60:
|
|
report.append(f" - … and {len(new_lines) - 60} more")
|
|
else:
|
|
report.append(f"- **Route PDF:** fingerprint recorded for the first time ({fp['pages']} pages).")
|
|
json.dump(fp, open(out, "w"), indent=1)
|
|
return True
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--report", default=None)
|
|
ap.add_argument("--form")
|
|
ap.add_argument("--register")
|
|
ap.add_argument("--pdf")
|
|
args = ap.parse_args()
|
|
sources = json.load(open(P("data", "sources.json")))
|
|
report, changed, failed = [], False, []
|
|
tmp = tempfile.mkdtemp(prefix="sources-")
|
|
steps = [("form", args.form, update_form), ("register", args.register, update_register), ("pdf", args.pdf, update_pdf)]
|
|
for key, local, fn in steps:
|
|
try:
|
|
path = local or download(sources[key]["download"], os.path.join(tmp, sources[key]["filename"]))
|
|
changed |= fn(os.path.abspath(path), report)
|
|
except Exception as e: # keep going: one broken link should not block the others
|
|
failed.append(key)
|
|
report.append(f"- **{sources[key]['name']}:** could not check ({e}).")
|
|
head = f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}"
|
|
text = "\n".join([f"## {head}", ""] + report) + "\n"
|
|
print(text)
|
|
if args.report:
|
|
open(args.report, "w", encoding="utf-8").write(text)
|
|
if os.environ.get("GITHUB_OUTPUT"):
|
|
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
|
|
f.write(f"changed={'true' if changed else 'false'}\nfailed={','.join(failed)}\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|