Refactor: lay the repo out by section (core, collection, analysis, planner, site) (#8)
Moves only; no behaviour changes. Every file now lives under the part of the project it belongs to, each section has a README, and docs/PRIVACY.md collects the rules all parts follow. core/ engine.js, graph.js, solver.js, python/clean.py collection/ sources/ (form, register, PDF check, daily update), bot/ (importer) analysis/ stats/ (Python statistics), network/ (build-graph.js), reports/ (POLICY.md) planner/ build-policy.js, policy-report.js site/ atlas/ (the page), build.js, qa/ tests/ core/ (unit), parity/ (JS vs Python), fixtures/ Checked against the previous layout: tests pass; coded runs, test outputs, graph.json and policy.json are byte-identical; the built site differs only in comments that name file paths. The published file names Ken's bot downloads (solver.js, graph.js, data/policy.json) are unchanged. Also removes the undefined/ folder of QA screenshots committed by mistake (the QA scripts now default to a temp folder) and test.txt, and drops the unused npm workspaces. Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
dcd5117373
commit
432e04c2c7
@@ -0,0 +1,183 @@
|
||||
"""update_sources.py: check the community spreadsheets and the route PDF for new information.
|
||||
|
||||
python collection/sources/update_sources.py [--report report.md] [--form FILE] [--register FILE] [--pdf FILE]
|
||||
|
||||
Run from the repo root. Without file arguments it downloads the current versions from the URLs in
|
||||
data/sources.json (the GitHub Action does this every day). With file arguments it uses local copies,
|
||||
which is handy for testing or when a download link changes.
|
||||
|
||||
What it does:
|
||||
* Threshing Routes form: re-exports data/public/form_responses.json and data/derived/runs_clean.csv
|
||||
when the responses changed, and reports how many rows were added or edited.
|
||||
* Dragon Register: re-exports data/public/register.json (rider names and photos dropped) when it changed.
|
||||
The raw register export is never written into the repo.
|
||||
* Route PDF: the PDF is not redistributed. Only a fingerprint is kept (data/derived/pdf_fingerprint.json):
|
||||
the file hash, per-page text hashes and per-line hashes. When it changes, the report lists the pages that
|
||||
changed and the lines that are new, so the coding rules and the scene crosswalk can be updated by hand.
|
||||
|
||||
Exit code is 0 either way; "changed" files are visible to git, and the report says what happened.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
||||
sys.path.insert(0, HERE)
|
||||
from export_raw import cell, load_raw # noqa: E402
|
||||
|
||||
P = lambda *a: os.path.join(ROOT, *a) # noqa: E731
|
||||
|
||||
|
||||
def sha(data, n=None):
|
||||
h = hashlib.sha256(data if isinstance(data, bytes) else data.encode("utf-8")).hexdigest()
|
||||
return h[:n] if n else h
|
||||
|
||||
|
||||
def download(url, dest):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "threshing-atlas-updater"})
|
||||
with urllib.request.urlopen(req, timeout=60) as r, open(dest, "wb") as f:
|
||||
data = r.read()
|
||||
f.write(data)
|
||||
head = data[:15].lower()
|
||||
if head.startswith(b"<!doctype html") or head.startswith(b"<html"):
|
||||
raise RuntimeError(f"{url} returned a web page, not a file (is the link still shared publicly?)")
|
||||
return dest
|
||||
|
||||
|
||||
def sheet_rows(path, drop=()):
|
||||
raw = load_raw(path)
|
||||
raw = raw[[c for c in raw.columns if not any(d in str(c).lower() for d in drop)]]
|
||||
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
|
||||
return [str(c) for c in raw.columns], rows
|
||||
|
||||
|
||||
def diff_rows(old, new):
|
||||
key = lambda r: json.dumps(r, sort_keys=True, ensure_ascii=False) # noqa: E731
|
||||
o, n = {key(r) for r in old}, {key(r) for r in new}
|
||||
return len(n - o), len(o - n)
|
||||
|
||||
|
||||
def update_form(src, report):
|
||||
out = P("data", "public", "form_responses.json")
|
||||
current = json.load(open(out, encoding="utf-8"))
|
||||
headers, rows = sheet_rows(src)
|
||||
added, removed = diff_rows(current["rows"], rows)
|
||||
if headers == current["headers"] and not added and not removed:
|
||||
report.append(f"- **Threshing Routes form:** no change ({len(rows)} responses).")
|
||||
return False
|
||||
subprocess.run([sys.executable, P("collection", "sources", "export_raw.py"), src, out], check=True, cwd=HERE)
|
||||
subprocess.run([sys.executable, P("core", "python", "clean.py"), out, P("data", "derived", "runs_clean.csv")], check=True, cwd=HERE)
|
||||
raw_copy = P("data", "raw", "form", "Threshing_Routes_Responses.xlsx")
|
||||
if os.path.abspath(src) != raw_copy: # local copy only; data/raw/form/*.xlsx is git-ignored
|
||||
os.makedirs(os.path.dirname(raw_copy), exist_ok=True)
|
||||
shutil.copyfile(src, raw_copy)
|
||||
new_q = [h for h in headers if h not in current["headers"]]
|
||||
gone_q = [h for h in current["headers"] if h not in headers]
|
||||
report.append(f"- **Threshing Routes form:** {len(current['rows'])} → {len(rows)} responses "
|
||||
f"({added} new or edited, {removed} removed or edited).")
|
||||
if new_q:
|
||||
report.append(f" - New questions (check `COLUMN_KEYS` in engine.js and clean.py): {', '.join(new_q)}")
|
||||
if gone_q:
|
||||
report.append(f" - Questions no longer in the sheet: {', '.join(gone_q)}")
|
||||
return True
|
||||
|
||||
|
||||
def update_register(src, report):
|
||||
out = P("data", "public", "register.json")
|
||||
current = json.load(open(out, encoding="utf-8"))
|
||||
headers, rows = sheet_rows(src, drop=("photo", "cadet name"))
|
||||
added, removed = diff_rows(current["rows"], rows)
|
||||
if headers == current["headers"] and not added and not removed:
|
||||
report.append(f"- **Dragon Register:** no change ({len(rows)} dragons).")
|
||||
return False
|
||||
subprocess.run([sys.executable, P("collection", "sources", "export_register.py"), src, out], check=True, cwd=HERE)
|
||||
report.append(f"- **Dragon Register:** {len(current['rows'])} → {len(rows)} entries ({added} new or edited, {removed} removed or edited).")
|
||||
return True
|
||||
|
||||
|
||||
def pdf_lines(path):
|
||||
try:
|
||||
from pypdf import PdfReader
|
||||
except ImportError:
|
||||
sys.exit("pypdf is required: pip install -r requirements.txt")
|
||||
pages = []
|
||||
for page in PdfReader(path).pages:
|
||||
text = page.extract_text() or ""
|
||||
pages.append([" ".join(l.split()) for l in text.splitlines() if l.strip()])
|
||||
return pages
|
||||
|
||||
|
||||
def update_pdf(src, report):
|
||||
out = P("data", "derived", "pdf_fingerprint.json")
|
||||
old = json.load(open(out)) if os.path.exists(out) else None
|
||||
data = open(src, "rb").read()
|
||||
pages = pdf_lines(src)
|
||||
fp = {
|
||||
"sha256": sha(data),
|
||||
"bytes": len(data),
|
||||
"pages": len(pages),
|
||||
"page_hashes": [sha("\n".join(p), 16) for p in pages],
|
||||
"line_hashes": sorted({sha(l.lower(), 12) for p in pages for l in p}),
|
||||
"checked": datetime.now(timezone.utc).strftime("%Y-%m-%d"),
|
||||
}
|
||||
if old and old["sha256"] == fp["sha256"]:
|
||||
report.append(f"- **Route PDF:** no change ({fp['pages']} pages).")
|
||||
return False
|
||||
if old and old.get("page_hashes") == fp["page_hashes"]:
|
||||
report.append("- **Route PDF:** the file changed but its text did not (layout or images only). Worth a look.")
|
||||
elif old:
|
||||
changed = [i + 1 for i, h in enumerate(fp["page_hashes"]) if i >= len(old["page_hashes"]) or old["page_hashes"][i] != h]
|
||||
old_lines = set(old["line_hashes"])
|
||||
new_lines = [l for p in pages for l in p if sha(l.lower(), 12) not in old_lines]
|
||||
removed = len(old_lines - set(fp["line_hashes"]))
|
||||
report.append(f"- **Route PDF:** changed. {old['pages']} → {fp['pages']} pages; pages with new text: "
|
||||
f"{', '.join(map(str, changed)) or 'none'}; {len(new_lines)} new lines, {removed} lines gone.")
|
||||
if new_lines:
|
||||
report.append(" - New lines (update the coding rules or the scene crosswalk if they add options):")
|
||||
report.extend(f" - {l}" for l in new_lines[:60])
|
||||
if len(new_lines) > 60:
|
||||
report.append(f" - … and {len(new_lines) - 60} more")
|
||||
else:
|
||||
report.append(f"- **Route PDF:** fingerprint recorded for the first time ({fp['pages']} pages).")
|
||||
json.dump(fp, open(out, "w"), indent=1)
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--report", default=None)
|
||||
ap.add_argument("--form")
|
||||
ap.add_argument("--register")
|
||||
ap.add_argument("--pdf")
|
||||
args = ap.parse_args()
|
||||
sources = json.load(open(P("data", "sources.json")))
|
||||
report, changed, failed = [], False, []
|
||||
tmp = tempfile.mkdtemp(prefix="sources-")
|
||||
steps = [("form", args.form, update_form), ("register", args.register, update_register), ("pdf", args.pdf, update_pdf)]
|
||||
for key, local, fn in steps:
|
||||
try:
|
||||
path = local or download(sources[key]["download"], os.path.join(tmp, sources[key]["filename"]))
|
||||
changed |= fn(os.path.abspath(path), report)
|
||||
except Exception as e: # keep going: one broken link should not block the others
|
||||
failed.append(key)
|
||||
report.append(f"- **{sources[key]['name']}:** could not check ({e}).")
|
||||
head = f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}"
|
||||
text = "\n".join([f"## {head}", ""] + report) + "\n"
|
||||
print(text)
|
||||
if args.report:
|
||||
open(args.report, "w", encoding="utf-8").write(text)
|
||||
if os.environ.get("GITHUB_OUTPUT"):
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
|
||||
f.write(f"changed={'true' if changed else 'false'}\nfailed={','.join(failed)}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user