Refactor: lay the repo out by section (core, collection, analysis, planner, site) (#8)

Moves only; no behaviour changes. Every file now lives under the part of the project it
belongs to, each section has a README, and docs/PRIVACY.md collects the rules all parts follow.

  core/        engine.js, graph.js, solver.js, python/clean.py
  collection/  sources/ (form, register, PDF check, daily update), bot/ (importer)
  analysis/    stats/ (Python statistics), network/ (build-graph.js), reports/ (POLICY.md)
  planner/     build-policy.js, policy-report.js
  site/        atlas/ (the page), build.js, qa/
  tests/       core/ (unit), parity/ (JS vs Python), fixtures/

Checked against the previous layout: tests pass; coded runs, test outputs, graph.json and
policy.json are byte-identical; the built site differs only in comments that name file paths.
The published file names Ken's bot downloads (solver.js, graph.js, data/policy.json) are unchanged.

Also removes the undefined/ folder of QA screenshots committed by mistake (the QA scripts now
default to a temp folder) and test.txt, and drops the unused npm workspaces.


Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ekcSchiano
2026-11-03 19:46:33 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent dcd5117373
commit 432e04c2c7
66 changed files with 201 additions and 2695 deletions
+183
View File
@@ -0,0 +1,183 @@
"""update_sources.py: check the community spreadsheets and the route PDF for new information.
python collection/sources/update_sources.py [--report report.md] [--form FILE] [--register FILE] [--pdf FILE]
Run from the repo root. Without file arguments it downloads the current versions from the URLs in
data/sources.json (the GitHub Action does this every day). With file arguments it uses local copies,
which is handy for testing or when a download link changes.
What it does:
* Threshing Routes form: re-exports data/public/form_responses.json and data/derived/runs_clean.csv
when the responses changed, and reports how many rows were added or edited.
* Dragon Register: re-exports data/public/register.json (rider names and photos dropped) when it changed.
The raw register export is never written into the repo.
* Route PDF: the PDF is not redistributed. Only a fingerprint is kept (data/derived/pdf_fingerprint.json):
the file hash, per-page text hashes and per-line hashes. When it changes, the report lists the pages that
changed and the lines that are new, so the coding rules and the scene crosswalk can be updated by hand.
Exit code is 0 either way; "changed" files are visible to git, and the report says what happened.
"""
import argparse
import hashlib
import json
import os
import shutil
import subprocess
import sys
import tempfile
import urllib.request
from datetime import datetime, timezone
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
sys.path.insert(0, HERE)
from export_raw import cell, load_raw # noqa: E402
P = lambda *a: os.path.join(ROOT, *a) # noqa: E731
def sha(data, n=None):
h = hashlib.sha256(data if isinstance(data, bytes) else data.encode("utf-8")).hexdigest()
return h[:n] if n else h
def download(url, dest):
req = urllib.request.Request(url, headers={"User-Agent": "threshing-atlas-updater"})
with urllib.request.urlopen(req, timeout=60) as r, open(dest, "wb") as f:
data = r.read()
f.write(data)
head = data[:15].lower()
if head.startswith(b"<!doctype html") or head.startswith(b"<html"):
raise RuntimeError(f"{url} returned a web page, not a file (is the link still shared publicly?)")
return dest
def sheet_rows(path, drop=()):
raw = load_raw(path)
raw = raw[[c for c in raw.columns if not any(d in str(c).lower() for d in drop)]]
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
return [str(c) for c in raw.columns], rows
def diff_rows(old, new):
key = lambda r: json.dumps(r, sort_keys=True, ensure_ascii=False) # noqa: E731
o, n = {key(r) for r in old}, {key(r) for r in new}
return len(n - o), len(o - n)
def update_form(src, report):
out = P("data", "public", "form_responses.json")
current = json.load(open(out, encoding="utf-8"))
headers, rows = sheet_rows(src)
added, removed = diff_rows(current["rows"], rows)
if headers == current["headers"] and not added and not removed:
report.append(f"- **Threshing Routes form:** no change ({len(rows)} responses).")
return False
subprocess.run([sys.executable, P("collection", "sources", "export_raw.py"), src, out], check=True, cwd=HERE)
subprocess.run([sys.executable, P("core", "python", "clean.py"), out, P("data", "derived", "runs_clean.csv")], check=True, cwd=HERE)
raw_copy = P("data", "raw", "form", "Threshing_Routes_Responses.xlsx")
if os.path.abspath(src) != raw_copy: # local copy only; data/raw/form/*.xlsx is git-ignored
os.makedirs(os.path.dirname(raw_copy), exist_ok=True)
shutil.copyfile(src, raw_copy)
new_q = [h for h in headers if h not in current["headers"]]
gone_q = [h for h in current["headers"] if h not in headers]
report.append(f"- **Threshing Routes form:** {len(current['rows'])} → {len(rows)} responses "
f"({added} new or edited, {removed} removed or edited).")
if new_q:
report.append(f" - New questions (check `COLUMN_KEYS` in engine.js and clean.py): {', '.join(new_q)}")
if gone_q:
report.append(f" - Questions no longer in the sheet: {', '.join(gone_q)}")
return True
def update_register(src, report):
out = P("data", "public", "register.json")
current = json.load(open(out, encoding="utf-8"))
headers, rows = sheet_rows(src, drop=("photo", "cadet name"))
added, removed = diff_rows(current["rows"], rows)
if headers == current["headers"] and not added and not removed:
report.append(f"- **Dragon Register:** no change ({len(rows)} dragons).")
return False
subprocess.run([sys.executable, P("collection", "sources", "export_register.py"), src, out], check=True, cwd=HERE)
report.append(f"- **Dragon Register:** {len(current['rows'])} → {len(rows)} entries ({added} new or edited, {removed} removed or edited).")
return True
def pdf_lines(path):
try:
from pypdf import PdfReader
except ImportError:
sys.exit("pypdf is required: pip install -r requirements.txt")
pages = []
for page in PdfReader(path).pages:
text = page.extract_text() or ""
pages.append([" ".join(l.split()) for l in text.splitlines() if l.strip()])
return pages
def update_pdf(src, report):
out = P("data", "derived", "pdf_fingerprint.json")
old = json.load(open(out)) if os.path.exists(out) else None
data = open(src, "rb").read()
pages = pdf_lines(src)
fp = {
"sha256": sha(data),
"bytes": len(data),
"pages": len(pages),
"page_hashes": [sha("\n".join(p), 16) for p in pages],
"line_hashes": sorted({sha(l.lower(), 12) for p in pages for l in p}),
"checked": datetime.now(timezone.utc).strftime("%Y-%m-%d"),
}
if old and old["sha256"] == fp["sha256"]:
report.append(f"- **Route PDF:** no change ({fp['pages']} pages).")
return False
if old and old.get("page_hashes") == fp["page_hashes"]:
report.append("- **Route PDF:** the file changed but its text did not (layout or images only). Worth a look.")
elif old:
changed = [i + 1 for i, h in enumerate(fp["page_hashes"]) if i >= len(old["page_hashes"]) or old["page_hashes"][i] != h]
old_lines = set(old["line_hashes"])
new_lines = [l for p in pages for l in p if sha(l.lower(), 12) not in old_lines]
removed = len(old_lines - set(fp["line_hashes"]))
report.append(f"- **Route PDF:** changed. {old['pages']} → {fp['pages']} pages; pages with new text: "
f"{', '.join(map(str, changed)) or 'none'}; {len(new_lines)} new lines, {removed} lines gone.")
if new_lines:
report.append(" - New lines (update the coding rules or the scene crosswalk if they add options):")
report.extend(f" - {l}" for l in new_lines[:60])
if len(new_lines) > 60:
report.append(f" - … and {len(new_lines) - 60} more")
else:
report.append(f"- **Route PDF:** fingerprint recorded for the first time ({fp['pages']} pages).")
json.dump(fp, open(out, "w"), indent=1)
return True
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--report", default=None)
ap.add_argument("--form")
ap.add_argument("--register")
ap.add_argument("--pdf")
args = ap.parse_args()
sources = json.load(open(P("data", "sources.json")))
report, changed, failed = [], False, []
tmp = tempfile.mkdtemp(prefix="sources-")
steps = [("form", args.form, update_form), ("register", args.register, update_register), ("pdf", args.pdf, update_pdf)]
for key, local, fn in steps:
try:
path = local or download(sources[key]["download"], os.path.join(tmp, sources[key]["filename"]))
changed |= fn(os.path.abspath(path), report)
except Exception as e: # keep going: one broken link should not block the others
failed.append(key)
report.append(f"- **{sources[key]['name']}:** could not check ({e}).")
head = f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}"
text = "\n".join([f"## {head}", ""] + report) + "\n"
print(text)
if args.report:
open(args.report, "w", encoding="utf-8").write(text)
if os.environ.get("GITHUB_OUTPUT"):
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
f.write(f"changed={'true' if changed else 'false'}\nfailed={','.join(failed)}\n")
if __name__ == "__main__":
main()