Refactor: lay the repo out by section (core, collection, analysis, planner, site) (#8)

Moves only; no behaviour changes. Every file now lives under the part of the project it
belongs to, each section has a README, and docs/PRIVACY.md collects the rules all parts follow.

  core/        engine.js, graph.js, solver.js, python/clean.py
  collection/  sources/ (form, register, PDF check, daily update), bot/ (importer)
  analysis/    stats/ (Python statistics), network/ (build-graph.js), reports/ (POLICY.md)
  planner/     build-policy.js, policy-report.js
  site/        atlas/ (the page), build.js, qa/
  tests/       core/ (unit), parity/ (JS vs Python), fixtures/

Checked against the previous layout: tests pass; coded runs, test outputs, graph.json and
policy.json are byte-identical; the built site differs only in comments that name file paths.
The published file names Ken's bot downloads (solver.js, graph.js, data/policy.json) are unchanged.

Also removes the undefined/ folder of QA screenshots committed by mistake (the QA scripts now
default to a temp folder) and test.txt, and drops the unused npm workspaces.


Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ekcSchiano
2026-11-03 19:46:33 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent dcd5117373
commit 432e04c2c7
66 changed files with 201 additions and 2695 deletions
+49
View File
@@ -0,0 +1,49 @@
"""export_raw.py — convert the Google Sheets .xlsx (or .csv) export into the raw JSON the web page loads.
python export_raw.py ../../data/raw/form/Threshing_Routes_Responses.xlsx ../../data/public/form_responses.json
The JSON is a list of objects keyed by the original question text (no recoding happens here, so the page
applies exactly the same cleaning rules to it as it would to a freshly uploaded export).
"""
import json
import sys
from datetime import datetime
import re
import pandas as pd
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
def load_raw(path):
if str(path).lower().endswith(".csv"):
return pd.read_csv(path)
xl = pd.ExcelFile(path)
sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0])
return pd.read_excel(xl, sheet)
def cell(v):
if v is None or v is pd.NaT or (not isinstance(v, str) and pd.isna(v)):
return None
if isinstance(v, (datetime, pd.Timestamp)):
return pd.Timestamp(v).strftime("%Y-%m-%d %H:%M:%S")
if isinstance(v, float) and v.is_integer():
return int(v)
if isinstance(v, str):
return EMAIL_RE.sub("[email removed]", v) # respondents sometimes leave an email in free text
return v
if __name__ == "__main__":
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/raw/form/Threshing_Routes_Responses.xlsx"
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/public/form_responses.json"
raw = load_raw(src)
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
# keep every header present even if a row is blank there, so column detection sees all questions
headers = [str(c) for c in raw.columns]
with open(out, "w", encoding="utf-8") as f:
json.dump({"source_file": str(src).split("/")[-1], "exported": datetime.now().strftime("%Y-%m-%d"),
"headers": headers, "rows": rows}, f, ensure_ascii=False, separators=(",", ":"))
print(f"{len(rows)} rows -> {out}")
+43
View File
@@ -0,0 +1,43 @@
"""export_register.py — convert the "Unofficial Dragon Register" .xlsx/.csv export into web/data/register.json.
python export_register.py ../../data/raw/register/Unofficial_Dragon_Register_Responses.xlsx ../../data/public/register.json
Raw rows are kept as-is, except that photo links and rider names are dropped. Dragons read from public dragonkind.com result pages
linked in the Reddit megathreads are stored separately under "extra" with their source URL.
"""
import json
import sys
from datetime import datetime
import pandas as pd
from export_raw import cell, load_raw
# Read from dragonkind.com/results/<id> pages linked in the Part 3 and Part 4 megathreads (2026-10-19).
REDDIT_RESULTS = [
("THRESH-VFRQVM5K3B0CP", "Thiduach", "Blue", "Swordtail", "Female", 51, "Gormfaileas"),
("THRESH-3M4P9MNFQGBS8", "Caenuis", "Blue", "Swordtail", "Female", 38, "Gormfaileas"),
("THRESH-5JQDJ21G1BYKR", "Oavuagh", "Green", "Morningstartail", "Female", 43, "Uaineloidsig"),
("THRESH-5KRAAVFB3D1MG", "Geagios", "Blue", "Clubtail", "Male", 99, "Gormfaileas"),
("THRESH-B5HSM00J4A2Z4", "Fiedheil", "Black", "Clubtail", "Male", 32, "Moerlaighen"),
("THRESH-BMPM2GAPK58JT", "Teirfua", "Green", "Morningstartail", "Female", 68, "Eifyrielle"),
("THRESH-MCQ8D6GAGF4K8", "Caollo", "Blue", "Morningstartail", "Female", 93, "Nyghmuiron"),
("THRESH-Q0XDBFMNBEB86", "Aenminn", "Red", "Swordtail", "Male", 59, "Berradhyr"),
("THRESH-W99DB29MG16QA", "Meinso", "Green", "Clubtail", "Female", 63, "Eifyrielle"),
("THRESH-XCJCY7HW7QSSP", "Einmial", "Blue", "Clubtail", "Male", 58, "Gormfaileas"),
("THRESH-XR6G68VPGT48M", "Recuagh", "Brown", "Morningstartail", "Female", 55, "Vormalgh"),
("THRESH-YQC97TK34214E", "Dannidh", "Orange", "Clubtail", "Male", 61, "Cariaslog"),
]
if __name__ == "__main__":
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/raw/register/Unofficial_Dragon_Register_Responses.xlsx"
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/public/register.json"
raw = load_raw(src)
raw = raw[[c for c in raw.columns if "photo" not in str(c).lower() and "cadet name" not in str(c).lower()]]
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
extra = [{"source": "dragonkind.com result page linked on Reddit", "url": f"https://dragonkind.com/results/{i}",
"name": n, "colour": c, "tail": t, "sex": s, "age": a, "lineage": l} for i, n, c, t, s, a, l in REDDIT_RESULTS]
json.dump({"source_file": str(src).split("/")[-1], "exported": datetime.now().strftime("%Y-%m-%d"),
"headers": [str(c) for c in raw.columns], "rows": rows, "extra": extra},
open(out, "w", encoding="utf-8"), ensure_ascii=False, separators=(",", ":"))
print(f"{len(rows)} register rows + {len(extra)} Reddit result pages -> {out}")
+183
View File
@@ -0,0 +1,183 @@
"""update_sources.py: check the community spreadsheets and the route PDF for new information.
python collection/sources/update_sources.py [--report report.md] [--form FILE] [--register FILE] [--pdf FILE]
Run from the repo root. Without file arguments it downloads the current versions from the URLs in
data/sources.json (the GitHub Action does this every day). With file arguments it uses local copies,
which is handy for testing or when a download link changes.
What it does:
* Threshing Routes form: re-exports data/public/form_responses.json and data/derived/runs_clean.csv
when the responses changed, and reports how many rows were added or edited.
* Dragon Register: re-exports data/public/register.json (rider names and photos dropped) when it changed.
The raw register export is never written into the repo.
* Route PDF: the PDF is not redistributed. Only a fingerprint is kept (data/derived/pdf_fingerprint.json):
the file hash, per-page text hashes and per-line hashes. When it changes, the report lists the pages that
changed and the lines that are new, so the coding rules and the scene crosswalk can be updated by hand.
Exit code is 0 either way; "changed" files are visible to git, and the report says what happened.
"""
import argparse
import hashlib
import json
import os
import shutil
import subprocess
import sys
import tempfile
import urllib.request
from datetime import datetime, timezone
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
sys.path.insert(0, HERE)
from export_raw import cell, load_raw # noqa: E402
P = lambda *a: os.path.join(ROOT, *a) # noqa: E731
def sha(data, n=None):
h = hashlib.sha256(data if isinstance(data, bytes) else data.encode("utf-8")).hexdigest()
return h[:n] if n else h
def download(url, dest):
req = urllib.request.Request(url, headers={"User-Agent": "threshing-atlas-updater"})
with urllib.request.urlopen(req, timeout=60) as r, open(dest, "wb") as f:
data = r.read()
f.write(data)
head = data[:15].lower()
if head.startswith(b"<!doctype html") or head.startswith(b"<html"):
raise RuntimeError(f"{url} returned a web page, not a file (is the link still shared publicly?)")
return dest
def sheet_rows(path, drop=()):
raw = load_raw(path)
raw = raw[[c for c in raw.columns if not any(d in str(c).lower() for d in drop)]]
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
return [str(c) for c in raw.columns], rows
def diff_rows(old, new):
key = lambda r: json.dumps(r, sort_keys=True, ensure_ascii=False) # noqa: E731
o, n = {key(r) for r in old}, {key(r) for r in new}
return len(n - o), len(o - n)
def update_form(src, report):
out = P("data", "public", "form_responses.json")
current = json.load(open(out, encoding="utf-8"))
headers, rows = sheet_rows(src)
added, removed = diff_rows(current["rows"], rows)
if headers == current["headers"] and not added and not removed:
report.append(f"- **Threshing Routes form:** no change ({len(rows)} responses).")
return False
subprocess.run([sys.executable, P("collection", "sources", "export_raw.py"), src, out], check=True, cwd=HERE)
subprocess.run([sys.executable, P("core", "python", "clean.py"), out, P("data", "derived", "runs_clean.csv")], check=True, cwd=HERE)
raw_copy = P("data", "raw", "form", "Threshing_Routes_Responses.xlsx")
if os.path.abspath(src) != raw_copy: # local copy only; data/raw/form/*.xlsx is git-ignored
os.makedirs(os.path.dirname(raw_copy), exist_ok=True)
shutil.copyfile(src, raw_copy)
new_q = [h for h in headers if h not in current["headers"]]
gone_q = [h for h in current["headers"] if h not in headers]
report.append(f"- **Threshing Routes form:** {len(current['rows'])} → {len(rows)} responses "
f"({added} new or edited, {removed} removed or edited).")
if new_q:
report.append(f" - New questions (check `COLUMN_KEYS` in engine.js and clean.py): {', '.join(new_q)}")
if gone_q:
report.append(f" - Questions no longer in the sheet: {', '.join(gone_q)}")
return True
def update_register(src, report):
out = P("data", "public", "register.json")
current = json.load(open(out, encoding="utf-8"))
headers, rows = sheet_rows(src, drop=("photo", "cadet name"))
added, removed = diff_rows(current["rows"], rows)
if headers == current["headers"] and not added and not removed:
report.append(f"- **Dragon Register:** no change ({len(rows)} dragons).")
return False
subprocess.run([sys.executable, P("collection", "sources", "export_register.py"), src, out], check=True, cwd=HERE)
report.append(f"- **Dragon Register:** {len(current['rows'])} → {len(rows)} entries ({added} new or edited, {removed} removed or edited).")
return True
def pdf_lines(path):
try:
from pypdf import PdfReader
except ImportError:
sys.exit("pypdf is required: pip install -r requirements.txt")
pages = []
for page in PdfReader(path).pages:
text = page.extract_text() or ""
pages.append([" ".join(l.split()) for l in text.splitlines() if l.strip()])
return pages
def update_pdf(src, report):
out = P("data", "derived", "pdf_fingerprint.json")
old = json.load(open(out)) if os.path.exists(out) else None
data = open(src, "rb").read()
pages = pdf_lines(src)
fp = {
"sha256": sha(data),
"bytes": len(data),
"pages": len(pages),
"page_hashes": [sha("\n".join(p), 16) for p in pages],
"line_hashes": sorted({sha(l.lower(), 12) for p in pages for l in p}),
"checked": datetime.now(timezone.utc).strftime("%Y-%m-%d"),
}
if old and old["sha256"] == fp["sha256"]:
report.append(f"- **Route PDF:** no change ({fp['pages']} pages).")
return False
if old and old.get("page_hashes") == fp["page_hashes"]:
report.append("- **Route PDF:** the file changed but its text did not (layout or images only). Worth a look.")
elif old:
changed = [i + 1 for i, h in enumerate(fp["page_hashes"]) if i >= len(old["page_hashes"]) or old["page_hashes"][i] != h]
old_lines = set(old["line_hashes"])
new_lines = [l for p in pages for l in p if sha(l.lower(), 12) not in old_lines]
removed = len(old_lines - set(fp["line_hashes"]))
report.append(f"- **Route PDF:** changed. {old['pages']} → {fp['pages']} pages; pages with new text: "
f"{', '.join(map(str, changed)) or 'none'}; {len(new_lines)} new lines, {removed} lines gone.")
if new_lines:
report.append(" - New lines (update the coding rules or the scene crosswalk if they add options):")
report.extend(f" - {l}" for l in new_lines[:60])
if len(new_lines) > 60:
report.append(f" - … and {len(new_lines) - 60} more")
else:
report.append(f"- **Route PDF:** fingerprint recorded for the first time ({fp['pages']} pages).")
json.dump(fp, open(out, "w"), indent=1)
return True
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--report", default=None)
ap.add_argument("--form")
ap.add_argument("--register")
ap.add_argument("--pdf")
args = ap.parse_args()
sources = json.load(open(P("data", "sources.json")))
report, changed, failed = [], False, []
tmp = tempfile.mkdtemp(prefix="sources-")
steps = [("form", args.form, update_form), ("register", args.register, update_register), ("pdf", args.pdf, update_pdf)]
for key, local, fn in steps:
try:
path = local or download(sources[key]["download"], os.path.join(tmp, sources[key]["filename"]))
changed |= fn(os.path.abspath(path), report)
except Exception as e: # keep going: one broken link should not block the others
failed.append(key)
report.append(f"- **{sources[key]['name']}:** could not check ({e}).")
head = f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}"
text = "\n".join([f"## {head}", ""] + report) + "\n"
print(text)
if args.report:
open(args.report, "w", encoding="utf-8").write(text)
if os.environ.get("GITHUB_OUTPUT"):
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
f.write(f"changed={'true' if changed else 'false'}\nfailed={','.join(failed)}\n")
if __name__ == "__main__":
main()