Refactor: lay the repo out by section (core, collection, analysis, planner, site) (#8)

Moves only; no behaviour changes. Every file now lives under the part of the project it
belongs to, each section has a README, and docs/PRIVACY.md collects the rules all parts follow.

  core/        engine.js, graph.js, solver.js, python/clean.py
  collection/  sources/ (form, register, PDF check, daily update), bot/ (importer)
  analysis/    stats/ (Python statistics), network/ (build-graph.js), reports/ (POLICY.md)
  planner/     build-policy.js, policy-report.js
  site/        atlas/ (the page), build.js, qa/
  tests/       core/ (unit), parity/ (JS vs Python), fixtures/

Checked against the previous layout: tests pass; coded runs, test outputs, graph.json and
policy.json are byte-identical; the built site differs only in comments that name file paths.
The published file names Ken's bot downloads (solver.js, graph.js, data/policy.json) are unchanged.

Also removes the undefined/ folder of QA screenshots committed by mistake (the QA scripts now
default to a temp folder) and test.txt, and drops the unused npm workspaces.


Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ekcSchiano
2026-11-03 19:46:33 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent dcd5117373
commit 432e04c2c7
66 changed files with 201 additions and 2695 deletions
+19
View File
@@ -0,0 +1,19 @@
# collection
Getting data in. Everything here turns a source into the shared run format (see `core/`) or into public files
under `data/`.
| Source | Code | Output |
|---|---|---|
| Community "Threshing Routes" form (Google Sheet) | `sources/export_raw.py`, then `core/python/clean.py` | `data/public/form_responses.json`, `data/derived/runs_clean.csv` |
| Unofficial Dragon Register (Google Sheet) | `sources/export_register.py` | `data/public/register.json` (rider names stay local) |
| Community route PDF | `sources/update_sources.py` | `data/derived/pdf_fingerprint.json` only; the PDF is never committed |
| DragonKind bot (Ken) | `bot/import-bot.js`, `bot/bot.js` | `data/bot/scenes.json`, `data/bot/trajectories-<tag>.jsonl` |
| Reddit megathreads | not collected yet | — |
`sources/update_sources.py` runs every day from `.github/workflows/update-sources.yml`. It downloads the sheets
and the PDF (links in `data/sources.json`), rebuilds the data if anything changed, and opens a pull request that
merges itself when CI passes. A broken link opens an issue.
The bot importer only ever writes fingerprints, short labels, choice text and outcomes. It refuses anything that
looks like game prose or an account identifier. What the bot should log is in `docs/BOT_DATA_PROTOCOL.md`.
+211
View File
@@ -0,0 +1,211 @@
'use strict';
/**
* Import DragonKind bot data into anonymised trajectories and a scene catalogue.
*
* Inputs (all from the PRIVATE bot repo, read locally, never committed):
* data/knowledge.json scene atlas: nodes (prose, choices, tint, backdrop) and edges
* data/runs.jsonl one record per attempt (newer bot versions)
* logs/encounters.jsonl every decision the bot made, with the options on screen
*
* Outputs (public, CC BY 4.0): no game prose, only fingerprints, short labels and choice text.
* scenes.json { sceneId: { stage, tint, backdrop, label, options: [{ id, text }] } }
* trajectories.jsonl one line per attempt: { id, source, meta, steps: [...], outcome }
*/
const crypto = require('node:crypto');
const fs = require('node:fs');
const path = require('node:path');
const sha1 = (s, n = 12) => crypto.createHash('sha1').update(String(s)).digest('hex').slice(0, n);
// Must match normalize() in the bot's src/strategy.js, so fingerprints line up.
function normalize(text) {
return String(text || '')
.toLowerCase()
.replace(/[‘’‛`']/g, "'")
.replace(/[“”"]/g, '"')
.replace(/[–—-]/g, ' ')
.replace(/…/g, ' ')
.replace(/[^a-z0-9'\s]/g, ' ')
.replace(/\s+/g, ' ')
.trim();
}
// Must match computeSceneFingerprint() in the bot's src/learner.js.
function sceneId(prose, choices) {
const proseNorm = normalize(prose).slice(0, 500);
const sorted = choices.map(normalize).filter(Boolean).sort();
return sha1(`${proseNorm} ### ${sorted.join(' || ')}`);
}
const optionId = (text) => sha1(normalize(text), 10);
const readJsonl = (file) => {
if (!file || !fs.existsSync(file)) return [];
return fs.readFileSync(file, 'utf8').split(/\r?\n/).filter((l) => l.trim()).map((l) => {
try { return JSON.parse(l); } catch { return null; }
}).filter(Boolean);
};
/**
* Short public labels. Keys are scene fingerprints; values are our own paraphrases, not game prose.
* Unknown scenes fall back to "<stage> · <tint>" and are listed by the importer so labels can be added.
*/
const LABELS = {
db7786392626: 'Start: choose company',
a68f681076c6: 'Valley edge: choose direction',
bc4ee4754b9c: 'First sighting: Red (forest centre)',
abe3011378ac: 'First sighting: Brown (forest centre)',
'84780dab01e3': 'Survived the first sighting',
d25f2759ace0: 'Deeper in the forest: a scream',
f72199f24b91: 'Clearing: squadmate vs two bullies',
b209bacacec3: 'Bully fight: first move',
fbe0f0bea700: 'Bully fight: smaller cadet has a dagger',
a2368b0dc212: 'Blocked the bigger cadet: bullies’ fate',
f4b4944d3f3b: 'Beat the smaller cadet: bullies’ fate',
'0cfd06cf8939': 'After the fight: choose a direction',
'0a561d53ad19': 'An hour later: valley or tree',
'96b68612eaf4': 'Top of the tree (shadow check)',
'10bc7353b8eb': 'Descended quickly: a Red below',
c62f1f1a2b3a: 'Let the Red pass: two Greens in the clearing',
'07e24681e3b3': 'Descended slowly: two Greens in the clearing'
};
function labelFor(id, node) {
if (LABELS[id]) return LABELS[id];
const bits = [node.stage || 'Scene'];
if (node.tint) bits.push(node.tint);
return bits.join(' · ');
}
/** Scene catalogue from knowledge.json (and, for scenes only in the log, from the log). */
function buildScenes(knowledge, encounters) {
const scenes = {};
for (const [id, n] of Object.entries((knowledge && knowledge.nodes) || {})) {
scenes[id] = {
stage: n.stage || 'UNKNOWN',
tint: n.tint || null,
backdrop: n.backdrop || null,
label: labelFor(id, n),
options: Object.values(n.choices || {}).map((c) => ({ id: optionId(c.label || c.choiceNorm), text: c.label || c.choiceNorm }))
};
}
for (const e of encounters) {
if (!Array.isArray(e.choices) || !e.prose) continue;
const id = e.nodeId || sceneId(e.prose, e.choices);
if (scenes[id]) continue;
const stage = (e.decision && e.decision.stage) || e.stage || 'UNKNOWN';
scenes[id] = { stage, tint: null, backdrop: null, label: labelFor(id, { stage }), options: e.choices.map((t) => ({ id: optionId(t), text: t })) };
}
return scenes;
}
const CHOICE_EVENTS = new Set(['DECISION', 'MANUAL_DECISION', 'MANUAL_EDGE_PICK']);
/**
* Trajectories from the encounter log: group events by attempt, keep the decisions that were acted on,
* and close each attempt at its first BURNED / DRAGON_BONDED event. Repeated BURNED events with no
* decisions in between (the cooldown screen being re-detected) are ignored.
*/
function trajectoriesFromEncounters(encounters, scenes) {
const byAttempt = new Map();
for (const e of encounters) {
const key = e.account ? `${e.account}:${e.attempt}` : String(e.attempt);
if (!byAttempt.has(key)) byAttempt.set(key, []);
byAttempt.get(key).push(e);
}
const out = [];
for (const [key, events] of byAttempt) {
events.sort((a, b) => String(a.ts).localeCompare(String(b.ts)));
const steps = [];
let outcome = null;
for (const e of events) {
if (outcome) break;
if (CHOICE_EVENTS.has(e.event)) {
const d = e.decision || {};
const text = d.choiceText || e.choiceText;
if (d.action && d.action !== 'choose') continue; // paused, nothing happened
if (!text || !Array.isArray(e.choices)) continue;
const scene = e.nodeId || sceneId(e.prose || '', e.choices);
const manual = e.event !== 'DECISION';
steps.push({
scene,
stage: d.stage || (scenes[scene] && scenes[scene].stage) || 'UNKNOWN',
tint: (scenes[scene] && scenes[scene].tint) || null,
option: optionId(text),
optionText: text,
offered: e.choices.map(optionId),
rule: d.rule || null,
decisionMode: manual ? 'manual' : d.source || null,
manual,
at: e.ts
});
} else if (e.event === 'BURNED' && steps.length) {
outcome = { type: 'BURNED', colour: null, intentional: Boolean(e.intentional) };
} else if (e.event === 'DRAGON_BONDED' && steps.length) {
outcome = { type: 'BONDED', colour: e.dragon ? String(e.dragon).toLowerCase() : null, intentional: false };
}
}
// A decision repeated on the same scene (the bot re-detecting the screen) counts once.
const dedup = steps.filter((s, i) => !(i > 0 && steps[i - 1].scene === s.scene && steps[i - 1].option === s.option));
for (let i = 0; i < dedup.length; i++) dedup[i].to = i + 1 < dedup.length ? dedup[i + 1].scene : null;
if (!dedup.length) continue;
const last = dedup[dedup.length - 1];
if (outcome) last.to = outcome.type === 'BURNED' ? 'BURNED' : `BONDED:${outcome.colour || 'unknown'}`;
out.push({
id: `bot-${sha1(`${key}:${dedup[0].at}`, 10)}`,
source: 'bot',
meta: { attempt: Number(String(key).split(':').pop()) || null, startedAt: dedup[0].at, accountKey: null, target: null, decisionMode: null },
steps: dedup,
outcome: outcome || { type: 'ABORTED', colour: null, intentional: false }
});
}
return out;
}
/** Fill run-level metadata (target colour, mode) from runs.jsonl where an attempt is covered by both. */
function attachRunMeta(trajs, runs, salt) {
for (const r of runs) {
const t = trajs.find((x) => x.meta.attempt === r.attempt && Math.abs(Date.parse(x.meta.startedAt) - Date.parse(r.startedAt)) < 6 * 3600e3);
if (!t) continue;
t.meta.target = r.targetDragon || null;
t.meta.decisionMode = r.decisionMode || null;
t.meta.policyId = r.policyId || null;
t.meta.experimentId = r.experimentId || null;
t.meta.arm = r.arm || null;
if (r.accountKey) t.meta.accountKey = r.accountKey;
else if (r.accountId && salt) t.meta.accountKey = sha1(`${salt}:${r.accountId}`, 10);
if (r.outcome === 'BONDED' && r.dragonColor) t.outcome = { type: 'BONDED', colour: String(r.dragonColor).toLowerCase(), intentional: false };
if (r.intentionalBurn) t.outcome.intentional = true;
}
return trajs;
}
function importBot({ dataDir, logsDir, salt = process.env.DK_ACCOUNT_SALT || null }) {
const knowledgeFile = path.join(dataDir, 'knowledge.json');
const knowledge = fs.existsSync(knowledgeFile) ? JSON.parse(fs.readFileSync(knowledgeFile, 'utf8')) : { nodes: {} };
const encounters = readJsonl(path.join(logsDir, 'encounters.jsonl'));
const runs = readJsonl(path.join(dataDir, 'runs.jsonl'));
const scenes = buildScenes(knowledge, encounters);
const trajectories = attachRunMeta(trajectoriesFromEncounters(encounters, scenes), runs, salt);
const unlabeled = Object.entries(scenes).filter(([id]) => !LABELS[id]).map(([id, s]) => `${id} (${s.stage}${s.tint ? `, ${s.tint}` : ''})`);
return { scenes, trajectories, unlabeled };
}
/** Guard: nothing that looks like game prose or an account identifier may leave this module. */
function assertPublicSafe(obj) {
const banned = ['prose', 'proseNorm', 'proseExcerpt', 'email', 'password', 'accountId', 'accountName'];
const walk = (v, p) => {
if (Array.isArray(v)) return v.forEach((x, i) => walk(x, `${p}[${i}]`));
if (v && typeof v === 'object') {
for (const [k, x] of Object.entries(v)) {
if (banned.includes(k)) throw new Error(`refusing to export field "${k}" at ${p}`);
walk(x, `${p}.${k}`);
}
} else if (typeof v === 'string' && v.length > 200) {
throw new Error(`refusing to export a ${v.length}-character string at ${p} (looks like prose)`);
}
};
walk(obj, '$');
}
module.exports = { normalize, sceneId, optionId, buildScenes, trajectoriesFromEncounters, attachRunMeta, importBot, assertPublicSafe, LABELS };
+54
View File
@@ -0,0 +1,54 @@
#!/usr/bin/env node
'use strict';
/**
* Convert a local copy of the DragonKind bot's data into the public, anonymised format.
*
* node collection/bot/import-bot.js --bot /path/to/DragonKind [--out data/bot] [--tag 2026-11-01]
*
* Reads <bot>/data/knowledge.json, <bot>/data/runs.jsonl and <bot>/logs/encounters.jsonl.
* Writes <out>/scenes.json (merged with any existing catalogue) and <out>/trajectories-<tag>.jsonl.
* Set DK_ACCOUNT_SALT in the environment to hash account ids; never commit the salt.
*/
const fs = require('node:fs');
const path = require('node:path');
const { importBot, assertPublicSafe } = require('./bot');
const args = Object.fromEntries(process.argv.slice(2).reduce((acc, a, i, all) => {
if (a.startsWith('--')) acc.push([a.slice(2), all[i + 1]]);
return acc;
}, []));
if (!args.bot) {
console.error('usage: import-bot.js --bot /path/to/DragonKind [--out data/bot] [--tag YYYY-MM-DD]');
process.exit(2);
}
const out = path.resolve(args.out || 'data/bot');
const tag = args.tag || new Date().toISOString().slice(0, 10);
const { scenes, trajectories, unlabeled } = importBot({
dataDir: path.join(args.bot, 'data'),
logsDir: path.join(args.bot, 'logs')
});
assertPublicSafe(scenes);
assertPublicSafe(trajectories);
fs.mkdirSync(out, { recursive: true });
const sceneFile = path.join(out, 'scenes.json');
const existing = fs.existsSync(sceneFile) ? JSON.parse(fs.readFileSync(sceneFile, 'utf8')) : {};
for (const [id, s] of Object.entries(scenes)) {
const old = existing[id];
if (!old) { existing[id] = s; continue; }
const ids = new Set(old.options.map((o) => o.id));
for (const o of s.options) if (!ids.has(o.id)) old.options.push(o);
old.tint = old.tint || s.tint;
old.backdrop = old.backdrop || s.backdrop;
if (old.label.includes('·') && !s.label.includes('·')) old.label = s.label;
}
const sorted = Object.fromEntries(Object.entries(existing).sort(([a], [b]) => a.localeCompare(b)));
fs.writeFileSync(sceneFile, `${JSON.stringify(sorted, null, 1)}\n`);
const trajFile = path.join(out, `trajectories-${tag}.jsonl`);
fs.writeFileSync(trajFile, trajectories.map((t) => JSON.stringify(t)).join('\n') + '\n');
const outcomes = trajectories.reduce((m, t) => ((m[t.outcome.type] = (m[t.outcome.type] || 0) + 1), m), {});
console.log(`${Object.keys(scenes).length} scenes, ${trajectories.length} attempts ${JSON.stringify(outcomes)}`);
console.log(`wrote ${path.relative(process.cwd(), sceneFile)} and ${path.relative(process.cwd(), trajFile)}`);
if (unlabeled.length) console.log(`scenes without a short label (add them to LABELS in collection/bot/bot.js):\n ${unlabeled.join('\n ')}`);
+49
View File
@@ -0,0 +1,49 @@
"""export_raw.py — convert the Google Sheets .xlsx (or .csv) export into the raw JSON the web page loads.
python export_raw.py ../../data/raw/form/Threshing_Routes_Responses.xlsx ../../data/public/form_responses.json
The JSON is a list of objects keyed by the original question text (no recoding happens here, so the page
applies exactly the same cleaning rules to it as it would to a freshly uploaded export).
"""
import json
import sys
from datetime import datetime
import re
import pandas as pd
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
def load_raw(path):
if str(path).lower().endswith(".csv"):
return pd.read_csv(path)
xl = pd.ExcelFile(path)
sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0])
return pd.read_excel(xl, sheet)
def cell(v):
if v is None or v is pd.NaT or (not isinstance(v, str) and pd.isna(v)):
return None
if isinstance(v, (datetime, pd.Timestamp)):
return pd.Timestamp(v).strftime("%Y-%m-%d %H:%M:%S")
if isinstance(v, float) and v.is_integer():
return int(v)
if isinstance(v, str):
return EMAIL_RE.sub("[email removed]", v) # respondents sometimes leave an email in free text
return v
if __name__ == "__main__":
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/raw/form/Threshing_Routes_Responses.xlsx"
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/public/form_responses.json"
raw = load_raw(src)
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
# keep every header present even if a row is blank there, so column detection sees all questions
headers = [str(c) for c in raw.columns]
with open(out, "w", encoding="utf-8") as f:
json.dump({"source_file": str(src).split("/")[-1], "exported": datetime.now().strftime("%Y-%m-%d"),
"headers": headers, "rows": rows}, f, ensure_ascii=False, separators=(",", ":"))
print(f"{len(rows)} rows -> {out}")
+43
View File
@@ -0,0 +1,43 @@
"""export_register.py — convert the "Unofficial Dragon Register" .xlsx/.csv export into web/data/register.json.
python export_register.py ../../data/raw/register/Unofficial_Dragon_Register_Responses.xlsx ../../data/public/register.json
Raw rows are kept as-is, except that photo links and rider names are dropped. Dragons read from public dragonkind.com result pages
linked in the Reddit megathreads are stored separately under "extra" with their source URL.
"""
import json
import sys
from datetime import datetime
import pandas as pd
from export_raw import cell, load_raw
# Read from dragonkind.com/results/<id> pages linked in the Part 3 and Part 4 megathreads (2026-10-19).
REDDIT_RESULTS = [
("THRESH-VFRQVM5K3B0CP", "Thiduach", "Blue", "Swordtail", "Female", 51, "Gormfaileas"),
("THRESH-3M4P9MNFQGBS8", "Caenuis", "Blue", "Swordtail", "Female", 38, "Gormfaileas"),
("THRESH-5JQDJ21G1BYKR", "Oavuagh", "Green", "Morningstartail", "Female", 43, "Uaineloidsig"),
("THRESH-5KRAAVFB3D1MG", "Geagios", "Blue", "Clubtail", "Male", 99, "Gormfaileas"),
("THRESH-B5HSM00J4A2Z4", "Fiedheil", "Black", "Clubtail", "Male", 32, "Moerlaighen"),
("THRESH-BMPM2GAPK58JT", "Teirfua", "Green", "Morningstartail", "Female", 68, "Eifyrielle"),
("THRESH-MCQ8D6GAGF4K8", "Caollo", "Blue", "Morningstartail", "Female", 93, "Nyghmuiron"),
("THRESH-Q0XDBFMNBEB86", "Aenminn", "Red", "Swordtail", "Male", 59, "Berradhyr"),
("THRESH-W99DB29MG16QA", "Meinso", "Green", "Clubtail", "Female", 63, "Eifyrielle"),
("THRESH-XCJCY7HW7QSSP", "Einmial", "Blue", "Clubtail", "Male", 58, "Gormfaileas"),
("THRESH-XR6G68VPGT48M", "Recuagh", "Brown", "Morningstartail", "Female", 55, "Vormalgh"),
("THRESH-YQC97TK34214E", "Dannidh", "Orange", "Clubtail", "Male", 61, "Cariaslog"),
]
if __name__ == "__main__":
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/raw/register/Unofficial_Dragon_Register_Responses.xlsx"
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/public/register.json"
raw = load_raw(src)
raw = raw[[c for c in raw.columns if "photo" not in str(c).lower() and "cadet name" not in str(c).lower()]]
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
extra = [{"source": "dragonkind.com result page linked on Reddit", "url": f"https://dragonkind.com/results/{i}",
"name": n, "colour": c, "tail": t, "sex": s, "age": a, "lineage": l} for i, n, c, t, s, a, l in REDDIT_RESULTS]
json.dump({"source_file": str(src).split("/")[-1], "exported": datetime.now().strftime("%Y-%m-%d"),
"headers": [str(c) for c in raw.columns], "rows": rows, "extra": extra},
open(out, "w", encoding="utf-8"), ensure_ascii=False, separators=(",", ":"))
print(f"{len(rows)} register rows + {len(extra)} Reddit result pages -> {out}")
+183
View File
@@ -0,0 +1,183 @@
"""update_sources.py: check the community spreadsheets and the route PDF for new information.
python collection/sources/update_sources.py [--report report.md] [--form FILE] [--register FILE] [--pdf FILE]
Run from the repo root. Without file arguments it downloads the current versions from the URLs in
data/sources.json (the GitHub Action does this every day). With file arguments it uses local copies,
which is handy for testing or when a download link changes.
What it does:
* Threshing Routes form: re-exports data/public/form_responses.json and data/derived/runs_clean.csv
when the responses changed, and reports how many rows were added or edited.
* Dragon Register: re-exports data/public/register.json (rider names and photos dropped) when it changed.
The raw register export is never written into the repo.
* Route PDF: the PDF is not redistributed. Only a fingerprint is kept (data/derived/pdf_fingerprint.json):
the file hash, per-page text hashes and per-line hashes. When it changes, the report lists the pages that
changed and the lines that are new, so the coding rules and the scene crosswalk can be updated by hand.
Exit code is 0 either way; "changed" files are visible to git, and the report says what happened.
"""
import argparse
import hashlib
import json
import os
import shutil
import subprocess
import sys
import tempfile
import urllib.request
from datetime import datetime, timezone
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
sys.path.insert(0, HERE)
from export_raw import cell, load_raw # noqa: E402
P = lambda *a: os.path.join(ROOT, *a) # noqa: E731
def sha(data, n=None):
h = hashlib.sha256(data if isinstance(data, bytes) else data.encode("utf-8")).hexdigest()
return h[:n] if n else h
def download(url, dest):
req = urllib.request.Request(url, headers={"User-Agent": "threshing-atlas-updater"})
with urllib.request.urlopen(req, timeout=60) as r, open(dest, "wb") as f:
data = r.read()
f.write(data)
head = data[:15].lower()
if head.startswith(b"<!doctype html") or head.startswith(b"<html"):
raise RuntimeError(f"{url} returned a web page, not a file (is the link still shared publicly?)")
return dest
def sheet_rows(path, drop=()):
raw = load_raw(path)
raw = raw[[c for c in raw.columns if not any(d in str(c).lower() for d in drop)]]
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
return [str(c) for c in raw.columns], rows
def diff_rows(old, new):
key = lambda r: json.dumps(r, sort_keys=True, ensure_ascii=False) # noqa: E731
o, n = {key(r) for r in old}, {key(r) for r in new}
return len(n - o), len(o - n)
def update_form(src, report):
out = P("data", "public", "form_responses.json")
current = json.load(open(out, encoding="utf-8"))
headers, rows = sheet_rows(src)
added, removed = diff_rows(current["rows"], rows)
if headers == current["headers"] and not added and not removed:
report.append(f"- **Threshing Routes form:** no change ({len(rows)} responses).")
return False
subprocess.run([sys.executable, P("collection", "sources", "export_raw.py"), src, out], check=True, cwd=HERE)
subprocess.run([sys.executable, P("core", "python", "clean.py"), out, P("data", "derived", "runs_clean.csv")], check=True, cwd=HERE)
raw_copy = P("data", "raw", "form", "Threshing_Routes_Responses.xlsx")
if os.path.abspath(src) != raw_copy: # local copy only; data/raw/form/*.xlsx is git-ignored
os.makedirs(os.path.dirname(raw_copy), exist_ok=True)
shutil.copyfile(src, raw_copy)
new_q = [h for h in headers if h not in current["headers"]]
gone_q = [h for h in current["headers"] if h not in headers]
report.append(f"- **Threshing Routes form:** {len(current['rows'])} → {len(rows)} responses "
f"({added} new or edited, {removed} removed or edited).")
if new_q:
report.append(f" - New questions (check `COLUMN_KEYS` in engine.js and clean.py): {', '.join(new_q)}")
if gone_q:
report.append(f" - Questions no longer in the sheet: {', '.join(gone_q)}")
return True
def update_register(src, report):
out = P("data", "public", "register.json")
current = json.load(open(out, encoding="utf-8"))
headers, rows = sheet_rows(src, drop=("photo", "cadet name"))
added, removed = diff_rows(current["rows"], rows)
if headers == current["headers"] and not added and not removed:
report.append(f"- **Dragon Register:** no change ({len(rows)} dragons).")
return False
subprocess.run([sys.executable, P("collection", "sources", "export_register.py"), src, out], check=True, cwd=HERE)
report.append(f"- **Dragon Register:** {len(current['rows'])} → {len(rows)} entries ({added} new or edited, {removed} removed or edited).")
return True
def pdf_lines(path):
try:
from pypdf import PdfReader
except ImportError:
sys.exit("pypdf is required: pip install -r requirements.txt")
pages = []
for page in PdfReader(path).pages:
text = page.extract_text() or ""
pages.append([" ".join(l.split()) for l in text.splitlines() if l.strip()])
return pages
def update_pdf(src, report):
out = P("data", "derived", "pdf_fingerprint.json")
old = json.load(open(out)) if os.path.exists(out) else None
data = open(src, "rb").read()
pages = pdf_lines(src)
fp = {
"sha256": sha(data),
"bytes": len(data),
"pages": len(pages),
"page_hashes": [sha("\n".join(p), 16) for p in pages],
"line_hashes": sorted({sha(l.lower(), 12) for p in pages for l in p}),
"checked": datetime.now(timezone.utc).strftime("%Y-%m-%d"),
}
if old and old["sha256"] == fp["sha256"]:
report.append(f"- **Route PDF:** no change ({fp['pages']} pages).")
return False
if old and old.get("page_hashes") == fp["page_hashes"]:
report.append("- **Route PDF:** the file changed but its text did not (layout or images only). Worth a look.")
elif old:
changed = [i + 1 for i, h in enumerate(fp["page_hashes"]) if i >= len(old["page_hashes"]) or old["page_hashes"][i] != h]
old_lines = set(old["line_hashes"])
new_lines = [l for p in pages for l in p if sha(l.lower(), 12) not in old_lines]
removed = len(old_lines - set(fp["line_hashes"]))
report.append(f"- **Route PDF:** changed. {old['pages']} → {fp['pages']} pages; pages with new text: "
f"{', '.join(map(str, changed)) or 'none'}; {len(new_lines)} new lines, {removed} lines gone.")
if new_lines:
report.append(" - New lines (update the coding rules or the scene crosswalk if they add options):")
report.extend(f" - {l}" for l in new_lines[:60])
if len(new_lines) > 60:
report.append(f" - … and {len(new_lines) - 60} more")
else:
report.append(f"- **Route PDF:** fingerprint recorded for the first time ({fp['pages']} pages).")
json.dump(fp, open(out, "w"), indent=1)
return True
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--report", default=None)
ap.add_argument("--form")
ap.add_argument("--register")
ap.add_argument("--pdf")
args = ap.parse_args()
sources = json.load(open(P("data", "sources.json")))
report, changed, failed = [], False, []
tmp = tempfile.mkdtemp(prefix="sources-")
steps = [("form", args.form, update_form), ("register", args.register, update_register), ("pdf", args.pdf, update_pdf)]
for key, local, fn in steps:
try:
path = local or download(sources[key]["download"], os.path.join(tmp, sources[key]["filename"]))
changed |= fn(os.path.abspath(path), report)
except Exception as e: # keep going: one broken link should not block the others
failed.append(key)
report.append(f"- **{sources[key]['name']}:** could not check ({e}).")
head = f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}"
text = "\n".join([f"## {head}", ""] + report) + "\n"
print(text)
if args.report:
open(args.report, "w", encoding="utf-8").write(text)
if os.environ.get("GITHUB_OUTPUT"):
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
f.write(f"changed={'true' if changed else 'false'}\nfailed={','.join(failed)}\n")
if __name__ == "__main__":
main()