Refactor: lay the repo out by section (core, collection, analysis, planner, site) (#8)
Moves only; no behaviour changes. Every file now lives under the part of the project it belongs to, each section has a README, and docs/PRIVACY.md collects the rules all parts follow. core/ engine.js, graph.js, solver.js, python/clean.py collection/ sources/ (form, register, PDF check, daily update), bot/ (importer) analysis/ stats/ (Python statistics), network/ (build-graph.js), reports/ (POLICY.md) planner/ build-policy.js, policy-report.js site/ atlas/ (the page), build.js, qa/ tests/ core/ (unit), parity/ (JS vs Python), fixtures/ Checked against the previous layout: tests pass; coded runs, test outputs, graph.json and policy.json are byte-identical; the built site differs only in comments that name file paths. The published file names Ken's bot downloads (solver.js, graph.js, data/policy.json) are unchanged. Also removes the undefined/ folder of QA screenshots committed by mistake (the QA scripts now default to a temp folder) and test.txt, and drops the unused npm workspaces. Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
dcd5117373
commit
432e04c2c7
@@ -0,0 +1,19 @@
|
||||
# collection
|
||||
|
||||
Getting data in. Everything here turns a source into the shared run format (see `core/`) or into public files
|
||||
under `data/`.
|
||||
|
||||
| Source | Code | Output |
|
||||
|---|---|---|
|
||||
| Community "Threshing Routes" form (Google Sheet) | `sources/export_raw.py`, then `core/python/clean.py` | `data/public/form_responses.json`, `data/derived/runs_clean.csv` |
|
||||
| Unofficial Dragon Register (Google Sheet) | `sources/export_register.py` | `data/public/register.json` (rider names stay local) |
|
||||
| Community route PDF | `sources/update_sources.py` | `data/derived/pdf_fingerprint.json` only; the PDF is never committed |
|
||||
| DragonKind bot (Ken) | `bot/import-bot.js`, `bot/bot.js` | `data/bot/scenes.json`, `data/bot/trajectories-<tag>.jsonl` |
|
||||
| Reddit megathreads | not collected yet | — |
|
||||
|
||||
`sources/update_sources.py` runs every day from `.github/workflows/update-sources.yml`. It downloads the sheets
|
||||
and the PDF (links in `data/sources.json`), rebuilds the data if anything changed, and opens a pull request that
|
||||
merges itself when CI passes. A broken link opens an issue.
|
||||
|
||||
The bot importer only ever writes fingerprints, short labels, choice text and outcomes. It refuses anything that
|
||||
looks like game prose or an account identifier. What the bot should log is in `docs/BOT_DATA_PROTOCOL.md`.
|
||||
@@ -0,0 +1,211 @@
|
||||
'use strict';
|
||||
/**
|
||||
* Import DragonKind bot data into anonymised trajectories and a scene catalogue.
|
||||
*
|
||||
* Inputs (all from the PRIVATE bot repo, read locally, never committed):
|
||||
* data/knowledge.json scene atlas: nodes (prose, choices, tint, backdrop) and edges
|
||||
* data/runs.jsonl one record per attempt (newer bot versions)
|
||||
* logs/encounters.jsonl every decision the bot made, with the options on screen
|
||||
*
|
||||
* Outputs (public, CC BY 4.0): no game prose, only fingerprints, short labels and choice text.
|
||||
* scenes.json { sceneId: { stage, tint, backdrop, label, options: [{ id, text }] } }
|
||||
* trajectories.jsonl one line per attempt: { id, source, meta, steps: [...], outcome }
|
||||
*/
|
||||
const crypto = require('node:crypto');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const sha1 = (s, n = 12) => crypto.createHash('sha1').update(String(s)).digest('hex').slice(0, n);
|
||||
|
||||
// Must match normalize() in the bot's src/strategy.js, so fingerprints line up.
|
||||
function normalize(text) {
|
||||
return String(text || '')
|
||||
.toLowerCase()
|
||||
.replace(/[‘’‛`']/g, "'")
|
||||
.replace(/[“”"]/g, '"')
|
||||
.replace(/[–—-]/g, ' ')
|
||||
.replace(/…/g, ' ')
|
||||
.replace(/[^a-z0-9'\s]/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
// Must match computeSceneFingerprint() in the bot's src/learner.js.
|
||||
function sceneId(prose, choices) {
|
||||
const proseNorm = normalize(prose).slice(0, 500);
|
||||
const sorted = choices.map(normalize).filter(Boolean).sort();
|
||||
return sha1(`${proseNorm} ### ${sorted.join(' || ')}`);
|
||||
}
|
||||
|
||||
const optionId = (text) => sha1(normalize(text), 10);
|
||||
|
||||
const readJsonl = (file) => {
|
||||
if (!file || !fs.existsSync(file)) return [];
|
||||
return fs.readFileSync(file, 'utf8').split(/\r?\n/).filter((l) => l.trim()).map((l) => {
|
||||
try { return JSON.parse(l); } catch { return null; }
|
||||
}).filter(Boolean);
|
||||
};
|
||||
|
||||
/**
|
||||
* Short public labels. Keys are scene fingerprints; values are our own paraphrases, not game prose.
|
||||
* Unknown scenes fall back to "<stage> · <tint>" and are listed by the importer so labels can be added.
|
||||
*/
|
||||
const LABELS = {
|
||||
db7786392626: 'Start: choose company',
|
||||
a68f681076c6: 'Valley edge: choose direction',
|
||||
bc4ee4754b9c: 'First sighting: Red (forest centre)',
|
||||
abe3011378ac: 'First sighting: Brown (forest centre)',
|
||||
'84780dab01e3': 'Survived the first sighting',
|
||||
d25f2759ace0: 'Deeper in the forest: a scream',
|
||||
f72199f24b91: 'Clearing: squadmate vs two bullies',
|
||||
b209bacacec3: 'Bully fight: first move',
|
||||
fbe0f0bea700: 'Bully fight: smaller cadet has a dagger',
|
||||
a2368b0dc212: 'Blocked the bigger cadet: bullies’ fate',
|
||||
f4b4944d3f3b: 'Beat the smaller cadet: bullies’ fate',
|
||||
'0cfd06cf8939': 'After the fight: choose a direction',
|
||||
'0a561d53ad19': 'An hour later: valley or tree',
|
||||
'96b68612eaf4': 'Top of the tree (shadow check)',
|
||||
'10bc7353b8eb': 'Descended quickly: a Red below',
|
||||
c62f1f1a2b3a: 'Let the Red pass: two Greens in the clearing',
|
||||
'07e24681e3b3': 'Descended slowly: two Greens in the clearing'
|
||||
};
|
||||
|
||||
function labelFor(id, node) {
|
||||
if (LABELS[id]) return LABELS[id];
|
||||
const bits = [node.stage || 'Scene'];
|
||||
if (node.tint) bits.push(node.tint);
|
||||
return bits.join(' · ');
|
||||
}
|
||||
|
||||
/** Scene catalogue from knowledge.json (and, for scenes only in the log, from the log). */
|
||||
function buildScenes(knowledge, encounters) {
|
||||
const scenes = {};
|
||||
for (const [id, n] of Object.entries((knowledge && knowledge.nodes) || {})) {
|
||||
scenes[id] = {
|
||||
stage: n.stage || 'UNKNOWN',
|
||||
tint: n.tint || null,
|
||||
backdrop: n.backdrop || null,
|
||||
label: labelFor(id, n),
|
||||
options: Object.values(n.choices || {}).map((c) => ({ id: optionId(c.label || c.choiceNorm), text: c.label || c.choiceNorm }))
|
||||
};
|
||||
}
|
||||
for (const e of encounters) {
|
||||
if (!Array.isArray(e.choices) || !e.prose) continue;
|
||||
const id = e.nodeId || sceneId(e.prose, e.choices);
|
||||
if (scenes[id]) continue;
|
||||
const stage = (e.decision && e.decision.stage) || e.stage || 'UNKNOWN';
|
||||
scenes[id] = { stage, tint: null, backdrop: null, label: labelFor(id, { stage }), options: e.choices.map((t) => ({ id: optionId(t), text: t })) };
|
||||
}
|
||||
return scenes;
|
||||
}
|
||||
|
||||
const CHOICE_EVENTS = new Set(['DECISION', 'MANUAL_DECISION', 'MANUAL_EDGE_PICK']);
|
||||
|
||||
/**
|
||||
* Trajectories from the encounter log: group events by attempt, keep the decisions that were acted on,
|
||||
* and close each attempt at its first BURNED / DRAGON_BONDED event. Repeated BURNED events with no
|
||||
* decisions in between (the cooldown screen being re-detected) are ignored.
|
||||
*/
|
||||
function trajectoriesFromEncounters(encounters, scenes) {
|
||||
const byAttempt = new Map();
|
||||
for (const e of encounters) {
|
||||
const key = e.account ? `${e.account}:${e.attempt}` : String(e.attempt);
|
||||
if (!byAttempt.has(key)) byAttempt.set(key, []);
|
||||
byAttempt.get(key).push(e);
|
||||
}
|
||||
const out = [];
|
||||
for (const [key, events] of byAttempt) {
|
||||
events.sort((a, b) => String(a.ts).localeCompare(String(b.ts)));
|
||||
const steps = [];
|
||||
let outcome = null;
|
||||
for (const e of events) {
|
||||
if (outcome) break;
|
||||
if (CHOICE_EVENTS.has(e.event)) {
|
||||
const d = e.decision || {};
|
||||
const text = d.choiceText || e.choiceText;
|
||||
if (d.action && d.action !== 'choose') continue; // paused, nothing happened
|
||||
if (!text || !Array.isArray(e.choices)) continue;
|
||||
const scene = e.nodeId || sceneId(e.prose || '', e.choices);
|
||||
const manual = e.event !== 'DECISION';
|
||||
steps.push({
|
||||
scene,
|
||||
stage: d.stage || (scenes[scene] && scenes[scene].stage) || 'UNKNOWN',
|
||||
tint: (scenes[scene] && scenes[scene].tint) || null,
|
||||
option: optionId(text),
|
||||
optionText: text,
|
||||
offered: e.choices.map(optionId),
|
||||
rule: d.rule || null,
|
||||
decisionMode: manual ? 'manual' : d.source || null,
|
||||
manual,
|
||||
at: e.ts
|
||||
});
|
||||
} else if (e.event === 'BURNED' && steps.length) {
|
||||
outcome = { type: 'BURNED', colour: null, intentional: Boolean(e.intentional) };
|
||||
} else if (e.event === 'DRAGON_BONDED' && steps.length) {
|
||||
outcome = { type: 'BONDED', colour: e.dragon ? String(e.dragon).toLowerCase() : null, intentional: false };
|
||||
}
|
||||
}
|
||||
// A decision repeated on the same scene (the bot re-detecting the screen) counts once.
|
||||
const dedup = steps.filter((s, i) => !(i > 0 && steps[i - 1].scene === s.scene && steps[i - 1].option === s.option));
|
||||
for (let i = 0; i < dedup.length; i++) dedup[i].to = i + 1 < dedup.length ? dedup[i + 1].scene : null;
|
||||
if (!dedup.length) continue;
|
||||
const last = dedup[dedup.length - 1];
|
||||
if (outcome) last.to = outcome.type === 'BURNED' ? 'BURNED' : `BONDED:${outcome.colour || 'unknown'}`;
|
||||
out.push({
|
||||
id: `bot-${sha1(`${key}:${dedup[0].at}`, 10)}`,
|
||||
source: 'bot',
|
||||
meta: { attempt: Number(String(key).split(':').pop()) || null, startedAt: dedup[0].at, accountKey: null, target: null, decisionMode: null },
|
||||
steps: dedup,
|
||||
outcome: outcome || { type: 'ABORTED', colour: null, intentional: false }
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Fill run-level metadata (target colour, mode) from runs.jsonl where an attempt is covered by both. */
|
||||
function attachRunMeta(trajs, runs, salt) {
|
||||
for (const r of runs) {
|
||||
const t = trajs.find((x) => x.meta.attempt === r.attempt && Math.abs(Date.parse(x.meta.startedAt) - Date.parse(r.startedAt)) < 6 * 3600e3);
|
||||
if (!t) continue;
|
||||
t.meta.target = r.targetDragon || null;
|
||||
t.meta.decisionMode = r.decisionMode || null;
|
||||
t.meta.policyId = r.policyId || null;
|
||||
t.meta.experimentId = r.experimentId || null;
|
||||
t.meta.arm = r.arm || null;
|
||||
if (r.accountKey) t.meta.accountKey = r.accountKey;
|
||||
else if (r.accountId && salt) t.meta.accountKey = sha1(`${salt}:${r.accountId}`, 10);
|
||||
if (r.outcome === 'BONDED' && r.dragonColor) t.outcome = { type: 'BONDED', colour: String(r.dragonColor).toLowerCase(), intentional: false };
|
||||
if (r.intentionalBurn) t.outcome.intentional = true;
|
||||
}
|
||||
return trajs;
|
||||
}
|
||||
|
||||
function importBot({ dataDir, logsDir, salt = process.env.DK_ACCOUNT_SALT || null }) {
|
||||
const knowledgeFile = path.join(dataDir, 'knowledge.json');
|
||||
const knowledge = fs.existsSync(knowledgeFile) ? JSON.parse(fs.readFileSync(knowledgeFile, 'utf8')) : { nodes: {} };
|
||||
const encounters = readJsonl(path.join(logsDir, 'encounters.jsonl'));
|
||||
const runs = readJsonl(path.join(dataDir, 'runs.jsonl'));
|
||||
const scenes = buildScenes(knowledge, encounters);
|
||||
const trajectories = attachRunMeta(trajectoriesFromEncounters(encounters, scenes), runs, salt);
|
||||
const unlabeled = Object.entries(scenes).filter(([id]) => !LABELS[id]).map(([id, s]) => `${id} (${s.stage}${s.tint ? `, ${s.tint}` : ''})`);
|
||||
return { scenes, trajectories, unlabeled };
|
||||
}
|
||||
|
||||
/** Guard: nothing that looks like game prose or an account identifier may leave this module. */
|
||||
function assertPublicSafe(obj) {
|
||||
const banned = ['prose', 'proseNorm', 'proseExcerpt', 'email', 'password', 'accountId', 'accountName'];
|
||||
const walk = (v, p) => {
|
||||
if (Array.isArray(v)) return v.forEach((x, i) => walk(x, `${p}[${i}]`));
|
||||
if (v && typeof v === 'object') {
|
||||
for (const [k, x] of Object.entries(v)) {
|
||||
if (banned.includes(k)) throw new Error(`refusing to export field "${k}" at ${p}`);
|
||||
walk(x, `${p}.${k}`);
|
||||
}
|
||||
} else if (typeof v === 'string' && v.length > 200) {
|
||||
throw new Error(`refusing to export a ${v.length}-character string at ${p} (looks like prose)`);
|
||||
}
|
||||
};
|
||||
walk(obj, '$');
|
||||
}
|
||||
|
||||
module.exports = { normalize, sceneId, optionId, buildScenes, trajectoriesFromEncounters, attachRunMeta, importBot, assertPublicSafe, LABELS };
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
/**
|
||||
* Convert a local copy of the DragonKind bot's data into the public, anonymised format.
|
||||
*
|
||||
* node collection/bot/import-bot.js --bot /path/to/DragonKind [--out data/bot] [--tag 2026-11-01]
|
||||
*
|
||||
* Reads <bot>/data/knowledge.json, <bot>/data/runs.jsonl and <bot>/logs/encounters.jsonl.
|
||||
* Writes <out>/scenes.json (merged with any existing catalogue) and <out>/trajectories-<tag>.jsonl.
|
||||
* Set DK_ACCOUNT_SALT in the environment to hash account ids; never commit the salt.
|
||||
*/
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { importBot, assertPublicSafe } = require('./bot');
|
||||
|
||||
const args = Object.fromEntries(process.argv.slice(2).reduce((acc, a, i, all) => {
|
||||
if (a.startsWith('--')) acc.push([a.slice(2), all[i + 1]]);
|
||||
return acc;
|
||||
}, []));
|
||||
if (!args.bot) {
|
||||
console.error('usage: import-bot.js --bot /path/to/DragonKind [--out data/bot] [--tag YYYY-MM-DD]');
|
||||
process.exit(2);
|
||||
}
|
||||
const out = path.resolve(args.out || 'data/bot');
|
||||
const tag = args.tag || new Date().toISOString().slice(0, 10);
|
||||
|
||||
const { scenes, trajectories, unlabeled } = importBot({
|
||||
dataDir: path.join(args.bot, 'data'),
|
||||
logsDir: path.join(args.bot, 'logs')
|
||||
});
|
||||
assertPublicSafe(scenes);
|
||||
assertPublicSafe(trajectories);
|
||||
|
||||
fs.mkdirSync(out, { recursive: true });
|
||||
const sceneFile = path.join(out, 'scenes.json');
|
||||
const existing = fs.existsSync(sceneFile) ? JSON.parse(fs.readFileSync(sceneFile, 'utf8')) : {};
|
||||
for (const [id, s] of Object.entries(scenes)) {
|
||||
const old = existing[id];
|
||||
if (!old) { existing[id] = s; continue; }
|
||||
const ids = new Set(old.options.map((o) => o.id));
|
||||
for (const o of s.options) if (!ids.has(o.id)) old.options.push(o);
|
||||
old.tint = old.tint || s.tint;
|
||||
old.backdrop = old.backdrop || s.backdrop;
|
||||
if (old.label.includes('·') && !s.label.includes('·')) old.label = s.label;
|
||||
}
|
||||
const sorted = Object.fromEntries(Object.entries(existing).sort(([a], [b]) => a.localeCompare(b)));
|
||||
fs.writeFileSync(sceneFile, `${JSON.stringify(sorted, null, 1)}\n`);
|
||||
const trajFile = path.join(out, `trajectories-${tag}.jsonl`);
|
||||
fs.writeFileSync(trajFile, trajectories.map((t) => JSON.stringify(t)).join('\n') + '\n');
|
||||
|
||||
const outcomes = trajectories.reduce((m, t) => ((m[t.outcome.type] = (m[t.outcome.type] || 0) + 1), m), {});
|
||||
console.log(`${Object.keys(scenes).length} scenes, ${trajectories.length} attempts ${JSON.stringify(outcomes)}`);
|
||||
console.log(`wrote ${path.relative(process.cwd(), sceneFile)} and ${path.relative(process.cwd(), trajFile)}`);
|
||||
if (unlabeled.length) console.log(`scenes without a short label (add them to LABELS in collection/bot/bot.js):\n ${unlabeled.join('\n ')}`);
|
||||
@@ -0,0 +1,49 @@
|
||||
"""export_raw.py — convert the Google Sheets .xlsx (or .csv) export into the raw JSON the web page loads.
|
||||
|
||||
python export_raw.py ../../data/raw/form/Threshing_Routes_Responses.xlsx ../../data/public/form_responses.json
|
||||
|
||||
The JSON is a list of objects keyed by the original question text (no recoding happens here, so the page
|
||||
applies exactly the same cleaning rules to it as it would to a freshly uploaded export).
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
import re
|
||||
|
||||
import pandas as pd
|
||||
|
||||
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
|
||||
|
||||
def load_raw(path):
|
||||
if str(path).lower().endswith(".csv"):
|
||||
return pd.read_csv(path)
|
||||
xl = pd.ExcelFile(path)
|
||||
sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0])
|
||||
return pd.read_excel(xl, sheet)
|
||||
|
||||
|
||||
def cell(v):
|
||||
if v is None or v is pd.NaT or (not isinstance(v, str) and pd.isna(v)):
|
||||
return None
|
||||
if isinstance(v, (datetime, pd.Timestamp)):
|
||||
return pd.Timestamp(v).strftime("%Y-%m-%d %H:%M:%S")
|
||||
if isinstance(v, float) and v.is_integer():
|
||||
return int(v)
|
||||
if isinstance(v, str):
|
||||
return EMAIL_RE.sub("[email removed]", v) # respondents sometimes leave an email in free text
|
||||
return v
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/raw/form/Threshing_Routes_Responses.xlsx"
|
||||
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/public/form_responses.json"
|
||||
raw = load_raw(src)
|
||||
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
|
||||
# keep every header present even if a row is blank there, so column detection sees all questions
|
||||
headers = [str(c) for c in raw.columns]
|
||||
with open(out, "w", encoding="utf-8") as f:
|
||||
json.dump({"source_file": str(src).split("/")[-1], "exported": datetime.now().strftime("%Y-%m-%d"),
|
||||
"headers": headers, "rows": rows}, f, ensure_ascii=False, separators=(",", ":"))
|
||||
print(f"{len(rows)} rows -> {out}")
|
||||
@@ -0,0 +1,43 @@
|
||||
"""export_register.py — convert the "Unofficial Dragon Register" .xlsx/.csv export into web/data/register.json.
|
||||
|
||||
python export_register.py ../../data/raw/register/Unofficial_Dragon_Register_Responses.xlsx ../../data/public/register.json
|
||||
|
||||
Raw rows are kept as-is, except that photo links and rider names are dropped. Dragons read from public dragonkind.com result pages
|
||||
linked in the Reddit megathreads are stored separately under "extra" with their source URL.
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from export_raw import cell, load_raw
|
||||
|
||||
# Read from dragonkind.com/results/<id> pages linked in the Part 3 and Part 4 megathreads (2026-10-19).
|
||||
REDDIT_RESULTS = [
|
||||
("THRESH-VFRQVM5K3B0CP", "Thiduach", "Blue", "Swordtail", "Female", 51, "Gormfaileas"),
|
||||
("THRESH-3M4P9MNFQGBS8", "Caenuis", "Blue", "Swordtail", "Female", 38, "Gormfaileas"),
|
||||
("THRESH-5JQDJ21G1BYKR", "Oavuagh", "Green", "Morningstartail", "Female", 43, "Uaineloidsig"),
|
||||
("THRESH-5KRAAVFB3D1MG", "Geagios", "Blue", "Clubtail", "Male", 99, "Gormfaileas"),
|
||||
("THRESH-B5HSM00J4A2Z4", "Fiedheil", "Black", "Clubtail", "Male", 32, "Moerlaighen"),
|
||||
("THRESH-BMPM2GAPK58JT", "Teirfua", "Green", "Morningstartail", "Female", 68, "Eifyrielle"),
|
||||
("THRESH-MCQ8D6GAGF4K8", "Caollo", "Blue", "Morningstartail", "Female", 93, "Nyghmuiron"),
|
||||
("THRESH-Q0XDBFMNBEB86", "Aenminn", "Red", "Swordtail", "Male", 59, "Berradhyr"),
|
||||
("THRESH-W99DB29MG16QA", "Meinso", "Green", "Clubtail", "Female", 63, "Eifyrielle"),
|
||||
("THRESH-XCJCY7HW7QSSP", "Einmial", "Blue", "Clubtail", "Male", 58, "Gormfaileas"),
|
||||
("THRESH-XR6G68VPGT48M", "Recuagh", "Brown", "Morningstartail", "Female", 55, "Vormalgh"),
|
||||
("THRESH-YQC97TK34214E", "Dannidh", "Orange", "Clubtail", "Male", 61, "Cariaslog"),
|
||||
]
|
||||
|
||||
if __name__ == "__main__":
|
||||
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/raw/register/Unofficial_Dragon_Register_Responses.xlsx"
|
||||
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/public/register.json"
|
||||
raw = load_raw(src)
|
||||
raw = raw[[c for c in raw.columns if "photo" not in str(c).lower() and "cadet name" not in str(c).lower()]]
|
||||
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
|
||||
extra = [{"source": "dragonkind.com result page linked on Reddit", "url": f"https://dragonkind.com/results/{i}",
|
||||
"name": n, "colour": c, "tail": t, "sex": s, "age": a, "lineage": l} for i, n, c, t, s, a, l in REDDIT_RESULTS]
|
||||
json.dump({"source_file": str(src).split("/")[-1], "exported": datetime.now().strftime("%Y-%m-%d"),
|
||||
"headers": [str(c) for c in raw.columns], "rows": rows, "extra": extra},
|
||||
open(out, "w", encoding="utf-8"), ensure_ascii=False, separators=(",", ":"))
|
||||
print(f"{len(rows)} register rows + {len(extra)} Reddit result pages -> {out}")
|
||||
@@ -0,0 +1,183 @@
|
||||
"""update_sources.py: check the community spreadsheets and the route PDF for new information.
|
||||
|
||||
python collection/sources/update_sources.py [--report report.md] [--form FILE] [--register FILE] [--pdf FILE]
|
||||
|
||||
Run from the repo root. Without file arguments it downloads the current versions from the URLs in
|
||||
data/sources.json (the GitHub Action does this every day). With file arguments it uses local copies,
|
||||
which is handy for testing or when a download link changes.
|
||||
|
||||
What it does:
|
||||
* Threshing Routes form: re-exports data/public/form_responses.json and data/derived/runs_clean.csv
|
||||
when the responses changed, and reports how many rows were added or edited.
|
||||
* Dragon Register: re-exports data/public/register.json (rider names and photos dropped) when it changed.
|
||||
The raw register export is never written into the repo.
|
||||
* Route PDF: the PDF is not redistributed. Only a fingerprint is kept (data/derived/pdf_fingerprint.json):
|
||||
the file hash, per-page text hashes and per-line hashes. When it changes, the report lists the pages that
|
||||
changed and the lines that are new, so the coding rules and the scene crosswalk can be updated by hand.
|
||||
|
||||
Exit code is 0 either way; "changed" files are visible to git, and the report says what happened.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
||||
sys.path.insert(0, HERE)
|
||||
from export_raw import cell, load_raw # noqa: E402
|
||||
|
||||
P = lambda *a: os.path.join(ROOT, *a) # noqa: E731
|
||||
|
||||
|
||||
def sha(data, n=None):
|
||||
h = hashlib.sha256(data if isinstance(data, bytes) else data.encode("utf-8")).hexdigest()
|
||||
return h[:n] if n else h
|
||||
|
||||
|
||||
def download(url, dest):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "threshing-atlas-updater"})
|
||||
with urllib.request.urlopen(req, timeout=60) as r, open(dest, "wb") as f:
|
||||
data = r.read()
|
||||
f.write(data)
|
||||
head = data[:15].lower()
|
||||
if head.startswith(b"<!doctype html") or head.startswith(b"<html"):
|
||||
raise RuntimeError(f"{url} returned a web page, not a file (is the link still shared publicly?)")
|
||||
return dest
|
||||
|
||||
|
||||
def sheet_rows(path, drop=()):
|
||||
raw = load_raw(path)
|
||||
raw = raw[[c for c in raw.columns if not any(d in str(c).lower() for d in drop)]]
|
||||
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
|
||||
return [str(c) for c in raw.columns], rows
|
||||
|
||||
|
||||
def diff_rows(old, new):
|
||||
key = lambda r: json.dumps(r, sort_keys=True, ensure_ascii=False) # noqa: E731
|
||||
o, n = {key(r) for r in old}, {key(r) for r in new}
|
||||
return len(n - o), len(o - n)
|
||||
|
||||
|
||||
def update_form(src, report):
|
||||
out = P("data", "public", "form_responses.json")
|
||||
current = json.load(open(out, encoding="utf-8"))
|
||||
headers, rows = sheet_rows(src)
|
||||
added, removed = diff_rows(current["rows"], rows)
|
||||
if headers == current["headers"] and not added and not removed:
|
||||
report.append(f"- **Threshing Routes form:** no change ({len(rows)} responses).")
|
||||
return False
|
||||
subprocess.run([sys.executable, P("collection", "sources", "export_raw.py"), src, out], check=True, cwd=HERE)
|
||||
subprocess.run([sys.executable, P("core", "python", "clean.py"), out, P("data", "derived", "runs_clean.csv")], check=True, cwd=HERE)
|
||||
raw_copy = P("data", "raw", "form", "Threshing_Routes_Responses.xlsx")
|
||||
if os.path.abspath(src) != raw_copy: # local copy only; data/raw/form/*.xlsx is git-ignored
|
||||
os.makedirs(os.path.dirname(raw_copy), exist_ok=True)
|
||||
shutil.copyfile(src, raw_copy)
|
||||
new_q = [h for h in headers if h not in current["headers"]]
|
||||
gone_q = [h for h in current["headers"] if h not in headers]
|
||||
report.append(f"- **Threshing Routes form:** {len(current['rows'])} → {len(rows)} responses "
|
||||
f"({added} new or edited, {removed} removed or edited).")
|
||||
if new_q:
|
||||
report.append(f" - New questions (check `COLUMN_KEYS` in engine.js and clean.py): {', '.join(new_q)}")
|
||||
if gone_q:
|
||||
report.append(f" - Questions no longer in the sheet: {', '.join(gone_q)}")
|
||||
return True
|
||||
|
||||
|
||||
def update_register(src, report):
|
||||
out = P("data", "public", "register.json")
|
||||
current = json.load(open(out, encoding="utf-8"))
|
||||
headers, rows = sheet_rows(src, drop=("photo", "cadet name"))
|
||||
added, removed = diff_rows(current["rows"], rows)
|
||||
if headers == current["headers"] and not added and not removed:
|
||||
report.append(f"- **Dragon Register:** no change ({len(rows)} dragons).")
|
||||
return False
|
||||
subprocess.run([sys.executable, P("collection", "sources", "export_register.py"), src, out], check=True, cwd=HERE)
|
||||
report.append(f"- **Dragon Register:** {len(current['rows'])} → {len(rows)} entries ({added} new or edited, {removed} removed or edited).")
|
||||
return True
|
||||
|
||||
|
||||
def pdf_lines(path):
|
||||
try:
|
||||
from pypdf import PdfReader
|
||||
except ImportError:
|
||||
sys.exit("pypdf is required: pip install -r requirements.txt")
|
||||
pages = []
|
||||
for page in PdfReader(path).pages:
|
||||
text = page.extract_text() or ""
|
||||
pages.append([" ".join(l.split()) for l in text.splitlines() if l.strip()])
|
||||
return pages
|
||||
|
||||
|
||||
def update_pdf(src, report):
|
||||
out = P("data", "derived", "pdf_fingerprint.json")
|
||||
old = json.load(open(out)) if os.path.exists(out) else None
|
||||
data = open(src, "rb").read()
|
||||
pages = pdf_lines(src)
|
||||
fp = {
|
||||
"sha256": sha(data),
|
||||
"bytes": len(data),
|
||||
"pages": len(pages),
|
||||
"page_hashes": [sha("\n".join(p), 16) for p in pages],
|
||||
"line_hashes": sorted({sha(l.lower(), 12) for p in pages for l in p}),
|
||||
"checked": datetime.now(timezone.utc).strftime("%Y-%m-%d"),
|
||||
}
|
||||
if old and old["sha256"] == fp["sha256"]:
|
||||
report.append(f"- **Route PDF:** no change ({fp['pages']} pages).")
|
||||
return False
|
||||
if old and old.get("page_hashes") == fp["page_hashes"]:
|
||||
report.append("- **Route PDF:** the file changed but its text did not (layout or images only). Worth a look.")
|
||||
elif old:
|
||||
changed = [i + 1 for i, h in enumerate(fp["page_hashes"]) if i >= len(old["page_hashes"]) or old["page_hashes"][i] != h]
|
||||
old_lines = set(old["line_hashes"])
|
||||
new_lines = [l for p in pages for l in p if sha(l.lower(), 12) not in old_lines]
|
||||
removed = len(old_lines - set(fp["line_hashes"]))
|
||||
report.append(f"- **Route PDF:** changed. {old['pages']} → {fp['pages']} pages; pages with new text: "
|
||||
f"{', '.join(map(str, changed)) or 'none'}; {len(new_lines)} new lines, {removed} lines gone.")
|
||||
if new_lines:
|
||||
report.append(" - New lines (update the coding rules or the scene crosswalk if they add options):")
|
||||
report.extend(f" - {l}" for l in new_lines[:60])
|
||||
if len(new_lines) > 60:
|
||||
report.append(f" - … and {len(new_lines) - 60} more")
|
||||
else:
|
||||
report.append(f"- **Route PDF:** fingerprint recorded for the first time ({fp['pages']} pages).")
|
||||
json.dump(fp, open(out, "w"), indent=1)
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--report", default=None)
|
||||
ap.add_argument("--form")
|
||||
ap.add_argument("--register")
|
||||
ap.add_argument("--pdf")
|
||||
args = ap.parse_args()
|
||||
sources = json.load(open(P("data", "sources.json")))
|
||||
report, changed, failed = [], False, []
|
||||
tmp = tempfile.mkdtemp(prefix="sources-")
|
||||
steps = [("form", args.form, update_form), ("register", args.register, update_register), ("pdf", args.pdf, update_pdf)]
|
||||
for key, local, fn in steps:
|
||||
try:
|
||||
path = local or download(sources[key]["download"], os.path.join(tmp, sources[key]["filename"]))
|
||||
changed |= fn(os.path.abspath(path), report)
|
||||
except Exception as e: # keep going: one broken link should not block the others
|
||||
failed.append(key)
|
||||
report.append(f"- **{sources[key]['name']}:** could not check ({e}).")
|
||||
head = f"Source check {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}"
|
||||
text = "\n".join([f"## {head}", ""] + report) + "\n"
|
||||
print(text)
|
||||
if args.report:
|
||||
open(args.report, "w", encoding="utf-8").write(text)
|
||||
if os.environ.get("GITHUB_OUTPUT"):
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
|
||||
f.write(f"changed={'true' if changed else 'false'}\nfailed={','.join(failed)}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user