Threshing Atlas: monorepo, route graph, bot importer, daily source checks

- apps/atlas: the web page (valley map, route flow, Dragon Register)
- packages/core: coding rules and statistics (engine.js) and the shared route
  graph (graph.js): scene graph, decision-variable graph, bot-to-form crosswalk,
  Beta credible intervals
- packages/ingest: DragonKind bot importer to anonymised trajectories (no prose,
  no account identifiers); data/bot holds Ken's sample
- research/python: independent reference implementation + update_sources.py
- tests: JS/Python parity checks and graph/importer unit tests
- CI, GitHub Pages deploy, daily check of the community sheets and route PDF
- Email addresses in free-text form answers are redacted; the raw spreadsheet
  is not committed
- MIT code licence, CC BY 4.0 data licence

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
ekcSchiano
2026-11-01 21:54:54 +00:00
co-authored by Claude Opus 5.5
parent 0052d1a5fd
commit c3d067b88c
48 changed files with 8334 additions and 0 deletions
+54
View File
@@ -0,0 +1,54 @@
#!/usr/bin/env node
'use strict';
/**
* Convert a local copy of the DragonKind bot's data into the public, anonymised format.
*
* node packages/ingest/bin/import-bot.js --bot /path/to/DragonKind [--out data/bot] [--tag 2026-11-01]
*
* Reads <bot>/data/knowledge.json, <bot>/data/runs.jsonl and <bot>/logs/encounters.jsonl.
* Writes <out>/scenes.json (merged with any existing catalogue) and <out>/trajectories-<tag>.jsonl.
* Set DK_ACCOUNT_SALT in the environment to hash account ids; never commit the salt.
*/
const fs = require('node:fs');
const path = require('node:path');
const { importBot, assertPublicSafe } = require('../src/bot');
const args = Object.fromEntries(process.argv.slice(2).reduce((acc, a, i, all) => {
if (a.startsWith('--')) acc.push([a.slice(2), all[i + 1]]);
return acc;
}, []));
if (!args.bot) {
console.error('usage: import-bot.js --bot /path/to/DragonKind [--out data/bot] [--tag YYYY-MM-DD]');
process.exit(2);
}
const out = path.resolve(args.out || 'data/bot');
const tag = args.tag || new Date().toISOString().slice(0, 10);
const { scenes, trajectories, unlabeled } = importBot({
dataDir: path.join(args.bot, 'data'),
logsDir: path.join(args.bot, 'logs')
});
assertPublicSafe(scenes);
assertPublicSafe(trajectories);
fs.mkdirSync(out, { recursive: true });
const sceneFile = path.join(out, 'scenes.json');
const existing = fs.existsSync(sceneFile) ? JSON.parse(fs.readFileSync(sceneFile, 'utf8')) : {};
for (const [id, s] of Object.entries(scenes)) {
const old = existing[id];
if (!old) { existing[id] = s; continue; }
const ids = new Set(old.options.map((o) => o.id));
for (const o of s.options) if (!ids.has(o.id)) old.options.push(o);
old.tint = old.tint || s.tint;
old.backdrop = old.backdrop || s.backdrop;
if (old.label.includes('·') && !s.label.includes('·')) old.label = s.label;
}
const sorted = Object.fromEntries(Object.entries(existing).sort(([a], [b]) => a.localeCompare(b)));
fs.writeFileSync(sceneFile, `${JSON.stringify(sorted, null, 1)}\n`);
const trajFile = path.join(out, `trajectories-${tag}.jsonl`);
fs.writeFileSync(trajFile, trajectories.map((t) => JSON.stringify(t)).join('\n') + '\n');
const outcomes = trajectories.reduce((m, t) => ((m[t.outcome.type] = (m[t.outcome.type] || 0) + 1), m), {});
console.log(`${Object.keys(scenes).length} scenes, ${trajectories.length} attempts ${JSON.stringify(outcomes)}`);
console.log(`wrote ${path.relative(process.cwd(), sceneFile)} and ${path.relative(process.cwd(), trajFile)}`);
if (unlabeled.length) console.log(`scenes without a short label (add them to LABELS in packages/ingest/src/bot.js):\n ${unlabeled.join('\n ')}`);
+8
View File
@@ -0,0 +1,8 @@
{
"name": "@threshing-atlas/ingest",
"version": "0.1.0",
"description": "Importers that turn DragonKind bot data and community sources into the shared graph format (no game prose).",
"main": "src/bot.js",
"bin": { "threshing-import-bot": "bin/import-bot.js" },
"license": "MIT"
}
+211
View File
@@ -0,0 +1,211 @@
'use strict';
/**
* Import DragonKind bot data into anonymised trajectories and a scene catalogue.
*
* Inputs (all from the PRIVATE bot repo, read locally, never committed):
* data/knowledge.json scene atlas: nodes (prose, choices, tint, backdrop) and edges
* data/runs.jsonl one record per attempt (newer bot versions)
* logs/encounters.jsonl every decision the bot made, with the options on screen
*
* Outputs (public, CC BY 4.0): no game prose, only fingerprints, short labels and choice text.
* scenes.json { sceneId: { stage, tint, backdrop, label, options: [{ id, text }] } }
* trajectories.jsonl one line per attempt: { id, source, meta, steps: [...], outcome }
*/
const crypto = require('node:crypto');
const fs = require('node:fs');
const path = require('node:path');
const sha1 = (s, n = 12) => crypto.createHash('sha1').update(String(s)).digest('hex').slice(0, n);
// Must match normalize() in the bot's src/strategy.js, so fingerprints line up.
function normalize(text) {
return String(text || '')
.toLowerCase()
.replace(/[‘’‛`']/g, "'")
.replace(/[“”"]/g, '"')
.replace(/[–—-]/g, ' ')
.replace(/…/g, ' ')
.replace(/[^a-z0-9'\s]/g, ' ')
.replace(/\s+/g, ' ')
.trim();
}
// Must match computeSceneFingerprint() in the bot's src/learner.js.
function sceneId(prose, choices) {
const proseNorm = normalize(prose).slice(0, 500);
const sorted = choices.map(normalize).filter(Boolean).sort();
return sha1(`${proseNorm} ### ${sorted.join(' || ')}`);
}
const optionId = (text) => sha1(normalize(text), 10);
const readJsonl = (file) => {
if (!file || !fs.existsSync(file)) return [];
return fs.readFileSync(file, 'utf8').split(/\r?\n/).filter((l) => l.trim()).map((l) => {
try { return JSON.parse(l); } catch { return null; }
}).filter(Boolean);
};
/**
* Short public labels. Keys are scene fingerprints; values are our own paraphrases, not game prose.
* Unknown scenes fall back to "<stage> · <tint>" and are listed by the importer so labels can be added.
*/
const LABELS = {
db7786392626: 'Start: choose company',
a68f681076c6: 'Valley edge: choose direction',
bc4ee4754b9c: 'First sighting: Red (forest centre)',
abe3011378ac: 'First sighting: Brown (forest centre)',
'84780dab01e3': 'Survived the first sighting',
d25f2759ace0: 'Deeper in the forest: a scream',
f72199f24b91: 'Clearing: squadmate vs two bullies',
b209bacacec3: 'Bully fight: first move',
fbe0f0bea700: 'Bully fight: smaller cadet has a dagger',
a2368b0dc212: 'Blocked the bigger cadet: bullies’ fate',
f4b4944d3f3b: 'Beat the smaller cadet: bullies’ fate',
'0cfd06cf8939': 'After the fight: choose a direction',
'0a561d53ad19': 'An hour later: valley or tree',
'96b68612eaf4': 'Top of the tree (shadow check)',
'10bc7353b8eb': 'Descended quickly: a Red below',
c62f1f1a2b3a: 'Let the Red pass: two Greens in the clearing',
'07e24681e3b3': 'Descended slowly: two Greens in the clearing'
};
function labelFor(id, node) {
if (LABELS[id]) return LABELS[id];
const bits = [node.stage || 'Scene'];
if (node.tint) bits.push(node.tint);
return bits.join(' · ');
}
/** Scene catalogue from knowledge.json (and, for scenes only in the log, from the log). */
function buildScenes(knowledge, encounters) {
const scenes = {};
for (const [id, n] of Object.entries((knowledge && knowledge.nodes) || {})) {
scenes[id] = {
stage: n.stage || 'UNKNOWN',
tint: n.tint || null,
backdrop: n.backdrop || null,
label: labelFor(id, n),
options: Object.values(n.choices || {}).map((c) => ({ id: optionId(c.label || c.choiceNorm), text: c.label || c.choiceNorm }))
};
}
for (const e of encounters) {
if (!Array.isArray(e.choices) || !e.prose) continue;
const id = e.nodeId || sceneId(e.prose, e.choices);
if (scenes[id]) continue;
const stage = (e.decision && e.decision.stage) || e.stage || 'UNKNOWN';
scenes[id] = { stage, tint: null, backdrop: null, label: labelFor(id, { stage }), options: e.choices.map((t) => ({ id: optionId(t), text: t })) };
}
return scenes;
}
const CHOICE_EVENTS = new Set(['DECISION', 'MANUAL_DECISION', 'MANUAL_EDGE_PICK']);
/**
* Trajectories from the encounter log: group events by attempt, keep the decisions that were acted on,
* and close each attempt at its first BURNED / DRAGON_BONDED event. Repeated BURNED events with no
* decisions in between (the cooldown screen being re-detected) are ignored.
*/
function trajectoriesFromEncounters(encounters, scenes) {
const byAttempt = new Map();
for (const e of encounters) {
const key = e.account ? `${e.account}:${e.attempt}` : String(e.attempt);
if (!byAttempt.has(key)) byAttempt.set(key, []);
byAttempt.get(key).push(e);
}
const out = [];
for (const [key, events] of byAttempt) {
events.sort((a, b) => String(a.ts).localeCompare(String(b.ts)));
const steps = [];
let outcome = null;
for (const e of events) {
if (outcome) break;
if (CHOICE_EVENTS.has(e.event)) {
const d = e.decision || {};
const text = d.choiceText || e.choiceText;
if (d.action && d.action !== 'choose') continue; // paused, nothing happened
if (!text || !Array.isArray(e.choices)) continue;
const scene = e.nodeId || sceneId(e.prose || '', e.choices);
const manual = e.event !== 'DECISION';
steps.push({
scene,
stage: d.stage || (scenes[scene] && scenes[scene].stage) || 'UNKNOWN',
tint: (scenes[scene] && scenes[scene].tint) || null,
option: optionId(text),
optionText: text,
offered: e.choices.map(optionId),
rule: d.rule || null,
decisionMode: manual ? 'manual' : d.source || null,
manual,
at: e.ts
});
} else if (e.event === 'BURNED' && steps.length) {
outcome = { type: 'BURNED', colour: null, intentional: Boolean(e.intentional) };
} else if (e.event === 'DRAGON_BONDED' && steps.length) {
outcome = { type: 'BONDED', colour: e.dragon ? String(e.dragon).toLowerCase() : null, intentional: false };
}
}
// A decision repeated on the same scene (the bot re-detecting the screen) counts once.
const dedup = steps.filter((s, i) => !(i > 0 && steps[i - 1].scene === s.scene && steps[i - 1].option === s.option));
for (let i = 0; i < dedup.length; i++) dedup[i].to = i + 1 < dedup.length ? dedup[i + 1].scene : null;
if (!dedup.length) continue;
const last = dedup[dedup.length - 1];
if (outcome) last.to = outcome.type === 'BURNED' ? 'BURNED' : `BONDED:${outcome.colour || 'unknown'}`;
out.push({
id: `bot-${sha1(`${key}:${dedup[0].at}`, 10)}`,
source: 'bot',
meta: { attempt: Number(String(key).split(':').pop()) || null, startedAt: dedup[0].at, accountKey: null, target: null, decisionMode: null },
steps: dedup,
outcome: outcome || { type: 'ABORTED', colour: null, intentional: false }
});
}
return out;
}
/** Fill run-level metadata (target colour, mode) from runs.jsonl where an attempt is covered by both. */
function attachRunMeta(trajs, runs, salt) {
for (const r of runs) {
const t = trajs.find((x) => x.meta.attempt === r.attempt && Math.abs(Date.parse(x.meta.startedAt) - Date.parse(r.startedAt)) < 6 * 3600e3);
if (!t) continue;
t.meta.target = r.targetDragon || null;
t.meta.decisionMode = r.decisionMode || null;
t.meta.policyId = r.policyId || null;
t.meta.experimentId = r.experimentId || null;
t.meta.arm = r.arm || null;
if (r.accountKey) t.meta.accountKey = r.accountKey;
else if (r.accountId && salt) t.meta.accountKey = sha1(`${salt}:${r.accountId}`, 10);
if (r.outcome === 'BONDED' && r.dragonColor) t.outcome = { type: 'BONDED', colour: String(r.dragonColor).toLowerCase(), intentional: false };
if (r.intentionalBurn) t.outcome.intentional = true;
}
return trajs;
}
function importBot({ dataDir, logsDir, salt = process.env.DK_ACCOUNT_SALT || null }) {
const knowledgeFile = path.join(dataDir, 'knowledge.json');
const knowledge = fs.existsSync(knowledgeFile) ? JSON.parse(fs.readFileSync(knowledgeFile, 'utf8')) : { nodes: {} };
const encounters = readJsonl(path.join(logsDir, 'encounters.jsonl'));
const runs = readJsonl(path.join(dataDir, 'runs.jsonl'));
const scenes = buildScenes(knowledge, encounters);
const trajectories = attachRunMeta(trajectoriesFromEncounters(encounters, scenes), runs, salt);
const unlabeled = Object.entries(scenes).filter(([id]) => !LABELS[id]).map(([id, s]) => `${id} (${s.stage}${s.tint ? `, ${s.tint}` : ''})`);
return { scenes, trajectories, unlabeled };
}
/** Guard: nothing that looks like game prose or an account identifier may leave this module. */
function assertPublicSafe(obj) {
const banned = ['prose', 'proseNorm', 'proseExcerpt', 'email', 'password', 'accountId', 'accountName'];
const walk = (v, p) => {
if (Array.isArray(v)) return v.forEach((x, i) => walk(x, `${p}[${i}]`));
if (v && typeof v === 'object') {
for (const [k, x] of Object.entries(v)) {
if (banned.includes(k)) throw new Error(`refusing to export field "${k}" at ${p}`);
walk(x, `${p}.${k}`);
}
} else if (typeof v === 'string' && v.length > 200) {
throw new Error(`refusing to export a ${v.length}-character string at ${p} (looks like prose)`);
}
};
walk(obj, '$');
}
module.exports = { normalize, sceneId, optionId, buildScenes, trajectoriesFromEncounters, attachRunMeta, importBot, assertPublicSafe, LABELS };