Threshing Atlas: monorepo, route graph, bot importer, daily source checks

- apps/atlas: the web page (valley map, route flow, Dragon Register)
- packages/core: coding rules and statistics (engine.js) and the shared route
  graph (graph.js): scene graph, decision-variable graph, bot-to-form crosswalk,
  Beta credible intervals
- packages/ingest: DragonKind bot importer to anonymised trajectories (no prose,
  no account identifiers); data/bot holds Ken's sample
- research/python: independent reference implementation + update_sources.py
- tests: JS/Python parity checks and graph/importer unit tests
- CI, GitHub Pages deploy, daily check of the community sheets and route PDF
- Email addresses in free-text form answers are redacted; the raw spreadsheet
  is not committed
- MIT code licence, CC BY 4.0 data licence

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
ekcSchiano
2026-11-01 21:54:54 +00:00
co-authored by Claude Opus 5.5
parent 0052d1a5fd
commit c3d067b88c
48 changed files with 8334 additions and 0 deletions
+54
View File
@@ -0,0 +1,54 @@
#!/usr/bin/env node
'use strict';
/**
* Convert a local copy of the DragonKind bot's data into the public, anonymised format.
*
* node packages/ingest/bin/import-bot.js --bot /path/to/DragonKind [--out data/bot] [--tag 2026-11-01]
*
* Reads <bot>/data/knowledge.json, <bot>/data/runs.jsonl and <bot>/logs/encounters.jsonl.
* Writes <out>/scenes.json (merged with any existing catalogue) and <out>/trajectories-<tag>.jsonl.
* Set DK_ACCOUNT_SALT in the environment to hash account ids; never commit the salt.
*/
const fs = require('node:fs');
const path = require('node:path');
const { importBot, assertPublicSafe } = require('../src/bot');
const args = Object.fromEntries(process.argv.slice(2).reduce((acc, a, i, all) => {
if (a.startsWith('--')) acc.push([a.slice(2), all[i + 1]]);
return acc;
}, []));
if (!args.bot) {
console.error('usage: import-bot.js --bot /path/to/DragonKind [--out data/bot] [--tag YYYY-MM-DD]');
process.exit(2);
}
const out = path.resolve(args.out || 'data/bot');
const tag = args.tag || new Date().toISOString().slice(0, 10);
const { scenes, trajectories, unlabeled } = importBot({
dataDir: path.join(args.bot, 'data'),
logsDir: path.join(args.bot, 'logs')
});
assertPublicSafe(scenes);
assertPublicSafe(trajectories);
fs.mkdirSync(out, { recursive: true });
const sceneFile = path.join(out, 'scenes.json');
const existing = fs.existsSync(sceneFile) ? JSON.parse(fs.readFileSync(sceneFile, 'utf8')) : {};
for (const [id, s] of Object.entries(scenes)) {
const old = existing[id];
if (!old) { existing[id] = s; continue; }
const ids = new Set(old.options.map((o) => o.id));
for (const o of s.options) if (!ids.has(o.id)) old.options.push(o);
old.tint = old.tint || s.tint;
old.backdrop = old.backdrop || s.backdrop;
if (old.label.includes('·') && !s.label.includes('·')) old.label = s.label;
}
const sorted = Object.fromEntries(Object.entries(existing).sort(([a], [b]) => a.localeCompare(b)));
fs.writeFileSync(sceneFile, `${JSON.stringify(sorted, null, 1)}\n`);
const trajFile = path.join(out, `trajectories-${tag}.jsonl`);
fs.writeFileSync(trajFile, trajectories.map((t) => JSON.stringify(t)).join('\n') + '\n');
const outcomes = trajectories.reduce((m, t) => ((m[t.outcome.type] = (m[t.outcome.type] || 0) + 1), m), {});
console.log(`${Object.keys(scenes).length} scenes, ${trajectories.length} attempts ${JSON.stringify(outcomes)}`);
console.log(`wrote ${path.relative(process.cwd(), sceneFile)} and ${path.relative(process.cwd(), trajFile)}`);
if (unlabeled.length) console.log(`scenes without a short label (add them to LABELS in packages/ingest/src/bot.js):\n ${unlabeled.join('\n ')}`);