Refactor: lay the repo out by section (core, collection, analysis, planner, site) (#8)

Moves only; no behaviour changes. Every file now lives under the part of the project it
belongs to, each section has a README, and docs/PRIVACY.md collects the rules all parts follow.

  core/        engine.js, graph.js, solver.js, python/clean.py
  collection/  sources/ (form, register, PDF check, daily update), bot/ (importer)
  analysis/    stats/ (Python statistics), network/ (build-graph.js), reports/ (POLICY.md)
  planner/     build-policy.js, policy-report.js
  site/        atlas/ (the page), build.js, qa/
  tests/       core/ (unit), parity/ (JS vs Python), fixtures/

Checked against the previous layout: tests pass; coded runs, test outputs, graph.json and
policy.json are byte-identical; the built site differs only in comments that name file paths.
The published file names Ken's bot downloads (solver.js, graph.js, data/policy.json) are unchanged.

Also removes the undefined/ folder of QA screenshots committed by mistake (the QA scripts now
default to a temp folder) and test.txt, and drops the unused npm workspaces.


Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ekcSchiano
2026-11-03 19:46:33 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent dcd5117373
commit 432e04c2c7
66 changed files with 201 additions and 2695 deletions
+211
View File
@@ -0,0 +1,211 @@
'use strict';
/**
* Import DragonKind bot data into anonymised trajectories and a scene catalogue.
*
* Inputs (all from the PRIVATE bot repo, read locally, never committed):
* data/knowledge.json scene atlas: nodes (prose, choices, tint, backdrop) and edges
* data/runs.jsonl one record per attempt (newer bot versions)
* logs/encounters.jsonl every decision the bot made, with the options on screen
*
* Outputs (public, CC BY 4.0): no game prose, only fingerprints, short labels and choice text.
* scenes.json { sceneId: { stage, tint, backdrop, label, options: [{ id, text }] } }
* trajectories.jsonl one line per attempt: { id, source, meta, steps: [...], outcome }
*/
const crypto = require('node:crypto');
const fs = require('node:fs');
const path = require('node:path');
const sha1 = (s, n = 12) => crypto.createHash('sha1').update(String(s)).digest('hex').slice(0, n);
// Must match normalize() in the bot's src/strategy.js, so fingerprints line up.
function normalize(text) {
return String(text || '')
.toLowerCase()
.replace(/[‘’‛`']/g, "'")
.replace(/[“”"]/g, '"')
.replace(/[–—-]/g, ' ')
.replace(/…/g, ' ')
.replace(/[^a-z0-9'\s]/g, ' ')
.replace(/\s+/g, ' ')
.trim();
}
// Must match computeSceneFingerprint() in the bot's src/learner.js.
function sceneId(prose, choices) {
const proseNorm = normalize(prose).slice(0, 500);
const sorted = choices.map(normalize).filter(Boolean).sort();
return sha1(`${proseNorm} ### ${sorted.join(' || ')}`);
}
const optionId = (text) => sha1(normalize(text), 10);
const readJsonl = (file) => {
if (!file || !fs.existsSync(file)) return [];
return fs.readFileSync(file, 'utf8').split(/\r?\n/).filter((l) => l.trim()).map((l) => {
try { return JSON.parse(l); } catch { return null; }
}).filter(Boolean);
};
/**
* Short public labels. Keys are scene fingerprints; values are our own paraphrases, not game prose.
* Unknown scenes fall back to "<stage> · <tint>" and are listed by the importer so labels can be added.
*/
const LABELS = {
db7786392626: 'Start: choose company',
a68f681076c6: 'Valley edge: choose direction',
bc4ee4754b9c: 'First sighting: Red (forest centre)',
abe3011378ac: 'First sighting: Brown (forest centre)',
'84780dab01e3': 'Survived the first sighting',
d25f2759ace0: 'Deeper in the forest: a scream',
f72199f24b91: 'Clearing: squadmate vs two bullies',
b209bacacec3: 'Bully fight: first move',
fbe0f0bea700: 'Bully fight: smaller cadet has a dagger',
a2368b0dc212: 'Blocked the bigger cadet: bullies’ fate',
f4b4944d3f3b: 'Beat the smaller cadet: bullies’ fate',
'0cfd06cf8939': 'After the fight: choose a direction',
'0a561d53ad19': 'An hour later: valley or tree',
'96b68612eaf4': 'Top of the tree (shadow check)',
'10bc7353b8eb': 'Descended quickly: a Red below',
c62f1f1a2b3a: 'Let the Red pass: two Greens in the clearing',
'07e24681e3b3': 'Descended slowly: two Greens in the clearing'
};
function labelFor(id, node) {
if (LABELS[id]) return LABELS[id];
const bits = [node.stage || 'Scene'];
if (node.tint) bits.push(node.tint);
return bits.join(' · ');
}
/** Scene catalogue from knowledge.json (and, for scenes only in the log, from the log). */
function buildScenes(knowledge, encounters) {
const scenes = {};
for (const [id, n] of Object.entries((knowledge && knowledge.nodes) || {})) {
scenes[id] = {
stage: n.stage || 'UNKNOWN',
tint: n.tint || null,
backdrop: n.backdrop || null,
label: labelFor(id, n),
options: Object.values(n.choices || {}).map((c) => ({ id: optionId(c.label || c.choiceNorm), text: c.label || c.choiceNorm }))
};
}
for (const e of encounters) {
if (!Array.isArray(e.choices) || !e.prose) continue;
const id = e.nodeId || sceneId(e.prose, e.choices);
if (scenes[id]) continue;
const stage = (e.decision && e.decision.stage) || e.stage || 'UNKNOWN';
scenes[id] = { stage, tint: null, backdrop: null, label: labelFor(id, { stage }), options: e.choices.map((t) => ({ id: optionId(t), text: t })) };
}
return scenes;
}
const CHOICE_EVENTS = new Set(['DECISION', 'MANUAL_DECISION', 'MANUAL_EDGE_PICK']);
/**
* Trajectories from the encounter log: group events by attempt, keep the decisions that were acted on,
* and close each attempt at its first BURNED / DRAGON_BONDED event. Repeated BURNED events with no
* decisions in between (the cooldown screen being re-detected) are ignored.
*/
function trajectoriesFromEncounters(encounters, scenes) {
const byAttempt = new Map();
for (const e of encounters) {
const key = e.account ? `${e.account}:${e.attempt}` : String(e.attempt);
if (!byAttempt.has(key)) byAttempt.set(key, []);
byAttempt.get(key).push(e);
}
const out = [];
for (const [key, events] of byAttempt) {
events.sort((a, b) => String(a.ts).localeCompare(String(b.ts)));
const steps = [];
let outcome = null;
for (const e of events) {
if (outcome) break;
if (CHOICE_EVENTS.has(e.event)) {
const d = e.decision || {};
const text = d.choiceText || e.choiceText;
if (d.action && d.action !== 'choose') continue; // paused, nothing happened
if (!text || !Array.isArray(e.choices)) continue;
const scene = e.nodeId || sceneId(e.prose || '', e.choices);
const manual = e.event !== 'DECISION';
steps.push({
scene,
stage: d.stage || (scenes[scene] && scenes[scene].stage) || 'UNKNOWN',
tint: (scenes[scene] && scenes[scene].tint) || null,
option: optionId(text),
optionText: text,
offered: e.choices.map(optionId),
rule: d.rule || null,
decisionMode: manual ? 'manual' : d.source || null,
manual,
at: e.ts
});
} else if (e.event === 'BURNED' && steps.length) {
outcome = { type: 'BURNED', colour: null, intentional: Boolean(e.intentional) };
} else if (e.event === 'DRAGON_BONDED' && steps.length) {
outcome = { type: 'BONDED', colour: e.dragon ? String(e.dragon).toLowerCase() : null, intentional: false };
}
}
// A decision repeated on the same scene (the bot re-detecting the screen) counts once.
const dedup = steps.filter((s, i) => !(i > 0 && steps[i - 1].scene === s.scene && steps[i - 1].option === s.option));
for (let i = 0; i < dedup.length; i++) dedup[i].to = i + 1 < dedup.length ? dedup[i + 1].scene : null;
if (!dedup.length) continue;
const last = dedup[dedup.length - 1];
if (outcome) last.to = outcome.type === 'BURNED' ? 'BURNED' : `BONDED:${outcome.colour || 'unknown'}`;
out.push({
id: `bot-${sha1(`${key}:${dedup[0].at}`, 10)}`,
source: 'bot',
meta: { attempt: Number(String(key).split(':').pop()) || null, startedAt: dedup[0].at, accountKey: null, target: null, decisionMode: null },
steps: dedup,
outcome: outcome || { type: 'ABORTED', colour: null, intentional: false }
});
}
return out;
}
/** Fill run-level metadata (target colour, mode) from runs.jsonl where an attempt is covered by both. */
function attachRunMeta(trajs, runs, salt) {
for (const r of runs) {
const t = trajs.find((x) => x.meta.attempt === r.attempt && Math.abs(Date.parse(x.meta.startedAt) - Date.parse(r.startedAt)) < 6 * 3600e3);
if (!t) continue;
t.meta.target = r.targetDragon || null;
t.meta.decisionMode = r.decisionMode || null;
t.meta.policyId = r.policyId || null;
t.meta.experimentId = r.experimentId || null;
t.meta.arm = r.arm || null;
if (r.accountKey) t.meta.accountKey = r.accountKey;
else if (r.accountId && salt) t.meta.accountKey = sha1(`${salt}:${r.accountId}`, 10);
if (r.outcome === 'BONDED' && r.dragonColor) t.outcome = { type: 'BONDED', colour: String(r.dragonColor).toLowerCase(), intentional: false };
if (r.intentionalBurn) t.outcome.intentional = true;
}
return trajs;
}
function importBot({ dataDir, logsDir, salt = process.env.DK_ACCOUNT_SALT || null }) {
const knowledgeFile = path.join(dataDir, 'knowledge.json');
const knowledge = fs.existsSync(knowledgeFile) ? JSON.parse(fs.readFileSync(knowledgeFile, 'utf8')) : { nodes: {} };
const encounters = readJsonl(path.join(logsDir, 'encounters.jsonl'));
const runs = readJsonl(path.join(dataDir, 'runs.jsonl'));
const scenes = buildScenes(knowledge, encounters);
const trajectories = attachRunMeta(trajectoriesFromEncounters(encounters, scenes), runs, salt);
const unlabeled = Object.entries(scenes).filter(([id]) => !LABELS[id]).map(([id, s]) => `${id} (${s.stage}${s.tint ? `, ${s.tint}` : ''})`);
return { scenes, trajectories, unlabeled };
}
/** Guard: nothing that looks like game prose or an account identifier may leave this module. */
function assertPublicSafe(obj) {
const banned = ['prose', 'proseNorm', 'proseExcerpt', 'email', 'password', 'accountId', 'accountName'];
const walk = (v, p) => {
if (Array.isArray(v)) return v.forEach((x, i) => walk(x, `${p}[${i}]`));
if (v && typeof v === 'object') {
for (const [k, x] of Object.entries(v)) {
if (banned.includes(k)) throw new Error(`refusing to export field "${k}" at ${p}`);
walk(x, `${p}.${k}`);
}
} else if (typeof v === 'string' && v.length > 200) {
throw new Error(`refusing to export a ${v.length}-character string at ${p} (looks like prose)`);
}
};
walk(obj, '$');
}
module.exports = { normalize, sceneId, optionId, buildScenes, trajectoriesFromEncounters, attachRunMeta, importBot, assertPublicSafe, LABELS };