Refactor: lay the repo out by section (core, collection, analysis, planner, site) (#8)
Moves only; no behaviour changes. Every file now lives under the part of the project it belongs to, each section has a README, and docs/PRIVACY.md collects the rules all parts follow. core/ engine.js, graph.js, solver.js, python/clean.py collection/ sources/ (form, register, PDF check, daily update), bot/ (importer) analysis/ stats/ (Python statistics), network/ (build-graph.js), reports/ (POLICY.md) planner/ build-policy.js, policy-report.js site/ atlas/ (the page), build.js, qa/ tests/ core/ (unit), parity/ (JS vs Python), fixtures/ Checked against the previous layout: tests pass; coded runs, test outputs, graph.json and policy.json are byte-identical; the built site differs only in comments that name file paths. The published file names Ken's bot downloads (solver.js, graph.js, data/policy.json) are unchanged. Also removes the undefined/ folder of QA screenshots committed by mistake (the QA scripts now default to a temp folder) and test.txt, and drops the unused npm workspaces. Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
dcd5117373
commit
432e04c2c7
@@ -0,0 +1,211 @@
|
||||
'use strict';
|
||||
/**
|
||||
* Import DragonKind bot data into anonymised trajectories and a scene catalogue.
|
||||
*
|
||||
* Inputs (all from the PRIVATE bot repo, read locally, never committed):
|
||||
* data/knowledge.json scene atlas: nodes (prose, choices, tint, backdrop) and edges
|
||||
* data/runs.jsonl one record per attempt (newer bot versions)
|
||||
* logs/encounters.jsonl every decision the bot made, with the options on screen
|
||||
*
|
||||
* Outputs (public, CC BY 4.0): no game prose, only fingerprints, short labels and choice text.
|
||||
* scenes.json { sceneId: { stage, tint, backdrop, label, options: [{ id, text }] } }
|
||||
* trajectories.jsonl one line per attempt: { id, source, meta, steps: [...], outcome }
|
||||
*/
|
||||
const crypto = require('node:crypto');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const sha1 = (s, n = 12) => crypto.createHash('sha1').update(String(s)).digest('hex').slice(0, n);
|
||||
|
||||
// Must match normalize() in the bot's src/strategy.js, so fingerprints line up.
|
||||
function normalize(text) {
|
||||
return String(text || '')
|
||||
.toLowerCase()
|
||||
.replace(/[‘’‛`']/g, "'")
|
||||
.replace(/[“”"]/g, '"')
|
||||
.replace(/[–—-]/g, ' ')
|
||||
.replace(/…/g, ' ')
|
||||
.replace(/[^a-z0-9'\s]/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
// Must match computeSceneFingerprint() in the bot's src/learner.js.
|
||||
function sceneId(prose, choices) {
|
||||
const proseNorm = normalize(prose).slice(0, 500);
|
||||
const sorted = choices.map(normalize).filter(Boolean).sort();
|
||||
return sha1(`${proseNorm} ### ${sorted.join(' || ')}`);
|
||||
}
|
||||
|
||||
const optionId = (text) => sha1(normalize(text), 10);
|
||||
|
||||
const readJsonl = (file) => {
|
||||
if (!file || !fs.existsSync(file)) return [];
|
||||
return fs.readFileSync(file, 'utf8').split(/\r?\n/).filter((l) => l.trim()).map((l) => {
|
||||
try { return JSON.parse(l); } catch { return null; }
|
||||
}).filter(Boolean);
|
||||
};
|
||||
|
||||
/**
|
||||
* Short public labels. Keys are scene fingerprints; values are our own paraphrases, not game prose.
|
||||
* Unknown scenes fall back to "<stage> · <tint>" and are listed by the importer so labels can be added.
|
||||
*/
|
||||
const LABELS = {
|
||||
db7786392626: 'Start: choose company',
|
||||
a68f681076c6: 'Valley edge: choose direction',
|
||||
bc4ee4754b9c: 'First sighting: Red (forest centre)',
|
||||
abe3011378ac: 'First sighting: Brown (forest centre)',
|
||||
'84780dab01e3': 'Survived the first sighting',
|
||||
d25f2759ace0: 'Deeper in the forest: a scream',
|
||||
f72199f24b91: 'Clearing: squadmate vs two bullies',
|
||||
b209bacacec3: 'Bully fight: first move',
|
||||
fbe0f0bea700: 'Bully fight: smaller cadet has a dagger',
|
||||
a2368b0dc212: 'Blocked the bigger cadet: bullies’ fate',
|
||||
f4b4944d3f3b: 'Beat the smaller cadet: bullies’ fate',
|
||||
'0cfd06cf8939': 'After the fight: choose a direction',
|
||||
'0a561d53ad19': 'An hour later: valley or tree',
|
||||
'96b68612eaf4': 'Top of the tree (shadow check)',
|
||||
'10bc7353b8eb': 'Descended quickly: a Red below',
|
||||
c62f1f1a2b3a: 'Let the Red pass: two Greens in the clearing',
|
||||
'07e24681e3b3': 'Descended slowly: two Greens in the clearing'
|
||||
};
|
||||
|
||||
function labelFor(id, node) {
|
||||
if (LABELS[id]) return LABELS[id];
|
||||
const bits = [node.stage || 'Scene'];
|
||||
if (node.tint) bits.push(node.tint);
|
||||
return bits.join(' · ');
|
||||
}
|
||||
|
||||
/** Scene catalogue from knowledge.json (and, for scenes only in the log, from the log). */
|
||||
function buildScenes(knowledge, encounters) {
|
||||
const scenes = {};
|
||||
for (const [id, n] of Object.entries((knowledge && knowledge.nodes) || {})) {
|
||||
scenes[id] = {
|
||||
stage: n.stage || 'UNKNOWN',
|
||||
tint: n.tint || null,
|
||||
backdrop: n.backdrop || null,
|
||||
label: labelFor(id, n),
|
||||
options: Object.values(n.choices || {}).map((c) => ({ id: optionId(c.label || c.choiceNorm), text: c.label || c.choiceNorm }))
|
||||
};
|
||||
}
|
||||
for (const e of encounters) {
|
||||
if (!Array.isArray(e.choices) || !e.prose) continue;
|
||||
const id = e.nodeId || sceneId(e.prose, e.choices);
|
||||
if (scenes[id]) continue;
|
||||
const stage = (e.decision && e.decision.stage) || e.stage || 'UNKNOWN';
|
||||
scenes[id] = { stage, tint: null, backdrop: null, label: labelFor(id, { stage }), options: e.choices.map((t) => ({ id: optionId(t), text: t })) };
|
||||
}
|
||||
return scenes;
|
||||
}
|
||||
|
||||
const CHOICE_EVENTS = new Set(['DECISION', 'MANUAL_DECISION', 'MANUAL_EDGE_PICK']);
|
||||
|
||||
/**
|
||||
* Trajectories from the encounter log: group events by attempt, keep the decisions that were acted on,
|
||||
* and close each attempt at its first BURNED / DRAGON_BONDED event. Repeated BURNED events with no
|
||||
* decisions in between (the cooldown screen being re-detected) are ignored.
|
||||
*/
|
||||
function trajectoriesFromEncounters(encounters, scenes) {
|
||||
const byAttempt = new Map();
|
||||
for (const e of encounters) {
|
||||
const key = e.account ? `${e.account}:${e.attempt}` : String(e.attempt);
|
||||
if (!byAttempt.has(key)) byAttempt.set(key, []);
|
||||
byAttempt.get(key).push(e);
|
||||
}
|
||||
const out = [];
|
||||
for (const [key, events] of byAttempt) {
|
||||
events.sort((a, b) => String(a.ts).localeCompare(String(b.ts)));
|
||||
const steps = [];
|
||||
let outcome = null;
|
||||
for (const e of events) {
|
||||
if (outcome) break;
|
||||
if (CHOICE_EVENTS.has(e.event)) {
|
||||
const d = e.decision || {};
|
||||
const text = d.choiceText || e.choiceText;
|
||||
if (d.action && d.action !== 'choose') continue; // paused, nothing happened
|
||||
if (!text || !Array.isArray(e.choices)) continue;
|
||||
const scene = e.nodeId || sceneId(e.prose || '', e.choices);
|
||||
const manual = e.event !== 'DECISION';
|
||||
steps.push({
|
||||
scene,
|
||||
stage: d.stage || (scenes[scene] && scenes[scene].stage) || 'UNKNOWN',
|
||||
tint: (scenes[scene] && scenes[scene].tint) || null,
|
||||
option: optionId(text),
|
||||
optionText: text,
|
||||
offered: e.choices.map(optionId),
|
||||
rule: d.rule || null,
|
||||
decisionMode: manual ? 'manual' : d.source || null,
|
||||
manual,
|
||||
at: e.ts
|
||||
});
|
||||
} else if (e.event === 'BURNED' && steps.length) {
|
||||
outcome = { type: 'BURNED', colour: null, intentional: Boolean(e.intentional) };
|
||||
} else if (e.event === 'DRAGON_BONDED' && steps.length) {
|
||||
outcome = { type: 'BONDED', colour: e.dragon ? String(e.dragon).toLowerCase() : null, intentional: false };
|
||||
}
|
||||
}
|
||||
// A decision repeated on the same scene (the bot re-detecting the screen) counts once.
|
||||
const dedup = steps.filter((s, i) => !(i > 0 && steps[i - 1].scene === s.scene && steps[i - 1].option === s.option));
|
||||
for (let i = 0; i < dedup.length; i++) dedup[i].to = i + 1 < dedup.length ? dedup[i + 1].scene : null;
|
||||
if (!dedup.length) continue;
|
||||
const last = dedup[dedup.length - 1];
|
||||
if (outcome) last.to = outcome.type === 'BURNED' ? 'BURNED' : `BONDED:${outcome.colour || 'unknown'}`;
|
||||
out.push({
|
||||
id: `bot-${sha1(`${key}:${dedup[0].at}`, 10)}`,
|
||||
source: 'bot',
|
||||
meta: { attempt: Number(String(key).split(':').pop()) || null, startedAt: dedup[0].at, accountKey: null, target: null, decisionMode: null },
|
||||
steps: dedup,
|
||||
outcome: outcome || { type: 'ABORTED', colour: null, intentional: false }
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Fill run-level metadata (target colour, mode) from runs.jsonl where an attempt is covered by both. */
|
||||
function attachRunMeta(trajs, runs, salt) {
|
||||
for (const r of runs) {
|
||||
const t = trajs.find((x) => x.meta.attempt === r.attempt && Math.abs(Date.parse(x.meta.startedAt) - Date.parse(r.startedAt)) < 6 * 3600e3);
|
||||
if (!t) continue;
|
||||
t.meta.target = r.targetDragon || null;
|
||||
t.meta.decisionMode = r.decisionMode || null;
|
||||
t.meta.policyId = r.policyId || null;
|
||||
t.meta.experimentId = r.experimentId || null;
|
||||
t.meta.arm = r.arm || null;
|
||||
if (r.accountKey) t.meta.accountKey = r.accountKey;
|
||||
else if (r.accountId && salt) t.meta.accountKey = sha1(`${salt}:${r.accountId}`, 10);
|
||||
if (r.outcome === 'BONDED' && r.dragonColor) t.outcome = { type: 'BONDED', colour: String(r.dragonColor).toLowerCase(), intentional: false };
|
||||
if (r.intentionalBurn) t.outcome.intentional = true;
|
||||
}
|
||||
return trajs;
|
||||
}
|
||||
|
||||
function importBot({ dataDir, logsDir, salt = process.env.DK_ACCOUNT_SALT || null }) {
|
||||
const knowledgeFile = path.join(dataDir, 'knowledge.json');
|
||||
const knowledge = fs.existsSync(knowledgeFile) ? JSON.parse(fs.readFileSync(knowledgeFile, 'utf8')) : { nodes: {} };
|
||||
const encounters = readJsonl(path.join(logsDir, 'encounters.jsonl'));
|
||||
const runs = readJsonl(path.join(dataDir, 'runs.jsonl'));
|
||||
const scenes = buildScenes(knowledge, encounters);
|
||||
const trajectories = attachRunMeta(trajectoriesFromEncounters(encounters, scenes), runs, salt);
|
||||
const unlabeled = Object.entries(scenes).filter(([id]) => !LABELS[id]).map(([id, s]) => `${id} (${s.stage}${s.tint ? `, ${s.tint}` : ''})`);
|
||||
return { scenes, trajectories, unlabeled };
|
||||
}
|
||||
|
||||
/** Guard: nothing that looks like game prose or an account identifier may leave this module. */
|
||||
function assertPublicSafe(obj) {
|
||||
const banned = ['prose', 'proseNorm', 'proseExcerpt', 'email', 'password', 'accountId', 'accountName'];
|
||||
const walk = (v, p) => {
|
||||
if (Array.isArray(v)) return v.forEach((x, i) => walk(x, `${p}[${i}]`));
|
||||
if (v && typeof v === 'object') {
|
||||
for (const [k, x] of Object.entries(v)) {
|
||||
if (banned.includes(k)) throw new Error(`refusing to export field "${k}" at ${p}`);
|
||||
walk(x, `${p}.${k}`);
|
||||
}
|
||||
} else if (typeof v === 'string' && v.length > 200) {
|
||||
throw new Error(`refusing to export a ${v.length}-character string at ${p} (looks like prose)`);
|
||||
}
|
||||
};
|
||||
walk(obj, '$');
|
||||
}
|
||||
|
||||
module.exports = { normalize, sceneId, optionId, buildScenes, trajectoriesFromEncounters, attachRunMeta, importBot, assertPublicSafe, LABELS };
|
||||
Reference in New Issue
Block a user