Refactor: lay the repo out by section (core, collection, analysis, planner, site) (#8)

Moves only; no behaviour changes. Every file now lives under the part of the project it
belongs to, each section has a README, and docs/PRIVACY.md collects the rules all parts follow.

  core/        engine.js, graph.js, solver.js, python/clean.py
  collection/  sources/ (form, register, PDF check, daily update), bot/ (importer)
  analysis/    stats/ (Python statistics), network/ (build-graph.js), reports/ (POLICY.md)
  planner/     build-policy.js, policy-report.js
  site/        atlas/ (the page), build.js, qa/
  tests/       core/ (unit), parity/ (JS vs Python), fixtures/

Checked against the previous layout: tests pass; coded runs, test outputs, graph.json and
policy.json are byte-identical; the built site differs only in comments that name file paths.
The published file names Ken's bot downloads (solver.js, graph.js, data/policy.json) are unchanged.

Also removes the undefined/ folder of QA screenshots committed by mistake (the QA scripts now
default to a temp folder) and test.txt, and drops the unused npm workspaces.


Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ekcSchiano
2026-11-03 19:46:33 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent dcd5117373
commit 432e04c2c7
66 changed files with 201 additions and 2695 deletions
+15
View File
@@ -0,0 +1,15 @@
# planner
The route planner: the best route to every dragon, and what goes back to Ken's bot.
| File | What it does |
|---|---|
| `build-policy.js` | Solves the decision model (`core/solver.js`) for every colour and both goals ("this attempt" and "eventually"), with 200 posterior draws and a cross-fit check. Writes `data/public/policy.json`, which has a content id so each bot run can record which policy it followed. About 4 minutes. |
| `policy-report.js` | Writes the readable version, `analysis/reports/POLICY.md`. |
Run both with `npm run policy`. The site's valley map and The Odds read `policy.json`; the bot downloads it with
`solver.js` and `graph.js` (see `docs/BOT_INTEGRATION.md`).
**The loop with the bot.** The planner's routes go out to the bot; the bot's runs come back through
`collection/bot/`. Experiments the bot should run to fill gaps the community form cannot (for example the
answers at each bonding encounter) belong here; `experiments.json` is on the roadmap.
+39
View File
@@ -0,0 +1,39 @@
// Build data/public/policy.json: the best decisions for each dragon colour, under both objectives.
// node planner/build-policy.js [--draws 200]
const fs = require("fs");
const path = require("path");
const E = require("../core/engine.js");
const S = require("../core/solver.js");
const root = path.resolve(__dirname, "..");
const arg = (k, d) => { const i = process.argv.indexOf("--" + k); return i > 0 ? process.argv[i + 1] : d; };
const draws = +arg("draws", 200);
const raw = JSON.parse(fs.readFileSync(path.join(root, "data/public/form_responses.json"), "utf8"));
const { runs } = E.codeRows(raw.rows, raw.headers);
const isRare = (g) => ["Blue", "Black", "Blue or Black"].includes(g);
const samples = {
// every run
all: runs,
// runs least affected by selective reporting: bulk logs, plus form runs from riders not chasing a rare dragon
lowBias: runs.filter((r) => r.source !== "Form" || !isRare(r.goal)),
};
const t0 = Date.now();
const out = { schema: 1, source: { form_rows: raw.rows.length }, samples: {} };
for (const [name, rs] of Object.entries(samples)) {
out.samples[name] = S.buildPolicy(rs, { draws });
console.log(`${name}: ${rs.length} runs, ${((Date.now() - t0) / 1000).toFixed(1)} s`);
}
// a content id, so the bot can record which policy produced each run (no timestamps: the build is reproducible)
out.id = require("crypto").createHash("sha1").update(JSON.stringify(out.samples)).digest("hex").slice(0, 12);
fs.writeFileSync(path.join(root, "data/public/policy.json"), JSON.stringify(out));
console.log("policy id", out.id);
const pct = (p) => (100 * p).toFixed(p < 0.01 ? 2 : 1) + "%";
for (const [name, s] of Object.entries(out.samples)) {
console.log(`\n${name} alphas ${JSON.stringify(s.model.alphas)}`);
for (const [X, t] of Object.entries(s.targets)) {
const a = t.thisAttempt, b = t.eventually;
console.log(` ${X.padEnd(6)} this attempt ${pct(a.pX.mean)} [${pct(a.pX.lo)}, ${pct(a.pX.hi)}] other bond ${pct(a.pOther.mean)}` +
` | eventually: ratio ${pct(b.ratio.mean)}, P(X) ${pct(b.pX.mean)}, ~${b.expectedAttempts.mean.toFixed(0)} attempts` +
` | cross-fit ${pct(t.crossFit.inSample)} -> held-out ${pct(t.crossFit.heldOut)}`);
}
}
+62
View File
@@ -0,0 +1,62 @@
// Write analysis/reports/POLICY.md: a readable summary of data/public/policy.json.
// node planner/policy-report.js
const fs = require("fs");
const path = require("path");
const root = path.resolve(__dirname, "..");
const P = JSON.parse(fs.readFileSync(path.join(root, "data/public/policy.json"), "utf8"));
const pct = (p) => (p == null ? "–" : (100 * p).toFixed(p < 0.1 ? 1 : 0) + "%");
const conf = (n) => (n.stillBest == null ? "" : ` (still best in ${pct(n.stillBest)} of posterior draws)`);
function walk(n, out, indent = "") {
if (!n) return;
if (n.type === "decision") {
const alts = Object.entries(n.options).sort((a, b) => b[1].pX - a[1].pX)
.map(([k, v]) => `${k} ${pct(v.pX)}`).join(", ");
if (n.tie) out.push(`${indent}- ${n.label}: any (no difference for this colour)`);
else out.push(`${indent}- **${n.label}: ${n.choice}**${conf(n)}${n.stillBest != null && n.stillBest < 0.75 ? " — no clear winner" : ""}. If chosen, then best play after: ${alts}`);
walk(n.next, out, indent);
} else if (n.type === "chance") {
const br = Object.entries(n.branches).sort((a, b) => b[1].p - a[1].p);
const live = br.filter(([k]) => !["die"].includes(k));
if (br.some(([k]) => k === "die")) {
const d = br.find(([k]) => k === "die");
out.push(`${indent}- ${n.label}: ${pct(1 - d[1].p)} survive`);
for (const [, b] of live) walk(b.next, out, indent);
return;
}
if (live.length === 1) return walk(live[0][1].next, out, indent);
out.push(`${indent}- ${n.label}: ${br.map(([k, b]) => `${k} ${pct(b.p)}`).join(", ")}`);
for (const [k, b] of br) {
out.push(`${indent} - if **${k}**:`);
walk(b.next, out, indent + " ");
}
} else if (n.outcomes) {
out.push(`${indent}- Outcome if you attempt: ${Object.entries(n.outcomes).sort((a, b) => b[1] - a[1]).map(([k, p]) => `${k === "burn" ? "incinerated" : "bond " + k} ${pct(p)}`).join(", ")}`);
} else out.push(`${indent}- ${n.what}`);
}
const s = P.samples.all, lb = P.samples.lowBias;
const L = [];
L.push("# Best routes by dragon colour", "");
L.push(`Generated from ${s.data.runs} community runs by \`planner/build-policy.js\`. Machine-readable version: \`data/public/policy.json\`.`, "");
L.push("**How to read this.** \"This attempt\" maximises the chance of bonding the colour on one run. \"Eventually\" assumes you will die on purpose and try again whenever the run goes wrong, so it avoids any choice that can end the account on a different dragon. Percentages are posterior means with 95% credible intervals. \"Held-out\" is the same policy chosen on half of the runs and scored on the other half: when it is much lower than the in-sample number, the route's advantage is mostly noise.", "");
L.push("**Caveats.** The form records choices only at the level of company, direction, first dragon, cadet encounter, fork, tree or valley, and which dragon you met. Choices inside each encounter are not recorded, so the bond chance at an encounter is the community's average with whatever answers people used. Riders chasing a rare dragon report sightings more often than riders who log every run; the \"low-bias\" columns use only bulk logs and non-chasers.", "");
L.push("| Colour | This attempt | Low-bias sample | Held-out | Eventually: chance the account ends on this colour | Attempts needed (about) |");
L.push("|---|---|---|---|---|---|");
for (const X of Object.keys(s.targets)) {
const t = s.targets[X], a = t.thisAttempt, e = t.eventually, l = lb.targets[X].thisAttempt;
L.push(`| ${X} | ${pct(a.pX.mean)} (${pct(a.pX.lo)}–${pct(a.pX.hi)}) | ${pct(l.pX.mean)} (${pct(l.pX.lo)}–${pct(l.pX.hi)}) | ${pct(t.crossFit.heldOut)} | ${pct(e.ratio.mean)} | ${Math.round(e.expectedAttempts.mean)} |`);
}
L.push("", "## How much do earlier choices matter?", "");
L.push("Each chance event can depend on the choices before it. The model learns how much from the data (empirical-Bayes concentration; larger means the history matters less):", "");
L.push("| Chance event | Depends on (most important first) | Concentration |", "|---|---|---|");
for (const [k, ctx] of Object.entries(s.model.chanceContext)) L.push(`| ${k} | ${ctx.join(", ") || "nothing"} | ${s.model.alphas[k]} |`);
for (const X of Object.keys(s.targets)) {
for (const [obj, title] of [["thisAttempt", "this attempt"], ["eventually", "eventually"]]) {
const out = [];
walk(s.targets[X][obj].tree, out);
L.push("", `## ${X}: ${title}`, "", ...out);
}
}
fs.writeFileSync(path.join(root, "analysis/reports/POLICY.md"), L.join("\n") + "\n");
console.log("wrote analysis/reports/POLICY.md", L.length, "lines");