Files
threshing-atlas/tests/core/advisor.test.js
T
ekcSchianoandClaude Fable 5.2 f8e4f575f4
CI / test (pull_request) Canceled after 0s
Screen tree, register rows in the master list, and bot runs coded from the picks that were played
- analysis/tree/build_tree.py -> data/public/tree.json, analysis/reports/TREE.md: every screen the bot has seen,
  every option, where each led (fixed or dice, with ranges), best known picks per colour chosen cautiously, where
  any picks end, and the untested options. solver.js screenPick(): least-tried and Thompson picks from the tree.
- The bot logs a pick it then overrode as a step: 37 runs were coded with choices they never made. Importer and
  graph.js now use the played steps only.
- runs_clean.csv: the Unofficial Dragon Register's 288 bonded dragons are in the master list as outcome-only
  rows, flagged (no analysis uses them).
- Cycle, refresh and the bot kit rebuild and carry the tree; tests check every generated report is current.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
2026-11-16 15:50:12 +00:00

255 lines
16 KiB
JavaScript

// The route advisor (core/solver.js): it must give exactly the Python route model's answers, and walk a run in the
// game's order. node --test tests/core/advisor.test.js
const test = require("node:test");
const assert = require("node:assert");
const path = require("node:path");
const fs = require("node:fs");
const os = require("node:os");
const { execFileSync } = require("node:child_process");
const S = require("../../core/solver");
const ROOT = path.join(__dirname, "../..");
const published = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/policy.json"), "utf8"));
// a synthetic world with known answers, built by the Python model (tests/fixtures/synthetic_policy.py)
const tmp = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "atlas-")), "synthetic.json");
execFileSync("python3", [path.join(ROOT, "tests/fixtures/synthetic_policy.py"), tmp], { stdio: "inherit" });
const policy = JSON.parse(fs.readFileSync(tmp, "utf8"));
const choice = (X, h, obj = "thisAttempt") => S.advisor({ model: policy.model }, X, obj).next(h).choice;
test("the advisor gives exactly the Python model's values and choices, for every colour and goal", () => {
let checked = 0;
for (const smp of Object.values(published.samples)) {
for (const [X, t] of Object.entries(smp.targets)) {
for (const obj of ["thisAttempt", "eventually"]) {
const adv = S.advisor(smp, X, obj);
assert.ok(Math.abs(adv.value.x - t[obj].pX.point) < 1e-12, `${X} ${obj}: ${adv.value.x} vs ${t[obj].pX.point}`);
// follow the published tree: every decision on the route must match, option by option
let n = t[obj].tree, h = {};
while (n && n.type !== "end") {
if (n.type === "decision") {
const s = adv.step(h);
assert.strictEqual(s.var, n.var); assert.strictEqual(s.choice, n.choice, `${X} ${obj} ${n.var}`);
for (const [o, v] of Object.entries(n.options)) assert.ok(Math.abs(s.options[o].pX - v.pX) < 1e-12, `${X} ${obj} ${n.var}=${o}`);
checked++;
if (n.var === "encounter") break;
h[n.var] = n.choice; n = n.next;
} else {
const [k, b] = Object.entries(n.branches).filter(([k]) => k !== "die").sort((a, c) => c[1].p - a[1].p)[0];
if (!n.var.startsWith("die")) h[n.var] = k;
n = b.next;
}
}
}
}
}
assert.ok(checked > 100, checked);
});
test("synthetic world: Blue goes to the valley, passes the Red, attempts only the Blue", () => {
const h = { company: "Alone", direction: "Center", fs: "Red", react: "Hold still", fate: "Killed the bullies yourself", post: "Forest" };
assert.strictEqual(choice("Blue", h), "Valley");
assert.strictEqual(choice("Blue", { ...h, hour: "Valley" }), "Pass the Red (meadow)");
const meadow = { ...h, hour: "Valley", valley: "Pass the Red (meadow)" };
assert.strictEqual(choice("Blue", { ...meadow, meadow: "Blue" }), "Attempt the bond");
assert.strictEqual(choice("Blue", { ...meadow, meadow: "Orange" }), "Die on purpose");
assert.strictEqual(choice("Red", { ...h, hour: "Valley" }), "Face the Red");
assert.strictEqual(choice("Black", h), "Tree");
});
test("routing to one encounter: Blue only at the meadow equals the Blue policy; only at the tree Greens gives 0", () => {
const smp = { model: policy.model };
const meadow = S.advisor(smp, "Blue", "thisAttempt", { only: ["meadow-Blue"] });
const all = S.advisor(smp, "Blue", "thisAttempt");
assert.ok(Math.abs(meadow.value.x - all.value.x) < 1e-9);
assert.ok(S.advisor(smp, "Blue", "thisAttempt", { only: ["tree-Greens"] }).value.x < 1e-9);
assert.ok(S.advisor(smp, "Brown", "thisAttempt", { only: ["tree-Greens"] }).value.x > 0.05); // the Greens rescue
assert.ok(meadow.attempts("meadow-Blue") > 100);
});
test("a run is walked in the game's order", () => {
const adv = S.advisor(published.samples.all, "Blue", "thisAttempt");
assert.strictEqual(adv.step({}).var, "company");
assert.strictEqual(adv.step({ company: "1 Squadmate" }).var, "leave");
assert.strictEqual(adv.step({ company: "1 Squadmate", leave: "Stay together" }).var, "direction");
assert.strictEqual(adv.step({ company: "Alone", direction: "Center" }).kind, "chance"); // the first dragon is a roll
const seen = { company: "Alone", direction: "Center", fs: "Red" };
assert.strictEqual(adv.step(seen).var, "react");
const base = { ...seen, react: "Hold still" };
const scream = adv.step(base);
assert.strictEqual(scream.var, "scream");
assert.strictEqual(scream.risk.label, "Killed at the first sighting");
const help = { ...base, scream: "Investigate the scream", clearing: "Help your squadmate" };
assert.strictEqual(adv.step({ ...base, scream: "Investigate the scream" }).var, "clearing");
assert.strictEqual(adv.step(help).var, "fight");
assert.strictEqual(adv.step({ ...help, fight: "Pause to assess" }).var, "dagger");
assert.strictEqual(adv.step({ ...help, fight: "Charge the smaller cadet" }).var, "fate"); // charging skips the dagger
const fate = adv.step({ ...help, fight: "Pause to assess", dagger: "Engage the smaller cadet" });
assert.strictEqual(fate.var, "fate");
assert.strictEqual(fate.risk.label, "Killed in the fight");
assert.strictEqual(adv.step({ ...base, scream: "Keep walking" }).var, "wounded");
assert.strictEqual(adv.step({ ...base, scream: "Investigate the scream", clearing: "Walk away" }).var, "hour");
// an older record that only knows the bullies' fate still lands in the right place (the skipped steps are
// assumed to be the recommended ones)
assert.strictEqual(adv.next({ ...base, fate: "Killed the bullies yourself" }).decision, "post");
});
test("recommendOption points the bot at the recommended option on its screen", () => {
const adv = S.advisor(published.samples.all, "Blue", "thisAttempt");
const h = { company: "Alone", direction: "Center", fs: "Red", react: "Hold still", scream: "Keep walking", wounded: "Tourniquet" };
const rec = S.recommendOption(adv, h, ["Continue toward the center of the valley", "Find the tallest tree and climb for a better view"]);
assert.strictEqual(rec.decision, "hour");
assert.strictEqual(rec.index, rec.choice === "Valley" ? 0 : 1);
assert.strictEqual(S.recommendOption(adv, {}, ["Something we have never seen"]).index, null);
});
test("solverHistory turns form variables into a history keyed by the game's steps", () => {
S.useGame(published.samples.all);
const h = S.solverHistory({ company: "Alone", direction: "Center", first_sighting: "Red", scream: "Ignore the scream", resolution: "Tourniquet", hour: "Tree", branch: "Descend slowly: two Greens" });
assert.deepStrictEqual(h, { company: "Alone", direction: "Center", fs: "Red", scream: "Keep walking", wounded: "Tourniquet", hour: "Tree", descend: "Slowly (two Greens)", shadow: "no" });
const l = S.solverHistory({ company: "1 Squadmate", left_squadmates: true, branch: "Descend quickly: let the Red pass", hour: "Tree" });
assert.strictEqual(l.leave, "Leave them");
assert.strictEqual(l.red, "Let it pass (two Greens)");
});
test("Thompson sampling: one posterior draw per seed, with the chance of each pick", () => {
const S = require("../../core/solver.js");
const P = require("../../data/public/policy.json");
const best = S.advisor(P.samples.all, "Blue", "eventually").next({});
assert.strictEqual(best.propensity, 1);
const a = S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: 3, draws: 8 });
const b = S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: 3, draws: 8 });
const ra = a.next({}), rb = b.next({});
assert.strictEqual(ra.choice, rb.choice); // same seed, same draw
assert.ok(ra.propensity >= 0 && ra.propensity <= 1);
const picks = new Set();
for (let s = 1; s <= 12; s++) picks.add(S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: s, draws: 1 }).next({}).choice);
assert.ok(picks.size > 1, "different seeds explore different options");
});
test("a first attempt has its own odds, exactly as the Python model publishes them", () => {
for (const smp of Object.values(published.samples)) {
for (const [X, t] of Object.entries(smp.targets)) {
for (const obj of ["thisAttempt", "eventually"]) {
const adv = S.advisor(smp, X, obj, { attempt: 1 });
assert.deepStrictEqual(adv.given, { attempt: "First attempt" });
assert.ok(Math.abs(adv.value.x - t.firstAttempt[obj].pX.point) < 1e-12, `${X} ${obj}`);
assert.strictEqual(adv.next({}).choice, t.firstAttempt[obj].tree.choice);
}
}
}
const all = published.samples.all;
const later = S.advisor(all, "Black", "thisAttempt", { attempt: 7 }), dflt = S.advisor(all, "Black", "thisAttempt");
assert.strictEqual(later.value.x, dflt.value.x); // no attempt given: a later attempt
assert.ok(S.advisor(all, "Black", "thisAttempt", { attempt: 1 }).value.x > 5 * later.value.x);
});
test("fix holds decisions to an experiment arm and solves the rest around it", () => {
const all = published.samples.all;
const fix = { direction: "East", scream: "Investigate the scream", clearing: "Help your squadmate", hour: "Tree" };
const adv = S.advisor(all, "Black", "eventually", { attempt: 1, fix });
assert.strictEqual(adv.step({ company: "Alone" }).choice, "East");
assert.deepStrictEqual(Object.keys(adv.step({ company: "Alone" }).options), ["East"]);
assert.ok(adv.value.x <= S.advisor(all, "Black", "eventually", { attempt: 1 }).value.x + 1e-12);
assert.throws(() => S.advisor(all, "Black", "eventually", { fix: { direction: "North" } }), /no option/);
});
test("mapping: uniform sampling picks among the options on the screen, with its propensity", () => {
const all = published.samples.all;
const screen = ["Enter the valley alone", "Enter the valley with one of your squadmates", "Enter the valley with two of your squadmates"];
const picks = new Set();
for (let seed = 1; seed <= 30; seed++) {
const rec = S.recommendOption(S.advisor(all, "Blue", "eventually", { sampling: "uniform", seed }), {}, screen);
assert.ok(Math.abs(rec.propensity - 1 / 3) < 1e-12);
assert.match(S.norm(screen[rec.index]), new RegExp(all.model.game.bot_text.company[rec.choice]));
picks.add(rec.choice);
}
assert.strictEqual(picks.size, 3);
const a = S.advisor(all, "Blue", "eventually", { sampling: "uniform", seed: 4 });
assert.strictEqual(S.recommendOption(a, {}, screen).index, S.recommendOption(a, {}, screen).index); // same seed, same pick
});
test("the experiment plan: what each attempt of each account is for", () => {
const plan = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/experiments.json"), "utf8"));
const all = published.samples.all;
// attempt 1: every new account plays the same exact path (the replication)
for (let acc = 0; acc < 30; acc++) {
const a = S.assign(plan, { accountSeq: acc, attempt: 1 });
assert.strictEqual(a.experimentId, "fresh-replicate");
assert.strictEqual(a.target, "Black");
assert.strictEqual(a.fix.direction, "West");
assert.deepStrictEqual(a, S.assign(plan, { accountSeq: acc, attempt: 1 })); // the same every time
}
const a1 = S.assign(plan, { accountSeq: 0, attempt: 1 });
const adv = S.advisor(all, a1.target, a1.objective, { attempt: 1, fix: a1.fix });
let h = {};
for (let i = 0; i < 20; i++) {
const s = adv.step(h);
if (s.kind === "decision") { if (s.var === "encounter") break; if (a1.fix[s.var]) assert.strictEqual(s.choice, a1.fix[s.var]); h[s.var] = s.choice; }
else if (s.kind === "chance") h[s.var] = s.var === "shadow" ? "no" : Object.keys(s.probs).find((k) => k !== "die");
else break;
}
assert.strictEqual(h.hour, "Tree");
// the ablation waits for the replication, then gives every arm the same number of accounts
const stage2 = { ...plan, experiments: plan.experiments.map((e) => (e.id === "fresh-replicate" ? { ...e, status: "complete" } : e.id === "fresh-ablate" ? { ...e, status: "running" } : e)) };
const count = {};
for (let acc = 0; acc < 12 * 5; acc++) { const a = S.assign(stage2, { accountSeq: acc, attempt: 1 }); assert.strictEqual(a.experimentId, "fresh-ablate"); count[a.arm] = (count[a.arm] || 0) + 1; }
assert.deepStrictEqual(Object.values(count), Array(12).fill(5));
// with squadmates, staying and the direction are one screen
const squad = stage2.experiments.find((e) => e.id === "fresh-ablate").arms.find((a) => a.id === "with two squadmates");
const rec = S.recommendOption(S.advisor(all, "Black", "eventually", { attempt: 1, fix: squad.fix }), { company: "2 Squadmates" },
["Go east with your squad", "Leave your squadmates", "Convince them to head west instead"]);
assert.strictEqual(rec.index, 2);
assert.deepStrictEqual(rec.also, { direction: "West" });
// attempt 2: an account that saw the shadow tries the same path again; half of the others do too
const who = (attempt, facts) => Array.from({ length: 40 }, (_, acc) => (S.assign(plan, { accountSeq: acc, attempt, facts }) || {}).experimentId);
assert.ok(who(2, { sawShadowBefore: true }).every((id) => id === "shadow-again"));
assert.strictEqual(who(2, { sawShadowBefore: false }).filter((id) => id === "second-attempt-same-path").length, 20);
// later: two accounts in ten are the long-lived ones, the rest share the research experiments
const later = who(6, { sawShadowBefore: false });
assert.strictEqual(later.filter((id) => id === "veteran-repeat").length, 4);
assert.strictEqual(later.filter((id) => id === "veteran-variety").length, 4);
assert.deepStrictEqual([...new Set(later)].sort(), ["mapping", "meadow-blue-route", "sentinel", "veteran-repeat", "veteran-variety"]);
// variety: a different combination on every attempt, both gates kept, every one playable
const combos = new Set();
for (let t = 3; t < 3 + 60; t++) {
const a = S.assign(plan, { accountSeq: 11, attempt: t, facts: { sawShadowBefore: false } });
assert.strictEqual(a.experimentId, "veteran-variety");
assert.strictEqual(a.fix.clearing, "Help your squadmate");
combos.add(a.arm);
if (t < 8) S.advisor(all, a.target, a.objective, { attempt: t, fix: a.fix });
}
assert.strictEqual(combos.size, 60);
// an experiment that is not running is not assigned; its share goes to the others
const done = { ...plan, experiments: plan.experiments.map((e) => (e.id === "meadow-blue-route" ? { ...e, status: "complete" } : e)) };
assert.notStrictEqual(S.assign(done, { accountSeq: 5, attempt: 5 }).experimentId, "meadow-blue-route");
assert.strictEqual(S.assign({ ...plan, experiments: [] }, { accountSeq: 1, attempt: 5 }), null);
});
test("screenPick: least-tried fills gaps, Thompson avoids what has killed", () => {
const tree = { screens: { s1: { options: {
a: { text: "Hold its gaze", n: 40, to: { next: { n: 38 }, DIED: { n: 2 } } },
b: { text: "Lower your eyes", n: 40, to: { DIED: { n: 40 } } },
c: { text: "Step back", n: 0, to: {} } } } } };
const screen = ["Lower your eyes", "Hold its gaze", "Step back", "An option the tree has never seen"];
const lt = S.screenPick(tree, "s1", screen, { mode: "least-tried", seed: 1 });
assert.ok([2, 3].includes(lt.index) && lt.n === 0);
assert.deepStrictEqual(lt, S.screenPick(tree, "s1", screen, { mode: "least-tried", seed: 1 })); // same seed, same pick
const picks = {};
for (let seed = 1; seed <= 300; seed++) { const p = S.screenPick(tree, "s1", screen, { mode: "thompson", seed }); picks[p.index] = (picks[p.index] || 0) + 1; }
assert.ok(!picks[0] || picks[0] < 3, "an option that died 40 times in 40 is almost never drawn");
assert.ok(picks[1] > 200 && picks[2] >= 5 && picks[3] >= 5, JSON.stringify(picks)); // mostly the good one; the untested ones still get tried
assert.strictEqual(S.screenPick(tree, "unknown-screen", ["x", "y"], { seed: 2 }).n, 0);
assert.strictEqual(S.screenPick(tree, "s1", []), null);
});
test("picks the bot logged and then overrode are not part of the run", () => {
const G = require("../../core/graph.js");
const steps = [
{ scene: "a68f681076c6", optionText: "Head west toward the scent", to: "BURNED" },
{ scene: "a68f681076c6", optionText: "Head toward the center of the forest", to: "bc4ee4754b9c" },
{ scene: "bc4ee4754b9c", optionText: "Lower your eyes in respect", to: "84780dab01e3", tint: "red" }];
assert.strictEqual(G.playedSteps(steps).length, 2);
assert.strictEqual(G.trajectoryVariables({ steps, outcome: {} }).vars.direction, "Center");
assert.strictEqual(G.playedSteps([{ scene: "x", optionText: "only" }]).length, 1); // a run in progress
});