Files
threshing-atlas/tests/core/advisor.test.js
T
ekcSchianoandClaude Fable 5.2 ab3664f369 Route model: first attempts are their own shadow roll; experiment plan and automated bot cycle
- game.yaml: `givens` (first or later attempt) and `distinct` levels that are not pooled; the shadow roll
  depends on the attempt first. Form submissions show the shadow on 9% of first attempts and 0.3% of later ones.
- Reporting bias: a roll whose bias cannot be measured can borrow another roll's (`borrow: {shadow: meadow}`),
  so form submissions inform the shadow again. An assumption, stated in planner/policy.yaml.
- policy.json: targets.<colour>.firstAttempt (this attempt, eventually). Black: about 1.4% on a first attempt,
  0.07% on a later one.
- solver.js: advisor options `attempt`, `fix`, sampling "uniform"; assign() and armFor() for the plan.
- planner/experiments.yaml + build_experiments.py -> data/public/experiments.json (progress and status per
  experiment, rebuilt from the bot's runs).
- python -m collection cycle: import bot runs, refit, re-plan, deliver the kit; .gitea/workflows/bot-cycle.yml.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
2026-11-12 03:31:33 +00:00

197 lines
12 KiB
JavaScript

// The route advisor (core/solver.js): it must give exactly the Python route model's answers, and walk a run in the
// game's order. node --test tests/core/advisor.test.js
const test = require("node:test");
const assert = require("node:assert");
const path = require("node:path");
const fs = require("node:fs");
const os = require("node:os");
const { execFileSync } = require("node:child_process");
const S = require("../../core/solver");
const ROOT = path.join(__dirname, "../..");
const published = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/policy.json"), "utf8"));
// a synthetic world with known answers, built by the Python model (tests/fixtures/synthetic_policy.py)
const tmp = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "atlas-")), "synthetic.json");
execFileSync("python3", [path.join(ROOT, "tests/fixtures/synthetic_policy.py"), tmp], { stdio: "inherit" });
const policy = JSON.parse(fs.readFileSync(tmp, "utf8"));
const choice = (X, h, obj = "thisAttempt") => S.advisor({ model: policy.model }, X, obj).next(h).choice;
test("the advisor gives exactly the Python model's values and choices, for every colour and goal", () => {
let checked = 0;
for (const smp of Object.values(published.samples)) {
for (const [X, t] of Object.entries(smp.targets)) {
for (const obj of ["thisAttempt", "eventually"]) {
const adv = S.advisor(smp, X, obj);
assert.ok(Math.abs(adv.value.x - t[obj].pX.point) < 1e-12, `${X} ${obj}: ${adv.value.x} vs ${t[obj].pX.point}`);
// follow the published tree: every decision on the route must match, option by option
let n = t[obj].tree, h = {};
while (n && n.type !== "end") {
if (n.type === "decision") {
const s = adv.step(h);
assert.strictEqual(s.var, n.var); assert.strictEqual(s.choice, n.choice, `${X} ${obj} ${n.var}`);
for (const [o, v] of Object.entries(n.options)) assert.ok(Math.abs(s.options[o].pX - v.pX) < 1e-12, `${X} ${obj} ${n.var}=${o}`);
checked++;
if (n.var === "encounter") break;
h[n.var] = n.choice; n = n.next;
} else {
const [k, b] = Object.entries(n.branches).filter(([k]) => k !== "die").sort((a, c) => c[1].p - a[1].p)[0];
if (!n.var.startsWith("die")) h[n.var] = k;
n = b.next;
}
}
}
}
}
assert.ok(checked > 100, checked);
});
test("synthetic world: Blue goes to the valley, passes the Red, attempts only the Blue", () => {
const h = { company: "Alone", direction: "Center", fs: "Red", react: "Hold still", fate: "Killed the bullies yourself", post: "Forest" };
assert.strictEqual(choice("Blue", h), "Valley");
assert.strictEqual(choice("Blue", { ...h, hour: "Valley" }), "Pass the Red (meadow)");
const meadow = { ...h, hour: "Valley", valley: "Pass the Red (meadow)" };
assert.strictEqual(choice("Blue", { ...meadow, meadow: "Blue" }), "Attempt the bond");
assert.strictEqual(choice("Blue", { ...meadow, meadow: "Orange" }), "Die on purpose");
assert.strictEqual(choice("Red", { ...h, hour: "Valley" }), "Face the Red");
assert.strictEqual(choice("Black", h), "Tree");
});
test("routing to one encounter: Blue only at the meadow equals the Blue policy; only at the tree Greens gives 0", () => {
const smp = { model: policy.model };
const meadow = S.advisor(smp, "Blue", "thisAttempt", { only: ["meadow-Blue"] });
const all = S.advisor(smp, "Blue", "thisAttempt");
assert.ok(Math.abs(meadow.value.x - all.value.x) < 1e-9);
assert.ok(S.advisor(smp, "Blue", "thisAttempt", { only: ["tree-Greens"] }).value.x < 1e-9);
assert.ok(S.advisor(smp, "Brown", "thisAttempt", { only: ["tree-Greens"] }).value.x > 0.05); // the Greens rescue
assert.ok(meadow.attempts("meadow-Blue") > 100);
});
test("a run is walked in the game's order", () => {
const adv = S.advisor(published.samples.all, "Blue", "thisAttempt");
assert.strictEqual(adv.step({}).var, "company");
assert.strictEqual(adv.step({ company: "1 Squadmate" }).var, "leave");
assert.strictEqual(adv.step({ company: "1 Squadmate", leave: "Stay together" }).var, "direction");
assert.strictEqual(adv.step({ company: "Alone", direction: "Center" }).kind, "chance"); // the first dragon is a roll
const seen = { company: "Alone", direction: "Center", fs: "Red" };
assert.strictEqual(adv.step(seen).var, "react");
const base = { ...seen, react: "Hold still" };
const scream = adv.step(base);
assert.strictEqual(scream.var, "scream");
assert.strictEqual(scream.risk.label, "Killed at the first sighting");
const help = { ...base, scream: "Investigate the scream", clearing: "Help your squadmate" };
assert.strictEqual(adv.step({ ...base, scream: "Investigate the scream" }).var, "clearing");
assert.strictEqual(adv.step(help).var, "fight");
assert.strictEqual(adv.step({ ...help, fight: "Pause to assess" }).var, "dagger");
assert.strictEqual(adv.step({ ...help, fight: "Charge the smaller cadet" }).var, "fate"); // charging skips the dagger
const fate = adv.step({ ...help, fight: "Pause to assess", dagger: "Engage the smaller cadet" });
assert.strictEqual(fate.var, "fate");
assert.strictEqual(fate.risk.label, "Killed in the fight");
assert.strictEqual(adv.step({ ...base, scream: "Keep walking" }).var, "wounded");
assert.strictEqual(adv.step({ ...base, scream: "Investigate the scream", clearing: "Walk away" }).var, "hour");
// an older record that only knows the bullies' fate still lands in the right place (the skipped steps are
// assumed to be the recommended ones)
assert.strictEqual(adv.next({ ...base, fate: "Killed the bullies yourself" }).decision, "post");
});
test("recommendOption points the bot at the recommended option on its screen", () => {
const adv = S.advisor(published.samples.all, "Blue", "thisAttempt");
const h = { company: "Alone", direction: "Center", fs: "Red", react: "Hold still", scream: "Keep walking", wounded: "Tourniquet" };
const rec = S.recommendOption(adv, h, ["Continue toward the center of the valley", "Find the tallest tree and climb for a better view"]);
assert.strictEqual(rec.decision, "hour");
assert.strictEqual(rec.index, rec.choice === "Valley" ? 0 : 1);
assert.strictEqual(S.recommendOption(adv, {}, ["Something we have never seen"]).index, null);
});
test("solverHistory turns form variables into a history keyed by the game's steps", () => {
S.useGame(published.samples.all);
const h = S.solverHistory({ company: "Alone", direction: "Center", first_sighting: "Red", scream: "Ignore the scream", resolution: "Tourniquet", hour: "Tree", branch: "Descend slowly: two Greens" });
assert.deepStrictEqual(h, { company: "Alone", direction: "Center", fs: "Red", scream: "Keep walking", wounded: "Tourniquet", hour: "Tree", descend: "Slowly (two Greens)", shadow: "no" });
const l = S.solverHistory({ company: "1 Squadmate", left_squadmates: true, branch: "Descend quickly: let the Red pass", hour: "Tree" });
assert.strictEqual(l.leave, "Leave them");
assert.strictEqual(l.red, "Let it pass (two Greens)");
});
test("Thompson sampling: one posterior draw per seed, with the chance of each pick", () => {
const S = require("../../core/solver.js");
const P = require("../../data/public/policy.json");
const best = S.advisor(P.samples.all, "Blue", "eventually").next({});
assert.strictEqual(best.propensity, 1);
const a = S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: 3, draws: 8 });
const b = S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: 3, draws: 8 });
const ra = a.next({}), rb = b.next({});
assert.strictEqual(ra.choice, rb.choice); // same seed, same draw
assert.ok(ra.propensity >= 0 && ra.propensity <= 1);
const picks = new Set();
for (let s = 1; s <= 12; s++) picks.add(S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: s, draws: 1 }).next({}).choice);
assert.ok(picks.size > 1, "different seeds explore different options");
});
test("a first attempt has its own odds, exactly as the Python model publishes them", () => {
for (const smp of Object.values(published.samples)) {
for (const [X, t] of Object.entries(smp.targets)) {
for (const obj of ["thisAttempt", "eventually"]) {
const adv = S.advisor(smp, X, obj, { attempt: 1 });
assert.deepStrictEqual(adv.given, { attempt: "First attempt" });
assert.ok(Math.abs(adv.value.x - t.firstAttempt[obj].pX.point) < 1e-12, `${X} ${obj}`);
assert.strictEqual(adv.next({}).choice, t.firstAttempt[obj].tree.choice);
}
}
}
const all = published.samples.all;
const later = S.advisor(all, "Black", "thisAttempt", { attempt: 7 }), dflt = S.advisor(all, "Black", "thisAttempt");
assert.strictEqual(later.value.x, dflt.value.x); // no attempt given: a later attempt
assert.ok(S.advisor(all, "Black", "thisAttempt", { attempt: 1 }).value.x > 5 * later.value.x);
});
test("fix holds decisions to an experiment arm and solves the rest around it", () => {
const all = published.samples.all;
const fix = { direction: "East", scream: "Investigate the scream", clearing: "Help your squadmate", hour: "Tree" };
const adv = S.advisor(all, "Black", "eventually", { attempt: 1, fix });
assert.strictEqual(adv.step({ company: "Alone" }).choice, "East");
assert.deepStrictEqual(Object.keys(adv.step({ company: "Alone" }).options), ["East"]);
assert.ok(adv.value.x <= S.advisor(all, "Black", "eventually", { attempt: 1 }).value.x + 1e-12);
assert.throws(() => S.advisor(all, "Black", "eventually", { fix: { direction: "North" } }), /no option/);
});
test("mapping: uniform sampling picks among the options on the screen, with its propensity", () => {
const all = published.samples.all;
const screen = ["Enter the valley alone", "Enter the valley with one of your squadmates", "Enter the valley with two of your squadmates"];
const picks = new Set();
for (let seed = 1; seed <= 30; seed++) {
const rec = S.recommendOption(S.advisor(all, "Blue", "eventually", { sampling: "uniform", seed }), {}, screen);
assert.ok(Math.abs(rec.propensity - 1 / 3) < 1e-12);
assert.match(S.norm(screen[rec.index]), new RegExp(all.model.game.bot_text.company[rec.choice]));
picks.add(rec.choice);
}
assert.strictEqual(picks.size, 3);
const a = S.advisor(all, "Blue", "eventually", { sampling: "uniform", seed: 4 });
assert.strictEqual(S.recommendOption(a, {}, screen).index, S.recommendOption(a, {}, screen).index); // same seed, same pick
});
test("the experiment plan: balanced arms across new accounts, shares after the first attempt", () => {
const plan = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/experiments.json"), "utf8"));
const first = plan.experiments.find((e) => e.id === "first-attempt-shadow");
const count = {};
for (let acc = 0; acc < 4 * 25; acc++) {
const a = S.assign(plan, { accountSeq: acc, attempt: 1 });
assert.strictEqual(a.experimentId, "first-attempt-shadow");
assert.strictEqual(a.target, "Black");
assert.strictEqual(a.propensity, 1 / first.arms.length);
assert.deepStrictEqual(a, S.assign(plan, { accountSeq: acc, attempt: 1 })); // the same every time
count[a.arm] = (count[a.arm] || 0) + 1;
S.advisor(published.samples.all, a.target, a.objective, { attempt: 1, fix: a.fix }); // every arm is playable
}
assert.deepStrictEqual(Object.values(count), [25, 25, 25, 25]);
// attempt 2: half the accounts check the shadow again; the others go to the shared experiments
const second = Array.from({ length: 40 }, (_, acc) => S.assign(plan, { accountSeq: acc, attempt: 2 }).experimentId);
assert.strictEqual(second.filter((id) => id === "attempt-2-shadow").length, 20);
const later = {};
for (let acc = 0; acc < 60; acc++) for (let t = 3; t <= 18; t++) { const a = S.assign(plan, { accountSeq: acc, attempt: t }); later[a.experimentId] = (later[a.experimentId] || 0) + 1; }
assert.deepStrictEqual(Object.keys(later).sort(), ["mapping", "meadow-blue-route", "sentinel"]);
assert.ok(Math.abs(later["meadow-blue-route"] / (60 * 16) - 0.6) < 0.06);
// an experiment that is no longer running is not assigned; its share goes to the others
const done = { ...plan, experiments: plan.experiments.map((e) => (e.id === "meadow-blue-route" ? { ...e, status: "complete" } : e)) };
assert.notStrictEqual(S.assign(done, { accountSeq: 1, attempt: 5 }).experimentId, "meadow-blue-route");
assert.strictEqual(S.assign({ ...plan, experiments: [] }, { accountSeq: 1, attempt: 5 }), null);
});