// The route advisor (core/solver.js): it must give exactly the Python route model's answers, and walk a run in the // game's order. node --test tests/core/advisor.test.js const test = require("node:test"); const assert = require("node:assert"); const path = require("node:path"); const fs = require("node:fs"); const os = require("node:os"); const { execFileSync } = require("node:child_process"); const S = require("../../core/solver"); const ROOT = path.join(__dirname, "../.."); const published = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/policy.json"), "utf8")); // a synthetic world with known answers, built by the Python model (tests/fixtures/synthetic_policy.py) const tmp = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "atlas-")), "synthetic.json"); execFileSync("python3", [path.join(ROOT, "tests/fixtures/synthetic_policy.py"), tmp], { stdio: "inherit" }); const policy = JSON.parse(fs.readFileSync(tmp, "utf8")); const choice = (X, h, obj = "thisAttempt") => S.advisor({ model: policy.model }, X, obj).next(h).choice; test("the advisor gives exactly the Python model's values and choices, for every colour and goal", () => { let checked = 0; for (const smp of Object.values(published.samples)) { for (const [X, t] of Object.entries(smp.targets)) { for (const obj of ["thisAttempt", "eventually"]) { const adv = S.advisor(smp, X, obj); assert.ok(Math.abs(adv.value.x - t[obj].pX.point) < 1e-12, `${X} ${obj}: ${adv.value.x} vs ${t[obj].pX.point}`); // follow the published tree: every decision on the route must match, option by option let n = t[obj].tree, h = {}; while (n && n.type !== "end") { if (n.type === "decision") { const s = adv.step(h); assert.strictEqual(s.var, n.var); assert.strictEqual(s.choice, n.choice, `${X} ${obj} ${n.var}`); for (const [o, v] of Object.entries(n.options)) assert.ok(Math.abs(s.options[o].pX - v.pX) < 1e-12, `${X} ${obj} ${n.var}=${o}`); checked++; if (n.var === "encounter") break; h[n.var] = n.choice; n = n.next; } else { const [k, b] = Object.entries(n.branches).filter(([k]) => k !== "die").sort((a, c) => c[1].p - a[1].p)[0]; if (!n.var.startsWith("die")) h[n.var] = k; n = b.next; } } } } } assert.ok(checked > 100, checked); }); test("synthetic world: Blue goes to the valley, passes the Red, attempts only the Blue", () => { const h = { company: "Alone", direction: "Center", fs: "Red", react: "Hold still", fate: "Killed the bullies yourself", post: "Forest" }; assert.strictEqual(choice("Blue", h), "Valley"); assert.strictEqual(choice("Blue", { ...h, hour: "Valley" }), "Pass the Red (meadow)"); const meadow = { ...h, hour: "Valley", valley: "Pass the Red (meadow)" }; assert.strictEqual(choice("Blue", { ...meadow, meadow: "Blue" }), "Attempt the bond"); assert.strictEqual(choice("Blue", { ...meadow, meadow: "Orange" }), "Die on purpose"); assert.strictEqual(choice("Red", { ...h, hour: "Valley" }), "Face the Red"); assert.strictEqual(choice("Black", h), "Tree"); }); test("routing to one encounter: Blue only at the meadow equals the Blue policy; only at the tree Greens gives 0", () => { const smp = { model: policy.model }; const meadow = S.advisor(smp, "Blue", "thisAttempt", { only: ["meadow-Blue"] }); const all = S.advisor(smp, "Blue", "thisAttempt"); assert.ok(Math.abs(meadow.value.x - all.value.x) < 1e-9); assert.ok(S.advisor(smp, "Blue", "thisAttempt", { only: ["tree-Greens"] }).value.x < 1e-9); assert.ok(S.advisor(smp, "Brown", "thisAttempt", { only: ["tree-Greens"] }).value.x > 0.05); // the Greens rescue assert.ok(meadow.attempts("meadow-Blue") > 100); }); test("a run is walked in the game's order", () => { const adv = S.advisor(published.samples.all, "Blue", "thisAttempt"); assert.strictEqual(adv.step({}).var, "company"); assert.strictEqual(adv.step({ company: "1 Squadmate" }).var, "leave"); assert.strictEqual(adv.step({ company: "1 Squadmate", leave: "Stay together" }).var, "direction"); assert.strictEqual(adv.step({ company: "Alone", direction: "Center" }).kind, "chance"); // the first dragon is a roll const seen = { company: "Alone", direction: "Center", fs: "Red" }; assert.strictEqual(adv.step(seen).var, "react"); const base = { ...seen, react: "Hold still" }; const scream = adv.step(base); assert.strictEqual(scream.var, "scream"); assert.strictEqual(scream.risk.label, "Killed at the first sighting"); const help = { ...base, scream: "Investigate the scream", clearing: "Help your squadmate" }; assert.strictEqual(adv.step({ ...base, scream: "Investigate the scream" }).var, "clearing"); assert.strictEqual(adv.step(help).var, "fight"); assert.strictEqual(adv.step({ ...help, fight: "Pause to assess" }).var, "dagger"); assert.strictEqual(adv.step({ ...help, fight: "Charge the smaller cadet" }).var, "fate"); // charging skips the dagger const fate = adv.step({ ...help, fight: "Pause to assess", dagger: "Engage the smaller cadet" }); assert.strictEqual(fate.var, "fate"); assert.strictEqual(fate.risk.label, "Killed in the fight"); assert.strictEqual(adv.step({ ...base, scream: "Keep walking" }).var, "wounded"); assert.strictEqual(adv.step({ ...base, scream: "Investigate the scream", clearing: "Walk away" }).var, "hour"); // an older record that only knows the bullies' fate still lands in the right place (the skipped steps are // assumed to be the recommended ones) assert.strictEqual(adv.next({ ...base, fate: "Killed the bullies yourself" }).decision, "post"); }); test("recommendOption points the bot at the recommended option on its screen", () => { const adv = S.advisor(published.samples.all, "Blue", "thisAttempt"); const h = { company: "Alone", direction: "Center", fs: "Red", react: "Hold still", scream: "Keep walking", wounded: "Tourniquet" }; const rec = S.recommendOption(adv, h, ["Continue toward the center of the valley", "Find the tallest tree and climb for a better view"]); assert.strictEqual(rec.decision, "hour"); assert.strictEqual(rec.index, rec.choice === "Valley" ? 0 : 1); assert.strictEqual(S.recommendOption(adv, {}, ["Something we have never seen"]).index, null); }); test("solverHistory turns form variables into a history keyed by the game's steps", () => { S.useGame(published.samples.all); const h = S.solverHistory({ company: "Alone", direction: "Center", first_sighting: "Red", scream: "Ignore the scream", resolution: "Tourniquet", hour: "Tree", branch: "Descend slowly: two Greens" }); assert.deepStrictEqual(h, { company: "Alone", direction: "Center", fs: "Red", scream: "Keep walking", wounded: "Tourniquet", hour: "Tree", descend: "Slowly (two Greens)", shadow: "no" }); const l = S.solverHistory({ company: "1 Squadmate", left_squadmates: true, branch: "Descend quickly: let the Red pass", hour: "Tree" }); assert.strictEqual(l.leave, "Leave them"); assert.strictEqual(l.red, "Let it pass (two Greens)"); }); test("Thompson sampling: one posterior draw per seed, with the chance of each pick", () => { const S = require("../../core/solver.js"); const P = require("../../data/public/policy.json"); const best = S.advisor(P.samples.all, "Blue", "eventually").next({}); assert.strictEqual(best.propensity, 1); const a = S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: 3, draws: 8 }); const b = S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: 3, draws: 8 }); const ra = a.next({}), rb = b.next({}); assert.strictEqual(ra.choice, rb.choice); // same seed, same draw assert.ok(ra.propensity >= 0 && ra.propensity <= 1); const picks = new Set(); for (let s = 1; s <= 12; s++) picks.add(S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: s, draws: 1 }).next({}).choice); assert.ok(picks.size > 1, "different seeds explore different options"); }); test("a first attempt has its own odds, exactly as the Python model publishes them", () => { for (const smp of Object.values(published.samples)) { for (const [X, t] of Object.entries(smp.targets)) { for (const obj of ["thisAttempt", "eventually"]) { const adv = S.advisor(smp, X, obj, { attempt: 1 }); assert.deepStrictEqual(adv.given, { attempt: "First attempt" }); assert.ok(Math.abs(adv.value.x - t.firstAttempt[obj].pX.point) < 1e-12, `${X} ${obj}`); assert.strictEqual(adv.next({}).choice, t.firstAttempt[obj].tree.choice); } } } const all = published.samples.all; const later = S.advisor(all, "Black", "thisAttempt", { attempt: 7 }), dflt = S.advisor(all, "Black", "thisAttempt"); assert.strictEqual(later.value.x, dflt.value.x); // no attempt given: a later attempt assert.ok(S.advisor(all, "Black", "thisAttempt", { attempt: 1 }).value.x > 5 * later.value.x); }); test("fix holds decisions to an experiment arm and solves the rest around it", () => { const all = published.samples.all; const fix = { direction: "East", scream: "Investigate the scream", clearing: "Help your squadmate", hour: "Tree" }; const adv = S.advisor(all, "Black", "eventually", { attempt: 1, fix }); assert.strictEqual(adv.step({ company: "Alone" }).choice, "East"); assert.deepStrictEqual(Object.keys(adv.step({ company: "Alone" }).options), ["East"]); assert.ok(adv.value.x <= S.advisor(all, "Black", "eventually", { attempt: 1 }).value.x + 1e-12); assert.throws(() => S.advisor(all, "Black", "eventually", { fix: { direction: "North" } }), /no option/); }); test("mapping: uniform sampling picks among the options on the screen, with its propensity", () => { const all = published.samples.all; const screen = ["Enter the valley alone", "Enter the valley with one of your squadmates", "Enter the valley with two of your squadmates"]; const picks = new Set(); for (let seed = 1; seed <= 30; seed++) { const rec = S.recommendOption(S.advisor(all, "Blue", "eventually", { sampling: "uniform", seed }), {}, screen); assert.ok(Math.abs(rec.propensity - 1 / 3) < 1e-12); assert.match(S.norm(screen[rec.index]), new RegExp(all.model.game.bot_text.company[rec.choice])); picks.add(rec.choice); } assert.strictEqual(picks.size, 3); const a = S.advisor(all, "Blue", "eventually", { sampling: "uniform", seed: 4 }); assert.strictEqual(S.recommendOption(a, {}, screen).index, S.recommendOption(a, {}, screen).index); // same seed, same pick }); test("the experiment plan: what each attempt of each account is for", () => { const plan = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/experiments.json"), "utf8")); const all = published.samples.all; // attempt 1: every new account plays the same exact path (the replication) for (let acc = 0; acc < 30; acc++) { const a = S.assign(plan, { accountSeq: acc, attempt: 1 }); assert.strictEqual(a.experimentId, "fresh-replicate"); assert.strictEqual(a.target, "Black"); assert.strictEqual(a.fix.direction, "West"); assert.deepStrictEqual(a, S.assign(plan, { accountSeq: acc, attempt: 1 })); // the same every time } const a1 = S.assign(plan, { accountSeq: 0, attempt: 1 }); const adv = S.advisor(all, a1.target, a1.objective, { attempt: 1, fix: a1.fix }); let h = {}; for (let i = 0; i < 20; i++) { const s = adv.step(h); if (s.kind === "decision") { if (s.var === "encounter") break; if (a1.fix[s.var]) assert.strictEqual(s.choice, a1.fix[s.var]); h[s.var] = s.choice; } else if (s.kind === "chance") h[s.var] = s.var === "shadow" ? "no" : Object.keys(s.probs).find((k) => k !== "die"); else break; } assert.strictEqual(h.hour, "Tree"); // the ablation waits for the replication, then gives every arm the same number of accounts const stage2 = { ...plan, experiments: plan.experiments.map((e) => (e.id === "fresh-replicate" ? { ...e, status: "complete" } : e.id === "fresh-ablate" ? { ...e, status: "running" } : e)) }; const count = {}; for (let acc = 0; acc < 12 * 5; acc++) { const a = S.assign(stage2, { accountSeq: acc, attempt: 1 }); assert.strictEqual(a.experimentId, "fresh-ablate"); count[a.arm] = (count[a.arm] || 0) + 1; } assert.deepStrictEqual(Object.values(count), Array(12).fill(5)); // with squadmates, staying and the direction are one screen const squad = stage2.experiments.find((e) => e.id === "fresh-ablate").arms.find((a) => a.id === "with two squadmates"); const rec = S.recommendOption(S.advisor(all, "Black", "eventually", { attempt: 1, fix: squad.fix }), { company: "2 Squadmates" }, ["Go east with your squad", "Leave your squadmates", "Convince them to head west instead"]); assert.strictEqual(rec.index, 2); assert.deepStrictEqual(rec.also, { direction: "West" }); // attempt 2: an account that saw the shadow tries the same path again; half of the others do too const who = (attempt, facts) => Array.from({ length: 40 }, (_, acc) => (S.assign(plan, { accountSeq: acc, attempt, facts }) || {}).experimentId); assert.ok(who(2, { sawShadowBefore: true }).every((id) => id === "shadow-again")); assert.strictEqual(who(2, { sawShadowBefore: false }).filter((id) => id === "second-attempt-same-path").length, 20); // later: two accounts in ten are the long-lived ones, the rest share the research experiments const later = who(6, { sawShadowBefore: false }); assert.strictEqual(later.filter((id) => id === "veteran-repeat").length, 4); assert.strictEqual(later.filter((id) => id === "veteran-variety").length, 4); assert.deepStrictEqual([...new Set(later)].sort(), ["mapping", "meadow-blue-route", "sentinel", "veteran-repeat", "veteran-variety"]); // variety: a different combination on every attempt, both gates kept, every one playable const combos = new Set(); for (let t = 3; t < 3 + 60; t++) { const a = S.assign(plan, { accountSeq: 11, attempt: t, facts: { sawShadowBefore: false } }); assert.strictEqual(a.experimentId, "veteran-variety"); assert.strictEqual(a.fix.clearing, "Help your squadmate"); combos.add(a.arm); if (t < 8) S.advisor(all, a.target, a.objective, { attempt: t, fix: a.fix }); } assert.strictEqual(combos.size, 60); // an experiment that is not running is not assigned; its share goes to the others const done = { ...plan, experiments: plan.experiments.map((e) => (e.id === "meadow-blue-route" ? { ...e, status: "complete" } : e)) }; assert.notStrictEqual(S.assign(done, { accountSeq: 5, attempt: 5 }).experimentId, "meadow-blue-route"); assert.strictEqual(S.assign({ ...plan, experiments: [] }, { accountSeq: 1, attempt: 5 }), null); }); test("screenPick: least-tried fills gaps, Thompson avoids what has killed", () => { const tree = { screens: { s1: { options: { a: { text: "Hold its gaze", n: 40, to: { next: { n: 38 }, DIED: { n: 2 } } }, b: { text: "Lower your eyes", n: 40, to: { DIED: { n: 40 } } }, c: { text: "Step back", n: 0, to: {} } } } } }; const screen = ["Lower your eyes", "Hold its gaze", "Step back", "An option the tree has never seen"]; const lt = S.screenPick(tree, "s1", screen, { mode: "least-tried", seed: 1 }); assert.ok([2, 3].includes(lt.index) && lt.n === 0); assert.deepStrictEqual(lt, S.screenPick(tree, "s1", screen, { mode: "least-tried", seed: 1 })); // same seed, same pick const picks = {}; for (let seed = 1; seed <= 300; seed++) { const p = S.screenPick(tree, "s1", screen, { mode: "thompson", seed }); picks[p.index] = (picks[p.index] || 0) + 1; } assert.ok(!picks[0] || picks[0] < 3, "an option that died 40 times in 40 is almost never drawn"); assert.ok(picks[1] > 200 && picks[2] >= 5 && picks[3] >= 5, JSON.stringify(picks)); // mostly the good one; the untested ones still get tried assert.strictEqual(S.screenPick(tree, "unknown-screen", ["x", "y"], { seed: 2 }).n, 0); assert.strictEqual(S.screenPick(tree, "s1", []), null); }); test("picks the bot logged and then overrode are not part of the run", () => { const G = require("../../core/graph.js"); const steps = [ { scene: "a68f681076c6", optionText: "Head west toward the scent", to: "BURNED" }, { scene: "a68f681076c6", optionText: "Head toward the center of the forest", to: "bc4ee4754b9c" }, { scene: "bc4ee4754b9c", optionText: "Lower your eyes in respect", to: "84780dab01e3", tint: "red" }]; assert.strictEqual(G.playedSteps(steps).length, 2); assert.strictEqual(G.trajectoryVariables({ steps, outcome: {} }).vars.direction, "Center"); assert.strictEqual(G.playedSteps([{ scene: "x", optionText: "only" }]).length, 1); // a run in progress });