CI / test (pull_request) Canceled after 0s
- analysis/tree/build_tree.py -> data/public/tree.json, analysis/reports/TREE.md: every screen the bot has seen, every option, where each led (fixed or dice, with ranges), best known picks per colour chosen cautiously, where any picks end, and the untested options. solver.js screenPick(): least-tried and Thompson picks from the tree. - The bot logs a pick it then overrode as a step: 37 runs were coded with choices they never made. Importer and graph.js now use the played steps only. - runs_clean.csv: the Unofficial Dragon Register's 288 bonded dragons are in the master list as outcome-only rows, flagged (no analysis uses them). - Cycle, refresh and the bot kit rebuild and carry the tree; tests check every generated report is current. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
255 lines
16 KiB
JavaScript
255 lines
16 KiB
JavaScript
// The route advisor (core/solver.js): it must give exactly the Python route model's answers, and walk a run in the
|
|
// game's order. node --test tests/core/advisor.test.js
|
|
const test = require("node:test");
|
|
const assert = require("node:assert");
|
|
const path = require("node:path");
|
|
const fs = require("node:fs");
|
|
const os = require("node:os");
|
|
const { execFileSync } = require("node:child_process");
|
|
const S = require("../../core/solver");
|
|
|
|
const ROOT = path.join(__dirname, "../..");
|
|
const published = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/policy.json"), "utf8"));
|
|
// a synthetic world with known answers, built by the Python model (tests/fixtures/synthetic_policy.py)
|
|
const tmp = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "atlas-")), "synthetic.json");
|
|
execFileSync("python3", [path.join(ROOT, "tests/fixtures/synthetic_policy.py"), tmp], { stdio: "inherit" });
|
|
const policy = JSON.parse(fs.readFileSync(tmp, "utf8"));
|
|
const choice = (X, h, obj = "thisAttempt") => S.advisor({ model: policy.model }, X, obj).next(h).choice;
|
|
|
|
test("the advisor gives exactly the Python model's values and choices, for every colour and goal", () => {
|
|
let checked = 0;
|
|
for (const smp of Object.values(published.samples)) {
|
|
for (const [X, t] of Object.entries(smp.targets)) {
|
|
for (const obj of ["thisAttempt", "eventually"]) {
|
|
const adv = S.advisor(smp, X, obj);
|
|
assert.ok(Math.abs(adv.value.x - t[obj].pX.point) < 1e-12, `${X} ${obj}: ${adv.value.x} vs ${t[obj].pX.point}`);
|
|
// follow the published tree: every decision on the route must match, option by option
|
|
let n = t[obj].tree, h = {};
|
|
while (n && n.type !== "end") {
|
|
if (n.type === "decision") {
|
|
const s = adv.step(h);
|
|
assert.strictEqual(s.var, n.var); assert.strictEqual(s.choice, n.choice, `${X} ${obj} ${n.var}`);
|
|
for (const [o, v] of Object.entries(n.options)) assert.ok(Math.abs(s.options[o].pX - v.pX) < 1e-12, `${X} ${obj} ${n.var}=${o}`);
|
|
checked++;
|
|
if (n.var === "encounter") break;
|
|
h[n.var] = n.choice; n = n.next;
|
|
} else {
|
|
const [k, b] = Object.entries(n.branches).filter(([k]) => k !== "die").sort((a, c) => c[1].p - a[1].p)[0];
|
|
if (!n.var.startsWith("die")) h[n.var] = k;
|
|
n = b.next;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
assert.ok(checked > 100, checked);
|
|
});
|
|
|
|
test("synthetic world: Blue goes to the valley, passes the Red, attempts only the Blue", () => {
|
|
const h = { company: "Alone", direction: "Center", fs: "Red", react: "Hold still", fate: "Killed the bullies yourself", post: "Forest" };
|
|
assert.strictEqual(choice("Blue", h), "Valley");
|
|
assert.strictEqual(choice("Blue", { ...h, hour: "Valley" }), "Pass the Red (meadow)");
|
|
const meadow = { ...h, hour: "Valley", valley: "Pass the Red (meadow)" };
|
|
assert.strictEqual(choice("Blue", { ...meadow, meadow: "Blue" }), "Attempt the bond");
|
|
assert.strictEqual(choice("Blue", { ...meadow, meadow: "Orange" }), "Die on purpose");
|
|
assert.strictEqual(choice("Red", { ...h, hour: "Valley" }), "Face the Red");
|
|
assert.strictEqual(choice("Black", h), "Tree");
|
|
});
|
|
|
|
test("routing to one encounter: Blue only at the meadow equals the Blue policy; only at the tree Greens gives 0", () => {
|
|
const smp = { model: policy.model };
|
|
const meadow = S.advisor(smp, "Blue", "thisAttempt", { only: ["meadow-Blue"] });
|
|
const all = S.advisor(smp, "Blue", "thisAttempt");
|
|
assert.ok(Math.abs(meadow.value.x - all.value.x) < 1e-9);
|
|
assert.ok(S.advisor(smp, "Blue", "thisAttempt", { only: ["tree-Greens"] }).value.x < 1e-9);
|
|
assert.ok(S.advisor(smp, "Brown", "thisAttempt", { only: ["tree-Greens"] }).value.x > 0.05); // the Greens rescue
|
|
assert.ok(meadow.attempts("meadow-Blue") > 100);
|
|
});
|
|
|
|
test("a run is walked in the game's order", () => {
|
|
const adv = S.advisor(published.samples.all, "Blue", "thisAttempt");
|
|
assert.strictEqual(adv.step({}).var, "company");
|
|
assert.strictEqual(adv.step({ company: "1 Squadmate" }).var, "leave");
|
|
assert.strictEqual(adv.step({ company: "1 Squadmate", leave: "Stay together" }).var, "direction");
|
|
assert.strictEqual(adv.step({ company: "Alone", direction: "Center" }).kind, "chance"); // the first dragon is a roll
|
|
const seen = { company: "Alone", direction: "Center", fs: "Red" };
|
|
assert.strictEqual(adv.step(seen).var, "react");
|
|
const base = { ...seen, react: "Hold still" };
|
|
const scream = adv.step(base);
|
|
assert.strictEqual(scream.var, "scream");
|
|
assert.strictEqual(scream.risk.label, "Killed at the first sighting");
|
|
const help = { ...base, scream: "Investigate the scream", clearing: "Help your squadmate" };
|
|
assert.strictEqual(adv.step({ ...base, scream: "Investigate the scream" }).var, "clearing");
|
|
assert.strictEqual(adv.step(help).var, "fight");
|
|
assert.strictEqual(adv.step({ ...help, fight: "Pause to assess" }).var, "dagger");
|
|
assert.strictEqual(adv.step({ ...help, fight: "Charge the smaller cadet" }).var, "fate"); // charging skips the dagger
|
|
const fate = adv.step({ ...help, fight: "Pause to assess", dagger: "Engage the smaller cadet" });
|
|
assert.strictEqual(fate.var, "fate");
|
|
assert.strictEqual(fate.risk.label, "Killed in the fight");
|
|
assert.strictEqual(adv.step({ ...base, scream: "Keep walking" }).var, "wounded");
|
|
assert.strictEqual(adv.step({ ...base, scream: "Investigate the scream", clearing: "Walk away" }).var, "hour");
|
|
// an older record that only knows the bullies' fate still lands in the right place (the skipped steps are
|
|
// assumed to be the recommended ones)
|
|
assert.strictEqual(adv.next({ ...base, fate: "Killed the bullies yourself" }).decision, "post");
|
|
});
|
|
|
|
test("recommendOption points the bot at the recommended option on its screen", () => {
|
|
const adv = S.advisor(published.samples.all, "Blue", "thisAttempt");
|
|
const h = { company: "Alone", direction: "Center", fs: "Red", react: "Hold still", scream: "Keep walking", wounded: "Tourniquet" };
|
|
const rec = S.recommendOption(adv, h, ["Continue toward the center of the valley", "Find the tallest tree and climb for a better view"]);
|
|
assert.strictEqual(rec.decision, "hour");
|
|
assert.strictEqual(rec.index, rec.choice === "Valley" ? 0 : 1);
|
|
assert.strictEqual(S.recommendOption(adv, {}, ["Something we have never seen"]).index, null);
|
|
});
|
|
|
|
test("solverHistory turns form variables into a history keyed by the game's steps", () => {
|
|
S.useGame(published.samples.all);
|
|
const h = S.solverHistory({ company: "Alone", direction: "Center", first_sighting: "Red", scream: "Ignore the scream", resolution: "Tourniquet", hour: "Tree", branch: "Descend slowly: two Greens" });
|
|
assert.deepStrictEqual(h, { company: "Alone", direction: "Center", fs: "Red", scream: "Keep walking", wounded: "Tourniquet", hour: "Tree", descend: "Slowly (two Greens)", shadow: "no" });
|
|
const l = S.solverHistory({ company: "1 Squadmate", left_squadmates: true, branch: "Descend quickly: let the Red pass", hour: "Tree" });
|
|
assert.strictEqual(l.leave, "Leave them");
|
|
assert.strictEqual(l.red, "Let it pass (two Greens)");
|
|
});
|
|
|
|
test("Thompson sampling: one posterior draw per seed, with the chance of each pick", () => {
|
|
const S = require("../../core/solver.js");
|
|
const P = require("../../data/public/policy.json");
|
|
const best = S.advisor(P.samples.all, "Blue", "eventually").next({});
|
|
assert.strictEqual(best.propensity, 1);
|
|
const a = S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: 3, draws: 8 });
|
|
const b = S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: 3, draws: 8 });
|
|
const ra = a.next({}), rb = b.next({});
|
|
assert.strictEqual(ra.choice, rb.choice); // same seed, same draw
|
|
assert.ok(ra.propensity >= 0 && ra.propensity <= 1);
|
|
const picks = new Set();
|
|
for (let s = 1; s <= 12; s++) picks.add(S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: s, draws: 1 }).next({}).choice);
|
|
assert.ok(picks.size > 1, "different seeds explore different options");
|
|
});
|
|
|
|
test("a first attempt has its own odds, exactly as the Python model publishes them", () => {
|
|
for (const smp of Object.values(published.samples)) {
|
|
for (const [X, t] of Object.entries(smp.targets)) {
|
|
for (const obj of ["thisAttempt", "eventually"]) {
|
|
const adv = S.advisor(smp, X, obj, { attempt: 1 });
|
|
assert.deepStrictEqual(adv.given, { attempt: "First attempt" });
|
|
assert.ok(Math.abs(adv.value.x - t.firstAttempt[obj].pX.point) < 1e-12, `${X} ${obj}`);
|
|
assert.strictEqual(adv.next({}).choice, t.firstAttempt[obj].tree.choice);
|
|
}
|
|
}
|
|
}
|
|
const all = published.samples.all;
|
|
const later = S.advisor(all, "Black", "thisAttempt", { attempt: 7 }), dflt = S.advisor(all, "Black", "thisAttempt");
|
|
assert.strictEqual(later.value.x, dflt.value.x); // no attempt given: a later attempt
|
|
assert.ok(S.advisor(all, "Black", "thisAttempt", { attempt: 1 }).value.x > 5 * later.value.x);
|
|
});
|
|
|
|
test("fix holds decisions to an experiment arm and solves the rest around it", () => {
|
|
const all = published.samples.all;
|
|
const fix = { direction: "East", scream: "Investigate the scream", clearing: "Help your squadmate", hour: "Tree" };
|
|
const adv = S.advisor(all, "Black", "eventually", { attempt: 1, fix });
|
|
assert.strictEqual(adv.step({ company: "Alone" }).choice, "East");
|
|
assert.deepStrictEqual(Object.keys(adv.step({ company: "Alone" }).options), ["East"]);
|
|
assert.ok(adv.value.x <= S.advisor(all, "Black", "eventually", { attempt: 1 }).value.x + 1e-12);
|
|
assert.throws(() => S.advisor(all, "Black", "eventually", { fix: { direction: "North" } }), /no option/);
|
|
});
|
|
|
|
test("mapping: uniform sampling picks among the options on the screen, with its propensity", () => {
|
|
const all = published.samples.all;
|
|
const screen = ["Enter the valley alone", "Enter the valley with one of your squadmates", "Enter the valley with two of your squadmates"];
|
|
const picks = new Set();
|
|
for (let seed = 1; seed <= 30; seed++) {
|
|
const rec = S.recommendOption(S.advisor(all, "Blue", "eventually", { sampling: "uniform", seed }), {}, screen);
|
|
assert.ok(Math.abs(rec.propensity - 1 / 3) < 1e-12);
|
|
assert.match(S.norm(screen[rec.index]), new RegExp(all.model.game.bot_text.company[rec.choice]));
|
|
picks.add(rec.choice);
|
|
}
|
|
assert.strictEqual(picks.size, 3);
|
|
const a = S.advisor(all, "Blue", "eventually", { sampling: "uniform", seed: 4 });
|
|
assert.strictEqual(S.recommendOption(a, {}, screen).index, S.recommendOption(a, {}, screen).index); // same seed, same pick
|
|
});
|
|
|
|
test("the experiment plan: what each attempt of each account is for", () => {
|
|
const plan = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/experiments.json"), "utf8"));
|
|
const all = published.samples.all;
|
|
// attempt 1: every new account plays the same exact path (the replication)
|
|
for (let acc = 0; acc < 30; acc++) {
|
|
const a = S.assign(plan, { accountSeq: acc, attempt: 1 });
|
|
assert.strictEqual(a.experimentId, "fresh-replicate");
|
|
assert.strictEqual(a.target, "Black");
|
|
assert.strictEqual(a.fix.direction, "West");
|
|
assert.deepStrictEqual(a, S.assign(plan, { accountSeq: acc, attempt: 1 })); // the same every time
|
|
}
|
|
const a1 = S.assign(plan, { accountSeq: 0, attempt: 1 });
|
|
const adv = S.advisor(all, a1.target, a1.objective, { attempt: 1, fix: a1.fix });
|
|
let h = {};
|
|
for (let i = 0; i < 20; i++) {
|
|
const s = adv.step(h);
|
|
if (s.kind === "decision") { if (s.var === "encounter") break; if (a1.fix[s.var]) assert.strictEqual(s.choice, a1.fix[s.var]); h[s.var] = s.choice; }
|
|
else if (s.kind === "chance") h[s.var] = s.var === "shadow" ? "no" : Object.keys(s.probs).find((k) => k !== "die");
|
|
else break;
|
|
}
|
|
assert.strictEqual(h.hour, "Tree");
|
|
// the ablation waits for the replication, then gives every arm the same number of accounts
|
|
const stage2 = { ...plan, experiments: plan.experiments.map((e) => (e.id === "fresh-replicate" ? { ...e, status: "complete" } : e.id === "fresh-ablate" ? { ...e, status: "running" } : e)) };
|
|
const count = {};
|
|
for (let acc = 0; acc < 12 * 5; acc++) { const a = S.assign(stage2, { accountSeq: acc, attempt: 1 }); assert.strictEqual(a.experimentId, "fresh-ablate"); count[a.arm] = (count[a.arm] || 0) + 1; }
|
|
assert.deepStrictEqual(Object.values(count), Array(12).fill(5));
|
|
// with squadmates, staying and the direction are one screen
|
|
const squad = stage2.experiments.find((e) => e.id === "fresh-ablate").arms.find((a) => a.id === "with two squadmates");
|
|
const rec = S.recommendOption(S.advisor(all, "Black", "eventually", { attempt: 1, fix: squad.fix }), { company: "2 Squadmates" },
|
|
["Go east with your squad", "Leave your squadmates", "Convince them to head west instead"]);
|
|
assert.strictEqual(rec.index, 2);
|
|
assert.deepStrictEqual(rec.also, { direction: "West" });
|
|
// attempt 2: an account that saw the shadow tries the same path again; half of the others do too
|
|
const who = (attempt, facts) => Array.from({ length: 40 }, (_, acc) => (S.assign(plan, { accountSeq: acc, attempt, facts }) || {}).experimentId);
|
|
assert.ok(who(2, { sawShadowBefore: true }).every((id) => id === "shadow-again"));
|
|
assert.strictEqual(who(2, { sawShadowBefore: false }).filter((id) => id === "second-attempt-same-path").length, 20);
|
|
// later: two accounts in ten are the long-lived ones, the rest share the research experiments
|
|
const later = who(6, { sawShadowBefore: false });
|
|
assert.strictEqual(later.filter((id) => id === "veteran-repeat").length, 4);
|
|
assert.strictEqual(later.filter((id) => id === "veteran-variety").length, 4);
|
|
assert.deepStrictEqual([...new Set(later)].sort(), ["mapping", "meadow-blue-route", "sentinel", "veteran-repeat", "veteran-variety"]);
|
|
// variety: a different combination on every attempt, both gates kept, every one playable
|
|
const combos = new Set();
|
|
for (let t = 3; t < 3 + 60; t++) {
|
|
const a = S.assign(plan, { accountSeq: 11, attempt: t, facts: { sawShadowBefore: false } });
|
|
assert.strictEqual(a.experimentId, "veteran-variety");
|
|
assert.strictEqual(a.fix.clearing, "Help your squadmate");
|
|
combos.add(a.arm);
|
|
if (t < 8) S.advisor(all, a.target, a.objective, { attempt: t, fix: a.fix });
|
|
}
|
|
assert.strictEqual(combos.size, 60);
|
|
// an experiment that is not running is not assigned; its share goes to the others
|
|
const done = { ...plan, experiments: plan.experiments.map((e) => (e.id === "meadow-blue-route" ? { ...e, status: "complete" } : e)) };
|
|
assert.notStrictEqual(S.assign(done, { accountSeq: 5, attempt: 5 }).experimentId, "meadow-blue-route");
|
|
assert.strictEqual(S.assign({ ...plan, experiments: [] }, { accountSeq: 1, attempt: 5 }), null);
|
|
});
|
|
|
|
test("screenPick: least-tried fills gaps, Thompson avoids what has killed", () => {
|
|
const tree = { screens: { s1: { options: {
|
|
a: { text: "Hold its gaze", n: 40, to: { next: { n: 38 }, DIED: { n: 2 } } },
|
|
b: { text: "Lower your eyes", n: 40, to: { DIED: { n: 40 } } },
|
|
c: { text: "Step back", n: 0, to: {} } } } } };
|
|
const screen = ["Lower your eyes", "Hold its gaze", "Step back", "An option the tree has never seen"];
|
|
const lt = S.screenPick(tree, "s1", screen, { mode: "least-tried", seed: 1 });
|
|
assert.ok([2, 3].includes(lt.index) && lt.n === 0);
|
|
assert.deepStrictEqual(lt, S.screenPick(tree, "s1", screen, { mode: "least-tried", seed: 1 })); // same seed, same pick
|
|
const picks = {};
|
|
for (let seed = 1; seed <= 300; seed++) { const p = S.screenPick(tree, "s1", screen, { mode: "thompson", seed }); picks[p.index] = (picks[p.index] || 0) + 1; }
|
|
assert.ok(!picks[0] || picks[0] < 3, "an option that died 40 times in 40 is almost never drawn");
|
|
assert.ok(picks[1] > 200 && picks[2] >= 5 && picks[3] >= 5, JSON.stringify(picks)); // mostly the good one; the untested ones still get tried
|
|
assert.strictEqual(S.screenPick(tree, "unknown-screen", ["x", "y"], { seed: 2 }).n, 0);
|
|
assert.strictEqual(S.screenPick(tree, "s1", []), null);
|
|
});
|
|
|
|
test("picks the bot logged and then overrode are not part of the run", () => {
|
|
const G = require("../../core/graph.js");
|
|
const steps = [
|
|
{ scene: "a68f681076c6", optionText: "Head west toward the scent", to: "BURNED" },
|
|
{ scene: "a68f681076c6", optionText: "Head toward the center of the forest", to: "bc4ee4754b9c" },
|
|
{ scene: "bc4ee4754b9c", optionText: "Lower your eyes in respect", to: "84780dab01e3", tint: "red" }];
|
|
assert.strictEqual(G.playedSteps(steps).length, 2);
|
|
assert.strictEqual(G.trajectoryVariables({ steps, outcome: {} }).vars.direction, "Center");
|
|
assert.strictEqual(G.playedSteps([{ scene: "x", optionText: "only" }]).length, 1); // a run in progress
|
|
});
|