Files
threshing-atlas/tests/core/bot_contract.test.js
T
kathleenandClaude Opus 5.5 d9f3c200b8
CI / test (pull_request) Successful in 57s
Advisor: never advise an option that makes the target impossible while another keeps it possible
On a later attempt the data put the Black near zero on every route, so every option
tied and the bot's tie rule (random among options within 0.2 points) walked away from
the squadmate or went to the valley (DragonKind research runs 90, 91 and 102). The
advisor now also solves the game with no data (every roll possible except through a
closed gate; an encounter bonds its own colour) and marks options after which the
target cannot happen at all as `ruledOut`. The choice is never one of them while
another option keeps the target possible, and recommendOption leaves them out of the
options it gives the bot. For the Black: always investigate, help and climb the tree.

Values are unchanged; the published route tree still matches option for option.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
2026-11-22 02:15:44 +00:00

171 lines
10 KiB
JavaScript

// The bot's side of the route advisor. Ken's bot (DragonKind/DragonKind src/atlasPolicy.js) loads policy.json,
// solver.js and graph.js from vendor/threshing-atlas/ and calls them in one fixed chain. This test copies that chain
// (AtlasPolicy.advise and buildTrajectory, as of DragonKind commit ebe134a2) and plays the bot's own recorded screens
// (data/public/tree.json) through it, for every colour, sample and goal, so a change here that would break the bot
// fails in this repo's CI first. node --test tests/core/bot_contract.test.js
const test = require("node:test");
const assert = require("node:assert");
const path = require("node:path");
const fs = require("node:fs");
const ROOT = path.join(__dirname, "../..");
const Solver = require("../../core/solver.js");
const Graph = require("../../core/graph.js");
const policy = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/policy.json"), "utf8"));
const tree = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/tree.json"), "utf8"));
const known = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/known_answers.json"), "utf8"));
// ---- copied from the bot (src/atlasPolicy.js): keep in step with it
const ATLAS_COLOURS = { black: "Black", blue: "Blue", green: "Green", red: "Red", brown: "Brown", orange: "Orange" };
const FIRST_SIGHTING_COLOURS = new Set(["red", "brown", "green", "orange"]);
function sceneColour({ tint = null, observed = {} } = {}) {
const t = String(tint || "").toLowerCase();
if (ATLAS_COLOURS[t]) return t;
const dragons = Array.isArray(observed?.dragons) ? observed.dragons : [];
return dragons.length === 1 ? dragons[0] : null;
}
function buildTrajectory(events = []) {
const steps = [];
const seen = {};
for (const ev of events) {
const colour = sceneColour(ev);
steps.push({ optionText: ev.optionText || "", tint: colour && FIRST_SIGHTING_COLOURS.has(colour) ? colour : null });
if (ev.observed?.shadow === true) seen.shadow = "yes";
else if (ev.observed?.shadow === false && seen.shadow !== "yes") seen.shadow = "no";
}
return { trajectory: { steps, outcome: {} }, seen };
}
const advisors = new Map();
function advise({ targetDragon, sample = "all", objective = "eventually", events = [], offered = [] }) {
const colour = ATLAS_COLOURS[targetDragon];
const { trajectory, seen } = buildTrajectory(events);
const { vars } = Graph.trajectoryVariables(trajectory);
const key = `${sample}|${colour}|${objective}`;
if (!advisors.has(key)) advisors.set(key, Solver.advisor(policy.samples[sample], colour, objective));
const adv = advisors.get(key);
const history = Solver.solverHistory(vars, seen, adv.game);
const raw = adv.next(history);
if (!raw) return null;
const withStep = (step) => ({ ...adv, next: (h) => (h === history ? step : adv.next(h)) });
return Solver.recommendOption(withStep(raw), history, offered);
}
// ---- end of the copy
test("the bot's modules have what it checks for when it loads them", () => {
assert.strictEqual(typeof Solver.advisor, "function");
assert.strictEqual(typeof Solver.recommendOption, "function");
assert.strictEqual(typeof Solver.solverHistory, "function");
assert.strictEqual(typeof Solver.knownAnswer, "function");
assert.strictEqual(typeof Graph.trajectoryVariables, "function");
assert.ok(policy.samples.all.model && policy.samples.lowBias.model && policy.id);
});
/**
* Play one run through the recorded screens, asking for advice on each. The run follows the advised option where the
* bot has played it (so the screen after it is on record), and otherwise the option the bot played most, going on to
* the screen it most often led to. Returns the advised option text on every screen that had one.
*/
function play(targetDragon, sample, objective) {
const events = [], advised = [];
let id = tree.start;
for (let guard = 0; id && tree.screens[id] && guard < 40; guard++) {
const sc = tree.screens[id];
const opts = Object.values(sc.options);
if (!opts.length) break;
const offered = opts.map((o) => o.text);
const here = { tint: sc.tint };
const rec = advise({ targetDragon, sample, objective, events: [...events, here], offered });
if (rec) {
assert.ok(rec.decision && rec.choice, `${targetDragon}: advice without a decision on ${sc.label}`);
assert.ok(rec.choice in rec.options, `${targetDragon}: ${rec.choice} is not one of its own options`);
assert.ok(rec.index === null || (Number.isInteger(rec.index) && rec.index >= 0 && rec.index < offered.length),
`${targetDragon}: index ${rec.index} on ${sc.label}`);
if (rec.index !== null) advised.push(offered[rec.index]);
}
const playedTo = (o) => Object.entries(o.to || {}).sort((a, b) => b[1].n - a[1].n)[0];
let i = rec && rec.index !== null && playedTo(opts[rec.index]) ? rec.index : -1;
if (i < 0) i = opts.reduce((b, o, j) => ((o.n || 0) > (opts[b].n || 0) ? j : b), 0);
events.push({ ...here, optionText: offered[i] });
const to = playedTo(opts[i]);
id = to ? to[0] : null;
}
return advised;
}
test("every colour, sample and goal plays the recorded screens without an error or an impossible pick", () => {
for (const sample of ["all", "lowBias"]) {
for (const target of Object.keys(ATLAS_COLOURS)) {
for (const objective of ["thisAttempt", "eventually"]) {
// the first screen (company) always has advice the bot can click
assert.ok(play(target, sample, objective).length >= 1, `${target} ${sample} ${objective}`);
}
}
}
});
test("for the Black the bot is pointed through both gates and up the tree", () => {
const picked = play("black", "all", "eventually").map((t) => t.toLowerCase());
assert.ok(picked.some((t) => t.startsWith("investigate the scream")), picked.join(" | "));
assert.ok(!picked.some((t) => t.startsWith("keep walking")), picked.join(" | "));
assert.ok(picked.some((t) => t.startsWith("find the tallest tree")), picked.join(" | "));
});
test("the experiment plan assigns every attempt something the advisor can play (docs/BOT_INTEGRATION.md)", () => {
const plan = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/experiments.json"), "utf8"));
const game = policy.samples.all.model.game;
let assigned = 0;
for (let accountSeq = 0; accountSeq < 20; accountSeq++) {
for (let attempt = 1; attempt <= 6; attempt++) {
for (const sawShadowBefore of [false, true]) {
const a = Solver.assign(plan, { accountSeq, attempt, facts: { sawShadowBefore } });
if (!a) continue;
assigned++;
assert.ok(Object.values(ATLAS_COLOURS).includes(a.target), `${a.experimentId}: target ${a.target}`);
assert.ok(a.propensity > 0 && a.propensity <= 1, `${a.experimentId}: propensity ${a.propensity}`);
for (const [k, v] of Object.entries(a.fix || {})) {
assert.ok(game.decisions[k] && game.decisions[k].options.includes(v), `${a.experimentId}: fix ${k} = ${v}`);
}
const adv = Solver.advisor(policy.samples.all, a.target, a.objective, { attempt, fix: a.fix, sampling: a.sampling, seed: a.seed });
const first = adv.next({});
assert.ok(first && first.choice, `${a.experimentId}: no advice on the first screen`);
if (a.fix && a.fix.company) assert.strictEqual(first.choice, a.fix.company, a.experimentId);
}
}
}
assert.ok(assigned > 0);
// a fresh account's first attempt is the Black attempt
assert.strictEqual(Solver.assign(plan, { accountSeq: 0, attempt: 1, facts: { sawShadowBefore: false } }).target, "Black");
});
test("known answers load and answer in the shape the bot expects", () => {
const sc = Object.values(tree.screens).find((s) => s.stage === "TARGET_BONDING" && Object.keys(s.options).length > 1);
const r = Solver.knownAnswer(known, Object.values(sc.options).map((o) => o.text), { target: "black", sceneColour: sc.tint ? ATLAS_COLOURS[sc.tint] || null : null });
if (r !== null) {
assert.ok(r.pick === null || Number.isInteger(r.pick));
assert.ok(Array.isArray(r.choices));
}
});
test("the bot's tie rule never takes an option that rules the Black out, even when every option's odds are about zero", () => {
// the bot (src/atlasPolicy.js) picks at random among options within tieMargin of the best. On a later attempt the
// data put the Black near zero everywhere, so before the advisor ruled such options out every option tied, and the
// bot walked away from the squadmate or went to the valley (DragonKind research runs 90, 91, 102, November 2026).
const TIE = 0.002; // DEFAULT_TIE_MARGIN in the bot
const tiedAt = (offered, history) => {
const rec = advise({ targetDragon: "black", events: history, offered });
if (!rec || rec.margin === null) return rec ? [rec.choice] : [];
const best = Math.max(...Object.values(rec.options).map((v) => v.pX));
return Object.entries(rec.options).filter(([, v]) => best - v.pX < TIE).map(([o]) => o);
};
// the screens exactly as the bot saw them (DragonKind research run 91)
const start = [{ optionText: "Enter the valley alone" }, { optionText: "Head toward the center of the forest" }, { tint: "red", optionText: "Lower your eyes in respect" }, { optionText: "Head deeper into the forest" }];
const t1 = tiedAt(["Investigate the scream", "Keep walking. The cadets are just as dangerous as the dragons today"], [...start, {}]);
assert.ok(t1.length && !t1.includes("Keep walking"), `scream: ${t1}`);
const t2 = tiedAt(["Play it safe and leave them to their own defense so you can find your dragon", "Help your squadmate"], [...start, { optionText: "Investigate the scream" }, {}]);
assert.ok(t2.length && !t2.includes("Walk away"), `clearing: ${t2}`);
const path = [...start, { optionText: "Investigate the scream" }, { optionText: "Help your squadmate" }, { optionText: "Pause to assess the situation even though it may risk your squadmate's life" },
{ optionText: "Engage the smaller cadet" }, { optionText: "Allow your squadmate to exact their own revenge" }, { optionText: "Walk farther into the forest" }, {}];
const t3 = tiedAt(["Find the tallest tree and climb for a better view", "Continue toward the center of the valley"], path);
assert.deepStrictEqual(t3, ["Tree"], `hour: ${t3}`);
});