From 77ce54acece5fdcaf59c19d3c438a8897f3e0ada Mon Sep 17 00:00:00 2001 From: kathleen Date: Thu, 19 Nov 2026 21:47:24 +0000 Subject: [PATCH] Test the bot's call chain in CI; send the experiment plan and screen tree to the bot (#26) tests/core/bot_contract.test.js copies the bot's AtlasPolicy.advise (DragonKind src/atlasPolicy.js at ebe134a2) and plays the bot's own recorded screens through it for every colour, sample and goal; checks the Black is routed through both gates and up the tree; checks every assignment of the experiment plan is playable; checks the known answers' shape. Added to npm run test:unit. push-policy now also sends data/public/experiments.json and tree.json, which docs/BOT_INTEGRATION.md says the bot reads (Solver.assign, screenPick) but which were never delivered. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ --- .gitea/workflows/push-policy.yml | 15 ++-- package.json | 2 +- tests/core/bot_contract.test.js | 147 +++++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 6 deletions(-) create mode 100644 tests/core/bot_contract.test.js diff --git a/.gitea/workflows/push-policy.yml b/.gitea/workflows/push-policy.yml index 36ba057..3203d99 100644 --- a/.gitea/workflows/push-policy.yml +++ b/.gitea/workflows/push-policy.yml @@ -1,7 +1,9 @@ # When the route model changes on main, send it to the bot: open (or update) a pull request on the bot's repo, -# DragonKind/DragonKind, that replaces vendor/threshing-atlas/{policy.json,solver.js,graph.js,known_answers.json} with -# this repo's data/public/policy.json, core/solver.js, core/graph.js and data/public/known_answers.json. They always travel together, so the policy and the -# solver that reads it never get out of step. Ken reviews and merges it; the running bot reloads the files after a pull. +# DragonKind/DragonKind, that replaces vendor/threshing-atlas/{policy.json,solver.js,graph.js,known_answers.json, +# experiments.json,tree.json} with this repo's data/public/policy.json, core/solver.js, core/graph.js, +# data/public/known_answers.json, data/public/experiments.json and data/public/tree.json. They always travel together, so +# the policy, the experiment plan and the solver that reads them never get out of step (docs/BOT_INTEGRATION.md). +# tests/core/bot_contract.test.js runs the bot's own call chain against them in CI. Ken reviews and merges it; the running bot reloads the files after a pull. # # Secrets: ATLAS_BOT_TOKEN (repository: read and write, with write access to DragonKind/DragonKind). name: Send the route model to the bot @@ -13,6 +15,8 @@ on: - core/solver.js - core/graph.js - data/public/known_answers.json + - data/public/experiments.json + - data/public/tree.json workflow_dispatch: jobs: send: @@ -35,7 +39,8 @@ jobs: GIT_CONFIG_VALUE_0="AUTHORIZATION: token $ATLAS_BOT_TOKEN" git clone --quiet --depth 1 https://git.kenschiano.net/DragonKind/DragonKind.git "$dst" mkdir -p "$dst/vendor/threshing-atlas" - cp data/public/policy.json core/solver.js core/graph.js data/public/known_answers.json "$dst/vendor/threshing-atlas/" + cp data/public/policy.json core/solver.js core/graph.js data/public/known_answers.json \ + data/public/experiments.json data/public/tree.json "$dst/vendor/threshing-atlas/" id=$(python3 -c "import json; print(json.load(open('data/public/policy.json')).get('id', ''))") sha=$(git rev-parse --short HEAD) cd "$dst" @@ -52,7 +57,7 @@ jobs: printf '%s\n' \ "The Threshing Atlas route model changed (policy \`$id\`, atlas commit \`$sha\`)." \ "" \ - "This replaces \`vendor/threshing-atlas/\` (policy.json, solver.js, graph.js) together, as \`npm run atlas:sync\` would." \ + "This replaces \`vendor/threshing-atlas/\` (policy.json, solver.js, graph.js, known_answers.json, experiments.json, tree.json) together, as \`npm run atlas:sync\` would." \ "This pull request is updated in place each time the model changes. After merging, pull on the bot's machine:" \ "the running bot reloads the files on its next decision." \ "" \ diff --git a/package.json b/package.json index 65e1a7c..13fa5de 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "build": "node site/build.js", "dev": "node site/build.js && python3 -m http.server 8000 -d dist", "test:py": "python3 -m collection build --check && python3 planner/build_experiments.py --check && python3 analysis/black/evidence.py --check && python3 analysis/tree/build_tree.py --check && python3 analysis/known/build_known.py --check && python3 analysis/overlap/overlap.py --check && PYTHONPATH=core python3 -m unittest discover -s tests/core -p 'test_*.py' && python3 -m unittest discover -s tests/collection -p 'test_*.py'", - "test:unit": "node --test tests/core/graph.test.js tests/core/advisor.test.js tests/core/known.test.js", + "test:unit": "node --test tests/core/graph.test.js tests/core/advisor.test.js tests/core/known.test.js tests/core/bot_contract.test.js", "test": "npm run test:py && npm run test:unit", "graph": "node analysis/network/build-graph.js", "policy": "python3 planner/build_policy.py && python3 planner/policy_report.py && python3 planner/build_experiments.py && python3 analysis/black/evidence.py && python3 analysis/tree/build_tree.py && python3 analysis/known/build_known.py && python3 analysis/overlap/overlap.py", diff --git a/tests/core/bot_contract.test.js b/tests/core/bot_contract.test.js new file mode 100644 index 0000000..fa7bf5b --- /dev/null +++ b/tests/core/bot_contract.test.js @@ -0,0 +1,147 @@ +// The bot's side of the route advisor. Ken's bot (DragonKind/DragonKind src/atlasPolicy.js) loads policy.json, +// solver.js and graph.js from vendor/threshing-atlas/ and calls them in one fixed chain. This test copies that chain +// (AtlasPolicy.advise and buildTrajectory, as of DragonKind commit ebe134a2) and plays the bot's own recorded screens +// (data/public/tree.json) through it, for every colour, sample and goal, so a change here that would break the bot +// fails in this repo's CI first. node --test tests/core/bot_contract.test.js +const test = require("node:test"); +const assert = require("node:assert"); +const path = require("node:path"); +const fs = require("node:fs"); + +const ROOT = path.join(__dirname, "../.."); +const Solver = require("../../core/solver.js"); +const Graph = require("../../core/graph.js"); +const policy = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/policy.json"), "utf8")); +const tree = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/tree.json"), "utf8")); +const known = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/known_answers.json"), "utf8")); + +// ---- copied from the bot (src/atlasPolicy.js): keep in step with it +const ATLAS_COLOURS = { black: "Black", blue: "Blue", green: "Green", red: "Red", brown: "Brown", orange: "Orange" }; +const FIRST_SIGHTING_COLOURS = new Set(["red", "brown", "green", "orange"]); +function sceneColour({ tint = null, observed = {} } = {}) { + const t = String(tint || "").toLowerCase(); + if (ATLAS_COLOURS[t]) return t; + const dragons = Array.isArray(observed?.dragons) ? observed.dragons : []; + return dragons.length === 1 ? dragons[0] : null; +} +function buildTrajectory(events = []) { + const steps = []; + const seen = {}; + for (const ev of events) { + const colour = sceneColour(ev); + steps.push({ optionText: ev.optionText || "", tint: colour && FIRST_SIGHTING_COLOURS.has(colour) ? colour : null }); + if (ev.observed?.shadow === true) seen.shadow = "yes"; + else if (ev.observed?.shadow === false && seen.shadow !== "yes") seen.shadow = "no"; + } + return { trajectory: { steps, outcome: {} }, seen }; +} +const advisors = new Map(); +function advise({ targetDragon, sample = "all", objective = "eventually", events = [], offered = [] }) { + const colour = ATLAS_COLOURS[targetDragon]; + const { trajectory, seen } = buildTrajectory(events); + const { vars } = Graph.trajectoryVariables(trajectory); + const key = `${sample}|${colour}|${objective}`; + if (!advisors.has(key)) advisors.set(key, Solver.advisor(policy.samples[sample], colour, objective)); + const adv = advisors.get(key); + const history = Solver.solverHistory(vars, seen, adv.game); + const raw = adv.next(history); + if (!raw) return null; + const withStep = (step) => ({ ...adv, next: (h) => (h === history ? step : adv.next(h)) }); + return Solver.recommendOption(withStep(raw), history, offered); +} +// ---- end of the copy + +test("the bot's modules have what it checks for when it loads them", () => { + assert.strictEqual(typeof Solver.advisor, "function"); + assert.strictEqual(typeof Solver.recommendOption, "function"); + assert.strictEqual(typeof Solver.solverHistory, "function"); + assert.strictEqual(typeof Solver.knownAnswer, "function"); + assert.strictEqual(typeof Graph.trajectoryVariables, "function"); + assert.ok(policy.samples.all.model && policy.samples.lowBias.model && policy.id); +}); + +/** + * Play one run through the recorded screens, asking for advice on each. The run follows the advised option where the + * bot has played it (so the screen after it is on record), and otherwise the option the bot played most, going on to + * the screen it most often led to. Returns the advised option text on every screen that had one. + */ +function play(targetDragon, sample, objective) { + const events = [], advised = []; + let id = tree.start; + for (let guard = 0; id && tree.screens[id] && guard < 40; guard++) { + const sc = tree.screens[id]; + const opts = Object.values(sc.options); + if (!opts.length) break; + const offered = opts.map((o) => o.text); + const here = { tint: sc.tint }; + const rec = advise({ targetDragon, sample, objective, events: [...events, here], offered }); + if (rec) { + assert.ok(rec.decision && rec.choice, `${targetDragon}: advice without a decision on ${sc.label}`); + assert.ok(rec.choice in rec.options, `${targetDragon}: ${rec.choice} is not one of its own options`); + assert.ok(rec.index === null || (Number.isInteger(rec.index) && rec.index >= 0 && rec.index < offered.length), + `${targetDragon}: index ${rec.index} on ${sc.label}`); + if (rec.index !== null) advised.push(offered[rec.index]); + } + const playedTo = (o) => Object.entries(o.to || {}).sort((a, b) => b[1].n - a[1].n)[0]; + let i = rec && rec.index !== null && playedTo(opts[rec.index]) ? rec.index : -1; + if (i < 0) i = opts.reduce((b, o, j) => ((o.n || 0) > (opts[b].n || 0) ? j : b), 0); + events.push({ ...here, optionText: offered[i] }); + const to = playedTo(opts[i]); + id = to ? to[0] : null; + } + return advised; +} + +test("every colour, sample and goal plays the recorded screens without an error or an impossible pick", () => { + for (const sample of ["all", "lowBias"]) { + for (const target of Object.keys(ATLAS_COLOURS)) { + for (const objective of ["thisAttempt", "eventually"]) { + // the first screen (company) always has advice the bot can click + assert.ok(play(target, sample, objective).length >= 1, `${target} ${sample} ${objective}`); + } + } + } +}); + +test("for the Black the bot is pointed through both gates and up the tree", () => { + const picked = play("black", "all", "eventually").map((t) => t.toLowerCase()); + assert.ok(picked.some((t) => t.startsWith("investigate the scream")), picked.join(" | ")); + assert.ok(!picked.some((t) => t.startsWith("keep walking")), picked.join(" | ")); + assert.ok(picked.some((t) => t.startsWith("find the tallest tree")), picked.join(" | ")); +}); + +test("the experiment plan assigns every attempt something the advisor can play (docs/BOT_INTEGRATION.md)", () => { + const plan = JSON.parse(fs.readFileSync(path.join(ROOT, "data/public/experiments.json"), "utf8")); + const game = policy.samples.all.model.game; + let assigned = 0; + for (let accountSeq = 0; accountSeq < 20; accountSeq++) { + for (let attempt = 1; attempt <= 6; attempt++) { + for (const sawShadowBefore of [false, true]) { + const a = Solver.assign(plan, { accountSeq, attempt, facts: { sawShadowBefore } }); + if (!a) continue; + assigned++; + assert.ok(Object.values(ATLAS_COLOURS).includes(a.target), `${a.experimentId}: target ${a.target}`); + assert.ok(a.propensity > 0 && a.propensity <= 1, `${a.experimentId}: propensity ${a.propensity}`); + for (const [k, v] of Object.entries(a.fix || {})) { + assert.ok(game.decisions[k] && game.decisions[k].options.includes(v), `${a.experimentId}: fix ${k} = ${v}`); + } + const adv = Solver.advisor(policy.samples.all, a.target, a.objective, { attempt, fix: a.fix, sampling: a.sampling, seed: a.seed }); + const first = adv.next({}); + assert.ok(first && first.choice, `${a.experimentId}: no advice on the first screen`); + if (a.fix && a.fix.company) assert.strictEqual(first.choice, a.fix.company, a.experimentId); + } + } + } + assert.ok(assigned > 0); + // a fresh account's first attempt is the Black attempt + assert.strictEqual(Solver.assign(plan, { accountSeq: 0, attempt: 1, facts: { sawShadowBefore: false } }).target, "Black"); +}); + +test("known answers load and answer in the shape the bot expects", () => { + const sc = Object.values(tree.screens).find((s) => s.stage === "TARGET_BONDING" && Object.keys(s.options).length > 1); + const r = Solver.knownAnswer(known, Object.values(sc.options).map((o) => o.text), { target: "black", sceneColour: sc.tint ? ATLAS_COLOURS[sc.tint] || null : null }); + if (r !== null) { + assert.ok(r.pick === null || Number.isInteger(r.pick)); + assert.ok(Array.isArray(r.choices)); + } +});