Route model learns from bot runs; Thompson sampling in the advisor (work in progress)
- planner/policy.yaml: which sources the route model learns from (spreadsheet,
reported, bot) and the low-bias sample
- bot runs: deaths coded by stage, Malek intent for runs with a target
- model: survival of the first dragon is not counted when nothing after it is
known; the meadow dragon falls back to what the bot saw
- solver.js: advisor(..., {sampling: "thompson", seed, draws}) follows one
posterior draw per attempt and reports each pick's propensity
Not shipped yet: mixing sources lowers the fitted shrinkage for the shadow roll
(alpha 500 -> 50 in the low-bias sample), see the conversation of 2026-11-09.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
d37eaa3378
commit
c0355616b6
+7
-2
@@ -69,6 +69,9 @@ def prepare_run(r):
|
||||
v["diedSight"] = r["scream"] == "DIED at first sighting"
|
||||
if v["diedSight"]:
|
||||
return v
|
||||
if r["scream"] not in ("Investigate & help", "Ignore the scream", "Investigate & leave"):
|
||||
v["sightUnknown"] = True # no answer after the first dragon: whether the rider survived it is not known
|
||||
return v
|
||||
res = r["resolution"]
|
||||
if r["scream"] == "Investigate & help":
|
||||
v["help"] = True
|
||||
@@ -113,7 +116,9 @@ def prepare_run(r):
|
||||
ended = lambda c: r.get("enc_" + c) and r.get("enc_" + c) != "Changed my mind" # noqa: E731
|
||||
v["meadow"] = "Blue" if (r.get("enc_Blue") or r.get("rare_reported") == "Blue (valley)") else \
|
||||
"Orange" if ended("Orange") else "Brown" if ended("Brown") else \
|
||||
"Orange" if r.get("enc_Orange") else "Brown" if r.get("enc_Brown") else None
|
||||
"Orange" if r.get("enc_Orange") else "Brown" if r.get("enc_Brown") else \
|
||||
"Blue" if r.get("saw_blue_meadow") is True else \
|
||||
r["outcome_color"] if r.get("saw_blue_meadow") is False and r.get("outcome_color") in ("Orange", "Brown") else None
|
||||
if v["meadow"]:
|
||||
v["encCtx"] = "meadow-" + v["meadow"]
|
||||
if v.get("encCtx"):
|
||||
@@ -127,7 +132,7 @@ def prepare_run(r):
|
||||
# what each chance event observed in a prepared run
|
||||
OBSERVE = {
|
||||
"fs": lambda v: v.get("fs"),
|
||||
"dieSight": lambda v: ("die" if v.get("diedSight") else "live") if v.get("fs") else None,
|
||||
"dieSight": lambda v: ("die" if v.get("diedSight") else "live") if v.get("fs") and not v.get("sightUnknown") else None,
|
||||
"dieCombat": lambda v: ("die" if v.get("diedCombat") else "live") if v.get("help") else None,
|
||||
"dieHour": lambda v: ("die" if v.get("diedHour") else "live") if v.get("survivedCadets") else None,
|
||||
"shadow": lambda v: None if "shadow" not in v else ("yes" if v["shadow"] else "no"),
|
||||
|
||||
+52
-14
@@ -24,8 +24,33 @@
|
||||
const encColour = (ctx) => (ctx === "tree-Greens" ? "Green" : ctx.split("-")[1]);
|
||||
const api = {};
|
||||
|
||||
/* ---------- the odds: posterior means with hierarchical back-off (as in core/atlas/model.py) ---------- */
|
||||
function makeParams(T, { alphas = {}, game }) {
|
||||
/* ---------- posterior draws, for Thompson sampling ---------- */
|
||||
function rng(seed) {
|
||||
let x = (seed >>> 0) || 0x9e3779b9;
|
||||
return () => { x ^= x << 13; x >>>= 0; x ^= x >>> 17; x ^= x << 5; x >>>= 0; return (x + 0.5) / 4294967296; };
|
||||
}
|
||||
function normal(rand) { return Math.sqrt(-2 * Math.log(rand())) * Math.cos(2 * Math.PI * rand()); }
|
||||
function gamma(k, rand) { // Marsaglia and Tsang
|
||||
if (k < 1) return gamma(k + 1, rand) * Math.pow(rand(), 1 / k);
|
||||
const d = k - 1 / 3, c = 1 / Math.sqrt(9 * d);
|
||||
for (;;) {
|
||||
let x, v;
|
||||
do { x = normal(rand); v = 1 + c * x; } while (v <= 0);
|
||||
v = v * v * v;
|
||||
const u = rand();
|
||||
if (u < 1 - 0.0331 * x ** 4 || Math.log(u) < 0.5 * x * x + d * (1 - v + Math.log(v))) return d * v;
|
||||
}
|
||||
}
|
||||
function dirichlet(a, rand) {
|
||||
const g = a.map((x) => Math.max(gamma(x, rand), 1e-300));
|
||||
let s = 0; for (const x of g) s += x;
|
||||
return g.map((x) => x / s);
|
||||
}
|
||||
|
||||
/* ---------- the odds: posterior means (or, with rand, one posterior draw) with hierarchical back-off ---------- */
|
||||
function makeParams(T, { alphas = {}, game, rand = null }) {
|
||||
const normalise = (a) => { let sum = 0; for (const x of a) sum += x; return a.map((x) => x / sum); };
|
||||
const estimate = rand ? (a) => dirichlet(a, rand) : normalise;
|
||||
const memo = new Map();
|
||||
const ctxKey = (s, names) => names.map((k) => (s[k] == null ? "?" : s[k])).join("|");
|
||||
function chance(node, state) {
|
||||
@@ -38,8 +63,7 @@
|
||||
if (memo.has(mk)) { p = memo.get(mk); continue; }
|
||||
const c = T[node][key] || {}, al = alphas[node] || ALPHA;
|
||||
const a = cats.map((k, i) => (c[k] || 0) + (p ? al * p[i] : 0.5));
|
||||
let sum = 0; for (const x of a) sum += x;
|
||||
p = a.map((x) => x / sum);
|
||||
p = estimate(a);
|
||||
memo.set(mk, p);
|
||||
}
|
||||
return Object.fromEntries(cats.map((k, i) => [k, p[i]]));
|
||||
@@ -49,9 +73,8 @@
|
||||
if (!memo.has(mk)) {
|
||||
const e = T.encounter[ctx] || { outcomes: {}, seen: {} };
|
||||
const cats = game.encounter_outcomes.filter((k) => k === "burn" || e.seen[k] || (e.outcomes[k] || 0) > 0 || k === encColour(ctx));
|
||||
const a = cats.map((k) => (e.outcomes[k] || 0) + 0.5);
|
||||
let sum = 0; for (const x of a) sum += x;
|
||||
memo.set(mk, Object.fromEntries(cats.map((k, i) => [k, a[i] / sum])));
|
||||
const a = estimate(cats.map((k) => (e.outcomes[k] || 0) + 0.5));
|
||||
memo.set(mk, Object.fromEntries(cats.map((k, i) => [k, a[i]])));
|
||||
}
|
||||
return memo.get(mk);
|
||||
}
|
||||
@@ -149,17 +172,31 @@
|
||||
* the deadly roll just passed on the step after it. When h already has a later step but skips an earlier
|
||||
* decision (an older form record, say), the recommended option is assumed for the skipped one.
|
||||
*/
|
||||
function advisor(sample, X, objective = "eventually", { only = null } = {}) {
|
||||
function advisor(sample, X, objective = "eventually", { only = null, sampling = "best", seed = 1, draws = 40 } = {}) {
|
||||
const m = sample.model, game = m.game;
|
||||
api.game = game; api.ENCOUNTERS = game.by_colour;
|
||||
const params = makeParams(m.tables, { alphas: m.alphas, game });
|
||||
const sol = objective === "eventually" ? solveRatio(params, game, m.support, X, only) : solve(params, game, m.support, X, 0, only);
|
||||
const solveWith = (params) => (objective === "eventually" ? solveRatio(params, game, m.support, X, only) : solve(params, game, m.support, X, 0, only));
|
||||
// "best": the best route under the posterior means. "thompson": the best route of one posterior draw of the odds
|
||||
// (one per attempt: make a new advisor with a new seed for each attempt), so every option is tried about as
|
||||
// often as it is likely to be the best one; as the data firm up, sampling settles on the best route.
|
||||
const thompson = sampling === "thompson";
|
||||
const params = makeParams(m.tables, { alphas: m.alphas, game, rand: thompson ? rng(seed) : null });
|
||||
const sol = solveWith(params);
|
||||
// the chance Thompson sampling picks each option at a history, estimated from `draws` other draws: the
|
||||
// propensity to log, so the runs can be re-weighted later
|
||||
let worlds = null;
|
||||
const propensityAt = (key, choice) => {
|
||||
if (!thompson) return 1;
|
||||
if (!worlds) worlds = Array.from({ length: draws }, (_, i) => solveWith(makeParams(m.tables, { alphas: m.alphas, game, rand: rng(seed * 7919 + i + 1) })).choices);
|
||||
return worlds.filter((c) => c.get(key) === choice).length / draws;
|
||||
};
|
||||
const VARS = new Set([...Object.keys(game.decisions), ...Object.keys(game.chance), "encounter"]);
|
||||
const fmt = (v) => ({ pX: v.x, pOther: v.o, pReach: v.r });
|
||||
const label = (k) => (game.decisions[k] || game.chance[k]).label;
|
||||
const decisionAt = (name, key, extra) => {
|
||||
const vals = sol.optVals.get(key) || {};
|
||||
return { kind: "decision", var: name, label: label(name), choice: sol.choices.get(key),
|
||||
const choice = sol.choices.get(key);
|
||||
return { kind: "decision", var: name, label: label(name), choice, propensity: propensityAt(key, choice),
|
||||
options: Object.fromEntries(Object.entries(vals).map(([o, v]) => [o, fmt(v)])), ...extra };
|
||||
};
|
||||
|
||||
@@ -197,13 +234,13 @@
|
||||
throw new Error("the game has no end");
|
||||
}
|
||||
return {
|
||||
colour: X, objective, only, value: sol.value, step, params, game,
|
||||
colour: X, objective, only, sampling, seed: thompson ? seed : null, value: sol.value, step, params, game,
|
||||
/** attempts recorded at an encounter (genuine attempts only), to show how much data a bond chance rests on */
|
||||
attempts: (ctx) => Object.values(((m.tables.encounter || {})[ctx] || { outcomes: {} }).outcomes).reduce((a, b) => a + b, 0),
|
||||
/** the next decision only (null when the next step is a roll or the run is over) */
|
||||
next(h) {
|
||||
const s = step(h);
|
||||
return s.kind === "decision" ? { decision: s.var, label: s.label, choice: s.choice, options: s.options } : null;
|
||||
return s.kind === "decision" ? { decision: s.var, label: s.label, choice: s.choice, propensity: s.propensity, options: s.options } : null;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -222,7 +259,8 @@
|
||||
const pat = ((adv.game.bot_text || {})[rec.decision] || {})[rec.choice];
|
||||
const idx = pat ? offered.map(norm).findIndex((t) => new RegExp(pat).test(t)) : -1;
|
||||
const values = Object.values(rec.options).map((v) => v.pX).sort((a, b) => b - a);
|
||||
return { index: idx < 0 ? null : idx, decision: rec.decision, choice: rec.choice, margin: values.length > 1 ? values[0] - values[1] : null,
|
||||
return { index: idx < 0 ? null : idx, decision: rec.decision, choice: rec.choice, propensity: rec.propensity,
|
||||
margin: values.length > 1 ? values[0] - values[1] : null,
|
||||
options: rec.options,
|
||||
reason: idx < 0 && rec.decision !== "encounter" ? "the recommended choice is not on this screen; use the bot's own rules" : `${rec.label}: ${rec.choice}` };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user