diff --git a/collection/etl/runs.py b/collection/etl/runs.py index 6a8af23..b1a90e8 100644 --- a/collection/etl/runs.py +++ b/collection/etl/runs.py @@ -198,6 +198,15 @@ def bot(name, spec): goal=target if target in COLOURS else None, outcome=outcome, outcome_color=colour, data_flags="; ".join(flags), **v) row["attempt_group"] = attempt_group(row["attempt"]) + if target: # the bot never bonds a dragon it was not sent for: it dies on purpose instead (Malek) + row["malek_intent"] = "Yes" + if outcome == "Died before the hour" and not o.get("intentional"): # where the game killed it + if row["scream"] == "Unknown": + row["scream"] = "DIED at first sighting" + elif row["scream"] == "Investigate & help" and row["resolution"] == "Unknown": + row["resolution"] = "DIED in combat" + else: + row["hour"] = "DIED before the hour" # what the bot saw: the shadow at the treetop, and which dragons waited in the meadow shadow = _seen_after(steps, 0, "shadow") if row["hour"] == "Tree" and shadow: diff --git a/core/atlas/model.py b/core/atlas/model.py index a745794..fb6f952 100644 --- a/core/atlas/model.py +++ b/core/atlas/model.py @@ -69,6 +69,9 @@ def prepare_run(r): v["diedSight"] = r["scream"] == "DIED at first sighting" if v["diedSight"]: return v + if r["scream"] not in ("Investigate & help", "Ignore the scream", "Investigate & leave"): + v["sightUnknown"] = True # no answer after the first dragon: whether the rider survived it is not known + return v res = r["resolution"] if r["scream"] == "Investigate & help": v["help"] = True @@ -113,7 +116,9 @@ def prepare_run(r): ended = lambda c: r.get("enc_" + c) and r.get("enc_" + c) != "Changed my mind" # noqa: E731 v["meadow"] = "Blue" if (r.get("enc_Blue") or r.get("rare_reported") == "Blue (valley)") else \ "Orange" if ended("Orange") else "Brown" if ended("Brown") else \ - "Orange" if r.get("enc_Orange") else "Brown" if r.get("enc_Brown") else None + "Orange" if r.get("enc_Orange") else "Brown" if r.get("enc_Brown") else \ + "Blue" if r.get("saw_blue_meadow") is True else \ + r["outcome_color"] if r.get("saw_blue_meadow") is False and r.get("outcome_color") in ("Orange", "Brown") else None if v["meadow"]: v["encCtx"] = "meadow-" + v["meadow"] if v.get("encCtx"): @@ -127,7 +132,7 @@ def prepare_run(r): # what each chance event observed in a prepared run OBSERVE = { "fs": lambda v: v.get("fs"), - "dieSight": lambda v: ("die" if v.get("diedSight") else "live") if v.get("fs") else None, + "dieSight": lambda v: ("die" if v.get("diedSight") else "live") if v.get("fs") and not v.get("sightUnknown") else None, "dieCombat": lambda v: ("die" if v.get("diedCombat") else "live") if v.get("help") else None, "dieHour": lambda v: ("die" if v.get("diedHour") else "live") if v.get("survivedCadets") else None, "shadow": lambda v: None if "shadow" not in v else ("yes" if v["shadow"] else "no"), diff --git a/core/solver.js b/core/solver.js index 4363335..baafe34 100644 --- a/core/solver.js +++ b/core/solver.js @@ -24,8 +24,33 @@ const encColour = (ctx) => (ctx === "tree-Greens" ? "Green" : ctx.split("-")[1]); const api = {}; - /* ---------- the odds: posterior means with hierarchical back-off (as in core/atlas/model.py) ---------- */ - function makeParams(T, { alphas = {}, game }) { + /* ---------- posterior draws, for Thompson sampling ---------- */ + function rng(seed) { + let x = (seed >>> 0) || 0x9e3779b9; + return () => { x ^= x << 13; x >>>= 0; x ^= x >>> 17; x ^= x << 5; x >>>= 0; return (x + 0.5) / 4294967296; }; + } + function normal(rand) { return Math.sqrt(-2 * Math.log(rand())) * Math.cos(2 * Math.PI * rand()); } + function gamma(k, rand) { // Marsaglia and Tsang + if (k < 1) return gamma(k + 1, rand) * Math.pow(rand(), 1 / k); + const d = k - 1 / 3, c = 1 / Math.sqrt(9 * d); + for (;;) { + let x, v; + do { x = normal(rand); v = 1 + c * x; } while (v <= 0); + v = v * v * v; + const u = rand(); + if (u < 1 - 0.0331 * x ** 4 || Math.log(u) < 0.5 * x * x + d * (1 - v + Math.log(v))) return d * v; + } + } + function dirichlet(a, rand) { + const g = a.map((x) => Math.max(gamma(x, rand), 1e-300)); + let s = 0; for (const x of g) s += x; + return g.map((x) => x / s); + } + + /* ---------- the odds: posterior means (or, with rand, one posterior draw) with hierarchical back-off ---------- */ + function makeParams(T, { alphas = {}, game, rand = null }) { + const normalise = (a) => { let sum = 0; for (const x of a) sum += x; return a.map((x) => x / sum); }; + const estimate = rand ? (a) => dirichlet(a, rand) : normalise; const memo = new Map(); const ctxKey = (s, names) => names.map((k) => (s[k] == null ? "?" : s[k])).join("|"); function chance(node, state) { @@ -38,8 +63,7 @@ if (memo.has(mk)) { p = memo.get(mk); continue; } const c = T[node][key] || {}, al = alphas[node] || ALPHA; const a = cats.map((k, i) => (c[k] || 0) + (p ? al * p[i] : 0.5)); - let sum = 0; for (const x of a) sum += x; - p = a.map((x) => x / sum); + p = estimate(a); memo.set(mk, p); } return Object.fromEntries(cats.map((k, i) => [k, p[i]])); @@ -49,9 +73,8 @@ if (!memo.has(mk)) { const e = T.encounter[ctx] || { outcomes: {}, seen: {} }; const cats = game.encounter_outcomes.filter((k) => k === "burn" || e.seen[k] || (e.outcomes[k] || 0) > 0 || k === encColour(ctx)); - const a = cats.map((k) => (e.outcomes[k] || 0) + 0.5); - let sum = 0; for (const x of a) sum += x; - memo.set(mk, Object.fromEntries(cats.map((k, i) => [k, a[i] / sum]))); + const a = estimate(cats.map((k) => (e.outcomes[k] || 0) + 0.5)); + memo.set(mk, Object.fromEntries(cats.map((k, i) => [k, a[i]]))); } return memo.get(mk); } @@ -149,17 +172,31 @@ * the deadly roll just passed on the step after it. When h already has a later step but skips an earlier * decision (an older form record, say), the recommended option is assumed for the skipped one. */ - function advisor(sample, X, objective = "eventually", { only = null } = {}) { + function advisor(sample, X, objective = "eventually", { only = null, sampling = "best", seed = 1, draws = 40 } = {}) { const m = sample.model, game = m.game; api.game = game; api.ENCOUNTERS = game.by_colour; - const params = makeParams(m.tables, { alphas: m.alphas, game }); - const sol = objective === "eventually" ? solveRatio(params, game, m.support, X, only) : solve(params, game, m.support, X, 0, only); + const solveWith = (params) => (objective === "eventually" ? solveRatio(params, game, m.support, X, only) : solve(params, game, m.support, X, 0, only)); + // "best": the best route under the posterior means. "thompson": the best route of one posterior draw of the odds + // (one per attempt: make a new advisor with a new seed for each attempt), so every option is tried about as + // often as it is likely to be the best one; as the data firm up, sampling settles on the best route. + const thompson = sampling === "thompson"; + const params = makeParams(m.tables, { alphas: m.alphas, game, rand: thompson ? rng(seed) : null }); + const sol = solveWith(params); + // the chance Thompson sampling picks each option at a history, estimated from `draws` other draws: the + // propensity to log, so the runs can be re-weighted later + let worlds = null; + const propensityAt = (key, choice) => { + if (!thompson) return 1; + if (!worlds) worlds = Array.from({ length: draws }, (_, i) => solveWith(makeParams(m.tables, { alphas: m.alphas, game, rand: rng(seed * 7919 + i + 1) })).choices); + return worlds.filter((c) => c.get(key) === choice).length / draws; + }; const VARS = new Set([...Object.keys(game.decisions), ...Object.keys(game.chance), "encounter"]); const fmt = (v) => ({ pX: v.x, pOther: v.o, pReach: v.r }); const label = (k) => (game.decisions[k] || game.chance[k]).label; const decisionAt = (name, key, extra) => { const vals = sol.optVals.get(key) || {}; - return { kind: "decision", var: name, label: label(name), choice: sol.choices.get(key), + const choice = sol.choices.get(key); + return { kind: "decision", var: name, label: label(name), choice, propensity: propensityAt(key, choice), options: Object.fromEntries(Object.entries(vals).map(([o, v]) => [o, fmt(v)])), ...extra }; }; @@ -197,13 +234,13 @@ throw new Error("the game has no end"); } return { - colour: X, objective, only, value: sol.value, step, params, game, + colour: X, objective, only, sampling, seed: thompson ? seed : null, value: sol.value, step, params, game, /** attempts recorded at an encounter (genuine attempts only), to show how much data a bond chance rests on */ attempts: (ctx) => Object.values(((m.tables.encounter || {})[ctx] || { outcomes: {} }).outcomes).reduce((a, b) => a + b, 0), /** the next decision only (null when the next step is a roll or the run is over) */ next(h) { const s = step(h); - return s.kind === "decision" ? { decision: s.var, label: s.label, choice: s.choice, options: s.options } : null; + return s.kind === "decision" ? { decision: s.var, label: s.label, choice: s.choice, propensity: s.propensity, options: s.options } : null; }, }; } @@ -222,7 +259,8 @@ const pat = ((adv.game.bot_text || {})[rec.decision] || {})[rec.choice]; const idx = pat ? offered.map(norm).findIndex((t) => new RegExp(pat).test(t)) : -1; const values = Object.values(rec.options).map((v) => v.pX).sort((a, b) => b - a); - return { index: idx < 0 ? null : idx, decision: rec.decision, choice: rec.choice, margin: values.length > 1 ? values[0] - values[1] : null, + return { index: idx < 0 ? null : idx, decision: rec.decision, choice: rec.choice, propensity: rec.propensity, + margin: values.length > 1 ? values[0] - values[1] : null, options: rec.options, reason: idx < 0 && rec.decision !== "encounter" ? "the recommended choice is not on this screen; use the bot's own rules" : `${rec.label}: ${rec.choice}` }; } diff --git a/planner/build_policy.py b/planner/build_policy.py index a7acabe..ad0a603 100644 --- a/planner/build_policy.py +++ b/planner/build_policy.py @@ -2,7 +2,8 @@ python planner/build_policy.py [--draws 200] [--out data/public/policy.json] -Two samples: every run, and a low-bias sample (bulk logs plus form runs from riders not chasing a rare dragon). +Two samples, from the runs and rules in planner/policy.yaml: every run, and a low-bias sample (bot runs, bulk logs, +and riders not chasing a rare dragon). The output has a content id, so each bot run can record which policy it followed. """ import argparse @@ -20,9 +21,19 @@ from atlas import runs as runs_io # noqa: E402 import yaml # noqa: E402 -# the same runs the site's statistics use (analysis/stats/measures.yaml `sources`) -with open(ROOT / "analysis/stats/measures.yaml", encoding="utf-8") as _f: - SOURCES = yaml.safe_load(_f)["sources"] +# which runs the model learns from, and the low-bias sample (planner/policy.yaml) +with open(ROOT / "planner/policy.yaml", encoding="utf-8") as _f: + CONFIG = yaml.safe_load(_f) +SOURCES = CONFIG["sources"] + + +def low_bias(r, lb=CONFIG["low_bias"]): + """Kept in the low-bias sample: every run of an always-kept source, bulk logs, and riders not chasing a rare dragon.""" + if r["source"] in lb["always"]: + return True + if r["source"] == "reddit spreadsheet" and "bulk" in lb["entries_kept"] and r["entry"] not in (None, "Form"): + return True + return r["goal"] not in lb["rare_goals"] def clean(o): @@ -49,10 +60,14 @@ def main(argv=None): rows = len(runs) samples = { "all": runs, - "lowBias": [r for r in runs if r["entry"] != "Form" or r["goal"] not in model.RARE_GOALS], + "lowBias": [r for r in runs if low_bias(r)], } t0 = time.time() - out = {"schema": 1, "source": {"form_rows": rows}, "samples": {}} + by_source = {} + for r in runs: + by_source[r["source"]] = by_source.get(r["source"], 0) + 1 + out = {"schema": 1, "source": {"form_rows": by_source.get("reddit spreadsheet", 0), "runs": rows, "by_source": by_source}, + "samples": {}} for name, rs in samples.items(): out["samples"][name] = clean(solver.build_policy(rs, draws=a.draws)) print(f"{name}: {len(rs)} runs, {time.time() - t0:.1f} s") diff --git a/planner/policy.yaml b/planner/policy.yaml new file mode 100644 index 0000000..08d5818 --- /dev/null +++ b/planner/policy.yaml @@ -0,0 +1,17 @@ +# Which runs the route model (data/public/policy.json) learns from. planner/build_policy.py reads this file. +# +# The model estimates the game's rolls (first dragon, deaths, shadow, meadow dragon, bond chances) given the choices +# made before them, so any run that records its choices and what happened can teach it, whoever chose the route. +# Deliberate deaths (Malek, the bot's "no shadow" / "no Blue" burns) are never counted as failed bond attempts. +version: 1 + +# the `source` values of data/public/runs_clean.csv the model learns from +sources: [reddit spreadsheet, reported, bot] + +# Two samples. "all" is every run above. "lowBias" leaves out runs where a rare sighting is more likely to have +# been reported than a miss: form submissions and reports from riders chasing Blue or Black. The bot logs every +# run, and bulk-pasted logs record every run of the players who kept them, so both stay in. +low_bias: + always: [bot] # sources whose every run is kept + entries_kept: [bulk] # spreadsheet rows whose `entry` is not "Form" (bulk-pasted logs) + rare_goals: [Blue, Black, Blue or Black] diff --git a/tests/core/advisor.test.js b/tests/core/advisor.test.js index cc728db..9673d6f 100644 --- a/tests/core/advisor.test.js +++ b/tests/core/advisor.test.js @@ -110,3 +110,18 @@ test("solverHistory turns form variables into a history keyed by the game's step assert.strictEqual(l.leave, "Leave them"); assert.strictEqual(l.red, "Let it pass (two Greens)"); }); + +test("Thompson sampling: one posterior draw per seed, with the chance of each pick", () => { + const S = require("../../core/solver.js"); + const P = require("../../data/public/policy.json"); + const best = S.advisor(P.samples.all, "Blue", "eventually").next({}); + assert.strictEqual(best.propensity, 1); + const a = S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: 3, draws: 8 }); + const b = S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: 3, draws: 8 }); + const ra = a.next({}), rb = b.next({}); + assert.strictEqual(ra.choice, rb.choice); // same seed, same draw + assert.ok(ra.propensity >= 0 && ra.propensity <= 1); + const picks = new Set(); + for (let s = 1; s <= 12; s++) picks.add(S.advisor(P.samples.all, "Blue", "eventually", { sampling: "thompson", seed: s, draws: 1 }).next({}).choice); + assert.ok(picks.size > 1, "different seeds explore different options"); +});