From 2271fe097609ac593c0e07d09fe52857bb0e2f17 Mon Sep 17 00:00:00 2001 From: ekcSchiano Date: Tue, 10 Nov 2026 02:28:28 +0000 Subject: [PATCH] Route model: correct for reporting bias - core/atlas/bias.py: for each rare roll (shadow, meadow Blue) and each source group, the factor by which the group over-reports the rare outcome relative to complete logging (bot, bulk logs), with a 90% range; rare outcomes from biased groups count 1/rho; a roll the complete logs have (almost) never shown is learned from them only - planner/policy.yaml: groups, rolls and thresholds; estimates published in policy.json under model.reportingBias - fit_alpha: a roll seen only one way is pooled fully - docs: goal accounts follow the best route, research accounts use Thompson sampling (decisionMode atlas_thompson) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ --- core/atlas/bias.py | 109 ++++++++++++++++++++++++++++++++++++++++ core/atlas/model.py | 21 ++++++-- core/atlas/solver.py | 17 +++++-- data/dictionary.yaml | 4 +- docs/BOT_INTEGRATION.md | 40 ++++++++++++--- planner/build_policy.py | 2 +- planner/policy.yaml | 17 +++++++ tests/core/test_bias.py | 52 +++++++++++++++++++ 8 files changed, 244 insertions(+), 18 deletions(-) create mode 100644 core/atlas/bias.py create mode 100644 tests/core/test_bias.py diff --git a/core/atlas/bias.py b/core/atlas/bias.py new file mode 100644 index 0000000..03bddaf --- /dev/null +++ b/core/atlas/bias.py @@ -0,0 +1,109 @@ +"""Reporting bias: some sources report a rare sighting more readily than a miss. + +The bot logs every run, and bulk-pasted logs record every run of the players who kept them ("complete" groups). +Form submissions and reports are chosen by the rider, and a run with a shadow or a meadow Blue is more likely to be +written up than one without. For each rare roll and each selective group, the bias is the factor by which the +group's odds of the rare outcome exceed the complete groups' odds: + + rho = odds(rare | group) / odds(rare | complete groups) + +Each rare outcome a group reports then counts 1/rho in the route model's tables. That keeps the group's pattern +across routes and moves its overall level to what complete logging shows. The assumption: the bias is the same on +every route (planner/policy.yaml says which groups, rolls and settings are used). +""" +import random + +from . import model + + +def group_of(r, groups, rare_goals): + """The first group in planner/policy.yaml whose condition the run meets.""" + for g in groups: + w = g.get("when", {}) + if "source" in w and r.get("source") != w["source"]: + continue + if "entry_not" in w and r.get("entry") in (None, w["entry_not"]): + continue + if w.get("goal") == "rare" and r.get("goal") not in rare_goals: + continue + return g["name"] + return None + + +def _odds(y, n): + p = (y + 0.5) / (n + 1) # Jeffreys + return p / (1 - p) + + +def estimate(runs, prepared, cfg, rare_goals, seed=20261109, sims=4000): + """{group: {roll: {rare, n, rho, lo, hi, used}}} and the group of every run.""" + groups = cfg["groups"] + names = [group_of(r, groups, rare_goals) for r in runs] + complete = {g["name"] for g in groups if g.get("complete")} + rolls = cfg["rolls"] + tally = {} + for name, v in zip(names, prepared): + for node, rare in rolls.items(): + y = model.OBSERVE[node](v) + if y is None: + continue + t = tally.setdefault(name, {}).setdefault(node, [0, 0]) + t[0] += y == rare + t[1] += 1 + ref = {node: [sum(tally.get(g, {}).get(node, [0, 0])[i] for g in complete) for i in (0, 1)] for node in rolls} + rnd = random.Random(seed) + out = {} + for g in groups: + name = g["name"] + out[name] = {} + for node in rolls: + y, n = tally.get(name, {}).get(node, [0, 0]) + ry, rn = ref[node] + e = {"rare": y, "n": n, "completeRare": ry, "completeN": rn} + if g.get("complete"): + e.update(rho=1.0, lo=1.0, hi=1.0, used="complete: counted as reported") + elif ry < cfg.get("min_complete_rare", 5): + # complete logging has (almost) never seen it: the bias cannot be estimated, so this roll is learned + # from complete logging only (its range then shows how little is known) + e.update(rho=None, lo=None, hi=None, exclude=True, + used=f"left out: complete logs have only {ry} rare outcomes, too few to estimate the bias") + elif n < cfg.get("min_runs", 30) or rn < cfg.get("min_runs", 30): + e.update(rho=None, used=f"too few runs: uses {g.get('same_as') or 'no correction'}") + else: + rho = _odds(y, n) / _odds(ry, rn) + sim = [] # a 90% range for rho from the two rates' posteriors + for _ in range(sims): + a = rnd.betavariate(y + 0.5, n - y + 0.5) + b = rnd.betavariate(ry + 0.5, rn - ry + 0.5) + sim.append((a / (1 - a)) / (b / (1 - b))) + sim.sort() + e.update(rho=rho, lo=sim[int(0.05 * sims)], hi=sim[int(0.95 * sims) - 1], used="estimated") + out[name][node] = e + # groups with too little data borrow another group's estimate + for g in groups: + for node in rolls: + e = out[g["name"]][node] + if e.get("rho") is None and not e.get("exclude"): + src = out.get(g.get("same_as") or "", {}).get(node, {}) + e["rho"] = src.get("rho") or 1.0 + e["lo"], e["hi"] = src.get("lo", e["rho"]), src.get("hi", e["rho"]) + return out, names + + +def weights(names, prepared, est, rolls): + """Per run: {roll: weight of the outcome it observed}: 1/rho for a rare outcome from a biased group, 0 for any + outcome of a roll its group is left out of. Only where it differs from 1.""" + out = [] + for name, v in zip(names, prepared): + w = {} + for node, rare in rolls.items(): + e = (est.get(name) or {}).get(node, {}) + y = model.OBSERVE[node](v) + if y is None: + continue + if e.get("exclude"): + w[node] = 0.0 + elif y == rare and (e.get("rho") or 1.0) != 1.0: + w[node] = 1.0 / e["rho"] + out.append(w) + return out diff --git a/core/atlas/model.py b/core/atlas/model.py index fb6f952..d9df446 100644 --- a/core/atlas/model.py +++ b/core/atlas/model.py @@ -148,21 +148,30 @@ def ctx_key(v, names): # ---------------------------------------------------------------- 2. counts -def count_tables(prepared): - """Counts for every chance event at every back-off level, plus each encounter's outcomes.""" +def count_tables(prepared, weights=None): + """Counts for every chance event at every back-off level, plus each encounter's outcomes. weights: per run, + {chance event: weight of the outcome it observed} (reporting-bias corrections, core/atlas/bias.py).""" T = {} for node, sp in CHANCE.items(): t = T[node] = {} - for v in prepared: + for i, v in enumerate(prepared): y = OBSERVE[node](v) if not y: continue + w = weights[i].get(node, 1) if weights else 1 + if w == 0: + continue for L in range(len(sp["context"]) + 1): names = sp["context"][:L] if any(v.get(k) is None for k in names): break c = t.setdefault(f"{L}:{ctx_key(v, names)}", {}) - c[y] = c.get(y, 0) + 1 + c[y] = c.get(y, 0) + w + for t in T.values(): # weighted counts are fractions: keep them short and stable + for c in t.values(): + for k, x in c.items(): + if isinstance(x, float): + c[k] = round(x, 4) enc = T["encounter"] = {} for v in prepared: if not v.get("encCtx"): @@ -261,6 +270,10 @@ def fit_alpha(T, node): pairs.append(([t[key].get(k, 0) for k in cats], pooled(parent))) if not pairs: return {"alpha": ALPHA, "logLik": None} + # a roll that has only ever come out one way says nothing about whether routes differ: pool fully (the fit + # would otherwise drive every route's chance of the unseen outcome to zero) + if sum(1 for k in cats if t.get("0:", {}).get(k, 0) > 0) < 2: + return {"alpha": max(ALPHA_GRID), "logLik": None} best_a, best_ll, lls = ALPHA, -math.inf, {} for a in ALPHA_GRID: ll = 0 diff --git a/core/atlas/solver.py b/core/atlas/solver.py index 01ebde9..7a32959 100644 --- a/core/atlas/solver.py +++ b/core/atlas/solver.py @@ -311,12 +311,18 @@ def summarise(xs): # ---------------------------------------------------------------- the published policy -def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, colours=None): - """The full policy for every colour. runs: coded runs (atlas.coding), as dicts.""" +def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, colours=None, bias_config=None): + """The full policy for every colour. runs: coded runs (data/public/runs_clean.csv), as dicts. bias_config: + planner/policy.yaml `reporting_bias` (None: every run counts as reported).""" colours = colours or model.COLOURS min_support = G["min_support"] if min_support is None else min_support prepared = [model.prepare_run(r) for r in runs] - T = model.count_tables(prepared) + bias_est, w = None, None + if bias_config: + from . import bias + bias_est, names = bias.estimate(runs, prepared, bias_config, set(model.RARE_GOALS)) + w = bias.weights(names, prepared, bias_est, bias_config["rolls"]) + T = model.count_tables(prepared, w) support, counts = model.support_from(prepared, min_support) fitted = {k: model.fit_alpha(T, k) for k in model.CHANCE} alphas = {k: alpha for k in model.CHANCE} if alpha else {k: f["alpha"] for k, f in fitted.items()} @@ -325,7 +331,8 @@ def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, c # Cross-fitting: choose the policy on one random half of the runs and score it on the other half. r = model.Rng(seed) fold = [0 if r() < 0.5 else 1 for _ in prepared] - halves = [model.count_tables([v for v, f in zip(prepared, fold) if f == h]) for h in (0, 1)] + halves = [model.count_tables([v for v, f in zip(prepared, fold) if f == h], + [x for x, f in zip(w, fold) if f == h] if w else None) for h in (0, 1)] half_params = [model.Params(Th, "mean", alphas=alphas) for Th in halves] targets = {} @@ -366,6 +373,8 @@ def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, c "chanceContext": {k: s["context"] for k, s in model.CHANCE.items()}, "decisions": model.DECISIONS, "support": support, "supportCounts": counts, "tables": T, + # how much each kind of source over-reports each rare roll, and the correction applied (bias.py) + "reportingBias": bias_est, # the game itself (core/spec/game.yaml), so the advisor in core/solver.js can re-solve from any point "game": {k: G[k] for k in ("version", "colours", "decisions", "chance", "encounters", "encounter_outcomes", "by_colour", "start", "steps", "from_form", "bot_text")}, diff --git a/data/dictionary.yaml b/data/dictionary.yaml index f0d746f..240659c 100644 --- a/data/dictionary.yaml +++ b/data/dictionary.yaml @@ -91,10 +91,10 @@ columns: type: text description: >- For bot runs, how the bot chose: "auto_learn" (its own rules and the route PDF), "atlas_policy" (the Atlas's - route model), "explore" (trying untested options), "experiment" (a randomised test), "manual" (a person + route model), "atlas_thompson" (the route model, sampling routes in proportion to how likely each is best), "explore" (trying untested options), "experiment" (a randomised test), "manual" (a person clicked). Matters for analysis: the bot follows a strategy, so its choices are not like a player's. Empty for other sources. - values: [auto_learn, atlas_policy, explore, experiment, manual] + values: [auto_learn, atlas_policy, atlas_thompson, explore, experiment, manual] date: { type: date, description: "Date of the run or its report (YYYY-MM-DD): the form submission, the issue, or the bot attempt. Empty for bulk-pasted rows and result pages." } wing: { type: text, description: The rider's wing, from the form., values: [First, Second, Third, Fourth, Multiple, Unknown] } section: { type: text, description: The rider's section within the wing (free text as answered). } diff --git a/docs/BOT_INTEGRATION.md b/docs/BOT_INTEGRATION.md index f69692e..44141a6 100644 --- a/docs/BOT_INTEGRATION.md +++ b/docs/BOT_INTEGRATION.md @@ -125,12 +125,33 @@ The atlas is advice. The bot's own safety rules stay in charge: Log every atlas pick with `rule: 'ATLAS_POLICY'`, `source: 'atlas'`, and the run-level `policyId: policy.id` (see `docs/BOT_DATA_PROTOCOL.md`), so we can tell which policy produced which runs. -### Ties are free experiments +### Learning: goal accounts follow the best route, research accounts sample -Many choices barely matter by the data we have (for Black, the first direction differs by a tenth of a percentage -point). When `margin` is below the tie threshold (start with 0.002, i.e. 0.2 percentage points), pick at random -among the tied options and log `propensity` (1 / number of tied options). That costs nothing in expected success, -and it is exactly the randomised data the atlas needs to tell real history effects from guide-copying. +The atlas learns from the bot's own runs: every zip of the bot's data that is imported adds its rolls (first dragon, +deaths, the shadow, the meadow dragon, bond results) to the route model, and the next `atlas-for-bot` zip carries +the updated model. Deliberate burns (Malek, "no shadow", "no Blue") never count as failed bond attempts. + +How the bot should choose depends on the account's role: + +- **Goal accounts** (`accountRole: "goal"`): the best route under the model, as before: + `Solver.advisor(policy.samples.all, target, 'eventually')`. +- **Research accounts** (`accountRole: "research"`): Thompson sampling. Make a new advisor for every attempt with a + new seed: + + ```js + const adv = Solver.advisor(policy.samples.all, target, 'eventually', { sampling: 'thompson', seed: attemptNumber }); + ``` + + Each attempt follows the best route of one plausible version of the odds, drawn from the model's uncertainty. An + option that might be the best one gets tried about as often as it is likely to be the best, so the attempts go + where the model is least sure; as the data firm up, sampling settles on the best route by itself. + `recommendOption()` then returns a `propensity` (the chance this option was picked, from 40 other draws; set + `draws` to change that). Log it on the step with `decisionMode: "atlas_thompson"`, so the runs can be re-weighted. + Making the advisor takes a few seconds per attempt (it solves the game once per draw). + +The tie rule below still works for goal accounts; research accounts don't need it. + +### Ties are free experiments ### Which numbers to trust @@ -140,8 +161,13 @@ and it is exactly the randomised data the atlas needs to tell real history effec - `samples.lowBias`: bulk logs and riders not chasing a rare dragon. Use it for absolute expectations, such as how many attempts a Blue should take. Rare sightings are over-reported in form posts, so `all` is optimistic for Blue. -Current headline (from `analysis/reports/POLICY.md`): Black about 1.7% per attempt (about 60 attempts), Blue 3.7–7% per attempt -(14–28 attempts), depending on the sample. +Current headline, with the bot's runs and the reporting-bias correction (November 2026): Black about 0.03% per +attempt (90% range 0–0.15%: the bot and bulk logs have never seen the shadow in about 700 tree runs), Blue about +1.8–2.5% per attempt (40–55 attempts), depending on the sample. + +Form submissions over-report rare sightings: riders report a meadow Blue about 4 times more readily than a meadow +without one. The model estimates that bias for each kind of source and corrects for it (`planner/policy.yaml`, +`reporting_bias`; the estimates are in `policy.json` under `model.reportingBias`). ## 2. From the bot to the atlas diff --git a/planner/build_policy.py b/planner/build_policy.py index ad0a603..a629b98 100644 --- a/planner/build_policy.py +++ b/planner/build_policy.py @@ -69,7 +69,7 @@ def main(argv=None): out = {"schema": 1, "source": {"form_rows": by_source.get("reddit spreadsheet", 0), "runs": rows, "by_source": by_source}, "samples": {}} for name, rs in samples.items(): - out["samples"][name] = clean(solver.build_policy(rs, draws=a.draws)) + out["samples"][name] = clean(solver.build_policy(rs, draws=a.draws, bias_config=CONFIG.get("reporting_bias"))) print(f"{name}: {len(rs)} runs, {time.time() - t0:.1f} s") # a content id (no timestamps: the build is reproducible) out["id"] = hashlib.sha1(json.dumps(out["samples"], separators=(",", ":"), sort_keys=True).encode()).hexdigest()[:12] diff --git a/planner/policy.yaml b/planner/policy.yaml index 08d5818..e6406d0 100644 --- a/planner/policy.yaml +++ b/planner/policy.yaml @@ -15,3 +15,20 @@ low_bias: always: [bot] # sources whose every run is kept entries_kept: [bulk] # spreadsheet rows whose `entry` is not "Form" (bulk-pasted logs) rare_goals: [Blue, Black, Blue or Black] + +# Reporting bias (core/atlas/bias.py). A rider is more likely to write up a run with a shadow or a meadow Blue than +# one without, so those sightings are over-represented in form submissions. For each rare roll below and each group +# that is not complete, the model estimates how many times more often the group reports the rare outcome than the +# complete groups do (rho, with a 90% range), and counts each rare outcome it reports as 1/rho. The group's pattern +# across routes is kept; its overall level comes from complete logging. Assumes the bias is the same on every route. +reporting_bias: + rolls: { shadow: "yes", meadow: Blue } # roll: its rare outcome + min_runs: 30 # fewer runs than this in a group (or the complete groups): borrow + min_complete_rare: 5 # fewer rare outcomes than this in complete logs: the bias cannot be + # estimated, so the roll is learned from complete logs only + groups: # first match wins + - { name: bot, when: { source: bot }, complete: true } + - { name: bulk logs, when: { source: reddit spreadsheet, entry_not: Form }, complete: true } + - { name: "form, chasing Blue/Black", when: { source: reddit spreadsheet, goal: rare } } + - { name: "form, other", when: { source: reddit spreadsheet } } + - { name: reported, when: { source: reported }, same_as: "form, other" } diff --git a/tests/core/test_bias.py b/tests/core/test_bias.py new file mode 100644 index 0000000..908795f --- /dev/null +++ b/tests/core/test_bias.py @@ -0,0 +1,52 @@ +"""Reporting-bias correction (core/atlas/bias.py).""" +import unittest + +from atlas import bias, model + +CFG = {"rolls": {"meadow": "Blue"}, "min_runs": 30, "min_complete_rare": 5, + "groups": [{"name": "bot", "when": {"source": "bot"}, "complete": True}, + {"name": "form", "when": {"source": "reddit spreadsheet"}}]} + + +def meadow_run(source, colour): + r = {"source": source, "entry": "Form", "goal": "Any (for fun)", "company": "Alone", "direction": "Center", + "first_sighting": "Red", "sighting_action": None, "left_squadmates": None, "scream": "Ignore the scream", + "resolution": "Tourniquet", "post_path": "(no fork)", "hour": "Valley", "branch": "Passed the Red: meadow", + "outcome": "Incinerated", "outcome_color": colour, "malek_intent": "No", "rare_reported": None, + "saw_blue_meadow": colour == "Blue"} + for c in model.COLOURS: + r["enc_" + c] = "Incinerated" if c == colour else None + return r + + +class Bias(unittest.TestCase): + def runs(self, bot_blue, bot_n, form_blue, form_n): + return ([meadow_run("bot", "Blue")] * bot_blue + [meadow_run("bot", "Orange")] * (bot_n - bot_blue) + + [meadow_run("reddit spreadsheet", "Blue")] * form_blue + [meadow_run("reddit spreadsheet", "Orange")] * (form_n - form_blue)) + + def test_over_reporting_is_estimated_and_corrected(self): + runs = self.runs(10, 100, 100, 300) # 10% when everything is logged, 33% in the form + prep = [model.prepare_run(r) for r in runs] + est, names = bias.estimate(runs, prep, CFG, set()) + rho = est["form"]["meadow"]["rho"] + self.assertAlmostEqual(rho, 4.4, delta=0.5) + self.assertLess(est["form"]["meadow"]["lo"], rho) + self.assertGreater(est["form"]["meadow"]["hi"], rho) + T = model.count_tables(prep, bias.weights(names, prep, est, CFG["rolls"])) + c = T["meadow"]["0:"] + # corrected: the form's Blue sightings count 1/rho each, so the pooled rate is close to what the bot sees + self.assertAlmostEqual(c["Blue"] / (c["Blue"] + c["Orange"]), 0.10, delta=0.02) + + def test_unestimable_bias_leaves_the_group_out(self): + runs = self.runs(0, 200, 60, 300) # complete logging never saw it + prep = [model.prepare_run(r) for r in runs] + est, names = bias.estimate(runs, prep, CFG, set()) + self.assertTrue(est["form"]["meadow"]["exclude"]) + T = model.count_tables(prep, bias.weights(names, prep, est, CFG["rolls"])) + self.assertEqual(T["meadow"]["0:"], {"Orange": 200}) + # a roll seen only one way: routes cannot be told apart, so it is pooled fully + self.assertEqual(model.fit_alpha(T, "meadow")["alpha"], max(model.ALPHA_GRID)) + + +if __name__ == "__main__": + unittest.main()