Route model: correct for reporting bias

- core/atlas/bias.py: for each rare roll (shadow, meadow Blue) and each source group,
  the factor by which the group over-reports the rare outcome relative to complete
  logging (bot, bulk logs), with a 90% range; rare outcomes from biased groups count
  1/rho; a roll the complete logs have (almost) never shown is learned from them only
- planner/policy.yaml: groups, rolls and thresholds; estimates published in
  policy.json under model.reportingBias
- fit_alpha: a roll seen only one way is pooled fully
- docs: goal accounts follow the best route, research accounts use Thompson sampling
  (decisionMode atlas_thompson)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
ekcSchiano
2026-11-10 02:29:09 +00:00
co-authored by Claude Opus 5.5
parent c0355616b6
commit 2271fe0976
8 changed files with 244 additions and 18 deletions
+109
View File
@@ -0,0 +1,109 @@
"""Reporting bias: some sources report a rare sighting more readily than a miss.
The bot logs every run, and bulk-pasted logs record every run of the players who kept them ("complete" groups).
Form submissions and reports are chosen by the rider, and a run with a shadow or a meadow Blue is more likely to be
written up than one without. For each rare roll and each selective group, the bias is the factor by which the
group's odds of the rare outcome exceed the complete groups' odds:
rho = odds(rare | group) / odds(rare | complete groups)
Each rare outcome a group reports then counts 1/rho in the route model's tables. That keeps the group's pattern
across routes and moves its overall level to what complete logging shows. The assumption: the bias is the same on
every route (planner/policy.yaml says which groups, rolls and settings are used).
"""
import random
from . import model
def group_of(r, groups, rare_goals):
"""The first group in planner/policy.yaml whose condition the run meets."""
for g in groups:
w = g.get("when", {})
if "source" in w and r.get("source") != w["source"]:
continue
if "entry_not" in w and r.get("entry") in (None, w["entry_not"]):
continue
if w.get("goal") == "rare" and r.get("goal") not in rare_goals:
continue
return g["name"]
return None
def _odds(y, n):
p = (y + 0.5) / (n + 1) # Jeffreys
return p / (1 - p)
def estimate(runs, prepared, cfg, rare_goals, seed=20261109, sims=4000):
"""{group: {roll: {rare, n, rho, lo, hi, used}}} and the group of every run."""
groups = cfg["groups"]
names = [group_of(r, groups, rare_goals) for r in runs]
complete = {g["name"] for g in groups if g.get("complete")}
rolls = cfg["rolls"]
tally = {}
for name, v in zip(names, prepared):
for node, rare in rolls.items():
y = model.OBSERVE[node](v)
if y is None:
continue
t = tally.setdefault(name, {}).setdefault(node, [0, 0])
t[0] += y == rare
t[1] += 1
ref = {node: [sum(tally.get(g, {}).get(node, [0, 0])[i] for g in complete) for i in (0, 1)] for node in rolls}
rnd = random.Random(seed)
out = {}
for g in groups:
name = g["name"]
out[name] = {}
for node in rolls:
y, n = tally.get(name, {}).get(node, [0, 0])
ry, rn = ref[node]
e = {"rare": y, "n": n, "completeRare": ry, "completeN": rn}
if g.get("complete"):
e.update(rho=1.0, lo=1.0, hi=1.0, used="complete: counted as reported")
elif ry < cfg.get("min_complete_rare", 5):
# complete logging has (almost) never seen it: the bias cannot be estimated, so this roll is learned
# from complete logging only (its range then shows how little is known)
e.update(rho=None, lo=None, hi=None, exclude=True,
used=f"left out: complete logs have only {ry} rare outcomes, too few to estimate the bias")
elif n < cfg.get("min_runs", 30) or rn < cfg.get("min_runs", 30):
e.update(rho=None, used=f"too few runs: uses {g.get('same_as') or 'no correction'}")
else:
rho = _odds(y, n) / _odds(ry, rn)
sim = [] # a 90% range for rho from the two rates' posteriors
for _ in range(sims):
a = rnd.betavariate(y + 0.5, n - y + 0.5)
b = rnd.betavariate(ry + 0.5, rn - ry + 0.5)
sim.append((a / (1 - a)) / (b / (1 - b)))
sim.sort()
e.update(rho=rho, lo=sim[int(0.05 * sims)], hi=sim[int(0.95 * sims) - 1], used="estimated")
out[name][node] = e
# groups with too little data borrow another group's estimate
for g in groups:
for node in rolls:
e = out[g["name"]][node]
if e.get("rho") is None and not e.get("exclude"):
src = out.get(g.get("same_as") or "", {}).get(node, {})
e["rho"] = src.get("rho") or 1.0
e["lo"], e["hi"] = src.get("lo", e["rho"]), src.get("hi", e["rho"])
return out, names
def weights(names, prepared, est, rolls):
"""Per run: {roll: weight of the outcome it observed}: 1/rho for a rare outcome from a biased group, 0 for any
outcome of a roll its group is left out of. Only where it differs from 1."""
out = []
for name, v in zip(names, prepared):
w = {}
for node, rare in rolls.items():
e = (est.get(name) or {}).get(node, {})
y = model.OBSERVE[node](v)
if y is None:
continue
if e.get("exclude"):
w[node] = 0.0
elif y == rare and (e.get("rho") or 1.0) != 1.0:
w[node] = 1.0 / e["rho"]
out.append(w)
return out
+17 -4
View File
@@ -148,21 +148,30 @@ def ctx_key(v, names):
# ---------------------------------------------------------------- 2. counts # ---------------------------------------------------------------- 2. counts
def count_tables(prepared): def count_tables(prepared, weights=None):
"""Counts for every chance event at every back-off level, plus each encounter's outcomes.""" """Counts for every chance event at every back-off level, plus each encounter's outcomes. weights: per run,
{chance event: weight of the outcome it observed} (reporting-bias corrections, core/atlas/bias.py)."""
T = {} T = {}
for node, sp in CHANCE.items(): for node, sp in CHANCE.items():
t = T[node] = {} t = T[node] = {}
for v in prepared: for i, v in enumerate(prepared):
y = OBSERVE[node](v) y = OBSERVE[node](v)
if not y: if not y:
continue continue
w = weights[i].get(node, 1) if weights else 1
if w == 0:
continue
for L in range(len(sp["context"]) + 1): for L in range(len(sp["context"]) + 1):
names = sp["context"][:L] names = sp["context"][:L]
if any(v.get(k) is None for k in names): if any(v.get(k) is None for k in names):
break break
c = t.setdefault(f"{L}:{ctx_key(v, names)}", {}) c = t.setdefault(f"{L}:{ctx_key(v, names)}", {})
c[y] = c.get(y, 0) + 1 c[y] = c.get(y, 0) + w
for t in T.values(): # weighted counts are fractions: keep them short and stable
for c in t.values():
for k, x in c.items():
if isinstance(x, float):
c[k] = round(x, 4)
enc = T["encounter"] = {} enc = T["encounter"] = {}
for v in prepared: for v in prepared:
if not v.get("encCtx"): if not v.get("encCtx"):
@@ -261,6 +270,10 @@ def fit_alpha(T, node):
pairs.append(([t[key].get(k, 0) for k in cats], pooled(parent))) pairs.append(([t[key].get(k, 0) for k in cats], pooled(parent)))
if not pairs: if not pairs:
return {"alpha": ALPHA, "logLik": None} return {"alpha": ALPHA, "logLik": None}
# a roll that has only ever come out one way says nothing about whether routes differ: pool fully (the fit
# would otherwise drive every route's chance of the unseen outcome to zero)
if sum(1 for k in cats if t.get("0:", {}).get(k, 0) > 0) < 2:
return {"alpha": max(ALPHA_GRID), "logLik": None}
best_a, best_ll, lls = ALPHA, -math.inf, {} best_a, best_ll, lls = ALPHA, -math.inf, {}
for a in ALPHA_GRID: for a in ALPHA_GRID:
ll = 0 ll = 0
+13 -4
View File
@@ -311,12 +311,18 @@ def summarise(xs):
# ---------------------------------------------------------------- the published policy # ---------------------------------------------------------------- the published policy
def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, colours=None): def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, colours=None, bias_config=None):
"""The full policy for every colour. runs: coded runs (atlas.coding), as dicts.""" """The full policy for every colour. runs: coded runs (data/public/runs_clean.csv), as dicts. bias_config:
planner/policy.yaml `reporting_bias` (None: every run counts as reported)."""
colours = colours or model.COLOURS colours = colours or model.COLOURS
min_support = G["min_support"] if min_support is None else min_support min_support = G["min_support"] if min_support is None else min_support
prepared = [model.prepare_run(r) for r in runs] prepared = [model.prepare_run(r) for r in runs]
T = model.count_tables(prepared) bias_est, w = None, None
if bias_config:
from . import bias
bias_est, names = bias.estimate(runs, prepared, bias_config, set(model.RARE_GOALS))
w = bias.weights(names, prepared, bias_est, bias_config["rolls"])
T = model.count_tables(prepared, w)
support, counts = model.support_from(prepared, min_support) support, counts = model.support_from(prepared, min_support)
fitted = {k: model.fit_alpha(T, k) for k in model.CHANCE} fitted = {k: model.fit_alpha(T, k) for k in model.CHANCE}
alphas = {k: alpha for k in model.CHANCE} if alpha else {k: f["alpha"] for k, f in fitted.items()} alphas = {k: alpha for k in model.CHANCE} if alpha else {k: f["alpha"] for k, f in fitted.items()}
@@ -325,7 +331,8 @@ def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, c
# Cross-fitting: choose the policy on one random half of the runs and score it on the other half. # Cross-fitting: choose the policy on one random half of the runs and score it on the other half.
r = model.Rng(seed) r = model.Rng(seed)
fold = [0 if r() < 0.5 else 1 for _ in prepared] fold = [0 if r() < 0.5 else 1 for _ in prepared]
halves = [model.count_tables([v for v, f in zip(prepared, fold) if f == h]) for h in (0, 1)] halves = [model.count_tables([v for v, f in zip(prepared, fold) if f == h],
[x for x, f in zip(w, fold) if f == h] if w else None) for h in (0, 1)]
half_params = [model.Params(Th, "mean", alphas=alphas) for Th in halves] half_params = [model.Params(Th, "mean", alphas=alphas) for Th in halves]
targets = {} targets = {}
@@ -366,6 +373,8 @@ def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, c
"chanceContext": {k: s["context"] for k, s in model.CHANCE.items()}, "chanceContext": {k: s["context"] for k, s in model.CHANCE.items()},
"decisions": model.DECISIONS, "support": support, "supportCounts": counts, "decisions": model.DECISIONS, "support": support, "supportCounts": counts,
"tables": T, "tables": T,
# how much each kind of source over-reports each rare roll, and the correction applied (bias.py)
"reportingBias": bias_est,
# the game itself (core/spec/game.yaml), so the advisor in core/solver.js can re-solve from any point # the game itself (core/spec/game.yaml), so the advisor in core/solver.js can re-solve from any point
"game": {k: G[k] for k in ("version", "colours", "decisions", "chance", "encounters", "encounter_outcomes", "game": {k: G[k] for k in ("version", "colours", "decisions", "chance", "encounters", "encounter_outcomes",
"by_colour", "start", "steps", "from_form", "bot_text")}, "by_colour", "start", "steps", "from_form", "bot_text")},
+2 -2
View File
@@ -91,10 +91,10 @@ columns:
type: text type: text
description: >- description: >-
For bot runs, how the bot chose: "auto_learn" (its own rules and the route PDF), "atlas_policy" (the Atlas's For bot runs, how the bot chose: "auto_learn" (its own rules and the route PDF), "atlas_policy" (the Atlas's
route model), "explore" (trying untested options), "experiment" (a randomised test), "manual" (a person route model), "atlas_thompson" (the route model, sampling routes in proportion to how likely each is best), "explore" (trying untested options), "experiment" (a randomised test), "manual" (a person
clicked). Matters for analysis: the bot follows a strategy, so its choices are not like a player's. Empty for clicked). Matters for analysis: the bot follows a strategy, so its choices are not like a player's. Empty for
other sources. other sources.
values: [auto_learn, atlas_policy, explore, experiment, manual] values: [auto_learn, atlas_policy, atlas_thompson, explore, experiment, manual]
date: { type: date, description: "Date of the run or its report (YYYY-MM-DD): the form submission, the issue, or the bot attempt. Empty for bulk-pasted rows and result pages." } date: { type: date, description: "Date of the run or its report (YYYY-MM-DD): the form submission, the issue, or the bot attempt. Empty for bulk-pasted rows and result pages." }
wing: { type: text, description: The rider's wing, from the form., values: [First, Second, Third, Fourth, Multiple, Unknown] } wing: { type: text, description: The rider's wing, from the form., values: [First, Second, Third, Fourth, Multiple, Unknown] }
section: { type: text, description: The rider's section within the wing (free text as answered). } section: { type: text, description: The rider's section within the wing (free text as answered). }
+33 -7
View File
@@ -125,12 +125,33 @@ The atlas is advice. The bot's own safety rules stay in charge:
Log every atlas pick with `rule: 'ATLAS_POLICY'`, `source: 'atlas'`, and the run-level `policyId: policy.id` Log every atlas pick with `rule: 'ATLAS_POLICY'`, `source: 'atlas'`, and the run-level `policyId: policy.id`
(see `docs/BOT_DATA_PROTOCOL.md`), so we can tell which policy produced which runs. (see `docs/BOT_DATA_PROTOCOL.md`), so we can tell which policy produced which runs.
### Ties are free experiments ### Learning: goal accounts follow the best route, research accounts sample
Many choices barely matter by the data we have (for Black, the first direction differs by a tenth of a percentage The atlas learns from the bot's own runs: every zip of the bot's data that is imported adds its rolls (first dragon,
point). When `margin` is below the tie threshold (start with 0.002, i.e. 0.2 percentage points), pick at random deaths, the shadow, the meadow dragon, bond results) to the route model, and the next `atlas-for-bot` zip carries
among the tied options and log `propensity` (1 / number of tied options). That costs nothing in expected success, the updated model. Deliberate burns (Malek, "no shadow", "no Blue") never count as failed bond attempts.
and it is exactly the randomised data the atlas needs to tell real history effects from guide-copying.
How the bot should choose depends on the account's role:
- **Goal accounts** (`accountRole: "goal"`): the best route under the model, as before:
`Solver.advisor(policy.samples.all, target, 'eventually')`.
- **Research accounts** (`accountRole: "research"`): Thompson sampling. Make a new advisor for every attempt with a
new seed:
```js
const adv = Solver.advisor(policy.samples.all, target, 'eventually', { sampling: 'thompson', seed: attemptNumber });
```
Each attempt follows the best route of one plausible version of the odds, drawn from the model's uncertainty. An
option that might be the best one gets tried about as often as it is likely to be the best, so the attempts go
where the model is least sure; as the data firm up, sampling settles on the best route by itself.
`recommendOption()` then returns a `propensity` (the chance this option was picked, from 40 other draws; set
`draws` to change that). Log it on the step with `decisionMode: "atlas_thompson"`, so the runs can be re-weighted.
Making the advisor takes a few seconds per attempt (it solves the game once per draw).
The tie rule below still works for goal accounts; research accounts don't need it.
### Ties are free experiments
### Which numbers to trust ### Which numbers to trust
@@ -140,8 +161,13 @@ and it is exactly the randomised data the atlas needs to tell real history effec
- `samples.lowBias`: bulk logs and riders not chasing a rare dragon. Use it for absolute expectations, such as how - `samples.lowBias`: bulk logs and riders not chasing a rare dragon. Use it for absolute expectations, such as how
many attempts a Blue should take. Rare sightings are over-reported in form posts, so `all` is optimistic for Blue. many attempts a Blue should take. Rare sightings are over-reported in form posts, so `all` is optimistic for Blue.
Current headline (from `analysis/reports/POLICY.md`): Black about 1.7% per attempt (about 60 attempts), Blue 3.7–7% per attempt Current headline, with the bot's runs and the reporting-bias correction (November 2026): Black about 0.03% per
(14–28 attempts), depending on the sample. attempt (90% range 0–0.15%: the bot and bulk logs have never seen the shadow in about 700 tree runs), Blue about
1.8–2.5% per attempt (40–55 attempts), depending on the sample.
Form submissions over-report rare sightings: riders report a meadow Blue about 4 times more readily than a meadow
without one. The model estimates that bias for each kind of source and corrects for it (`planner/policy.yaml`,
`reporting_bias`; the estimates are in `policy.json` under `model.reportingBias`).
## 2. From the bot to the atlas ## 2. From the bot to the atlas
+1 -1
View File
@@ -69,7 +69,7 @@ def main(argv=None):
out = {"schema": 1, "source": {"form_rows": by_source.get("reddit spreadsheet", 0), "runs": rows, "by_source": by_source}, out = {"schema": 1, "source": {"form_rows": by_source.get("reddit spreadsheet", 0), "runs": rows, "by_source": by_source},
"samples": {}} "samples": {}}
for name, rs in samples.items(): for name, rs in samples.items():
out["samples"][name] = clean(solver.build_policy(rs, draws=a.draws)) out["samples"][name] = clean(solver.build_policy(rs, draws=a.draws, bias_config=CONFIG.get("reporting_bias")))
print(f"{name}: {len(rs)} runs, {time.time() - t0:.1f} s") print(f"{name}: {len(rs)} runs, {time.time() - t0:.1f} s")
# a content id (no timestamps: the build is reproducible) # a content id (no timestamps: the build is reproducible)
out["id"] = hashlib.sha1(json.dumps(out["samples"], separators=(",", ":"), sort_keys=True).encode()).hexdigest()[:12] out["id"] = hashlib.sha1(json.dumps(out["samples"], separators=(",", ":"), sort_keys=True).encode()).hexdigest()[:12]
+17
View File
@@ -15,3 +15,20 @@ low_bias:
always: [bot] # sources whose every run is kept always: [bot] # sources whose every run is kept
entries_kept: [bulk] # spreadsheet rows whose `entry` is not "Form" (bulk-pasted logs) entries_kept: [bulk] # spreadsheet rows whose `entry` is not "Form" (bulk-pasted logs)
rare_goals: [Blue, Black, Blue or Black] rare_goals: [Blue, Black, Blue or Black]
# Reporting bias (core/atlas/bias.py). A rider is more likely to write up a run with a shadow or a meadow Blue than
# one without, so those sightings are over-represented in form submissions. For each rare roll below and each group
# that is not complete, the model estimates how many times more often the group reports the rare outcome than the
# complete groups do (rho, with a 90% range), and counts each rare outcome it reports as 1/rho. The group's pattern
# across routes is kept; its overall level comes from complete logging. Assumes the bias is the same on every route.
reporting_bias:
rolls: { shadow: "yes", meadow: Blue } # roll: its rare outcome
min_runs: 30 # fewer runs than this in a group (or the complete groups): borrow
min_complete_rare: 5 # fewer rare outcomes than this in complete logs: the bias cannot be
# estimated, so the roll is learned from complete logs only
groups: # first match wins
- { name: bot, when: { source: bot }, complete: true }
- { name: bulk logs, when: { source: reddit spreadsheet, entry_not: Form }, complete: true }
- { name: "form, chasing Blue/Black", when: { source: reddit spreadsheet, goal: rare } }
- { name: "form, other", when: { source: reddit spreadsheet } }
- { name: reported, when: { source: reported }, same_as: "form, other" }
+52
View File
@@ -0,0 +1,52 @@
"""Reporting-bias correction (core/atlas/bias.py)."""
import unittest
from atlas import bias, model
CFG = {"rolls": {"meadow": "Blue"}, "min_runs": 30, "min_complete_rare": 5,
"groups": [{"name": "bot", "when": {"source": "bot"}, "complete": True},
{"name": "form", "when": {"source": "reddit spreadsheet"}}]}
def meadow_run(source, colour):
r = {"source": source, "entry": "Form", "goal": "Any (for fun)", "company": "Alone", "direction": "Center",
"first_sighting": "Red", "sighting_action": None, "left_squadmates": None, "scream": "Ignore the scream",
"resolution": "Tourniquet", "post_path": "(no fork)", "hour": "Valley", "branch": "Passed the Red: meadow",
"outcome": "Incinerated", "outcome_color": colour, "malek_intent": "No", "rare_reported": None,
"saw_blue_meadow": colour == "Blue"}
for c in model.COLOURS:
r["enc_" + c] = "Incinerated" if c == colour else None
return r
class Bias(unittest.TestCase):
def runs(self, bot_blue, bot_n, form_blue, form_n):
return ([meadow_run("bot", "Blue")] * bot_blue + [meadow_run("bot", "Orange")] * (bot_n - bot_blue) +
[meadow_run("reddit spreadsheet", "Blue")] * form_blue + [meadow_run("reddit spreadsheet", "Orange")] * (form_n - form_blue))
def test_over_reporting_is_estimated_and_corrected(self):
runs = self.runs(10, 100, 100, 300) # 10% when everything is logged, 33% in the form
prep = [model.prepare_run(r) for r in runs]
est, names = bias.estimate(runs, prep, CFG, set())
rho = est["form"]["meadow"]["rho"]
self.assertAlmostEqual(rho, 4.4, delta=0.5)
self.assertLess(est["form"]["meadow"]["lo"], rho)
self.assertGreater(est["form"]["meadow"]["hi"], rho)
T = model.count_tables(prep, bias.weights(names, prep, est, CFG["rolls"]))
c = T["meadow"]["0:"]
# corrected: the form's Blue sightings count 1/rho each, so the pooled rate is close to what the bot sees
self.assertAlmostEqual(c["Blue"] / (c["Blue"] + c["Orange"]), 0.10, delta=0.02)
def test_unestimable_bias_leaves_the_group_out(self):
runs = self.runs(0, 200, 60, 300) # complete logging never saw it
prep = [model.prepare_run(r) for r in runs]
est, names = bias.estimate(runs, prep, CFG, set())
self.assertTrue(est["form"]["meadow"]["exclude"])
T = model.count_tables(prep, bias.weights(names, prep, est, CFG["rolls"]))
self.assertEqual(T["meadow"]["0:"], {"Orange": 200})
# a roll seen only one way: routes cannot be told apart, so it is pooled fully
self.assertEqual(model.fit_alpha(T, "meadow")["alpha"], max(model.ALPHA_GRID))
if __name__ == "__main__":
unittest.main()