Route model: correct for reporting bias

- core/atlas/bias.py: for each rare roll (shadow, meadow Blue) and each source group,
  the factor by which the group over-reports the rare outcome relative to complete
  logging (bot, bulk logs), with a 90% range; rare outcomes from biased groups count
  1/rho; a roll the complete logs have (almost) never shown is learned from them only
- planner/policy.yaml: groups, rolls and thresholds; estimates published in
  policy.json under model.reportingBias
- fit_alpha: a roll seen only one way is pooled fully
- docs: goal accounts follow the best route, research accounts use Thompson sampling
  (decisionMode atlas_thompson)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
ekcSchiano
2026-11-10 02:29:09 +00:00
co-authored by Claude Opus 5.5
parent c0355616b6
commit 2271fe0976
8 changed files with 244 additions and 18 deletions
+109
View File
@@ -0,0 +1,109 @@
"""Reporting bias: some sources report a rare sighting more readily than a miss.
The bot logs every run, and bulk-pasted logs record every run of the players who kept them ("complete" groups).
Form submissions and reports are chosen by the rider, and a run with a shadow or a meadow Blue is more likely to be
written up than one without. For each rare roll and each selective group, the bias is the factor by which the
group's odds of the rare outcome exceed the complete groups' odds:
rho = odds(rare | group) / odds(rare | complete groups)
Each rare outcome a group reports then counts 1/rho in the route model's tables. That keeps the group's pattern
across routes and moves its overall level to what complete logging shows. The assumption: the bias is the same on
every route (planner/policy.yaml says which groups, rolls and settings are used).
"""
import random
from . import model
def group_of(r, groups, rare_goals):
"""The first group in planner/policy.yaml whose condition the run meets."""
for g in groups:
w = g.get("when", {})
if "source" in w and r.get("source") != w["source"]:
continue
if "entry_not" in w and r.get("entry") in (None, w["entry_not"]):
continue
if w.get("goal") == "rare" and r.get("goal") not in rare_goals:
continue
return g["name"]
return None
def _odds(y, n):
p = (y + 0.5) / (n + 1) # Jeffreys
return p / (1 - p)
def estimate(runs, prepared, cfg, rare_goals, seed=20261109, sims=4000):
"""{group: {roll: {rare, n, rho, lo, hi, used}}} and the group of every run."""
groups = cfg["groups"]
names = [group_of(r, groups, rare_goals) for r in runs]
complete = {g["name"] for g in groups if g.get("complete")}
rolls = cfg["rolls"]
tally = {}
for name, v in zip(names, prepared):
for node, rare in rolls.items():
y = model.OBSERVE[node](v)
if y is None:
continue
t = tally.setdefault(name, {}).setdefault(node, [0, 0])
t[0] += y == rare
t[1] += 1
ref = {node: [sum(tally.get(g, {}).get(node, [0, 0])[i] for g in complete) for i in (0, 1)] for node in rolls}
rnd = random.Random(seed)
out = {}
for g in groups:
name = g["name"]
out[name] = {}
for node in rolls:
y, n = tally.get(name, {}).get(node, [0, 0])
ry, rn = ref[node]
e = {"rare": y, "n": n, "completeRare": ry, "completeN": rn}
if g.get("complete"):
e.update(rho=1.0, lo=1.0, hi=1.0, used="complete: counted as reported")
elif ry < cfg.get("min_complete_rare", 5):
# complete logging has (almost) never seen it: the bias cannot be estimated, so this roll is learned
# from complete logging only (its range then shows how little is known)
e.update(rho=None, lo=None, hi=None, exclude=True,
used=f"left out: complete logs have only {ry} rare outcomes, too few to estimate the bias")
elif n < cfg.get("min_runs", 30) or rn < cfg.get("min_runs", 30):
e.update(rho=None, used=f"too few runs: uses {g.get('same_as') or 'no correction'}")
else:
rho = _odds(y, n) / _odds(ry, rn)
sim = [] # a 90% range for rho from the two rates' posteriors
for _ in range(sims):
a = rnd.betavariate(y + 0.5, n - y + 0.5)
b = rnd.betavariate(ry + 0.5, rn - ry + 0.5)
sim.append((a / (1 - a)) / (b / (1 - b)))
sim.sort()
e.update(rho=rho, lo=sim[int(0.05 * sims)], hi=sim[int(0.95 * sims) - 1], used="estimated")
out[name][node] = e
# groups with too little data borrow another group's estimate
for g in groups:
for node in rolls:
e = out[g["name"]][node]
if e.get("rho") is None and not e.get("exclude"):
src = out.get(g.get("same_as") or "", {}).get(node, {})
e["rho"] = src.get("rho") or 1.0
e["lo"], e["hi"] = src.get("lo", e["rho"]), src.get("hi", e["rho"])
return out, names
def weights(names, prepared, est, rolls):
"""Per run: {roll: weight of the outcome it observed}: 1/rho for a rare outcome from a biased group, 0 for any
outcome of a roll its group is left out of. Only where it differs from 1."""
out = []
for name, v in zip(names, prepared):
w = {}
for node, rare in rolls.items():
e = (est.get(name) or {}).get(node, {})
y = model.OBSERVE[node](v)
if y is None:
continue
if e.get("exclude"):
w[node] = 0.0
elif y == rare and (e.get("rho") or 1.0) != 1.0:
w[node] = 1.0 / e["rho"]
out.append(w)
return out
+17 -4
View File
@@ -148,21 +148,30 @@ def ctx_key(v, names):
# ---------------------------------------------------------------- 2. counts
def count_tables(prepared):
"""Counts for every chance event at every back-off level, plus each encounter's outcomes."""
def count_tables(prepared, weights=None):
"""Counts for every chance event at every back-off level, plus each encounter's outcomes. weights: per run,
{chance event: weight of the outcome it observed} (reporting-bias corrections, core/atlas/bias.py)."""
T = {}
for node, sp in CHANCE.items():
t = T[node] = {}
for v in prepared:
for i, v in enumerate(prepared):
y = OBSERVE[node](v)
if not y:
continue
w = weights[i].get(node, 1) if weights else 1
if w == 0:
continue
for L in range(len(sp["context"]) + 1):
names = sp["context"][:L]
if any(v.get(k) is None for k in names):
break
c = t.setdefault(f"{L}:{ctx_key(v, names)}", {})
c[y] = c.get(y, 0) + 1
c[y] = c.get(y, 0) + w
for t in T.values(): # weighted counts are fractions: keep them short and stable
for c in t.values():
for k, x in c.items():
if isinstance(x, float):
c[k] = round(x, 4)
enc = T["encounter"] = {}
for v in prepared:
if not v.get("encCtx"):
@@ -261,6 +270,10 @@ def fit_alpha(T, node):
pairs.append(([t[key].get(k, 0) for k in cats], pooled(parent)))
if not pairs:
return {"alpha": ALPHA, "logLik": None}
# a roll that has only ever come out one way says nothing about whether routes differ: pool fully (the fit
# would otherwise drive every route's chance of the unseen outcome to zero)
if sum(1 for k in cats if t.get("0:", {}).get(k, 0) > 0) < 2:
return {"alpha": max(ALPHA_GRID), "logLik": None}
best_a, best_ll, lls = ALPHA, -math.inf, {}
for a in ALPHA_GRID:
ll = 0
+13 -4
View File
@@ -311,12 +311,18 @@ def summarise(xs):
# ---------------------------------------------------------------- the published policy
def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, colours=None):
"""The full policy for every colour. runs: coded runs (atlas.coding), as dicts."""
def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, colours=None, bias_config=None):
"""The full policy for every colour. runs: coded runs (data/public/runs_clean.csv), as dicts. bias_config:
planner/policy.yaml `reporting_bias` (None: every run counts as reported)."""
colours = colours or model.COLOURS
min_support = G["min_support"] if min_support is None else min_support
prepared = [model.prepare_run(r) for r in runs]
T = model.count_tables(prepared)
bias_est, w = None, None
if bias_config:
from . import bias
bias_est, names = bias.estimate(runs, prepared, bias_config, set(model.RARE_GOALS))
w = bias.weights(names, prepared, bias_est, bias_config["rolls"])
T = model.count_tables(prepared, w)
support, counts = model.support_from(prepared, min_support)
fitted = {k: model.fit_alpha(T, k) for k in model.CHANCE}
alphas = {k: alpha for k in model.CHANCE} if alpha else {k: f["alpha"] for k, f in fitted.items()}
@@ -325,7 +331,8 @@ def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, c
# Cross-fitting: choose the policy on one random half of the runs and score it on the other half.
r = model.Rng(seed)
fold = [0 if r() < 0.5 else 1 for _ in prepared]
halves = [model.count_tables([v for v, f in zip(prepared, fold) if f == h]) for h in (0, 1)]
halves = [model.count_tables([v for v, f in zip(prepared, fold) if f == h],
[x for x, f in zip(w, fold) if f == h] if w else None) for h in (0, 1)]
half_params = [model.Params(Th, "mean", alphas=alphas) for Th in halves]
targets = {}
@@ -366,6 +373,8 @@ def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, c
"chanceContext": {k: s["context"] for k, s in model.CHANCE.items()},
"decisions": model.DECISIONS, "support": support, "supportCounts": counts,
"tables": T,
# how much each kind of source over-reports each rare roll, and the correction applied (bias.py)
"reportingBias": bias_est,
# the game itself (core/spec/game.yaml), so the advisor in core/solver.js can re-solve from any point
"game": {k: G[k] for k in ("version", "colours", "decisions", "chance", "encounters", "encounter_outcomes",
"by_colour", "start", "steps", "from_form", "bot_text")},