Route model: correct for reporting bias
- core/atlas/bias.py: for each rare roll (shadow, meadow Blue) and each source group, the factor by which the group over-reports the rare outcome relative to complete logging (bot, bulk logs), with a 90% range; rare outcomes from biased groups count 1/rho; a roll the complete logs have (almost) never shown is learned from them only - planner/policy.yaml: groups, rolls and thresholds; estimates published in policy.json under model.reportingBias - fit_alpha: a roll seen only one way is pooled fully - docs: goal accounts follow the best route, research accounts use Thompson sampling (decisionMode atlas_thompson) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c0355616b6
commit
2271fe0976
@@ -0,0 +1,109 @@
|
||||
"""Reporting bias: some sources report a rare sighting more readily than a miss.
|
||||
|
||||
The bot logs every run, and bulk-pasted logs record every run of the players who kept them ("complete" groups).
|
||||
Form submissions and reports are chosen by the rider, and a run with a shadow or a meadow Blue is more likely to be
|
||||
written up than one without. For each rare roll and each selective group, the bias is the factor by which the
|
||||
group's odds of the rare outcome exceed the complete groups' odds:
|
||||
|
||||
rho = odds(rare | group) / odds(rare | complete groups)
|
||||
|
||||
Each rare outcome a group reports then counts 1/rho in the route model's tables. That keeps the group's pattern
|
||||
across routes and moves its overall level to what complete logging shows. The assumption: the bias is the same on
|
||||
every route (planner/policy.yaml says which groups, rolls and settings are used).
|
||||
"""
|
||||
import random
|
||||
|
||||
from . import model
|
||||
|
||||
|
||||
def group_of(r, groups, rare_goals):
|
||||
"""The first group in planner/policy.yaml whose condition the run meets."""
|
||||
for g in groups:
|
||||
w = g.get("when", {})
|
||||
if "source" in w and r.get("source") != w["source"]:
|
||||
continue
|
||||
if "entry_not" in w and r.get("entry") in (None, w["entry_not"]):
|
||||
continue
|
||||
if w.get("goal") == "rare" and r.get("goal") not in rare_goals:
|
||||
continue
|
||||
return g["name"]
|
||||
return None
|
||||
|
||||
|
||||
def _odds(y, n):
|
||||
p = (y + 0.5) / (n + 1) # Jeffreys
|
||||
return p / (1 - p)
|
||||
|
||||
|
||||
def estimate(runs, prepared, cfg, rare_goals, seed=20261109, sims=4000):
|
||||
"""{group: {roll: {rare, n, rho, lo, hi, used}}} and the group of every run."""
|
||||
groups = cfg["groups"]
|
||||
names = [group_of(r, groups, rare_goals) for r in runs]
|
||||
complete = {g["name"] for g in groups if g.get("complete")}
|
||||
rolls = cfg["rolls"]
|
||||
tally = {}
|
||||
for name, v in zip(names, prepared):
|
||||
for node, rare in rolls.items():
|
||||
y = model.OBSERVE[node](v)
|
||||
if y is None:
|
||||
continue
|
||||
t = tally.setdefault(name, {}).setdefault(node, [0, 0])
|
||||
t[0] += y == rare
|
||||
t[1] += 1
|
||||
ref = {node: [sum(tally.get(g, {}).get(node, [0, 0])[i] for g in complete) for i in (0, 1)] for node in rolls}
|
||||
rnd = random.Random(seed)
|
||||
out = {}
|
||||
for g in groups:
|
||||
name = g["name"]
|
||||
out[name] = {}
|
||||
for node in rolls:
|
||||
y, n = tally.get(name, {}).get(node, [0, 0])
|
||||
ry, rn = ref[node]
|
||||
e = {"rare": y, "n": n, "completeRare": ry, "completeN": rn}
|
||||
if g.get("complete"):
|
||||
e.update(rho=1.0, lo=1.0, hi=1.0, used="complete: counted as reported")
|
||||
elif ry < cfg.get("min_complete_rare", 5):
|
||||
# complete logging has (almost) never seen it: the bias cannot be estimated, so this roll is learned
|
||||
# from complete logging only (its range then shows how little is known)
|
||||
e.update(rho=None, lo=None, hi=None, exclude=True,
|
||||
used=f"left out: complete logs have only {ry} rare outcomes, too few to estimate the bias")
|
||||
elif n < cfg.get("min_runs", 30) or rn < cfg.get("min_runs", 30):
|
||||
e.update(rho=None, used=f"too few runs: uses {g.get('same_as') or 'no correction'}")
|
||||
else:
|
||||
rho = _odds(y, n) / _odds(ry, rn)
|
||||
sim = [] # a 90% range for rho from the two rates' posteriors
|
||||
for _ in range(sims):
|
||||
a = rnd.betavariate(y + 0.5, n - y + 0.5)
|
||||
b = rnd.betavariate(ry + 0.5, rn - ry + 0.5)
|
||||
sim.append((a / (1 - a)) / (b / (1 - b)))
|
||||
sim.sort()
|
||||
e.update(rho=rho, lo=sim[int(0.05 * sims)], hi=sim[int(0.95 * sims) - 1], used="estimated")
|
||||
out[name][node] = e
|
||||
# groups with too little data borrow another group's estimate
|
||||
for g in groups:
|
||||
for node in rolls:
|
||||
e = out[g["name"]][node]
|
||||
if e.get("rho") is None and not e.get("exclude"):
|
||||
src = out.get(g.get("same_as") or "", {}).get(node, {})
|
||||
e["rho"] = src.get("rho") or 1.0
|
||||
e["lo"], e["hi"] = src.get("lo", e["rho"]), src.get("hi", e["rho"])
|
||||
return out, names
|
||||
|
||||
|
||||
def weights(names, prepared, est, rolls):
|
||||
"""Per run: {roll: weight of the outcome it observed}: 1/rho for a rare outcome from a biased group, 0 for any
|
||||
outcome of a roll its group is left out of. Only where it differs from 1."""
|
||||
out = []
|
||||
for name, v in zip(names, prepared):
|
||||
w = {}
|
||||
for node, rare in rolls.items():
|
||||
e = (est.get(name) or {}).get(node, {})
|
||||
y = model.OBSERVE[node](v)
|
||||
if y is None:
|
||||
continue
|
||||
if e.get("exclude"):
|
||||
w[node] = 0.0
|
||||
elif y == rare and (e.get("rho") or 1.0) != 1.0:
|
||||
w[node] = 1.0 / e["rho"]
|
||||
out.append(w)
|
||||
return out
|
||||
+17
-4
@@ -148,21 +148,30 @@ def ctx_key(v, names):
|
||||
|
||||
# ---------------------------------------------------------------- 2. counts
|
||||
|
||||
def count_tables(prepared):
|
||||
"""Counts for every chance event at every back-off level, plus each encounter's outcomes."""
|
||||
def count_tables(prepared, weights=None):
|
||||
"""Counts for every chance event at every back-off level, plus each encounter's outcomes. weights: per run,
|
||||
{chance event: weight of the outcome it observed} (reporting-bias corrections, core/atlas/bias.py)."""
|
||||
T = {}
|
||||
for node, sp in CHANCE.items():
|
||||
t = T[node] = {}
|
||||
for v in prepared:
|
||||
for i, v in enumerate(prepared):
|
||||
y = OBSERVE[node](v)
|
||||
if not y:
|
||||
continue
|
||||
w = weights[i].get(node, 1) if weights else 1
|
||||
if w == 0:
|
||||
continue
|
||||
for L in range(len(sp["context"]) + 1):
|
||||
names = sp["context"][:L]
|
||||
if any(v.get(k) is None for k in names):
|
||||
break
|
||||
c = t.setdefault(f"{L}:{ctx_key(v, names)}", {})
|
||||
c[y] = c.get(y, 0) + 1
|
||||
c[y] = c.get(y, 0) + w
|
||||
for t in T.values(): # weighted counts are fractions: keep them short and stable
|
||||
for c in t.values():
|
||||
for k, x in c.items():
|
||||
if isinstance(x, float):
|
||||
c[k] = round(x, 4)
|
||||
enc = T["encounter"] = {}
|
||||
for v in prepared:
|
||||
if not v.get("encCtx"):
|
||||
@@ -261,6 +270,10 @@ def fit_alpha(T, node):
|
||||
pairs.append(([t[key].get(k, 0) for k in cats], pooled(parent)))
|
||||
if not pairs:
|
||||
return {"alpha": ALPHA, "logLik": None}
|
||||
# a roll that has only ever come out one way says nothing about whether routes differ: pool fully (the fit
|
||||
# would otherwise drive every route's chance of the unseen outcome to zero)
|
||||
if sum(1 for k in cats if t.get("0:", {}).get(k, 0) > 0) < 2:
|
||||
return {"alpha": max(ALPHA_GRID), "logLik": None}
|
||||
best_a, best_ll, lls = ALPHA, -math.inf, {}
|
||||
for a in ALPHA_GRID:
|
||||
ll = 0
|
||||
|
||||
+13
-4
@@ -311,12 +311,18 @@ def summarise(xs):
|
||||
|
||||
# ---------------------------------------------------------------- the published policy
|
||||
|
||||
def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, colours=None):
|
||||
"""The full policy for every colour. runs: coded runs (atlas.coding), as dicts."""
|
||||
def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, colours=None, bias_config=None):
|
||||
"""The full policy for every colour. runs: coded runs (data/public/runs_clean.csv), as dicts. bias_config:
|
||||
planner/policy.yaml `reporting_bias` (None: every run counts as reported)."""
|
||||
colours = colours or model.COLOURS
|
||||
min_support = G["min_support"] if min_support is None else min_support
|
||||
prepared = [model.prepare_run(r) for r in runs]
|
||||
T = model.count_tables(prepared)
|
||||
bias_est, w = None, None
|
||||
if bias_config:
|
||||
from . import bias
|
||||
bias_est, names = bias.estimate(runs, prepared, bias_config, set(model.RARE_GOALS))
|
||||
w = bias.weights(names, prepared, bias_est, bias_config["rolls"])
|
||||
T = model.count_tables(prepared, w)
|
||||
support, counts = model.support_from(prepared, min_support)
|
||||
fitted = {k: model.fit_alpha(T, k) for k in model.CHANCE}
|
||||
alphas = {k: alpha for k in model.CHANCE} if alpha else {k: f["alpha"] for k, f in fitted.items()}
|
||||
@@ -325,7 +331,8 @@ def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, c
|
||||
# Cross-fitting: choose the policy on one random half of the runs and score it on the other half.
|
||||
r = model.Rng(seed)
|
||||
fold = [0 if r() < 0.5 else 1 for _ in prepared]
|
||||
halves = [model.count_tables([v for v, f in zip(prepared, fold) if f == h]) for h in (0, 1)]
|
||||
halves = [model.count_tables([v for v, f in zip(prepared, fold) if f == h],
|
||||
[x for x, f in zip(w, fold) if f == h] if w else None) for h in (0, 1)]
|
||||
half_params = [model.Params(Th, "mean", alphas=alphas) for Th in halves]
|
||||
|
||||
targets = {}
|
||||
@@ -366,6 +373,8 @@ def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, c
|
||||
"chanceContext": {k: s["context"] for k, s in model.CHANCE.items()},
|
||||
"decisions": model.DECISIONS, "support": support, "supportCounts": counts,
|
||||
"tables": T,
|
||||
# how much each kind of source over-reports each rare roll, and the correction applied (bias.py)
|
||||
"reportingBias": bias_est,
|
||||
# the game itself (core/spec/game.yaml), so the advisor in core/solver.js can re-solve from any point
|
||||
"game": {k: G[k] for k in ("version", "colours", "decisions", "chance", "encounters", "encounter_outcomes",
|
||||
"by_colour", "start", "steps", "from_form", "bot_text")},
|
||||
|
||||
Reference in New Issue
Block a user