- game.yaml: `givens` (first or later attempt) and `distinct` levels that are not pooled; the shadow roll
depends on the attempt first. Form submissions show the shadow on 9% of first attempts and 0.3% of later ones.
- Reporting bias: a roll whose bias cannot be measured can borrow another roll's (`borrow: {shadow: meadow}`),
so form submissions inform the shadow again. An assumption, stated in planner/policy.yaml.
- policy.json: targets.<colour>.firstAttempt (this attempt, eventually). Black: about 1.4% on a first attempt,
0.07% on a later one.
- solver.js: advisor options `attempt`, `fix`, sampling "uniform"; assign() and armFor() for the plan.
- planner/experiments.yaml + build_experiments.py -> data/public/experiments.json (progress and status per
experiment, rebuilt from the bot's runs).
- python -m collection cycle: import bot runs, refit, re-plan, deliver the kit; .gitea/workflows/bot-cycle.yml.
Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
126 lines
5.9 KiB
Python
126 lines
5.9 KiB
Python
"""Reporting bias: some sources report a rare sighting more readily than a miss.
|
|
|
|
The bot logs every run, and bulk-pasted logs record every run of the players who kept them ("complete" groups).
|
|
Form submissions and reports are chosen by the rider, and a run with a shadow or a meadow Blue is more likely to be
|
|
written up than one without. For each rare roll and each selective group, the bias is the factor by which the
|
|
group's odds of the rare outcome exceed the complete groups' odds:
|
|
|
|
rho = odds(rare | group) / odds(rare | complete groups)
|
|
|
|
Each rare outcome a group reports then counts 1/rho in the route model's tables. That keeps the group's pattern
|
|
across routes and moves its overall level to what complete logging shows. The assumption: the bias is the same on
|
|
every route (planner/policy.yaml says which groups, rolls and settings are used).
|
|
"""
|
|
import random
|
|
|
|
from . import model
|
|
|
|
|
|
def group_of(r, groups, rare_goals):
|
|
"""The first group in planner/policy.yaml whose condition the run meets."""
|
|
for g in groups:
|
|
w = g.get("when", {})
|
|
if "source" in w and r.get("source") != w["source"]:
|
|
continue
|
|
if "entry_not" in w and r.get("entry") in (None, w["entry_not"]):
|
|
continue
|
|
if w.get("goal") == "rare" and r.get("goal") not in rare_goals:
|
|
continue
|
|
return g["name"]
|
|
return None
|
|
|
|
|
|
def _odds(y, n):
|
|
p = (y + 0.5) / (n + 1) # Jeffreys
|
|
return p / (1 - p)
|
|
|
|
|
|
def estimate(runs, prepared, cfg, rare_goals, seed=20261109, sims=4000):
|
|
"""{group: {roll: {rare, n, rho, lo, hi, used}}} and the group of every run."""
|
|
groups = cfg["groups"]
|
|
names = [group_of(r, groups, rare_goals) for r in runs]
|
|
complete = {g["name"] for g in groups if g.get("complete")}
|
|
rolls = cfg["rolls"]
|
|
tally = {}
|
|
for name, v in zip(names, prepared):
|
|
for node, rare in rolls.items():
|
|
y = model.OBSERVE[node](v)
|
|
if y is None:
|
|
continue
|
|
t = tally.setdefault(name, {}).setdefault(node, [0, 0])
|
|
t[0] += y == rare
|
|
t[1] += 1
|
|
ref = {node: [sum(tally.get(g, {}).get(node, [0, 0])[i] for g in complete) for i in (0, 1)] for node in rolls}
|
|
rnd = random.Random(seed)
|
|
out = {}
|
|
for g in groups:
|
|
name = g["name"]
|
|
out[name] = {}
|
|
for node in rolls:
|
|
y, n = tally.get(name, {}).get(node, [0, 0])
|
|
ry, rn = ref[node]
|
|
e = {"rare": y, "n": n, "completeRare": ry, "completeN": rn}
|
|
if g.get("complete"):
|
|
e.update(rho=1.0, lo=1.0, hi=1.0, used="complete: counted as reported")
|
|
elif ry < cfg.get("min_complete_rare", 5) and (cfg.get("borrow") or {}).get(node):
|
|
# complete logging has (almost) never seen it, so the bias cannot be estimated for this roll: assume
|
|
# the group over-reports it as much as it over-reports another rare roll (filled in below)
|
|
e.update(rho=None, borrow=cfg["borrow"][node])
|
|
elif ry < cfg.get("min_complete_rare", 5):
|
|
# complete logging has (almost) never seen it: the bias cannot be estimated, so this roll is learned
|
|
# from complete logging only (its range then shows how little is known)
|
|
e.update(rho=None, lo=None, hi=None, exclude=True,
|
|
used=f"left out: complete logs have only {ry} rare outcomes, too few to estimate the bias")
|
|
elif n < cfg.get("min_runs", 30) or rn < cfg.get("min_runs", 30):
|
|
e.update(rho=None, used=f"too few runs: uses {g.get('same_as') or 'no correction'}")
|
|
else:
|
|
rho = _odds(y, n) / _odds(ry, rn)
|
|
sim = [] # a 90% range for rho from the two rates' posteriors
|
|
for _ in range(sims):
|
|
a = rnd.betavariate(y + 0.5, n - y + 0.5)
|
|
b = rnd.betavariate(ry + 0.5, rn - ry + 0.5)
|
|
sim.append((a / (1 - a)) / (b / (1 - b)))
|
|
sim.sort()
|
|
e.update(rho=rho, lo=sim[int(0.05 * sims)], hi=sim[int(0.95 * sims) - 1], used="estimated")
|
|
out[name][node] = e
|
|
# groups with too little data borrow another group's estimate
|
|
for g in groups:
|
|
for node in rolls:
|
|
e = out[g["name"]][node]
|
|
if e.get("rho") is None and not e.get("exclude") and not e.get("borrow"):
|
|
src = out.get(g.get("same_as") or "", {}).get(node, {})
|
|
e["rho"] = src.get("rho") or 1.0
|
|
e["lo"], e["hi"] = src.get("lo", e["rho"]), src.get("hi", e["rho"])
|
|
# a roll whose bias cannot be estimated borrows the same group's estimate for another roll (`borrow`)
|
|
for g in groups:
|
|
for node in rolls:
|
|
e = out[g["name"]][node]
|
|
if e.get("borrow"):
|
|
src = out[g["name"]].get(e["borrow"], {})
|
|
if src.get("rho"):
|
|
e.update(rho=src["rho"], lo=src.get("lo"), hi=src.get("hi"),
|
|
used=f"assumed equal to this group's bias for the {e['borrow']} roll: complete logs have only "
|
|
f"{e['completeRare']} rare outcomes, too few to estimate it")
|
|
else:
|
|
e.update(lo=None, hi=None, exclude=True, used="left out: no bias estimate to borrow")
|
|
return out, names
|
|
|
|
|
|
def weights(names, prepared, est, rolls):
|
|
"""Per run: {roll: weight of the outcome it observed}: 1/rho for a rare outcome from a biased group, 0 for any
|
|
outcome of a roll its group is left out of. Only where it differs from 1."""
|
|
out = []
|
|
for name, v in zip(names, prepared):
|
|
w = {}
|
|
for node, rare in rolls.items():
|
|
e = (est.get(name) or {}).get(node, {})
|
|
y = model.OBSERVE[node](v)
|
|
if y is None:
|
|
continue
|
|
if e.get("exclude"):
|
|
w[node] = 0.0
|
|
elif y == rare and (e.get("rho") or 1.0) != 1.0:
|
|
w[node] = 1.0 / e["rho"]
|
|
out.append(w)
|
|
return out
|