Files
ekcSchianoandClaude Fable 5.2 ab3664f369 Route model: first attempts are their own shadow roll; experiment plan and automated bot cycle
- game.yaml: `givens` (first or later attempt) and `distinct` levels that are not pooled; the shadow roll
  depends on the attempt first. Form submissions show the shadow on 9% of first attempts and 0.3% of later ones.
- Reporting bias: a roll whose bias cannot be measured can borrow another roll's (`borrow: {shadow: meadow}`),
  so form submissions inform the shadow again. An assumption, stated in planner/policy.yaml.
- policy.json: targets.<colour>.firstAttempt (this attempt, eventually). Black: about 1.4% on a first attempt,
  0.07% on a later one.
- solver.js: advisor options `attempt`, `fix`, sampling "uniform"; assign() and armFor() for the plan.
- planner/experiments.yaml + build_experiments.py -> data/public/experiments.json (progress and status per
  experiment, rebuilt from the bot's runs).
- python -m collection cycle: import bot runs, refit, re-plan, deliver the kit; .gitea/workflows/bot-cycle.yml.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
2026-11-12 03:31:33 +00:00

126 lines
5.9 KiB
Python

"""Reporting bias: some sources report a rare sighting more readily than a miss.
The bot logs every run, and bulk-pasted logs record every run of the players who kept them ("complete" groups).
Form submissions and reports are chosen by the rider, and a run with a shadow or a meadow Blue is more likely to be
written up than one without. For each rare roll and each selective group, the bias is the factor by which the
group's odds of the rare outcome exceed the complete groups' odds:
rho = odds(rare | group) / odds(rare | complete groups)
Each rare outcome a group reports then counts 1/rho in the route model's tables. That keeps the group's pattern
across routes and moves its overall level to what complete logging shows. The assumption: the bias is the same on
every route (planner/policy.yaml says which groups, rolls and settings are used).
"""
import random
from . import model
def group_of(r, groups, rare_goals):
"""The first group in planner/policy.yaml whose condition the run meets."""
for g in groups:
w = g.get("when", {})
if "source" in w and r.get("source") != w["source"]:
continue
if "entry_not" in w and r.get("entry") in (None, w["entry_not"]):
continue
if w.get("goal") == "rare" and r.get("goal") not in rare_goals:
continue
return g["name"]
return None
def _odds(y, n):
p = (y + 0.5) / (n + 1) # Jeffreys
return p / (1 - p)
def estimate(runs, prepared, cfg, rare_goals, seed=20261109, sims=4000):
"""{group: {roll: {rare, n, rho, lo, hi, used}}} and the group of every run."""
groups = cfg["groups"]
names = [group_of(r, groups, rare_goals) for r in runs]
complete = {g["name"] for g in groups if g.get("complete")}
rolls = cfg["rolls"]
tally = {}
for name, v in zip(names, prepared):
for node, rare in rolls.items():
y = model.OBSERVE[node](v)
if y is None:
continue
t = tally.setdefault(name, {}).setdefault(node, [0, 0])
t[0] += y == rare
t[1] += 1
ref = {node: [sum(tally.get(g, {}).get(node, [0, 0])[i] for g in complete) for i in (0, 1)] for node in rolls}
rnd = random.Random(seed)
out = {}
for g in groups:
name = g["name"]
out[name] = {}
for node in rolls:
y, n = tally.get(name, {}).get(node, [0, 0])
ry, rn = ref[node]
e = {"rare": y, "n": n, "completeRare": ry, "completeN": rn}
if g.get("complete"):
e.update(rho=1.0, lo=1.0, hi=1.0, used="complete: counted as reported")
elif ry < cfg.get("min_complete_rare", 5) and (cfg.get("borrow") or {}).get(node):
# complete logging has (almost) never seen it, so the bias cannot be estimated for this roll: assume
# the group over-reports it as much as it over-reports another rare roll (filled in below)
e.update(rho=None, borrow=cfg["borrow"][node])
elif ry < cfg.get("min_complete_rare", 5):
# complete logging has (almost) never seen it: the bias cannot be estimated, so this roll is learned
# from complete logging only (its range then shows how little is known)
e.update(rho=None, lo=None, hi=None, exclude=True,
used=f"left out: complete logs have only {ry} rare outcomes, too few to estimate the bias")
elif n < cfg.get("min_runs", 30) or rn < cfg.get("min_runs", 30):
e.update(rho=None, used=f"too few runs: uses {g.get('same_as') or 'no correction'}")
else:
rho = _odds(y, n) / _odds(ry, rn)
sim = [] # a 90% range for rho from the two rates' posteriors
for _ in range(sims):
a = rnd.betavariate(y + 0.5, n - y + 0.5)
b = rnd.betavariate(ry + 0.5, rn - ry + 0.5)
sim.append((a / (1 - a)) / (b / (1 - b)))
sim.sort()
e.update(rho=rho, lo=sim[int(0.05 * sims)], hi=sim[int(0.95 * sims) - 1], used="estimated")
out[name][node] = e
# groups with too little data borrow another group's estimate
for g in groups:
for node in rolls:
e = out[g["name"]][node]
if e.get("rho") is None and not e.get("exclude") and not e.get("borrow"):
src = out.get(g.get("same_as") or "", {}).get(node, {})
e["rho"] = src.get("rho") or 1.0
e["lo"], e["hi"] = src.get("lo", e["rho"]), src.get("hi", e["rho"])
# a roll whose bias cannot be estimated borrows the same group's estimate for another roll (`borrow`)
for g in groups:
for node in rolls:
e = out[g["name"]][node]
if e.get("borrow"):
src = out[g["name"]].get(e["borrow"], {})
if src.get("rho"):
e.update(rho=src["rho"], lo=src.get("lo"), hi=src.get("hi"),
used=f"assumed equal to this group's bias for the {e['borrow']} roll: complete logs have only "
f"{e['completeRare']} rare outcomes, too few to estimate it")
else:
e.update(lo=None, hi=None, exclude=True, used="left out: no bias estimate to borrow")
return out, names
def weights(names, prepared, est, rolls):
"""Per run: {roll: weight of the outcome it observed}: 1/rho for a rare outcome from a biased group, 0 for any
outcome of a roll its group is left out of. Only where it differs from 1."""
out = []
for name, v in zip(names, prepared):
w = {}
for node, rare in rolls.items():
e = (est.get(name) or {}).get(node, {})
y = model.OBSERVE[node](v)
if y is None:
continue
if e.get("exclude"):
w[node] = 0.0
elif y == rare and (e.get("rho") or 1.0) != 1.0:
w[node] = 1.0 / e["rho"]
out.append(w)
return out