- core/atlas/bias.py: for each rare roll (shadow, meadow Blue) and each source group, the factor by which the group over-reports the rare outcome relative to complete logging (bot, bulk logs), with a 90% range; rare outcomes from biased groups count 1/rho; a roll the complete logs have (almost) never shown is learned from them only - planner/policy.yaml: groups, rolls and thresholds; estimates published in policy.json under model.reportingBias - fit_alpha: a roll seen only one way is pooled fully - docs: goal accounts follow the best route, research accounts use Thompson sampling (decisionMode atlas_thompson) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
385 lines
15 KiB
Python
385 lines
15 KiB
Python
"""The route model: best decisions for bonding each dragon colour.
|
|
|
|
The game tree is core/spec/game.yaml. One interpreter (Walker) walks it; three kinds of walk reuse it:
|
|
|
|
Solver backward induction: the best option at every decision (vectorised over posterior draws)
|
|
Evaluator the value of a fixed policy (the one chosen on the posterior means) in every draw
|
|
Exporter the published decision tree, following the chosen options
|
|
|
|
Two objectives per target colour X:
|
|
1. "this attempt": maximise P(bond X on this attempt).
|
|
2. "eventually": deaths are free (you can try again), so what matters is not ending the account on the wrong
|
|
dragon: maximise P(X) / (P(X) + P(another colour)), by Dinkelbach iteration; ties go to the higher P(X).
|
|
|
|
Values are triples (x, o, r) of arrays over worlds: P(target), P(another colour), P(reaching an encounter alive).
|
|
"""
|
|
import numpy as np
|
|
|
|
from . import model, spec
|
|
|
|
G = spec.game()
|
|
STEPS = G["steps"]
|
|
LABEL = {**{k: v["label"] for k, v in G["decisions"].items()}, **{k: v["label"] for k, v in G["chance"].items()}}
|
|
|
|
|
|
def canon(state):
|
|
return tuple(sorted(state.items()))
|
|
|
|
|
|
def apply(then, state, value):
|
|
"""The state after an option or outcome: keep / drop, then set ($ = the value just taken)."""
|
|
s = {k: state[k] for k in then["keep"] if k in state} if "keep" in then else dict(state)
|
|
for k in then.get("drop", ()):
|
|
s.pop(k, None)
|
|
for k, v in (then.get("set") or {}).items():
|
|
s[k] = value if v == "$" else v
|
|
return s
|
|
|
|
|
|
class Walker:
|
|
"""Walks the game tree of game.yaml. Subclasses say what a decision, a roll, an ending and an encounter mean."""
|
|
|
|
def __init__(self, params, X, support, only=None):
|
|
self.params, self.X, self.support, self.only = params, X, support, only
|
|
|
|
def options(self, name, state):
|
|
by = G["decisions"][name].get("support_by")
|
|
if by:
|
|
return self.support.get(f"{name}:{state.get(by)}") or model.DECISIONS[name]
|
|
return self.support[name]
|
|
|
|
def root(self):
|
|
return self.run(G["start"], {})
|
|
|
|
def run(self, step_id, state):
|
|
st = STEPS[step_id]
|
|
if "decision" in st:
|
|
name = st["decision"]
|
|
return self.decision(name, state, self.options(name, state), lambda o: self.follow(st["then"], o, state))
|
|
node = st["chance"]
|
|
return self.chance(node, state, lambda k: self.follow(st["then"], k, state))
|
|
|
|
def follow(self, then, value, state):
|
|
t = then.get(value) or then["default"]
|
|
if "end" in t:
|
|
return self.end(t["end"])
|
|
s = apply(t, state, value)
|
|
if "encounter" in t:
|
|
ctx = t["encounter"].replace("$", value)
|
|
opts = ["Die on purpose"] if self.only and ctx not in self.only else self.support["encounter"]
|
|
return self.decision("encounter", {**s, "enc": ctx}, opts,
|
|
lambda o: self.die_on_purpose() if o == "Die on purpose" else self.attempt(ctx))
|
|
return self.run(t["go"], s)
|
|
|
|
|
|
class _Values(Walker):
|
|
"""Shared by Solver and Evaluator: values are (x, o, r) arrays over the worlds in params."""
|
|
|
|
def __init__(self, params, X, support, only=None):
|
|
super().__init__(params, X, support, only)
|
|
D = params.D
|
|
self.ZERO = (np.zeros(D), np.zeros(D), np.zeros(D))
|
|
self.REACHED = (np.zeros(D), np.zeros(D), np.ones(D))
|
|
|
|
def end(self, what):
|
|
return self.ZERO
|
|
|
|
def die_on_purpose(self):
|
|
return self.REACHED
|
|
|
|
def attempt(self, ctx):
|
|
cache = self.__dict__.setdefault("_attempt", {})
|
|
if ctx not in cache:
|
|
cache[ctx] = self._attempt_value(ctx)
|
|
return cache[ctx]
|
|
|
|
def _attempt_value(self, ctx):
|
|
x = 0
|
|
oth = 0
|
|
for k, p in self.params.encounter(ctx).items():
|
|
if k != "burn":
|
|
if k == self.X:
|
|
x = x + p
|
|
else:
|
|
oth = oth + p
|
|
D = self.params.D
|
|
return (np.broadcast_to(x, D).astype(float), np.broadcast_to(oth, D).astype(float), np.ones(D))
|
|
|
|
def chance(self, node, state, cont):
|
|
acc = self.ZERO
|
|
for k, p in self.params.chance(node, state).items():
|
|
if not np.any(p):
|
|
continue
|
|
v = cont(k)
|
|
acc = (acc[0] + p * v[0], acc[1] + p * v[1], acc[2] + p * v[2])
|
|
return acc
|
|
|
|
|
|
class Solver(_Values):
|
|
"""Backward induction. lam: scalar or array (one per world). After root(): .value, .choices, .option_values."""
|
|
|
|
def __init__(self, params, X, support, lam=0.0, record=False, only=None):
|
|
super().__init__(params, X, support, only)
|
|
self.lam = lam
|
|
self.memo, self.choices = {}, {}
|
|
self.option_values = {} if record else None
|
|
|
|
def score(self, v):
|
|
lam = self.lam
|
|
return (1 - lam) * v[0] - lam * v[1] + 1e-9 * v[0] - 1e-12 * v[1]
|
|
|
|
def decision(self, name, state, options, cont):
|
|
key = (name, canon(state))
|
|
if key in self.memo:
|
|
return self.memo[key]
|
|
top = arg = None
|
|
vals = {}
|
|
for i, o in enumerate(options):
|
|
v = cont(o)
|
|
vals[o] = v
|
|
if top is None:
|
|
top, arg = v, np.zeros(self.params.D, dtype=int)
|
|
else:
|
|
better = self.score(v) > self.score(top)
|
|
top = tuple(np.where(better, v[j], top[j]) for j in range(3))
|
|
arg = np.where(better, i, arg)
|
|
self.memo[key] = top
|
|
self.choices[key] = (list(options), arg)
|
|
if self.option_values is not None:
|
|
self.option_values[key] = vals
|
|
return top
|
|
|
|
def solve(self):
|
|
self.value = self.root()
|
|
self.memo.clear() # the values are no longer needed; posterior draws keep only their choices
|
|
return self
|
|
|
|
def decide(self, name, state, world=0):
|
|
opts, arg = self.choices[(name, canon(state))]
|
|
return opts[int(arg[world])]
|
|
|
|
|
|
class Evaluator(_Values):
|
|
"""The value of a fixed policy: decide(name, state) -> option, in every world of params."""
|
|
|
|
def __init__(self, params, X, support, decide, only=None):
|
|
super().__init__(params, X, support, only)
|
|
self.decide = decide
|
|
|
|
def decision(self, name, state, options, cont):
|
|
return cont(self.decide(name, state))
|
|
|
|
|
|
class Exporter(Walker):
|
|
"""The published tree: at decisions the chosen option, every option's value (each followed by the best play
|
|
afterwards) and how often it stays best across posterior draws; at rolls every branch and its probability.
|
|
Only the chosen option's subtree is expanded."""
|
|
|
|
def __init__(self, params, X, support, sol, draws, min_show=1e-4, only=None):
|
|
super().__init__(params, X, support, only)
|
|
self.sol, self.draws, self.min_show = sol, draws, min_show
|
|
|
|
def decision(self, name, state, options, cont):
|
|
key = (name, canon(state))
|
|
chosen = self.sol.decide(name, state)
|
|
ov = self.sol.option_values.get(key, {})
|
|
xs = sorted((float(v[0][0]) for v in ov.values()), reverse=True)
|
|
tie = len(xs) > 1 and xs[0] - xs[1] < 1e-12 # nothing to choose between (e.g. every option ends the run)
|
|
still = None
|
|
if not tie and self.draws is not None:
|
|
opts, arg = self.draws.choices[key]
|
|
ok = arg == opts.index(chosen)
|
|
reached = getattr(self.draws.params, "reached", {}).get(key)
|
|
if reached is not None:
|
|
ok = ok & reached
|
|
still = int(np.sum(ok)) / len(arg)
|
|
return {
|
|
"type": "decision", "var": name, "label": LABEL[name], "choice": chosen, "tie": tie, "stillBest": still,
|
|
"options": {o: {"pX": float(v[0][0]), "pOther": float(v[1][0]), "pReach": float(v[2][0])} for o, v in ov.items()},
|
|
"next": cont(chosen),
|
|
}
|
|
|
|
def chance(self, node, state, cont):
|
|
dist = self.params.chance(node, state)
|
|
return {"type": "chance", "var": node, "label": LABEL[node],
|
|
"branches": {k: {"p": float(p[0]), "next": cont(k)} for k, p in dist.items() if float(p[0]) >= self.min_show}}
|
|
|
|
def end(self, what):
|
|
return {"type": "end", "what": what}
|
|
|
|
def die_on_purpose(self):
|
|
return self.end("die on purpose")
|
|
|
|
def attempt(self, ctx):
|
|
return {"type": "end", "what": f"encounter: {ctx}", "outcomes": {k: float(p[0]) for k, p in self.params.encounter(ctx).items()}}
|
|
|
|
|
|
class _Drawer(Walker):
|
|
"""Walks the game in one world only, drawing its distributions in the order the walk first needs them and
|
|
skipping branches with probability exactly zero in that world. Records which decisions the world reaches."""
|
|
|
|
def __init__(self, params, support, world, reached):
|
|
super().__init__(params, None, support)
|
|
self.d, self.seen, self.reached = world, set(), reached
|
|
|
|
def decision(self, name, state, options, cont):
|
|
key = (name, canon(state))
|
|
if key in self.seen:
|
|
return
|
|
self.seen.add(key)
|
|
self.reached.setdefault(key, np.zeros(self.params.D, dtype=bool))[self.d] = True
|
|
for o in options:
|
|
cont(o)
|
|
|
|
def chance(self, node, state, cont):
|
|
for k, p in self.params.chance(node, state).items():
|
|
if p[self.d]:
|
|
cont(k)
|
|
|
|
def attempt(self, ctx):
|
|
self.params.encounter(ctx)
|
|
|
|
def end(self, what):
|
|
return None
|
|
|
|
def die_on_purpose(self):
|
|
return None
|
|
|
|
|
|
def draw_worlds(T, seeds, alphas, support):
|
|
"""Posterior draws, one world per seed, each drawn in its own walk of the game (see Params)."""
|
|
p = model.Params(T, "draw", seeds=seeds, alphas=alphas)
|
|
p.reached = {}
|
|
for d in range(p.D):
|
|
p.world = d
|
|
_Drawer(p, support, d, p.reached).root()
|
|
p.world = None
|
|
return p
|
|
|
|
|
|
# ---------------------------------------------------------------- objectives
|
|
|
|
def solve(params, X, support, lam=0.0, record=False, only=None):
|
|
return Solver(params, X, support, lam=lam, record=record, only=only).solve()
|
|
|
|
|
|
def solve_ratio(params, X, support, record=False, only=None):
|
|
"""Dinkelbach: lam* = max P(X) / (P(X) + P(other)), separately in every world (at most 30 updates each)."""
|
|
D = params.D
|
|
lam = np.zeros(D)
|
|
s = solve(params, X, support, lam=lam, only=only)
|
|
done = np.zeros(D, dtype=bool)
|
|
for _ in range(30):
|
|
x, o = s.value[0], s.value[1]
|
|
tot = x + o
|
|
r = np.where(tot > 0, x / np.where(tot > 0, tot, 1), 0.0)
|
|
done = done | (np.abs(r - lam) < 1e-10)
|
|
if done.all():
|
|
break
|
|
lam = np.where(done, lam, r)
|
|
s = solve(params, X, support, lam=lam, only=only)
|
|
if record:
|
|
s = solve(params, X, support, lam=lam, record=True, only=only)
|
|
s.lam = lam
|
|
return s
|
|
|
|
|
|
def evaluate(params, X, support, decide, only=None):
|
|
return Evaluator(params, X, support, decide, only=only).root()
|
|
|
|
|
|
# ---------------------------------------------------------------- summaries
|
|
|
|
def _sum(xs):
|
|
s = 0
|
|
for x in xs:
|
|
s += x
|
|
return s
|
|
|
|
|
|
def quantile(xs, q):
|
|
s = sorted(xs)
|
|
i = (len(s) - 1) * q
|
|
lo, hi = int(np.floor(i)), int(np.ceil(i))
|
|
return s[lo] + (s[hi] - s[lo]) * (i - lo)
|
|
|
|
|
|
def summarise(xs):
|
|
xs = [float(x) for x in xs]
|
|
return {"mean": _sum(xs) / len(xs), "lo": quantile(xs, 0.025), "hi": quantile(xs, 0.975)}
|
|
|
|
|
|
# ---------------------------------------------------------------- the published policy
|
|
|
|
def build_policy(runs, draws=200, seed=20261101, min_support=None, alpha=None, colours=None, bias_config=None):
|
|
"""The full policy for every colour. runs: coded runs (data/public/runs_clean.csv), as dicts. bias_config:
|
|
planner/policy.yaml `reporting_bias` (None: every run counts as reported)."""
|
|
colours = colours or model.COLOURS
|
|
min_support = G["min_support"] if min_support is None else min_support
|
|
prepared = [model.prepare_run(r) for r in runs]
|
|
bias_est, w = None, None
|
|
if bias_config:
|
|
from . import bias
|
|
bias_est, names = bias.estimate(runs, prepared, bias_config, set(model.RARE_GOALS))
|
|
w = bias.weights(names, prepared, bias_est, bias_config["rolls"])
|
|
T = model.count_tables(prepared, w)
|
|
support, counts = model.support_from(prepared, min_support)
|
|
fitted = {k: model.fit_alpha(T, k) for k in model.CHANCE}
|
|
alphas = {k: alpha for k in model.CHANCE} if alpha else {k: f["alpha"] for k, f in fitted.items()}
|
|
mean = model.Params(T, "mean", alphas=alphas)
|
|
drawp = draw_worlds(T, [seed + i * 7919 for i in range(draws)], alphas, support)
|
|
# Cross-fitting: choose the policy on one random half of the runs and score it on the other half.
|
|
r = model.Rng(seed)
|
|
fold = [0 if r() < 0.5 else 1 for _ in prepared]
|
|
halves = [model.count_tables([v for v, f in zip(prepared, fold) if f == h],
|
|
[x for x, f in zip(w, fold) if f == h] if w else None) for h in (0, 1)]
|
|
half_params = [model.Params(Th, "mean", alphas=alphas) for Th in halves]
|
|
|
|
targets = {}
|
|
for X in colours:
|
|
one = solve(mean, X, support, record=True)
|
|
ratio = solve_ratio(mean, X, support, record=True)
|
|
d_one = solve(drawp, X, support)
|
|
d_ratio = solve_ratio(drawp, X, support)
|
|
ev_one = evaluate(drawp, X, support, one.decide)
|
|
ev_ratio = evaluate(drawp, X, support, ratio.decide)
|
|
|
|
def block(sol, ev, dr):
|
|
x, o = ev[0], ev[1]
|
|
return {
|
|
"pX": {**summarise(x), "point": float(sol.value[0][0])},
|
|
"pOther": {**summarise(o), "point": float(sol.value[1][0])},
|
|
"ratio": summarise([xi / (xi + oi) if xi + oi else 0 for xi, oi in zip(x, o)]),
|
|
"expectedAttempts": summarise([1 / xi if xi > 0 else float("inf") for xi in x]),
|
|
"tree": Exporter(mean, X, support, sol, dr).root(),
|
|
}
|
|
|
|
cross = []
|
|
for f in (0, 1):
|
|
pol = solve(half_params[f], X, support)
|
|
cross.append({"inSample": float(pol.value[0][0]),
|
|
"heldOut": float(evaluate(half_params[1 - f], X, support, pol.decide)[0][0])})
|
|
targets[X] = {
|
|
"thisAttempt": block(one, ev_one, d_one), "eventually": block(ratio, ev_ratio, d_ratio),
|
|
"crossFit": {"inSample": (cross[0]["inSample"] + cross[1]["inSample"]) / 2,
|
|
"heldOut": (cross[0]["heldOut"] + cross[1]["heldOut"]) / 2},
|
|
}
|
|
return {
|
|
"schema": 1,
|
|
"model": {
|
|
"description": "Decision tree with perfect recall; chance probabilities depend on earlier choices via hierarchical Dirichlet back-off.",
|
|
"alphas": alphas, "alphaFit": {k: f["logLik"] for k, f in fitted.items()}, "draws": draws, "seed": seed,
|
|
"minSupport": min_support,
|
|
"chanceContext": {k: s["context"] for k, s in model.CHANCE.items()},
|
|
"decisions": model.DECISIONS, "support": support, "supportCounts": counts,
|
|
"tables": T,
|
|
# how much each kind of source over-reports each rare roll, and the correction applied (bias.py)
|
|
"reportingBias": bias_est,
|
|
# the game itself (core/spec/game.yaml), so the advisor in core/solver.js can re-solve from any point
|
|
"game": {k: G[k] for k in ("version", "colours", "decisions", "chance", "encounters", "encounter_outcomes",
|
|
"by_colour", "start", "steps", "from_form", "bot_text")},
|
|
},
|
|
"data": {"runs": len(runs)},
|
|
"targets": targets,
|
|
}
|