Files
threshing-atlas/planner/build_experiments.py
T
ekcSchianoandClaude Fable 5.2 d01ec46fcc The Black is not dice: developer statements, evidence report, and a plan that looks for the rule
- planner/knowledge.yaml: what the game's developer has said (paraphrased, dated, graded), what the Atlas does
  with each statement, and the open questions with the experiment that tests each.
- analysis/black/evidence.py -> analysis/reports/BLACK.md: counts from the runs. On the same recorded route the
  shadow shows on 21 of 79 first attempts and 1 of 80 later ones; no single route; Blue then Black after the shadow.
- game.yaml: the shadow is a guard (not a roll); what follows it is the Blue, then the Black; on-screen text for
  the squad's direction screen.
- planner/experiments.yaml v2: one exact path on every fresh account (fresh-replicate), then one choice changed
  at a time (fresh-ablate); shadow-again, second-attempt-same-path, veteran-repeat, veteran-variety; what the bot
  records about the account; the sequence to follow once the shadow shows.
- solver.js: assign() takes account facts, account offsets and cycled arms; recommendOption() answers the
  squad screen (stay + direction) in one pick.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
2026-11-16 15:24:03 +00:00

238 lines
11 KiB
Python

"""Build data/public/experiments.json: the experiment plan the bot follows, updated from the bot's own runs.
python planner/build_experiments.py [--out data/public/experiments.json] [--check]
Reads the design (planner/experiments.yaml), checks it against the game (core/spec/game.yaml), counts what the bot
has logged for every arm (data/bot/trajectories-*.jsonl, matched to data/public/runs_clean.csv), and decides each
experiment's status:
running still being assigned
complete every arm has its planned number of observed rolls
paused its stop_early rule was met (the plan needs a person)
The bot reads the result with assign() in core/solver.js; an experiment that is not running is no longer assigned
and its share goes to the others. The plan has a content id (logged by the bot as planId).
"""
import argparse
import glob
import hashlib
import itertools
import json
import random
import re
import sys
from pathlib import Path
import yaml
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "core"))
from atlas import model # noqa: E402
from atlas import runs as runs_io # noqa: E402
DESIGN = ROOT / "planner/experiments.yaml"
OUT = ROOT / "data/public/experiments.json"
SAMPLING = ("best", "thompson", "uniform")
def design():
with open(DESIGN, encoding="utf-8") as f:
return yaml.safe_load(f)
def check_options(where, fixed):
for k, o in (fixed or {}).items():
if k not in model.DECISIONS or k in ("encounter", "cadet"):
raise ValueError(f"{where}: '{k}' is not a decision of the game")
if o not in model.DECISIONS[k]:
raise ValueError(f"{where}: '{k}' has no option '{o}' (game.yaml: {model.DECISIONS[k]})")
def arms(exp, routes=None):
"""The arms of an experiment, each with the decisions it holds (`fix`): every combination of the factors, or
the explicit `arms` (one `change` to the route each), or a single arm. A `cycle` arm also carries the decisions
it varies from attempt to attempt."""
route = exp.get("route") or {}
if isinstance(route, str):
if route not in (routes or {}):
raise ValueError(f"{exp['id']}: unknown route '{route}'")
route = routes[route]
check_options(exp["id"] + " route", route)
if exp.get("arms"):
out = []
for a in exp["arms"]:
check_options(f"{exp['id']} arm {a['id']}", a.get("change"))
out.append({"id": a["id"], "change": a.get("change") or {}, "fix": {**route, **(a.get("change") or {})}})
if len({a["id"] for a in out}) != len(out):
raise ValueError(f"{exp['id']}: arm ids must be unique")
return out
if exp.get("cycle"):
for k, opts in exp["cycle"].items():
if k in route:
raise ValueError(f"{exp['id']}: '{k}' is both cycled and part of the route")
for o in opts:
check_options(exp["id"] + " cycle", {k: o})
return [{"id": "all", "fix": dict(route), "cycle": exp["cycle"]}]
factors = exp.get("factors") or {}
for k, opts in factors.items():
if k in route:
raise ValueError(f"{exp['id']}: '{k}' is both a factor and part of the route")
for o in opts:
check_options(exp["id"] + " factors", {k: o})
if not factors:
return [{"id": "all", "fix": dict(route)}]
out = []
for combo in itertools.product(*factors.values()):
levels = dict(zip(factors, combo))
out.append({"id": " / ".join(f"{k}={o}" for k, o in levels.items()), "levels": levels, "fix": {**route, **levels}})
return out
def validate(d):
ids = [e["id"] for e in d["experiments"]]
if len(set(ids)) != len(ids):
raise ValueError("experiment ids must be unique")
for e in d["experiments"]:
if e.get("starts_after") and e["starts_after"] not in ids:
raise ValueError(f"{e['id']}: starts_after names no experiment")
for k in (e.get("when") or {}):
if k not in (d.get("record") or []):
raise ValueError(f"{e['id']}: when.{k} is not something the bot records (`record`)")
if e["target"] not in model.COLOURS:
raise ValueError(f"{e['id']}: unknown target {e['target']}")
if e.get("sampling", "best") not in SAMPLING:
raise ValueError(f"{e['id']}: sampling must be one of {SAMPLING}")
if e["unit"] not in ("account", "attempt"):
raise ValueError(f"{e['id']}: unit must be account or attempt")
if not e.get("share", 0) > 0:
raise ValueError(f"{e['id']}: share must be positive")
m = e.get("measures")
if m and (m["roll"] not in model.CHANCE or m["rare"] not in model.CHANCE[m["roll"]]["outcomes"]):
raise ValueError(f"{e['id']}: measures {m} is not a roll and outcome of the game")
if not e.get("ongoing") and not e.get("planned_per_arm"):
raise ValueError(f"{e['id']}: needs planned_per_arm (or ongoing: true)")
def interval(y, n, seed=1, sims=4000):
"""Jeffreys 90% range for a rate."""
if not n:
return None
r = random.Random(seed)
s = sorted(r.betavariate(y + 0.5, n - y + 0.5) for _ in range(sims))
return {"rate": (y + 0.5) / (n + 1), "lo": s[int(0.05 * sims)], "hi": s[int(0.95 * sims) - 1]}
def logged(bot_dir=ROOT / "data/bot", runs_path=None):
"""Bot runs that carry an experiment: [(experimentId, arm, prepared run)], each run once."""
prepared = {r["run_id"]: model.prepare_run(r) for r in runs_io.read(runs_path or runs_io.PATH, sources=["bot"], drop=())}
seen, out = set(), []
for f in sorted(glob.glob(str(Path(bot_dir) / "trajectories-*.jsonl"))):
if "sample" in Path(f).name:
continue
with open(f, encoding="utf-8") as fh:
for line in fh:
t = json.loads(line)
meta = t.get("meta") or {}
if not meta.get("experimentId") or t["id"] in seen:
continue
seen.add(t["id"])
out.append((meta["experimentId"], meta.get("arm"), prepared.get(t["id"])))
return out
def build(d=None, log=None):
d = d or design()
validate(d)
log = logged() if log is None else log
exps, status_of = [], {}
for e in d["experiments"]:
m = e.get("measures")
a = arms(e, d.get("routes"))
tally = {x["id"]: {"runs": 0, "observed": 0, "rare": 0} for x in a}
for exp_id, arm, v in log:
if exp_id != e["id"]:
continue
if a[0].get("cycle"): # a cycled arm is logged under the combination it played: all one arm here
arm = a[0]["id"]
t = tally.setdefault(arm or "?", {"runs": 0, "observed": 0, "rare": 0})
t["runs"] += 1
y = model.OBSERVE[m["roll"]](v) if m and v else None
if y is not None:
t["observed"] += 1
t["rare"] += y == m["rare"]
planned = e.get("planned_per_arm")
# mapping has nothing to measure: its runs are its count
done = {k: (t["observed"] if m else t["runs"]) for k, t in tally.items() if k in {x["id"] for x in a}}
total_obs = sum(t["observed"] for t in tally.values())
total_rare = sum(t["rare"] for t in tally.values())
status, note = "running", None
se = e.get("stop_early")
if e.get("starts_after") and status_of.get(e["starts_after"]) != "complete":
status, note = "waiting", f"starts when {e['starts_after']} is complete"
elif not e.get("ongoing") and all(n >= planned for n in done.values()):
status, note = "complete", f"every arm has its planned {planned}"
elif se and total_obs >= se["after"] and total_rare <= se["rare_at_most"]:
status = "paused"
note = f"stop_early: {total_rare} rare outcome(s) in {total_obs} observed rolls (rule: at most {se['rare_at_most']} after {se['after']})"
for x in a:
t = tally[x["id"]]
x["progress"] = {**t, "planned": planned, **({"estimate": interval(t["rare"], t["observed"])} if m and t["observed"] else {})}
status_of[e["id"]] = status
exps.append({
"id": e["id"], "status": status, **({"note": note} if note else {}),
"question": e["question"], "attempts": {"from": e["attempts"]["from"], "to": e["attempts"].get("to")},
"unit": e["unit"], "share": e["share"], **({"accounts": e["accounts"]} if e.get("accounts") else {}),
**({"exclusive": True} if e.get("exclusive") else {}),
**({"when": e["when"]} if e.get("when") else {}),
"target": e["target"], "objective": e.get("objective", "eventually"), "sampling": e.get("sampling", "best"),
"seed": d["seed"], "measures": m, "burn": e.get("burn"), "arms": a,
"progress": {"runs": sum(t["runs"] for t in tally.values()), "observed": total_obs, "rare": total_rare,
**({"estimate": interval(total_rare, total_obs)} if m and total_obs else {}),
"unknownArms": sorted(k for k in tally if k not in {x["id"] for x in a})},
})
for s in (d.get("scripts") or {}).values():
for st in s:
for k in ("take", "never"):
re.compile(st.get(k) or "")
plan = {"schema": 2, "seed": d["seed"], "lifecycle": d["lifecycle"], "record": d.get("record") or [],
"scripts": d.get("scripts") or {}, "experiments": exps}
plan["id"] = hashlib.sha1(json.dumps(plan, sort_keys=True, separators=(",", ":")).encode()).hexdigest()[:12]
return plan
def render(plan):
return json.dumps(plan, indent=1, ensure_ascii=False) + "\n"
def summary(plan):
L = [f"experiment plan {plan['id']}"]
for e in plan["experiments"]:
p = e["progress"]
est = p.get("estimate")
L.append(f" {e['id']:<26} {e['status']:<9} {p['observed']} observed, {p['rare']} rare"
+ (f" ({100 * est['rate']:.1f}%, 90% range {100 * est['lo']:.1f}-{100 * est['hi']:.1f})" if est else "")
+ (f" [{e['note']}]" if e.get("note") else ""))
return "\n".join(L)
def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--out", default=str(OUT))
ap.add_argument("--check", action="store_true", help="write nothing; exit 1 if the file is out of date")
a = ap.parse_args(argv)
plan = build()
text = render(plan)
out = Path(a.out)
if a.check:
if not out.exists() or out.read_text(encoding="utf-8") != text:
print(f"{out} is out of date (run python planner/build_experiments.py)", file=sys.stderr)
return 1
return 0
out.write_text(text, encoding="utf-8")
print(summary(plan))
return 0
if __name__ == "__main__":
sys.exit(main())