Route model: first attempts are their own shadow roll; experiment plan and automated bot cycle
- game.yaml: `givens` (first or later attempt) and `distinct` levels that are not pooled; the shadow roll
depends on the attempt first. Form submissions show the shadow on 9% of first attempts and 0.3% of later ones.
- Reporting bias: a roll whose bias cannot be measured can borrow another roll's (`borrow: {shadow: meadow}`),
so form submissions inform the shadow again. An assumption, stated in planner/policy.yaml.
- policy.json: targets.<colour>.firstAttempt (this attempt, eventually). Black: about 1.4% on a first attempt,
0.07% on a later one.
- solver.js: advisor options `attempt`, `fix`, sampling "uniform"; assign() and armFor() for the plan.
- planner/experiments.yaml + build_experiments.py -> data/public/experiments.json (progress and status per
experiment, rebuilt from the bot's runs).
- python -m collection cycle: import bot runs, refit, re-plan, deliver the kit; .gitea/workflows/bot-cycle.yml.
Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
This commit is contained in:
co-authored by
Claude Fable 5.2
parent
ee07ee143e
commit
ab3664f369
@@ -0,0 +1,199 @@
|
||||
"""Build data/public/experiments.json: the experiment plan the bot follows, updated from the bot's own runs.
|
||||
|
||||
python planner/build_experiments.py [--out data/public/experiments.json] [--check]
|
||||
|
||||
Reads the design (planner/experiments.yaml), checks it against the game (core/spec/game.yaml), counts what the bot
|
||||
has logged for every arm (data/bot/trajectories-*.jsonl, matched to data/public/runs_clean.csv), and decides each
|
||||
experiment's status:
|
||||
|
||||
running still being assigned
|
||||
complete every arm has its planned number of observed rolls
|
||||
paused its stop_early rule was met (the plan needs a person)
|
||||
|
||||
The bot reads the result with assign() in core/solver.js; an experiment that is not running is no longer assigned
|
||||
and its share goes to the others. The plan has a content id (logged by the bot as planId).
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "core"))
|
||||
from atlas import model # noqa: E402
|
||||
from atlas import runs as runs_io # noqa: E402
|
||||
|
||||
DESIGN = ROOT / "planner/experiments.yaml"
|
||||
OUT = ROOT / "data/public/experiments.json"
|
||||
SAMPLING = ("best", "thompson", "uniform")
|
||||
|
||||
|
||||
def design():
|
||||
with open(DESIGN, encoding="utf-8") as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
|
||||
def check_options(where, fixed):
|
||||
for k, o in (fixed or {}).items():
|
||||
if k not in model.DECISIONS or k in ("encounter", "cadet"):
|
||||
raise ValueError(f"{where}: '{k}' is not a decision of the game")
|
||||
if o not in model.DECISIONS[k]:
|
||||
raise ValueError(f"{where}: '{k}' has no option '{o}' (game.yaml: {model.DECISIONS[k]})")
|
||||
|
||||
|
||||
def arms(exp):
|
||||
"""Every combination of the factors, each with the route held on every arm."""
|
||||
route = exp.get("route") or {}
|
||||
check_options(exp["id"] + " route", route)
|
||||
factors = exp.get("factors") or {}
|
||||
for k, opts in factors.items():
|
||||
if k in route:
|
||||
raise ValueError(f"{exp['id']}: '{k}' is both a factor and part of the route")
|
||||
for o in opts:
|
||||
check_options(exp["id"] + " factors", {k: o})
|
||||
if not factors:
|
||||
return [{"id": "all", "fix": dict(route)}]
|
||||
out = []
|
||||
for combo in itertools.product(*factors.values()):
|
||||
levels = dict(zip(factors, combo))
|
||||
out.append({"id": " / ".join(f"{k}={o}" for k, o in levels.items()), "levels": levels, "fix": {**route, **levels}})
|
||||
return out
|
||||
|
||||
|
||||
def validate(d):
|
||||
ids = [e["id"] for e in d["experiments"]]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise ValueError("experiment ids must be unique")
|
||||
for e in d["experiments"]:
|
||||
if e["target"] not in model.COLOURS:
|
||||
raise ValueError(f"{e['id']}: unknown target {e['target']}")
|
||||
if e.get("sampling", "best") not in SAMPLING:
|
||||
raise ValueError(f"{e['id']}: sampling must be one of {SAMPLING}")
|
||||
if e["unit"] not in ("account", "attempt"):
|
||||
raise ValueError(f"{e['id']}: unit must be account or attempt")
|
||||
if not e.get("share", 0) > 0:
|
||||
raise ValueError(f"{e['id']}: share must be positive")
|
||||
m = e.get("measures")
|
||||
if m and (m["roll"] not in model.CHANCE or m["rare"] not in model.CHANCE[m["roll"]]["outcomes"]):
|
||||
raise ValueError(f"{e['id']}: measures {m} is not a roll and outcome of the game")
|
||||
if not e.get("ongoing") and not e.get("planned_per_arm"):
|
||||
raise ValueError(f"{e['id']}: needs planned_per_arm (or ongoing: true)")
|
||||
|
||||
|
||||
def interval(y, n, seed=1, sims=4000):
|
||||
"""Jeffreys 90% range for a rate."""
|
||||
if not n:
|
||||
return None
|
||||
r = random.Random(seed)
|
||||
s = sorted(r.betavariate(y + 0.5, n - y + 0.5) for _ in range(sims))
|
||||
return {"rate": (y + 0.5) / (n + 1), "lo": s[int(0.05 * sims)], "hi": s[int(0.95 * sims) - 1]}
|
||||
|
||||
|
||||
def logged(bot_dir=ROOT / "data/bot", runs_path=None):
|
||||
"""Bot runs that carry an experiment: [(experimentId, arm, prepared run)], each run once."""
|
||||
prepared = {r["run_id"]: model.prepare_run(r) for r in runs_io.read(runs_path or runs_io.PATH, sources=["bot"], drop=())}
|
||||
seen, out = set(), []
|
||||
for f in sorted(glob.glob(str(Path(bot_dir) / "trajectories-*.jsonl"))):
|
||||
if "sample" in Path(f).name:
|
||||
continue
|
||||
with open(f, encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
t = json.loads(line)
|
||||
meta = t.get("meta") or {}
|
||||
if not meta.get("experimentId") or t["id"] in seen:
|
||||
continue
|
||||
seen.add(t["id"])
|
||||
out.append((meta["experimentId"], meta.get("arm"), prepared.get(t["id"])))
|
||||
return out
|
||||
|
||||
|
||||
def build(d=None, log=None):
|
||||
d = d or design()
|
||||
validate(d)
|
||||
log = logged() if log is None else log
|
||||
exps = []
|
||||
for e in d["experiments"]:
|
||||
m = e.get("measures")
|
||||
a = arms(e)
|
||||
tally = {x["id"]: {"runs": 0, "observed": 0, "rare": 0} for x in a}
|
||||
for exp_id, arm, v in log:
|
||||
if exp_id != e["id"]:
|
||||
continue
|
||||
t = tally.setdefault(arm or "?", {"runs": 0, "observed": 0, "rare": 0})
|
||||
t["runs"] += 1
|
||||
y = model.OBSERVE[m["roll"]](v) if m and v else None
|
||||
if y is not None:
|
||||
t["observed"] += 1
|
||||
t["rare"] += y == m["rare"]
|
||||
planned = e.get("planned_per_arm")
|
||||
# mapping has nothing to measure: its runs are its count
|
||||
done = {k: (t["observed"] if m else t["runs"]) for k, t in tally.items() if k in {x["id"] for x in a}}
|
||||
total_obs = sum(t["observed"] for t in tally.values())
|
||||
total_rare = sum(t["rare"] for t in tally.values())
|
||||
status, note = "running", None
|
||||
se = e.get("stop_early")
|
||||
if not e.get("ongoing") and all(n >= planned for n in done.values()):
|
||||
status, note = "complete", f"every arm has its planned {planned}"
|
||||
elif se and total_obs >= se["after"] and total_rare <= se["rare_at_most"]:
|
||||
status = "paused"
|
||||
note = f"stop_early: {total_rare} rare outcome(s) in {total_obs} observed rolls (rule: at most {se['rare_at_most']} after {se['after']})"
|
||||
for x in a:
|
||||
t = tally[x["id"]]
|
||||
x["progress"] = {**t, "planned": planned, **({"estimate": interval(t["rare"], t["observed"])} if m and t["observed"] else {})}
|
||||
exps.append({
|
||||
"id": e["id"], "status": status, **({"note": note} if note else {}),
|
||||
"question": e["question"], "attempts": {"from": e["attempts"]["from"], "to": e["attempts"].get("to")},
|
||||
"unit": e["unit"], "share": e["share"], **({"accounts": e["accounts"]} if e.get("accounts") else {}),
|
||||
**({"exclusive": True} if e.get("exclusive") else {}),
|
||||
"target": e["target"], "objective": e.get("objective", "eventually"), "sampling": e.get("sampling", "best"),
|
||||
"seed": d["seed"], "measures": m, "burn": e.get("burn"), "arms": a,
|
||||
"progress": {"runs": sum(t["runs"] for t in tally.values()), "observed": total_obs, "rare": total_rare,
|
||||
**({"estimate": interval(total_rare, total_obs)} if m and total_obs else {}),
|
||||
"unknownArms": sorted(k for k in tally if k not in {x["id"] for x in a})},
|
||||
})
|
||||
plan = {"schema": 1, "seed": d["seed"], "lifecycle": d["lifecycle"], "experiments": exps}
|
||||
plan["id"] = hashlib.sha1(json.dumps(plan, sort_keys=True, separators=(",", ":")).encode()).hexdigest()[:12]
|
||||
return plan
|
||||
|
||||
|
||||
def render(plan):
|
||||
return json.dumps(plan, indent=1, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def summary(plan):
|
||||
L = [f"experiment plan {plan['id']}"]
|
||||
for e in plan["experiments"]:
|
||||
p = e["progress"]
|
||||
est = p.get("estimate")
|
||||
L.append(f" {e['id']:<22} {e['status']:<9} {p['observed']} observed, {p['rare']} rare"
|
||||
+ (f" ({100 * est['rate']:.1f}%, 90% range {100 * est['lo']:.1f}-{100 * est['hi']:.1f})" if est else "")
|
||||
+ (f" [{e['note']}]" if e.get("note") else ""))
|
||||
return "\n".join(L)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--out", default=str(OUT))
|
||||
ap.add_argument("--check", action="store_true", help="write nothing; exit 1 if the file is out of date")
|
||||
a = ap.parse_args(argv)
|
||||
plan = build()
|
||||
text = render(plan)
|
||||
out = Path(a.out)
|
||||
if a.check:
|
||||
if not out.exists() or out.read_text(encoding="utf-8") != text:
|
||||
print(f"{out} is out of date (run python planner/build_experiments.py)", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
out.write_text(text, encoding="utf-8")
|
||||
print(summary(plan))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user