- game.yaml: `givens` (first or later attempt) and `distinct` levels that are not pooled; the shadow roll
depends on the attempt first. Form submissions show the shadow on 9% of first attempts and 0.3% of later ones.
- Reporting bias: a roll whose bias cannot be measured can borrow another roll's (`borrow: {shadow: meadow}`),
so form submissions inform the shadow again. An assumption, stated in planner/policy.yaml.
- policy.json: targets.<colour>.firstAttempt (this attempt, eventually). Black: about 1.4% on a first attempt,
0.07% on a later one.
- solver.js: advisor options `attempt`, `fix`, sampling "uniform"; assign() and armFor() for the plan.
- planner/experiments.yaml + build_experiments.py -> data/public/experiments.json (progress and status per
experiment, rebuilt from the bot's runs).
- python -m collection cycle: import bot runs, refit, re-plan, deliver the kit; .gitea/workflows/bot-cycle.yml.
Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014jqCC2LZ4x1WHQ1V57nTsv
67 lines
2.9 KiB
Python
67 lines
2.9 KiB
Python
"""The experiment plan (planner/experiments.yaml -> data/public/experiments.json)."""
|
|
import copy
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "planner"))
|
|
import build_experiments as B # noqa: E402
|
|
from atlas import model # noqa: E402
|
|
|
|
D = B.design()
|
|
GATED_TREE = {"attempt": "First attempt", "scream": "Investigate the scream", "clearing": "Help your squadmate"}
|
|
|
|
|
|
class Plan(unittest.TestCase):
|
|
def test_design_is_valid_and_arms_are_every_combination(self):
|
|
plan = B.build(D, log=[])
|
|
by = {e["id"]: e for e in plan["experiments"]}
|
|
self.assertEqual(len(by["first-attempt-shadow"]["arms"]), 4)
|
|
self.assertEqual(len(by["meadow-blue-route"]["arms"]), 6)
|
|
for e in plan["experiments"]:
|
|
self.assertEqual(e["status"], "running")
|
|
for a in e["arms"]:
|
|
B.check_options(e["id"], a["fix"])
|
|
# the Black attempt keeps the shadow's gate open on every arm
|
|
for a in by["first-attempt-shadow"]["arms"]:
|
|
self.assertTrue(model.gate_open("shadow", a["fix"]))
|
|
|
|
def test_an_unknown_option_is_refused(self):
|
|
d = copy.deepcopy(D)
|
|
d["experiments"][0]["factors"]["direction"] = ["North"]
|
|
with self.assertRaises(ValueError):
|
|
B.build(d, log=[])
|
|
|
|
def test_progress_completes_and_stops_early(self):
|
|
e0 = D["experiments"][0]
|
|
arms = [a["id"] for a in B.arms(e0)]
|
|
seen = {**GATED_TREE, "shadow": True}
|
|
none = {**GATED_TREE, "shadow": False}
|
|
# every arm at its planned count, with shadows: complete
|
|
log = [(e0["id"], a, seen if i < 3 else none) for a in arms for i in range(e0["planned_per_arm"])]
|
|
e = B.build(D, log=log)["experiments"][0]
|
|
self.assertEqual(e["status"], "complete")
|
|
self.assertEqual(e["progress"]["rare"], 12)
|
|
self.assertAlmostEqual(e["arms"][0]["progress"]["estimate"]["rate"], 3.5 / 61)
|
|
# no shadow at all after the stop_early count: paused, for a person to look at
|
|
log = [(e0["id"], arms[i % 4], none) for i in range(e0["stop_early"]["after"])]
|
|
e = B.build(D, log=log)["experiments"][0]
|
|
self.assertEqual(e["status"], "paused")
|
|
# a run whose roll was not observed (died before the tree) is a run, not an observation
|
|
e = B.build(D, log=[(e0["id"], arms[0], {"attempt": "First attempt"})])["experiments"][0]
|
|
self.assertEqual((e["progress"]["runs"], e["progress"]["observed"]), (1, 0))
|
|
|
|
def test_the_plan_id_follows_its_content(self):
|
|
a, b = B.build(D, log=[]), B.build(D, log=[])
|
|
self.assertEqual(a["id"], b["id"])
|
|
c = B.build(D, log=[(D["experiments"][0]["id"], "x", None)])
|
|
self.assertNotEqual(a["id"], c["id"])
|
|
|
|
def test_published_plan_is_up_to_date(self):
|
|
self.assertEqual(B.main(["--check"]), 0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|