"""The experiment plan (planner/experiments.yaml -> data/public/experiments.json).""" import copy import sys import unittest from pathlib import Path ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "planner")) import build_experiments as B # noqa: E402 from atlas import model # noqa: E402 D = B.design() GATED_TREE = {"attempt": "First attempt", "scream": "Investigate the scream", "clearing": "Help your squadmate"} class Plan(unittest.TestCase): def test_design_is_valid_and_arms_are_every_combination(self): plan = B.build(D, log=[]) by = {e["id"]: e for e in plan["experiments"]} self.assertEqual(len(by["first-attempt-shadow"]["arms"]), 4) self.assertEqual(len(by["meadow-blue-route"]["arms"]), 6) for e in plan["experiments"]: self.assertEqual(e["status"], "running") for a in e["arms"]: B.check_options(e["id"], a["fix"]) # the Black attempt keeps the shadow's gate open on every arm for a in by["first-attempt-shadow"]["arms"]: self.assertTrue(model.gate_open("shadow", a["fix"])) def test_an_unknown_option_is_refused(self): d = copy.deepcopy(D) d["experiments"][0]["factors"]["direction"] = ["North"] with self.assertRaises(ValueError): B.build(d, log=[]) def test_progress_completes_and_stops_early(self): e0 = D["experiments"][0] arms = [a["id"] for a in B.arms(e0)] seen = {**GATED_TREE, "shadow": True} none = {**GATED_TREE, "shadow": False} # every arm at its planned count, with shadows: complete log = [(e0["id"], a, seen if i < 3 else none) for a in arms for i in range(e0["planned_per_arm"])] e = B.build(D, log=log)["experiments"][0] self.assertEqual(e["status"], "complete") self.assertEqual(e["progress"]["rare"], 12) self.assertAlmostEqual(e["arms"][0]["progress"]["estimate"]["rate"], 3.5 / 61) # no shadow at all after the stop_early count: paused, for a person to look at log = [(e0["id"], arms[i % 4], none) for i in range(e0["stop_early"]["after"])] e = B.build(D, log=log)["experiments"][0] self.assertEqual(e["status"], "paused") # a run whose roll was not observed (died before the tree) is a run, not an observation e = B.build(D, log=[(e0["id"], arms[0], {"attempt": "First attempt"})])["experiments"][0] self.assertEqual((e["progress"]["runs"], e["progress"]["observed"]), (1, 0)) def test_the_plan_id_follows_its_content(self): a, b = B.build(D, log=[]), B.build(D, log=[]) self.assertEqual(a["id"], b["id"]) c = B.build(D, log=[(D["experiments"][0]["id"], "x", None)]) self.assertNotEqual(a["id"], c["id"]) def test_published_plan_is_up_to_date(self): self.assertEqual(B.main(["--check"]), 0) if __name__ == "__main__": unittest.main()