"""The experiment plan (planner/experiments.yaml -> data/public/experiments.json).""" import copy import sys import unittest from pathlib import Path ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "planner")) import build_experiments as B # noqa: E402 from atlas import model # noqa: E402 D = B.design() GATED_TREE = {"attempt": "First attempt", "scream": "Investigate the scream", "clearing": "Help your squadmate"} class Plan(unittest.TestCase): def test_design_is_valid(self): plan = B.build(D, log=[]) by = {e["id"]: e for e in plan["experiments"]} # the Black attempt is one exact path, and it keeps the shadow's gate open (arm,) = by["fresh-replicate"]["arms"] self.assertEqual(arm["fix"], D["routes"]["best-reported"]) self.assertTrue(model.gate_open("shadow", arm["fix"])) self.assertEqual(len(by["meadow-blue-route"]["arms"]), 6) for e in plan["experiments"]: for a in e["arms"]: B.check_options(e["id"], a["fix"]) B.check_options(e["id"], {k: v[0] for k, v in (a.get("cycle") or {}).items()}) self.assertEqual(plan["record"], D["record"]) self.assertEqual(len(plan["scripts"]["shadow"]), 12) def test_ablation_changes_one_thing_at_a_time_and_waits_for_the_replication(self): plan = B.build(D, log=[]) by = {e["id"]: e for e in plan["experiments"]} base = D["routes"]["best-reported"] self.assertEqual(by["fresh-ablate"]["status"], "waiting") for a in by["fresh-ablate"]["arms"]: changed = {k for k in a["fix"] if a["fix"][k] != base.get(k)} self.assertEqual(changed, set(a["change"]), a["id"]) self.assertLessEqual(len(changed - {"leave"}), 1, a["id"]) # leaving goes with the company self.assertTrue(model.gate_open("shadow", a["fix"])) # once the replication has its planned count, the ablation starts e0 = next(e for e in D["experiments"] if e["id"] == "fresh-replicate") log = [(e0["id"], "all", {**GATED_TREE, "shadow": i < 9}) for i in range(e0["planned_per_arm"])] by = {e["id"]: e for e in B.build(D, log=log)["experiments"]} self.assertEqual(by["fresh-replicate"]["status"], "complete") self.assertEqual(by["fresh-replicate"]["progress"]["rare"], 9) self.assertEqual(by["fresh-ablate"]["status"], "running") def test_an_unknown_option_route_or_fact_is_refused(self): for edit in (lambda d: d["routes"]["best-reported"].update(direction="North"), lambda d: d["experiments"][0].update(route="nowhere"), lambda d: d["experiments"][1]["arms"][0]["change"].update(company="Three"), lambda d: d["experiments"][2].update(when={"luck": True}), lambda d: d["experiments"][1].update(starts_after="nothing")): d = copy.deepcopy(D) edit(d) with self.assertRaises(ValueError): B.build(d, log=[]) def test_progress(self): e0 = D["experiments"][0] none = {**GATED_TREE, "shadow": False} e = B.build(D, log=[(e0["id"], "all", {**GATED_TREE, "shadow": True})] * 3 + [(e0["id"], "all", none)] * 7)["experiments"][0] self.assertEqual((e["status"], e["progress"]["observed"], e["progress"]["rare"]), ("running", 10, 3)) self.assertAlmostEqual(e["arms"][0]["progress"]["estimate"]["rate"], 3.5 / 11) # a run whose shadow was not observed (died before the tree) is a run, not an observation e = B.build(D, log=[(e0["id"], "all", {"attempt": "First attempt"})])["experiments"][0] self.assertEqual((e["progress"]["runs"], e["progress"]["observed"]), (1, 0)) # a cycled experiment logs the combination it played; they all count toward its one arm v = B.build(D, log=[("veteran-variety", "company=Alone / direction=East", none)] * 4) v = next(e for e in v["experiments"] if e["id"] == "veteran-variety") self.assertEqual((v["arms"][0]["progress"]["observed"], v["progress"]["unknownArms"]), (4, [])) # stop_early pauses an experiment for a person to look at d = copy.deepcopy(D) d["experiments"][0]["stop_early"] = {"after": 5, "rare_at_most": 0} self.assertEqual(B.build(d, log=[(e0["id"], "all", none)] * 5)["experiments"][0]["status"], "paused") def test_the_plan_id_follows_its_content(self): a, b = B.build(D, log=[]), B.build(D, log=[]) self.assertEqual(a["id"], b["id"]) c = B.build(D, log=[(D["experiments"][0]["id"], "x", None)]) self.assertNotEqual(a["id"], c["id"]) def test_published_plan_is_up_to_date(self): self.assertEqual(B.main(["--check"]), 0) if __name__ == "__main__": unittest.main()