"""Reporting-bias correction (core/atlas/bias.py).""" import unittest from atlas import bias, model CFG = {"rolls": {"meadow": "Blue"}, "min_runs": 30, "min_complete_rare": 5, "groups": [{"name": "bot", "when": {"source": "bot"}, "complete": True}, {"name": "form", "when": {"source": "reddit spreadsheet"}}]} def meadow_run(source, colour): r = {"source": source, "entry": "Form", "goal": "Any (for fun)", "company": "Alone", "direction": "Center", "first_sighting": "Red", "sighting_action": None, "left_squadmates": None, "scream": "Ignore the scream", "resolution": "Tourniquet", "post_path": "(no fork)", "hour": "Valley", "branch": "Passed the Red: meadow", "outcome": "Incinerated", "outcome_color": colour, "malek_intent": "No", "rare_reported": None, "saw_blue_meadow": colour == "Blue"} for c in model.COLOURS: r["enc_" + c] = "Incinerated" if c == colour else None return r class Bias(unittest.TestCase): def runs(self, bot_blue, bot_n, form_blue, form_n): return ([meadow_run("bot", "Blue")] * bot_blue + [meadow_run("bot", "Orange")] * (bot_n - bot_blue) + [meadow_run("reddit spreadsheet", "Blue")] * form_blue + [meadow_run("reddit spreadsheet", "Orange")] * (form_n - form_blue)) def test_over_reporting_is_estimated_and_corrected(self): runs = self.runs(10, 100, 100, 300) # 10% when everything is logged, 33% in the form prep = [model.prepare_run(r) for r in runs] est, names = bias.estimate(runs, prep, CFG, set()) rho = est["form"]["meadow"]["rho"] self.assertAlmostEqual(rho, 4.4, delta=0.5) self.assertLess(est["form"]["meadow"]["lo"], rho) self.assertGreater(est["form"]["meadow"]["hi"], rho) T = model.count_tables(prep, bias.weights(names, prep, est, CFG["rolls"])) c = T["meadow"]["0:"] # corrected: the form's Blue sightings count 1/rho each, so the pooled rate is close to what the bot sees self.assertAlmostEqual(c["Blue"] / (c["Blue"] + c["Orange"]), 0.10, delta=0.02) def test_unestimable_bias_leaves_the_group_out(self): runs = self.runs(0, 200, 60, 300) # complete logging never saw it prep = [model.prepare_run(r) for r in runs] est, names = bias.estimate(runs, prep, CFG, set()) self.assertTrue(est["form"]["meadow"]["exclude"]) T = model.count_tables(prep, bias.weights(names, prep, est, CFG["rolls"])) self.assertEqual(T["meadow"]["0:"], {"Orange": 200}) # a roll seen only one way: routes cannot be told apart, so it is pooled fully self.assertEqual(model.fit_alpha(T, "meadow")["alpha"], max(model.ALPHA_GRID)) def test_unestimable_bias_can_borrow_another_rolls_estimate(self): # the shadow was never seen in complete logs, the meadow Blue was: the form's shadow reports count 1/rho(meadow) cfg = {**CFG, "rolls": {"shadow": "yes", "meadow": "Blue"}, "borrow": {"shadow": "meadow"}} runs = self.runs(10, 100, 100, 300) + [tree_run("bot", False)] * 100 + \ [tree_run("reddit spreadsheet", True)] * 8 + [tree_run("reddit spreadsheet", False)] * 92 prep = [model.prepare_run(r) for r in runs] est, names = bias.estimate(runs, prep, cfg, set()) e = est["form"]["shadow"] self.assertAlmostEqual(e["rho"], est["form"]["meadow"]["rho"]) self.assertIn("assumed", e["used"]) self.assertFalse(e.get("exclude")) T = model.count_tables(prep, bias.weights(names, prep, est, cfg["rolls"])) self.assertAlmostEqual(T["shadow"]["1:First attempt"]["yes"], 8 / e["rho"], places=3) # without `borrow`, the group is left out of the roll est2, _ = bias.estimate(runs, prep, {**cfg, "borrow": {}}, set()) self.assertTrue(est2["form"]["shadow"]["exclude"]) def tree_run(source, shadow, attempt_group="First attempt"): r = meadow_run(source, "Orange") r.update(scream="Investigate & help", resolution="Squadmate's revenge", post_path="Forest", hour="Tree", branch="Shadow (rare)" if shadow else "Descend slowly: two Greens", outcome_color="Black" if shadow else "Green", enc_Orange=None, enc_Black="Incinerated" if shadow else None, enc_Green=None if shadow else "Incinerated", saw_blue_meadow=None, attempt_group=attempt_group, combat_first=None, combat_second=None) return r class FirstAttempt(unittest.TestCase): """game.yaml `givens`: first and later attempts are different shadow rolls (`distinct`), not pooled.""" def setUp(self): runs = [tree_run("bot", True)] * 9 + [tree_run("bot", False)] * 91 + [tree_run("bot", False, "Later attempt")] * 900 + \ [tree_run("bot", True, "Later attempt")] * 1 self.T = model.count_tables([model.prepare_run(r) for r in runs]) def test_each_attempt_kind_keeps_its_own_rate(self): p = model.Params(self.T, "mean", alphas={"shadow": 100000}) gated = {"scream": "Investigate the scream", "clearing": "Help your squadmate"} later = float(p.chance("shadow", gated)["yes"][0]) first = float(p.having(attempt="First attempt").chance("shadow", gated)["yes"][0]) self.assertAlmostEqual(first, 0.09, delta=0.01) # not dragged to the pooled 1% self.assertAlmostEqual(later, 0.001, delta=0.001) self.assertEqual(p.given, {"attempt": "Later attempt"}) # the default: a later attempt def test_policy_reports_the_first_attempt_separately(self): from atlas import solver runs = [tree_run("bot", True)] * 9 + [tree_run("bot", False)] * 91 + [tree_run("bot", False, "Later attempt")] * 900 pol = solver.build_policy([dict(r, enc_Black="Bonded!", outcome="Bonded") if r["branch"] == "Shadow (rare)" else r for r in runs], draws=6, colours=["Black"], min_support=1) t = pol["targets"]["Black"] f = t["firstAttempt"] self.assertEqual(f["given"], {"attempt": "First attempt"}) self.assertGreater(f["thisAttempt"]["pX"]["point"], 20 * t["thisAttempt"]["pX"]["point"]) self.assertLessEqual(f["thisAttempt"]["pX"]["lo"], f["thisAttempt"]["pX"]["hi"]) self.assertIn("pEventually", f["eventually"]) if __name__ == "__main__": unittest.main() class Gate(unittest.TestCase): def test_closed_gate_fixes_the_roll_and_is_not_counted(self): spec = model.CHANCE["shadow"] self.assertTrue(spec.get("requires")) state = {"scream": "Keep walking"} self.assertFalse(model.gate_open("shadow", state)) p = model.Params({"shadow": {"0:": {"yes": 5, "no": 5}}}, "mean").chance("shadow", state) self.assertEqual((float(p["yes"][0]), float(p["no"][0])), (0.0, 1.0)) # a tree run that ignored the scream says nothing about the shadow run = meadow_run("bot", "Orange") run.update(hour="Tree", branch="Descend slowly: two Greens", scream="Ignore the scream") self.assertIsNone(model.OBSERVE["shadow"](model.prepare_run(run))) run.update(scream="Investigate & help", resolution="Killed the bullies yourself", post_path="Forest", combat_first="Assess the situation", combat_second="Engage the smaller cadet") self.assertEqual(model.OBSERVE["shadow"](model.prepare_run(run)), "no") class PossibleBonds(unittest.TestCase): def test_impossible_bond_is_not_counted(self): good = meadow_run("reddit spreadsheet", "Orange") good.update(hour="Tree", branch="Shadow (rare)", outcome="Bonded", outcome_color="Black", enc_Black="Bonded!", enc_Orange=None, goal="Black", scream="Investigate & help", resolution="Killed the bullies yourself", post_path="Forest", combat_first="Assess the situation", combat_second="Engage the smaller cadet") bad = dict(good, enc_Black=None, enc_Blue="Incinerated", outcome_color="Green", goal="Any (for fun)") T = model.count_tables([model.prepare_run(good), model.prepare_run(bad)]) self.assertEqual(T["encounter"]["shadow-Black"]["outcomes"], {"Black": 1}) self.assertEqual(T["encounter"]["shadow-Blue"]["outcomes"], {}) # a Green at the Blue: not counted self.assertTrue(model.can_bond("tree-Greens", "Brown"))