Files
threshing-atlas/tests/core/test_bias.py
T
ekcSchianoandClaude Opus 5.5 2271fe0976 Route model: correct for reporting bias
- core/atlas/bias.py: for each rare roll (shadow, meadow Blue) and each source group,
  the factor by which the group over-reports the rare outcome relative to complete
  logging (bot, bulk logs), with a 90% range; rare outcomes from biased groups count
  1/rho; a roll the complete logs have (almost) never shown is learned from them only
- planner/policy.yaml: groups, rolls and thresholds; estimates published in
  policy.json under model.reportingBias
- fit_alpha: a roll seen only one way is pooled fully
- docs: goal accounts follow the best route, research accounts use Thompson sampling
  (decisionMode atlas_thompson)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
2026-11-10 02:29:09 +00:00

53 lines
2.7 KiB
Python

"""Reporting-bias correction (core/atlas/bias.py)."""
import unittest
from atlas import bias, model
CFG = {"rolls": {"meadow": "Blue"}, "min_runs": 30, "min_complete_rare": 5,
"groups": [{"name": "bot", "when": {"source": "bot"}, "complete": True},
{"name": "form", "when": {"source": "reddit spreadsheet"}}]}
def meadow_run(source, colour):
r = {"source": source, "entry": "Form", "goal": "Any (for fun)", "company": "Alone", "direction": "Center",
"first_sighting": "Red", "sighting_action": None, "left_squadmates": None, "scream": "Ignore the scream",
"resolution": "Tourniquet", "post_path": "(no fork)", "hour": "Valley", "branch": "Passed the Red: meadow",
"outcome": "Incinerated", "outcome_color": colour, "malek_intent": "No", "rare_reported": None,
"saw_blue_meadow": colour == "Blue"}
for c in model.COLOURS:
r["enc_" + c] = "Incinerated" if c == colour else None
return r
class Bias(unittest.TestCase):
def runs(self, bot_blue, bot_n, form_blue, form_n):
return ([meadow_run("bot", "Blue")] * bot_blue + [meadow_run("bot", "Orange")] * (bot_n - bot_blue) +
[meadow_run("reddit spreadsheet", "Blue")] * form_blue + [meadow_run("reddit spreadsheet", "Orange")] * (form_n - form_blue))
def test_over_reporting_is_estimated_and_corrected(self):
runs = self.runs(10, 100, 100, 300) # 10% when everything is logged, 33% in the form
prep = [model.prepare_run(r) for r in runs]
est, names = bias.estimate(runs, prep, CFG, set())
rho = est["form"]["meadow"]["rho"]
self.assertAlmostEqual(rho, 4.4, delta=0.5)
self.assertLess(est["form"]["meadow"]["lo"], rho)
self.assertGreater(est["form"]["meadow"]["hi"], rho)
T = model.count_tables(prep, bias.weights(names, prep, est, CFG["rolls"]))
c = T["meadow"]["0:"]
# corrected: the form's Blue sightings count 1/rho each, so the pooled rate is close to what the bot sees
self.assertAlmostEqual(c["Blue"] / (c["Blue"] + c["Orange"]), 0.10, delta=0.02)
def test_unestimable_bias_leaves_the_group_out(self):
runs = self.runs(0, 200, 60, 300) # complete logging never saw it
prep = [model.prepare_run(r) for r in runs]
est, names = bias.estimate(runs, prep, CFG, set())
self.assertTrue(est["form"]["meadow"]["exclude"])
T = model.count_tables(prep, bias.weights(names, prep, est, CFG["rolls"]))
self.assertEqual(T["meadow"]["0:"], {"Orange": 200})
# a roll seen only one way: routes cannot be told apart, so it is pooled fully
self.assertEqual(model.fit_alpha(T, "meadow")["alpha"], max(model.ALPHA_GRID))
if __name__ == "__main__":
unittest.main()