Files
threshing-atlas/core/atlas/coding.py
T
963b08c522 Core: fix three coding bugs from the core review, with rule tests (#11)
1. Letting the Red pass. Only climbing down quickly puts you above the Red, so a tree run that
   passed on the Red and then met the Greens let it pass. Those 16 runs were coded as slow
   descents; they are now "Descend quickly: let the Red pass" and count for the Greens'
   encounter, so the route model's "let it pass" option rests on its own runs. A tree run with
   both the Red and the Greens attempted (impossible in one run) is now flagged.
2. Leaving your squadmates. The form only offered "> Left them" from 2026-10-27 (0 of 504 squad
   runs before chose it, 40 of 144 after). Earlier and undated squad runs are now unknown rather
   than "stayed together" (form.yaml: left_squadmates_asked_from).
3. Bot text. The wounded cadet's other two choices map onto the form; a test checks every choice
   the bot has logged maps onto a form variable (encounter answers excepted, not modelled yet).

Each rule has a test in tests/core/test_coding.py. Data rebuilt (runs, graph, policy, report);
the headline odds move by at most 0.02 points. The Methods tab describes both rules.


Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 21:12:34 +00:00

315 lines
13 KiB
Python

"""Community form rows -> one coded run per row.
The tables (how each question is found, what every answer means, which answers count as missing) live in
core/spec/form.yaml. This module holds the logic that combines answers into a run: which way the rider went,
which branch they took after the hour, how the run ended, and which rows look inconsistent (data_flags).
python -m atlas.coding [form_responses.json|.xlsx|.csv] [runs_clean.csv]
Defaults: data/public/form_responses.json -> data/derived/runs_clean.csv
"""
import json
import re
import sys
from datetime import datetime
from pathlib import Path
import pandas as pd
from . import spec
ROOT = Path(__file__).resolve().parents[2]
F = spec.form()
A = F["answers"]
COLORS = F["colours"]
MISSING = frozenset(F["missing_values"])
ENC = F["encounter_answers"]
_EMAIL = re.compile(F["redact"]["pattern"])
def is_missing(v):
"""True for answers that carry no information about the choice (blank, "Unsure", not asked...)."""
return v is None or v == "" or v in MISSING or (isinstance(v, float) and v != v)
def redact(v):
"""Replace email addresses in free text."""
return v if v is None else _EMAIL.sub(F["redact"]["replacement"], str(v))
# ---------------------------------------------------------------- reading cells
def find_columns(cols):
"""Map each question key in form.yaml to the export's column header. Returns (columns, missing keys)."""
out = {}
low = {c: str(c).lower() for c in cols}
for key, col in F["columns"].items():
words = col["match"]
hits = [c for c in cols if all((w[1:] not in low[c]) if w[0] == "!" else (w in low[c]) for w in words)]
if hits:
out[key] = hits[0]
missing = [k for k, col in F["columns"].items()
if k not in out and not col.get("optional") and col.get("replaced_by") not in out]
return out, missing
def blank(v):
if v is None or v is pd.NaT:
return True
if not isinstance(v, str):
try:
if pd.isna(v):
return True
except (TypeError, ValueError):
pass
return str(v).strip() in ("", "nan", "NaN", "NaT")
def text(v):
return None if blank(v) else str(v).strip()
def parse_timestamp(v):
"""Form submissions carry a timestamp; bulk-pasted rows carry a label such as 'Data Import'.
Returns 'YYYY-MM-DD' or None."""
if isinstance(v, (datetime, pd.Timestamp)) and not pd.isna(v):
return pd.Timestamp(v).strftime("%Y-%m-%d")
if isinstance(v, (int, float)) and not isinstance(v, bool) and 40000 < v < 80000: # Excel serial date
return (pd.Timestamp("1899-12-30") + pd.Timedelta(days=int(v))).strftime("%Y-%m-%d")
t = text(v)
if not t:
return None
m = re.match(r"^(\d{4})-(\d{2})-(\d{2})", t)
if m:
return f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
m = re.match(r"^(\d{1,2})/(\d{1,2})/(\d{4})", t) # Google Sheets CSV export: M/D/YYYY
if m:
return f"{m.group(3)}-{int(m.group(1)):02d}-{int(m.group(2)):02d}"
return None
def parse_attempt(v):
v = text(v)
if v is None:
return None
t = v.lower()
if "lost count" in t:
return None
for w, n in F["attempt_words"].items():
if t.startswith(w):
return n
m = re.search(r"\d+", t)
return int(m.group()) if m else None
# ---------------------------------------------------------------- one row -> one run
def code_row(r, C):
"""Code one export row. r: mapping header -> cell; C: from find_columns()."""
g = lambda k: text(r[C[k]]) if k in C else None # noqa: E731
flags = []
ts = r[C["timestamp"]] if "timestamp" in C else None
d = parse_timestamp(ts)
source, date = ("Form", d) if d else ((text(ts) or "Unknown import"), None)
wings = [w for w in ("First", "Second", "Third", "Fourth") if g("wing_" + w)]
wing = wings[0] if len(wings) == 1 else ("Multiple" if wings else "Unknown")
section = g("wing_" + wings[0]) if len(wings) == 1 else None
if len(wings) > 1:
flags.append("multiple wings")
if "first_dir" in C:
# Redesigned form: one direction question and one colour question. The older per-direction colour
# columns are still in the sheet for some rows and are used only to fill gaps and to cross-check.
direction = A["direction"].get(g("first_dir"))
first = g("first_colour")
legacy = [d for d in ("West", "Center") if g("leg_" + d)]
if not direction and len(legacy) == 1:
direction = legacy[0]
flags.append("direction taken from the older sighting column")
leg = g("leg_" + direction) if direction else None
if not first and leg:
first = leg
elif first and leg and first != leg and first != "Unsure" and leg != "Unsure":
flags.append("first sighting differs between the old and new columns")
if not direction:
direction = "Unknown"
flags.append("direction missing")
else:
dirs = [d for d in ("East", "West", "Center") if g("dir_" + d)]
if len(dirs) == 1:
direction, first = dirs[0], g("dir_" + dirs[0])
else:
direction, first = ("Multiple" if dirs else "Unknown"), None
flags.append("direction " + ("multiple" if dirs else "missing"))
company_raw = g("company") or "Unknown"
suffix = F["company_left_suffix"]
left_squad = company_raw.endswith(suffix)
company = company_raw[: -len(suffix)] if left_squad else company_raw
if not left_squad and company in ("1 Squadmate", "2 Squadmates") and not (date and date >= F["left_squadmates_asked_from"]):
left_squad = None # the form did not offer "left them" yet: unknown, not "stayed together"
pools = F["sighting_pool"]
pool = list(pools.get(company, {}).get(direction, [])) + (list(pools["Alone"].get(direction, [])) if left_squad else [])
if first and first != "Unsure" and pool and first not in pool:
flags.append(f"first sighting {first} not in documented pool for {company}/{direction}")
cadet_raw = g("cadet")
combat = combat2 = None
if "cadet_actions" in C:
act = g("cadet_actions")
a = A["cadet_actions"].get(act, ["Unknown", None, None, "Unknown"])
if act and act not in A["cadet_actions"]:
flags.append("unrecognised cadet answer")
scream, combat, combat2 = a[0], a[1], a[2]
out = A["cadet_outcome"].get(cadet_raw)
if a[3] is not None:
resolution = a[3]
if out == "DIED in combat" and not resolution.startswith("DIED"):
flags.append("died in combat on a path without combat")
else:
resolution = out or "Unknown"
else:
scream, resolution = A["cadet_legacy"].get(cadet_raw, ["Unknown", "Unknown"])
if cadet_raw and cadet_raw not in A["cadet_legacy"]:
flags.append("unrecognised cadet answer")
combat = A["combat_legacy"].get(g("combat")) if g("combat") else None
post_raw = g("post")
if scream == "Investigate & help" and not resolution.startswith("DIED"):
post = post_raw if post_raw in ("Forest", "West", "East") else "Unknown"
elif scream in ("Ignore the scream", "Investigate & leave"):
post = "(no fork)" # the form allowed an answer here; the game has no fork on this branch
else:
post = None
hour = A["hour"].get(g("hour"))
died_early = scream.startswith("DIED") or resolution.startswith("DIED")
if died_early:
hour = None
elif hour is None:
hour = "Unknown"
enc = {c: g("enc_" + c) for c in COLORS}
rare = A["rare"].get(g("rare"))
bonded = [c for c in COLORS if enc[c] == ENC["bonded"]]
burned = [c for c in COLORS if enc[c] == ENC["burned"]]
# Which encounter branch did the run take after the hour?
branch = None
if hour == "Tree":
if enc["Blue"] or enc["Black"] or rare == "Shadow":
branch = "Shadow (rare)"
elif enc["Red"] == ENC["passed"] and enc["Green"]:
# only climbing down quickly puts you above the Red: passing on it and meeting the Greens means the
# rider let it pass
branch = "Descend quickly: let the Red pass"
elif enc["Green"]:
branch = "Descend slowly: two Greens"
if enc["Red"]:
flags.append("tree run with both the Red and the two Greens attempted")
elif enc["Red"]:
branch = "Descend quickly: Red"
elif enc["Orange"] or enc["Brown"]:
branch = "Descend slowly: two Greens"
flags.append("tree run with Orange/Brown but no Green recorded")
elif hour == "Valley":
if enc["Green"]:
flags.append("Green recorded on a Valley run (no Greens in the valley)")
meadow = [c for c in ("Orange", "Brown", "Blue") if enc[c]] or rare == "Blue (valley)"
if meadow or enc["Red"] == ENC["passed"]:
branch = "Passed the Red: meadow"
elif enc["Red"]:
branch = "Valley Red"
# How the run ended
outcome, outcome_color = None, None
if died_early:
outcome = "Died before the hour"
elif len(bonded) == 1:
outcome, outcome_color = "Bonded", bonded[0]
elif len(bonded) > 1:
outcome = "Ambiguous"
flags.append("more than one dragon marked Bonded")
elif burned:
order = F["burn_order"]["valley" if hour == "Valley" else "tree"]
outcome, outcome_color = "Incinerated", [c for c in order if c in burned][-1]
if len(burned) > 1:
flags.append("more than one dragon marked Incinerated")
elif hour == "DIED before the hour":
outcome = "Died before the hour"
else:
outcome = "No outcome recorded"
rescued = enc["Green"] == ENC["rescued"] and outcome == "Bonded" and outcome_color in ("Orange", "Brown")
attempt = parse_attempt(g("attempt"))
if (rare == "Blue (valley)" and hour == "Tree") or (rare == "Shadow" and hour == "Valley"):
flags.append("rare-dragon answer does not match the path")
goal_raw = g("goal_v2")
goal = g("goal") or (A["goal_v2"].get(goal_raw, goal_raw) if goal_raw and goal_raw != "Unknown" else None) or "(not asked/blank)"
return {
"source": source, "date": date, "wing": wing, "section": section,
"squad": g("squad") or "Unknown", "attempt": attempt,
# "I've lost count" means many attempts, so it is a later attempt, not missing
"attempt_group": "First attempt" if attempt == 1 else ("Later attempt" if (attempt and attempt > 1) or "lost count" in (g("attempt") or "").lower() else "Unknown"),
"goal": goal, "company": company, "left_squadmates": left_squad,
"direction": direction, "first_sighting": first or "Unknown",
"sighting_action": (A["sighting_action"].get(g("sighting_action")) or g("sighting_action") or "Unknown") if "sighting_action" in C else "(not asked)",
"scream": scream, "resolution": resolution, "combat_first": combat or "(not recorded)",
"combat_second": combat2 or "(not recorded)",
"post_path": post, "hour": hour, "branch": branch,
**{"enc_" + c: enc[c] for c in COLORS},
"outcome": outcome, "outcome_color": outcome_color, "rescued": rescued,
"saw_blue_meadow": bool(hour == "Valley" and (enc["Blue"] or rare == "Blue (valley)")),
"saw_shadow": bool(hour == "Tree" and (enc["Blue"] or enc["Black"] or rare == "Shadow")),
"rare_reported": (rare or "Unknown") if "rare" in C else "(not asked)",
"malek_intent": (g("malek") or "Unknown") if "malek" in C else "(not asked)",
"tail": g("tail"),
"data_flags": "; ".join(flags),
"death_note": redact(g("death_note")), "bond_note": redact(g("bond_note")),
}
# ---------------------------------------------------------------- files
def read_export(path):
"""An export as a DataFrame: the bundled JSON (data/public/form_responses.json), a CSV or the .xlsx."""
p = str(path).lower()
if p.endswith(".json"):
d = json.load(open(path, encoding="utf-8"))
return pd.DataFrame(d["rows"], columns=d["headers"])
if p.endswith(".csv"):
return pd.read_csv(path)
xl = pd.ExcelFile(path)
sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0])
return pd.read_excel(xl, sheet)
def code_frame(raw):
"""Code every row of an export. Returns (runs DataFrame with run_id, missing question keys)."""
C, missing = find_columns(raw.columns)
df = pd.DataFrame([code_row(r, C) for _, r in raw.iterrows()])
df.insert(0, "run_id", range(1, len(df) + 1))
return df, missing
def load(path):
return code_frame(read_export(path))
def main(argv=None):
argv = sys.argv[1:] if argv is None else argv
src = argv[0] if argv else ROOT / "data/public/form_responses.json"
out = argv[1] if len(argv) > 1 else ROOT / "data/derived/runs_clean.csv"
df, missing = load(src)
if missing:
print("WARNING: columns not found:", missing)
df.to_csv(out, index=False)
print(f"{len(df)} runs -> {out}")
print(df.outcome.value_counts().to_string())
print("rows with flags:", (df["data_flags"] != "").sum())
if __name__ == "__main__":
main()