""" clean.py — turn the raw "Threshing Routes Responses" Google Form export into one coded row per threshing run. Reference implementation in Python. The web page (packages/core/src/engine.js) implements the same rules in JavaScript; tests/compare.py checks that both give identical output. Usage: python clean.py ../../data/raw/form/Threshing_Routes_Responses.xlsx -> writes ../../data/derived/runs_clean.csv Every coding rule is listed in RULES below and in the "Methods" section of the page. """ import re import sys from datetime import datetime import pandas as pd EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}") def redact(v): """Free-text answers sometimes contain a respondent's email address; never display or export it.""" return v if v is None else EMAIL_RE.sub("[email removed]", str(v)) COLORS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"] # Columns are found by keywords in the question text, so the script keeps working # if the form owner reorders or adds questions. COLUMN_KEYS = { "timestamp": ["timestamp"], "wing_First": ["wing and section", "[first]"], "wing_Second": ["wing and section", "[second]"], "wing_Third": ["wing and section", "[third]"], "wing_Fourth": ["wing and section", "[fourth]"], "squad": ["what squad"], "attempt": ["attempt number"], "company": ["who accompanied"], "dir_East": ["initial direction", "[east"], "dir_West": ["initial direction", "[west"], "dir_Center": ["initial direction", "[forest center"], "cadet": ["outcome of your cadet encounter"], "post": ["path did you take after the bully"], "hour": ["after walking an hour"], "enc_Red": ["attempt to bond", "[red"], "enc_Green": ["attempt to bond", "[green"], "enc_Orange": ["attempt to bond", "[orange"], "enc_Brown": ["attempt to bond", "[brown"], "enc_Blue": ["attempt to bond", "[blue"], "enc_Black": ["attempt to bond", "[black"], "death_note": ["those who died"], "bond_note": ["those who bonded"], "goal": ["colour dragon are you trying"], "combat": ["combat choice"], } CADET_MAP = { "Bullies > You dealt with them": ("Investigate & help", "Killed the bullies yourself"), "Bullies > Squadmate's revenge": ("Investigate & help", "Squadmate's revenge"), "Bullies > Left to the Dragons": ("Investigate & help", "Left bullies to the dragons"), "Bullies > Assessed the situation": ("Investigate & help", "Unknown"), "Injured Cadet > Tourniquet": ("Ignore the scream", "Tourniquet"), "Injured Cadet > Left them there": ("Ignore the scream", "Left injured cadet"), "Injured Cadet > Took their weapons": ("Ignore the scream", "Took their weapons"), "Avoided combat (Investigated scream > Left them to it)": ("Investigate & leave", "Left them to it"), "You died in combat": ("Investigate & help", "DIED in combat"), "N/A - You died with initial dragon sighting": ("DIED at first sighting", "DIED at first sighting"), } COMBAT_MAP = { "Bullies > Assessed the situation": "Assess the situation", "Bullies > Even the numbers (Rushed the smaller cadet)": "Even the numbers (smaller cadet)", "Bullies > Put yourself in the middle (Rushed the bigger cadet attacking your squadmate)": "Put yourself in the middle (bigger cadet)", } HOUR_MAP = { "Continued to Valley Centre": "Valley", "Climbed the tallest tree": "Tree", "N/A - You're dead": "DIED before the hour", } # First-sighting pools per (company, direction) according to the community PDF map. # Used only to FLAG rows that report a dragon outside the documented pool. SIGHTING_POOL = { ("Alone", "East"): {"Red", "Orange"}, ("Alone", "West"): {"Red", "Brown", "Green"}, ("Alone", "Center"): {"Red", "Brown"}, ("1 Squadmate", "East"): {"Green", "Orange", "Red"}, # 'leave squadmate' -> alone pools also possible ("1 Squadmate", "West"): {"Red", "Brown", "Green"}, ("1 Squadmate", "Center"): {"Red", "Brown"}, ("2 Squadmates", "East"): {"Red", "Orange"}, ("2 Squadmates", "West"): {"Green", "Brown", "Red"}, ("2 Squadmates", "Center"): {"Red", "Brown"}, } def find_columns(cols): out = {} low = {c: str(c).lower() for c in cols} for key, words in COLUMN_KEYS.items(): hits = [c for c in cols if all(w in low[c] for w in words)] if hits: out[key] = hits[0] missing = [k for k in COLUMN_KEYS if k not in out] return out, missing def blank(v): if v is None or v is pd.NaT: return True if not isinstance(v, str): try: if pd.isna(v): return True except (TypeError, ValueError): pass return str(v).strip() in ("", "nan", "NaN", "NaT") def s(v): return None if blank(v) else str(v).strip() def parse_timestamp(v): """Form submissions carry a timestamp; bulk-pasted rows carry a label such as 'Data Import'. Returns 'YYYY-MM-DD' or None.""" if isinstance(v, (datetime, pd.Timestamp)) and not pd.isna(v): return pd.Timestamp(v).strftime("%Y-%m-%d") if isinstance(v, (int, float)) and not isinstance(v, bool) and 40000 < v < 80000: # Excel serial date return (pd.Timestamp("1899-12-30") + pd.Timedelta(days=int(v))).strftime("%Y-%m-%d") t = s(v) if not t: return None m = re.match(r"^(\d{4})-(\d{2})-(\d{2})", t) if m: return f"{m.group(1)}-{m.group(2)}-{m.group(3)}" m = re.match(r"^(\d{1,2})/(\d{1,2})/(\d{4})", t) # Google Sheets CSV export: M/D/YYYY if m: return f"{m.group(3)}-{int(m.group(1)):02d}-{int(m.group(2)):02d}" return None def parse_attempt(v): v = s(v) if v is None: return None t = v.lower() if "lost count" in t: return None words = {"first": 1, "second": 2, "third": 3, "fourth": 4, "fifth": 5} for w, n in words.items(): if t.startswith(w): return n m = re.search(r"\d+", t) return int(m.group()) if m else None def code_row(r, C): g = lambda k: s(r[C[k]]) if k in C else None flags = [] ts = r[C["timestamp"]] if "timestamp" in C else None d = parse_timestamp(ts) if d: source, date = "Form", d else: source, date = (s(ts) or "Unknown import"), None wings = [w for w in ("First", "Second", "Third", "Fourth") if g("wing_" + w)] wing = wings[0] if len(wings) == 1 else ("Multiple" if wings else "Unknown") section = g("wing_" + wings[0]) if len(wings) == 1 else None if len(wings) > 1: flags.append("multiple wings") dirs = [d for d in ("East", "West", "Center") if g("dir_" + d)] if len(dirs) == 1: direction, first = dirs[0], g("dir_" + dirs[0]) else: direction, first = ("Multiple" if dirs else "Unknown"), None flags.append("direction " + ("multiple" if dirs else "missing")) company = g("company") or "Unknown" if first and first != "Unsure" and (company, direction) in SIGHTING_POOL: if first not in SIGHTING_POOL[(company, direction)]: flags.append(f"first sighting {first} not in documented pool for {company}/{direction}") cadet_raw = g("cadet") scream, resolution = CADET_MAP.get(cadet_raw, ("Unknown", "Unknown")) if cadet_raw and cadet_raw not in CADET_MAP: flags.append("unrecognised cadet answer") combat = COMBAT_MAP.get(g("combat")) if g("combat") else None post_raw = g("post") if scream == "Investigate & help" and not resolution.startswith("DIED"): post = post_raw if post_raw in ("Forest", "West", "East") else "Unknown" elif scream in ("Ignore the scream", "Investigate & leave"): post = "(no fork)" if post_raw in ("Forest", "West", "East"): post = "(no fork)" # the form allowed an answer here; the game has no fork on this branch else: post = None hour = HOUR_MAP.get(g("hour")) died_early = scream.startswith("DIED") or resolution.startswith("DIED") if died_early: hour = None elif hour is None: hour = "Unknown" enc = {c: g("enc_" + c) for c in COLORS} bonded = [c for c in COLORS if enc[c] == "Bonded!"] burned = [c for c in COLORS if enc[c] == "Incinerated"] # Which encounter branch did the run take after the hour? branch = None if hour == "Tree": if enc["Blue"] or enc["Black"]: branch = "Shadow (rare)" elif enc["Green"]: branch = "Descend slowly: two Greens" elif enc["Red"]: branch = "Descend quickly: Red" elif enc["Orange"] or enc["Brown"]: branch = "Descend slowly: two Greens" flags.append("tree run with Orange/Brown but no Green recorded") elif hour == "Valley": if enc["Green"]: flags.append("Green recorded on a Valley run (no Greens in the valley)") red_passed = enc["Red"] in ("Changed my mind",) meadow = [c for c in ("Orange", "Brown", "Blue") if enc[c]] if meadow or red_passed: branch = "Passed the Red: meadow" elif enc["Red"]: branch = "Valley Red" # Final outcome of the run outcome, outcome_color = None, None if died_early: outcome = "Died before the hour" elif len(bonded) == 1: outcome, outcome_color = "Bonded", bonded[0] elif len(bonded) > 1: outcome = "Ambiguous" flags.append("more than one dragon marked Bonded") elif burned: order = ["Red", "Green", "Orange", "Brown", "Blue", "Black"] if hour != "Valley" else \ ["Red", "Orange", "Brown", "Blue", "Green", "Black"] last = [c for c in order if c in burned][-1] outcome, outcome_color = "Incinerated", last if len(burned) > 1: flags.append("more than one dragon marked Incinerated") elif hour == "DIED before the hour": outcome = "Died before the hour" else: outcome = "No outcome recorded" # The dragon whose bonding encounter decided the run (attempted = Bonded or Incinerated) rescued = enc["Green"] == "Saved by another" and outcome == "Bonded" and outcome_color in ("Orange", "Brown") attempt = parse_attempt(g("attempt")) return { "source": source, "date": date, "wing": wing, "section": section, "squad": g("squad") or "Unknown", "attempt": attempt, "attempt_group": "First attempt" if attempt == 1 else ("Later attempt" if attempt and attempt > 1 else "Unknown/lost count"), "goal": g("goal") or "(not asked/blank)", "company": company, "direction": direction, "first_sighting": first or "Unknown", "scream": scream, "resolution": resolution, "combat_first": combat or "(not recorded)", "post_path": post, "hour": hour, "branch": branch, **{"enc_" + c: enc[c] for c in COLORS}, "outcome": outcome, "outcome_color": outcome_color, "rescued": rescued, "saw_blue_meadow": bool(hour == "Valley" and enc["Blue"]), "saw_shadow": bool(hour == "Tree" and (enc["Blue"] or enc["Black"])), "data_flags": "; ".join(flags), "death_note": redact(g("death_note")), "bond_note": redact(g("bond_note")), } def load(path): if str(path).lower().endswith(".json"): # the page's bundled export (data/public/form_responses.json) import json d = json.load(open(path, encoding="utf-8")) raw = pd.DataFrame(d["rows"], columns=d["headers"]) elif str(path).lower().endswith(".csv"): raw = pd.read_csv(path) else: xl = pd.ExcelFile(path) sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0]) raw = pd.read_excel(xl, sheet) C, missing = find_columns(raw.columns) rows = [code_row(r, C) for _, r in raw.iterrows()] df = pd.DataFrame(rows) df.insert(0, "run_id", range(1, len(df) + 1)) return df, missing if __name__ == "__main__": src = sys.argv[1] if len(sys.argv) > 1 else "../../data/public/form_responses.json" out = sys.argv[2] if len(sys.argv) > 2 else "../../data/derived/runs_clean.csv" df, missing = load(src) if missing: print("WARNING: columns not found:", missing) df.to_csv(out, index=False) print(f"{len(df)} runs -> {out}") print(df.outcome.value_counts().to_string()) print("rows with flags:", (df["data_flags"] != "").sum())