Files
threshing-atlas/research/python/clean.py
T
ekcSchianoandClaude Opus 5.5 46ee08a56b Coding rules for the redesigned community form; import the new data
The Threshing Routes form was redesigned (5,651 responses). Both implementations
(engine.js and clean.py) now read either version:
- one direction question + one colour question, with the older per-direction
  columns used only to fill gaps and cross-check
- cadet encounter as an action path (scream, first and second combat move)
  plus a separate outcome
- new fields: sighting_action, combat_second, rare_reported, malek_intent,
  left_squadmates, tail; "Why are you doing this run?" mapped onto the goal
  ("Blue or Black" for rare chasers)
- the rare-dragon question feeds saw_blue_meadow / saw_shadow and the branch
The old export still codes identically. JS/Python parity: all match.

Data: form 2,573 -> 5,651 rows, Dragon Register 216 -> 273, new route PDF
fingerprint. Supersedes the automated pull request #2.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
2026-11-01 22:10:51 +00:00

402 lines
18 KiB
Python

"""
clean.py — turn the raw "Threshing Routes Responses" Google Form export into one
coded row per threshing run.
Reference implementation in Python. The web page (packages/core/src/engine.js) implements the
same rules in JavaScript; tests/compare.py checks that both give identical output.
Usage:
python clean.py ../../data/raw/form/Threshing_Routes_Responses.xlsx -> writes ../../data/derived/runs_clean.csv
Every coding rule is listed in RULES below and in the "Methods" section of the page.
"""
import re
import sys
from datetime import datetime
import pandas as pd
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
def redact(v):
"""Free-text answers sometimes contain a respondent's email address; never display or export it."""
return v if v is None else EMAIL_RE.sub("[email removed]", str(v))
COLORS = ["Red", "Green", "Orange", "Brown", "Blue", "Black"]
# Columns are found by keywords in the question text, so the script keeps working
# if the form owner reorders or adds questions.
COLUMN_KEYS = {
"timestamp": ["timestamp"],
"wing_First": ["wing and section", "[first]"],
"wing_Second": ["wing and section", "[second]"],
"wing_Third": ["wing and section", "[third]"],
"wing_Fourth": ["wing and section", "[fourth]"],
"squad": ["what squad"],
"attempt": ["attempt number"],
"company": ["who accompanied"],
"dir_East": ["initial direction", "[east"],
"dir_West": ["initial direction", "[west"],
"dir_Center": ["initial direction", "[forest center"],
"cadet": ["outcome of your cadet encounter"],
"post": ["path did you take after the bully"],
"hour": ["after walking an hour"],
"enc_Red": ["attempt to bond", "[red"],
"enc_Green": ["attempt to bond", "[green"],
"enc_Orange": ["attempt to bond", "[orange"],
"enc_Brown": ["attempt to bond", "[brown"],
"enc_Blue": ["attempt to bond", "[blue"],
"enc_Black": ["attempt to bond", "[black"],
"death_note": ["those who died"],
"bond_note": ["those who bonded"],
"goal": ["colour dragon are you trying"],
"combat": ["combat choice"],
# Questions added in the redesigned form (late October 2026). "!x" means the header must NOT contain x.
"first_dir": ["what was your first direction"],
"first_colour": ["colour was your initial dragon sighting", "!["],
"leg_West": ["colour was your initial dragon sighting", "[west"],
"leg_Center": ["colour was your initial dragon sighting", "[forest center"],
"sighting_action": ["action did you take during the first dragon sighting"],
"cadet_actions": ["during your cadet encounter"],
"goal_v2": ["why are you doing this run"],
"malek": ["seek malek"],
"rare": ["encounter a rare dragon"],
"tail": ["what tail does your dragon have"],
}
# A key is only reported missing when neither it nor its replacement in the other form version was found.
ALTERNATIVE = {
"dir_East": "first_dir", "dir_West": "first_dir", "dir_Center": "first_dir", "first_dir": "dir_Center",
"first_colour": "dir_Center", "goal": "goal_v2", "goal_v2": "goal", "combat": "cadet_actions", "cadet_actions": "combat",
}
OPTIONAL = {"leg_West", "leg_Center", "sighting_action", "malek", "rare", "tail"}
CADET_MAP = {
"Bullies > You dealt with them": ("Investigate & help", "Killed the bullies yourself"),
"Bullies > Squadmate's revenge": ("Investigate & help", "Squadmate's revenge"),
"Bullies > Left to the Dragons": ("Investigate & help", "Left bullies to the dragons"),
"Bullies > Assessed the situation": ("Investigate & help", "Unknown"),
"Injured Cadet > Tourniquet": ("Ignore the scream", "Tourniquet"),
"Injured Cadet > Left them there": ("Ignore the scream", "Left injured cadet"),
"Injured Cadet > Took their weapons": ("Ignore the scream", "Took their weapons"),
"Avoided combat (Investigated scream > Left them to it)": ("Investigate & leave", "Left them to it"),
"You died in combat": ("Investigate & help", "DIED in combat"),
"N/A - You died with initial dragon sighting": ("DIED at first sighting", "DIED at first sighting"),
}
COMBAT_MAP = {
"Bullies > Assessed the situation": "Assess the situation",
"Bullies > Even the numbers (Rushed the smaller cadet)": "Even the numbers (smaller cadet)",
"Bullies > Put yourself in the middle (Rushed the bigger cadet attacking your squadmate)": "Put yourself in the middle (bigger cadet)",
}
# Redesigned form: the cadet encounter is asked as a path of actions plus a separate outcome.
# value: (scream, first combat move, second combat move, resolution when the action decides it)
CADET_ACTIONS = {
"Bullies > Assess > Unknown": ("Investigate & help", "Assess the situation", None, None),
"Bullies > Assess > Smaller Cadet": ("Investigate & help", "Assess the situation", "Engage the smaller cadet", None),
"Bullies > Assess > Daggers or not": ("Investigate & help", "Assess the situation", "Daggers or not (protect squadmate)", None),
"Bullies > Unknown": ("Investigate & help", None, None, None),
"Bullies > Smaller Cadet (Did not assess)": ("Investigate & help", "Even the numbers (smaller cadet)", None, None),
"Bullies > Bigger Cadet (Did not assess)": ("Investigate & help", "Put yourself in the middle (bigger cadet)", None, None),
"Injured Cadet > Tourniquet": ("Ignore the scream", None, None, "Tourniquet"),
"Injured Cadet > Left them there": ("Ignore the scream", None, None, "Left injured cadet"),
"Injured Cadet > Took their weapons": ("Ignore the scream", None, None, "Took their weapons"),
"Avoided combat (Investigated scream > Left them to it)": ("Investigate & leave", None, None, "Left them to it"),
"N/A - You died with initial dragon sighting": ("DIED at first sighting", None, None, "DIED at first sighting"),
"Unsure": ("Unknown", None, None, "Unknown"),
}
CADET_OUTCOME = {
"Squadmate's revenge": "Squadmate's revenge",
"You ended the bullies yourself": "Killed the bullies yourself",
"Left to the Dragons": "Left bullies to the dragons",
"You died in combat": "DIED in combat",
}
DIRECTION_MAP = {"Forest Center": "Center", "West / Scent": "West", "East / Roar": "East"}
GOAL_V2 = {
"Hoping for a Rare Dragon (Blue or Black)": "Blue or Black",
"Trying to bond the Red dragon under the tree": "Red (tree)",
"Daily Run Assignment data": "Daily run data",
"Just here for fun, and maybe a dragon": "Any (for fun)",
}
SIGHTING_ACTION = {
"Hold Ground / Stay Still": "Hold still", "Respect / Face it": "Respect / face it",
"Retreat / Run / Back Away": "Retreat / run", "Hide / Duck": "Hide", "Draw / Drop Weapon": "Weapon",
"Meet Gaze": "Meet its gaze", "Protect / Help Sqaudmate": "Protect squadmate", "Approach": "Approach",
"Stand on your own": "Stand alone", "Unsure": "Unknown", "Unsure - You don't remember": "Unknown",
}
RARE_MAP = {"Yes - A blue dragon in the valley": "Blue (valley)", "Yes - Shadow!!!": "Shadow", "No": "No"}
HOUR_MAP = {
"Continued to Valley Centre": "Valley",
"Climbed the tallest tree": "Tree",
"N/A - You're dead": "DIED before the hour",
}
# First-sighting pools per (company, direction) according to the community PDF map.
# Used only to FLAG rows that report a dragon outside the documented pool.
SIGHTING_POOL = {
("Alone", "East"): {"Red", "Orange"},
("Alone", "West"): {"Red", "Brown", "Green"},
("Alone", "Center"): {"Red", "Brown"},
("1 Squadmate", "East"): {"Green", "Orange", "Red"}, # 'leave squadmate' -> alone pools also possible
("1 Squadmate", "West"): {"Red", "Brown", "Green"},
("1 Squadmate", "Center"): {"Red", "Brown"},
("2 Squadmates", "East"): {"Red", "Orange"},
("2 Squadmates", "West"): {"Green", "Brown", "Red"},
("2 Squadmates", "Center"): {"Red", "Brown"},
}
def find_columns(cols):
out = {}
low = {c: str(c).lower() for c in cols}
for key, words in COLUMN_KEYS.items():
hits = [c for c in cols if all((w[1:] not in low[c]) if w[0] == "!" else (w in low[c]) for w in words)]
if hits:
out[key] = hits[0]
missing = [k for k in COLUMN_KEYS if k not in out and k not in OPTIONAL and ALTERNATIVE.get(k) not in out]
return out, missing
def blank(v):
if v is None or v is pd.NaT:
return True
if not isinstance(v, str):
try:
if pd.isna(v):
return True
except (TypeError, ValueError):
pass
return str(v).strip() in ("", "nan", "NaN", "NaT")
def s(v):
return None if blank(v) else str(v).strip()
def parse_timestamp(v):
"""Form submissions carry a timestamp; bulk-pasted rows carry a label such as 'Data Import'.
Returns 'YYYY-MM-DD' or None."""
if isinstance(v, (datetime, pd.Timestamp)) and not pd.isna(v):
return pd.Timestamp(v).strftime("%Y-%m-%d")
if isinstance(v, (int, float)) and not isinstance(v, bool) and 40000 < v < 80000: # Excel serial date
return (pd.Timestamp("1899-12-30") + pd.Timedelta(days=int(v))).strftime("%Y-%m-%d")
t = s(v)
if not t:
return None
m = re.match(r"^(\d{4})-(\d{2})-(\d{2})", t)
if m:
return f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
m = re.match(r"^(\d{1,2})/(\d{1,2})/(\d{4})", t) # Google Sheets CSV export: M/D/YYYY
if m:
return f"{m.group(3)}-{int(m.group(1)):02d}-{int(m.group(2)):02d}"
return None
def parse_attempt(v):
v = s(v)
if v is None:
return None
t = v.lower()
if "lost count" in t:
return None
words = {"first": 1, "second": 2, "third": 3, "fourth": 4, "fifth": 5}
for w, n in words.items():
if t.startswith(w):
return n
m = re.search(r"\d+", t)
return int(m.group()) if m else None
def code_row(r, C):
g = lambda k: s(r[C[k]]) if k in C else None
flags = []
ts = r[C["timestamp"]] if "timestamp" in C else None
d = parse_timestamp(ts)
if d:
source, date = "Form", d
else:
source, date = (s(ts) or "Unknown import"), None
wings = [w for w in ("First", "Second", "Third", "Fourth") if g("wing_" + w)]
wing = wings[0] if len(wings) == 1 else ("Multiple" if wings else "Unknown")
section = g("wing_" + wings[0]) if len(wings) == 1 else None
if len(wings) > 1:
flags.append("multiple wings")
if "first_dir" in C:
# Redesigned form: one direction question and one colour question. The older per-direction colour
# columns are still in the sheet for some rows and are used only to fill gaps and to cross-check.
direction = DIRECTION_MAP.get(g("first_dir"))
first = g("first_colour")
legacy = [d for d in ("West", "Center") if g("leg_" + d)]
if not direction and len(legacy) == 1:
direction = legacy[0]
flags.append("direction taken from the older sighting column")
leg = g("leg_" + direction) if direction else None
if not first and leg:
first = leg
elif first and leg and first != leg and first != "Unsure" and leg != "Unsure":
flags.append("first sighting differs between the old and new columns")
if not direction:
direction = "Unknown"
flags.append("direction missing")
else:
dirs = [d for d in ("East", "West", "Center") if g("dir_" + d)]
if len(dirs) == 1:
direction, first = dirs[0], g("dir_" + dirs[0])
else:
direction, first = ("Multiple" if dirs else "Unknown"), None
flags.append("direction " + ("multiple" if dirs else "missing"))
company_raw = g("company") or "Unknown"
left_squad = company_raw.endswith(" > Left them")
company = company_raw[: -len(" > Left them")] if left_squad else company_raw
# Leaving your squadmates sends you into the solo pools, so either pool is plausible.
pool = list(SIGHTING_POOL.get((company, direction), [])) + (list(SIGHTING_POOL.get(("Alone", direction), [])) if left_squad else [])
if first and first != "Unsure" and pool and first not in pool:
flags.append(f"first sighting {first} not in documented pool for {company}/{direction}")
cadet_raw = g("cadet")
combat = combat2 = None
if "cadet_actions" in C:
act = g("cadet_actions")
a = CADET_ACTIONS.get(act, ("Unknown", None, None, "Unknown"))
if act and act not in CADET_ACTIONS:
flags.append("unrecognised cadet answer")
scream, combat, combat2 = a[0], a[1], a[2]
out = CADET_OUTCOME.get(cadet_raw)
if a[3] is not None:
resolution = a[3]
if out == "DIED in combat" and not resolution.startswith("DIED"):
flags.append("died in combat on a path without combat")
else:
resolution = out or "Unknown"
else:
scream, resolution = CADET_MAP.get(cadet_raw, ("Unknown", "Unknown"))
if cadet_raw and cadet_raw not in CADET_MAP:
flags.append("unrecognised cadet answer")
combat = COMBAT_MAP.get(g("combat")) if g("combat") else None
post_raw = g("post")
if scream == "Investigate & help" and not resolution.startswith("DIED"):
post = post_raw if post_raw in ("Forest", "West", "East") else "Unknown"
elif scream in ("Ignore the scream", "Investigate & leave"):
post = "(no fork)"
if post_raw in ("Forest", "West", "East"):
post = "(no fork)" # the form allowed an answer here; the game has no fork on this branch
else:
post = None
hour = HOUR_MAP.get(g("hour"))
died_early = scream.startswith("DIED") or resolution.startswith("DIED")
if died_early:
hour = None
elif hour is None:
hour = "Unknown"
enc = {c: g("enc_" + c) for c in COLORS}
rare = RARE_MAP.get(g("rare"))
bonded = [c for c in COLORS if enc[c] == "Bonded!"]
burned = [c for c in COLORS if enc[c] == "Incinerated"]
# Which encounter branch did the run take after the hour?
branch = None
if hour == "Tree":
if enc["Blue"] or enc["Black"] or rare == "Shadow":
branch = "Shadow (rare)"
elif enc["Green"]:
branch = "Descend slowly: two Greens"
elif enc["Red"]:
branch = "Descend quickly: Red"
elif enc["Orange"] or enc["Brown"]:
branch = "Descend slowly: two Greens"
flags.append("tree run with Orange/Brown but no Green recorded")
elif hour == "Valley":
if enc["Green"]:
flags.append("Green recorded on a Valley run (no Greens in the valley)")
red_passed = enc["Red"] in ("Changed my mind",)
meadow = [c for c in ("Orange", "Brown", "Blue") if enc[c]] or rare == "Blue (valley)"
if meadow or red_passed:
branch = "Passed the Red: meadow"
elif enc["Red"]:
branch = "Valley Red"
# Final outcome of the run
outcome, outcome_color = None, None
if died_early:
outcome = "Died before the hour"
elif len(bonded) == 1:
outcome, outcome_color = "Bonded", bonded[0]
elif len(bonded) > 1:
outcome = "Ambiguous"
flags.append("more than one dragon marked Bonded")
elif burned:
order = ["Red", "Green", "Orange", "Brown", "Blue", "Black"] if hour != "Valley" else \
["Red", "Orange", "Brown", "Blue", "Green", "Black"]
last = [c for c in order if c in burned][-1]
outcome, outcome_color = "Incinerated", last
if len(burned) > 1:
flags.append("more than one dragon marked Incinerated")
elif hour == "DIED before the hour":
outcome = "Died before the hour"
else:
outcome = "No outcome recorded"
# The dragon whose bonding encounter decided the run (attempted = Bonded or Incinerated)
rescued = enc["Green"] == "Saved by another" and outcome == "Bonded" and outcome_color in ("Orange", "Brown")
attempt = parse_attempt(g("attempt"))
if (rare == "Blue (valley)" and hour == "Tree") or (rare == "Shadow" and hour == "Valley"):
flags.append("rare-dragon answer does not match the path")
goal_raw = g("goal_v2")
goal = g("goal") or (GOAL_V2.get(goal_raw, goal_raw) if goal_raw and goal_raw != "Unknown" else None) or "(not asked/blank)"
return {
"source": source, "date": date, "wing": wing, "section": section,
"squad": g("squad") or "Unknown", "attempt": attempt,
"attempt_group": "First attempt" if attempt == 1 else ("Later attempt" if attempt and attempt > 1 else "Unknown/lost count"),
"goal": goal, "company": company, "left_squadmates": left_squad,
"direction": direction, "first_sighting": first or "Unknown",
"sighting_action": (SIGHTING_ACTION.get(g("sighting_action")) or g("sighting_action") or "Unknown") if "sighting_action" in C else "(not asked)",
"scream": scream, "resolution": resolution, "combat_first": combat or "(not recorded)",
"combat_second": combat2 or "(not recorded)",
"post_path": post, "hour": hour, "branch": branch,
**{"enc_" + c: enc[c] for c in COLORS},
"outcome": outcome, "outcome_color": outcome_color, "rescued": rescued,
"saw_blue_meadow": bool(hour == "Valley" and (enc["Blue"] or rare == "Blue (valley)")),
"saw_shadow": bool(hour == "Tree" and (enc["Blue"] or enc["Black"] or rare == "Shadow")),
"rare_reported": (rare or "Unknown") if "rare" in C else "(not asked)",
"malek_intent": (g("malek") or "Unknown") if "malek" in C else "(not asked)",
"tail": g("tail"),
"data_flags": "; ".join(flags),
"death_note": redact(g("death_note")), "bond_note": redact(g("bond_note")),
}
def load(path):
if str(path).lower().endswith(".json"): # the page's bundled export (data/public/form_responses.json)
import json
d = json.load(open(path, encoding="utf-8"))
raw = pd.DataFrame(d["rows"], columns=d["headers"])
elif str(path).lower().endswith(".csv"):
raw = pd.read_csv(path)
else:
xl = pd.ExcelFile(path)
sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0])
raw = pd.read_excel(xl, sheet)
C, missing = find_columns(raw.columns)
rows = [code_row(r, C) for _, r in raw.iterrows()]
df = pd.DataFrame(rows)
df.insert(0, "run_id", range(1, len(df) + 1))
return df, missing
if __name__ == "__main__":
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/public/form_responses.json"
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/derived/runs_clean.csv"
df, missing = load(src)
if missing:
print("WARNING: columns not found:", missing)
df.to_csv(out, index=False)
print(f"{len(df)} runs -> {out}")
print(df.outcome.value_counts().to_string())
print("rows with flags:", (df["data_flags"] != "").sum())