"""Community form rows -> one coded run per row. The tables (how each question is found, what every answer means, which answers count as missing) live in core/spec/form.yaml. This module holds the logic that combines answers into a run: which way the rider went, which branch they took after the hour, how the run ended, and which rows look inconsistent (data_flags). python -m atlas.coding [form_responses.json|.xlsx|.csv] [out.csv] Prints a summary of the coded export (default: data/public/form_responses.json), and writes the coded runs to out.csv when given. The published runs (every source) are written by `python -m collection build`. """ import json import re import sys from datetime import datetime from pathlib import Path import pandas as pd from . import spec ROOT = Path(__file__).resolve().parents[2] F = spec.form() A = F["answers"] COLORS = F["colours"] MISSING = frozenset(F["missing_values"]) ENC = F["encounter_answers"] _EMAIL = re.compile(F["redact"]["pattern"]) def is_missing(v): """True for answers that carry no information about the choice (blank, "Unsure", not asked...).""" return v is None or v == "" or v in MISSING or (isinstance(v, float) and v != v) def redact(v): """Replace email addresses in free text.""" return v if v is None else _EMAIL.sub(F["redact"]["replacement"], str(v)) # ---------------------------------------------------------------- reading cells def find_columns(cols): """Map each question key in form.yaml to the export's column header. Returns (columns, missing keys).""" out = {} low = {c: str(c).lower() for c in cols} for key, col in F["columns"].items(): words = col["match"] hits = [c for c in cols if all((w[1:] not in low[c]) if w[0] == "!" else (w in low[c]) for w in words)] if hits: out[key] = hits[0] missing = [k for k, col in F["columns"].items() if k not in out and not col.get("optional") and col.get("replaced_by") not in out] return out, missing def blank(v): if v is None or v is pd.NaT: return True if not isinstance(v, str): try: if pd.isna(v): return True except (TypeError, ValueError): pass return str(v).strip() in ("", "nan", "NaN", "NaT") def text(v): return None if blank(v) else str(v).strip() def parse_timestamp(v): """Form submissions carry a timestamp; bulk-pasted rows carry a label such as 'Data Import'. Returns 'YYYY-MM-DD' or None.""" if isinstance(v, (datetime, pd.Timestamp)) and not pd.isna(v): return pd.Timestamp(v).strftime("%Y-%m-%d") if isinstance(v, (int, float)) and not isinstance(v, bool) and 40000 < v < 80000: # Excel serial date return (pd.Timestamp("1899-12-30") + pd.Timedelta(days=int(v))).strftime("%Y-%m-%d") t = text(v) if not t: return None m = re.match(r"^(\d{4})-(\d{2})-(\d{2})", t) if m: return f"{m.group(1)}-{m.group(2)}-{m.group(3)}" m = re.match(r"^(\d{1,2})/(\d{1,2})/(\d{4})", t) # Google Sheets CSV export: M/D/YYYY if m: return f"{m.group(3)}-{int(m.group(1)):02d}-{int(m.group(2)):02d}" return None def parse_attempt(v): v = text(v) if v is None: return None t = v.lower() if "lost count" in t: return None for w, n in F["attempt_words"].items(): if t.startswith(w): return n m = re.search(r"\d+", t) return int(m.group()) if m else None # ---------------------------------------------------------------- one row -> one run def code_row(r, C): """Code one export row. r: mapping header -> cell; C: from find_columns().""" g = lambda k: text(r[C[k]]) if k in C else None # noqa: E731 flags = [] ts = r[C["timestamp"]] if "timestamp" in C else None d = parse_timestamp(ts) source, date = ("Form", d) if d else ((text(ts) or "Unknown import"), None) wings = [w for w in ("First", "Second", "Third", "Fourth") if g("wing_" + w)] wing = wings[0] if len(wings) == 1 else ("Multiple" if wings else "Unknown") section = g("wing_" + wings[0]) if len(wings) == 1 else None if len(wings) > 1: flags.append("multiple wings") if "first_dir" in C: # Redesigned form: one direction question and one colour question. The older per-direction colour # columns are still in the sheet for some rows and are used only to fill gaps and to cross-check. direction = A["direction"].get(g("first_dir")) first = g("first_colour") legacy = [d for d in ("West", "Center") if g("leg_" + d)] if not direction and len(legacy) == 1: direction = legacy[0] flags.append("direction taken from the older sighting column") leg = g("leg_" + direction) if direction else None if not first and leg: first = leg elif first and leg and first != leg and first != "Unsure" and leg != "Unsure": flags.append("first sighting differs between the old and new columns") if not direction: direction = "Unknown" flags.append("direction missing") else: dirs = [d for d in ("East", "West", "Center") if g("dir_" + d)] if len(dirs) == 1: direction, first = dirs[0], g("dir_" + dirs[0]) else: direction, first = ("Multiple" if dirs else "Unknown"), None flags.append("direction " + ("multiple" if dirs else "missing")) company_raw = g("company") or "Unknown" suffix = F["company_left_suffix"] left_squad = company_raw.endswith(suffix) company = company_raw[: -len(suffix)] if left_squad else company_raw if not left_squad and company in ("1 Squadmate", "2 Squadmates") and not (date and date >= F["left_squadmates_asked_from"]): left_squad = None # the form did not offer "left them" yet: unknown, not "stayed together" pools = F["sighting_pool"] pool = list(pools.get(company, {}).get(direction, [])) + (list(pools["Alone"].get(direction, [])) if left_squad else []) if first and first != "Unsure" and pool and first not in pool: flags.append(f"first sighting {first} not in documented pool for {company}/{direction}") cadet_raw = g("cadet") combat = combat2 = None if "cadet_actions" in C: act = g("cadet_actions") a = A["cadet_actions"].get(act, ["Unknown", None, None, "Unknown"]) if act and act not in A["cadet_actions"]: flags.append("unrecognised cadet answer") scream, combat, combat2 = a[0], a[1], a[2] out = A["cadet_outcome"].get(cadet_raw) if a[3] is not None: resolution = a[3] if out == "DIED in combat" and not resolution.startswith("DIED"): flags.append("died in combat on a path without combat") else: resolution = out or "Unknown" else: scream, resolution = A["cadet_legacy"].get(cadet_raw, ["Unknown", "Unknown"]) if cadet_raw and cadet_raw not in A["cadet_legacy"]: flags.append("unrecognised cadet answer") combat = A["combat_legacy"].get(g("combat")) if g("combat") else None post_raw = g("post") if scream == "Investigate & help" and not resolution.startswith("DIED"): post = post_raw if post_raw in ("Forest", "West", "East") else "Unknown" elif scream in ("Ignore the scream", "Investigate & leave"): post = "(no fork)" # the form allowed an answer here; the game has no fork on this branch else: post = None hour = A["hour"].get(g("hour")) died_early = scream.startswith("DIED") or resolution.startswith("DIED") if died_early: hour = None elif hour is None: hour = "Unknown" enc = {c: g("enc_" + c) for c in COLORS} rare = A["rare"].get(g("rare")) bonded = [c for c in COLORS if enc[c] == ENC["bonded"]] burned = [c for c in COLORS if enc[c] == ENC["burned"]] # Which encounter branch did the run take after the hour? branch = None if hour == "Tree": if enc["Blue"] or enc["Black"] or rare == "Shadow": branch = "Shadow (rare)" elif enc["Red"] == ENC["passed"] and enc["Green"]: # only climbing down quickly puts you above the Red: passing on it and meeting the Greens means the # rider let it pass branch = "Descend quickly: let the Red pass" elif enc["Green"]: branch = "Descend slowly: two Greens" if enc["Red"]: flags.append("tree run with both the Red and the two Greens attempted") elif enc["Red"]: branch = "Descend quickly: Red" elif enc["Orange"] or enc["Brown"]: branch = "Descend slowly: two Greens" flags.append("tree run with Orange/Brown but no Green recorded") elif hour == "Valley": if enc["Green"]: flags.append("Green recorded on a Valley run (no Greens in the valley)") meadow = [c for c in ("Orange", "Brown", "Blue") if enc[c]] or rare == "Blue (valley)" if meadow or enc["Red"] == ENC["passed"]: branch = "Passed the Red: meadow" elif enc["Red"]: branch = "Valley Red" # How the run ended outcome, outcome_color = None, None if died_early: outcome = "Died before the hour" elif len(bonded) == 1: outcome, outcome_color = "Bonded", bonded[0] elif len(bonded) > 1: outcome = "Ambiguous" flags.append("more than one dragon marked Bonded") elif burned: order = F["burn_order"]["valley" if hour == "Valley" else "tree"] outcome, outcome_color = "Incinerated", [c for c in order if c in burned][-1] if len(burned) > 1: flags.append("more than one dragon marked Incinerated") elif hour == "DIED before the hour": outcome = "Died before the hour" else: outcome = "No outcome recorded" rescued = enc["Green"] == ENC["rescued"] and outcome == "Bonded" and outcome_color in ("Orange", "Brown") attempt = parse_attempt(g("attempt")) if (rare == "Blue (valley)" and hour == "Tree") or (rare == "Shadow" and hour == "Valley"): flags.append("rare-dragon answer does not match the path") goal_raw = g("goal_v2") goal = g("goal") or (A["goal_v2"].get(goal_raw, goal_raw) if goal_raw and goal_raw != "Unknown" else None) or "(not asked/blank)" return { "source": source, "date": date, "wing": wing, "section": section, "squad": g("squad") or "Unknown", "attempt": attempt, # "I've lost count" means many attempts, so it is a later attempt, not missing "attempt_group": "First attempt" if attempt == 1 else ("Later attempt" if (attempt and attempt > 1) or "lost count" in (g("attempt") or "").lower() else "Unknown"), "goal": goal, "company": company, "left_squadmates": left_squad, "direction": direction, "first_sighting": first or "Unknown", "sighting_action": (A["sighting_action"].get(g("sighting_action")) or g("sighting_action") or "Unknown") if "sighting_action" in C else "(not asked)", "scream": scream, "resolution": resolution, "combat_first": combat or "(not recorded)", "combat_second": combat2 or "(not recorded)", "post_path": post, "hour": hour, "branch": branch, **{"enc_" + c: enc[c] for c in COLORS}, "outcome": outcome, "outcome_color": outcome_color, "rescued": rescued, "saw_blue_meadow": bool(hour == "Valley" and (enc["Blue"] or rare == "Blue (valley)")), "saw_shadow": bool(hour == "Tree" and (enc["Blue"] or enc["Black"] or rare == "Shadow")), "rare_reported": (rare or "Unknown") if "rare" in C else "(not asked)", "malek_intent": (g("malek") or "Unknown") if "malek" in C else "(not asked)", "tail": g("tail"), "data_flags": "; ".join(flags), "death_note": redact(g("death_note")), "bond_note": redact(g("bond_note")), } # ---------------------------------------------------------------- files def read_export(path): """An export as a DataFrame: the bundled JSON (data/public/form_responses.json), a CSV or the .xlsx.""" p = str(path).lower() if p.endswith(".json"): with open(path, encoding="utf-8") as f: d = json.load(f) return pd.DataFrame(d["rows"], columns=d["headers"]) if p.endswith(".csv"): return pd.read_csv(path) xl = pd.ExcelFile(path) sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0]) return pd.read_excel(xl, sheet) def code_frame(raw): """Code every row of an export. Returns (runs DataFrame with run_id, missing question keys).""" C, missing = find_columns(raw.columns) df = pd.DataFrame([code_row(r, C) for _, r in raw.iterrows()]) df.insert(0, "run_id", range(1, len(df) + 1)) return df, missing def load(path): return code_frame(read_export(path)) def main(argv=None): argv = sys.argv[1:] if argv is None else argv src = argv[0] if argv else ROOT / "data/public/form_responses.json" out = argv[1] if len(argv) > 1 else None df, missing = load(src) if missing: print("WARNING: columns not found:", missing) # a whole-number column with gaps would otherwise be written as 1.0, 2.0, ... (data/dictionary.yaml: integer) df["attempt"] = df["attempt"].astype("Int64") if out: df.to_csv(out, index=False) print(f"{len(df)} runs" + (f" -> {out}" if out else "")) print(df.outcome.value_counts().to_string()) print("rows with flags:", (df["data_flags"] != "").sum()) if __name__ == "__main__": main()