site built from d7befc8e
This commit is contained in:
@@ -0,0 +1,323 @@
|
||||
"""Community form rows -> one coded run per row.
|
||||
|
||||
The tables (how each question is found, what every answer means, which answers count as missing) live in
|
||||
core/spec/form.yaml. This module holds the logic that combines answers into a run: which way the rider went,
|
||||
which branch they took after the hour, how the run ended, and which rows look inconsistent (data_flags).
|
||||
|
||||
python -m atlas.coding [form_responses.json|.xlsx|.csv] [out.csv]
|
||||
|
||||
Prints a summary of the coded export (default: data/public/form_responses.json), and writes the coded runs to
|
||||
out.csv when given. The published runs (every source) are written by `python -m collection build`.
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from . import spec
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
F = spec.form()
|
||||
A = F["answers"]
|
||||
COLORS = F["colours"]
|
||||
MISSING = frozenset(F["missing_values"])
|
||||
ENC = F["encounter_answers"]
|
||||
_EMAIL = re.compile(F["redact"]["pattern"])
|
||||
|
||||
|
||||
def is_missing(v):
|
||||
"""True for answers that carry no information about the choice (blank, "Unsure", not asked...)."""
|
||||
return v is None or v == "" or v in MISSING or (isinstance(v, float) and v != v)
|
||||
|
||||
|
||||
def redact(v):
|
||||
"""Replace email addresses in free text."""
|
||||
return v if v is None else _EMAIL.sub(F["redact"]["replacement"], str(v))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- reading cells
|
||||
|
||||
def find_columns(cols):
|
||||
"""Map each question key in form.yaml to the export's column header. Returns (columns, missing keys)."""
|
||||
out = {}
|
||||
low = {c: str(c).lower() for c in cols}
|
||||
for key, col in F["columns"].items():
|
||||
words = col["match"]
|
||||
hits = [c for c in cols if all((w[1:] not in low[c]) if w[0] == "!" else (w in low[c]) for w in words)]
|
||||
if hits:
|
||||
out[key] = hits[0]
|
||||
missing = [k for k, col in F["columns"].items()
|
||||
if k not in out and not col.get("optional") and col.get("replaced_by") not in out]
|
||||
return out, missing
|
||||
|
||||
|
||||
def blank(v):
|
||||
if v is None or v is pd.NaT:
|
||||
return True
|
||||
if not isinstance(v, str):
|
||||
try:
|
||||
if pd.isna(v):
|
||||
return True
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return str(v).strip() in ("", "nan", "NaN", "NaT")
|
||||
|
||||
|
||||
def text(v):
|
||||
return None if blank(v) else str(v).strip()
|
||||
|
||||
|
||||
def parse_timestamp(v):
|
||||
"""Form submissions carry a timestamp; bulk-pasted rows carry a label such as 'Data Import'.
|
||||
Returns 'YYYY-MM-DD' or None."""
|
||||
if isinstance(v, (datetime, pd.Timestamp)) and not pd.isna(v):
|
||||
return pd.Timestamp(v).strftime("%Y-%m-%d")
|
||||
if isinstance(v, (int, float)) and not isinstance(v, bool) and 40000 < v < 80000: # Excel serial date
|
||||
return (pd.Timestamp("1899-12-30") + pd.Timedelta(days=int(v))).strftime("%Y-%m-%d")
|
||||
t = text(v)
|
||||
if not t:
|
||||
return None
|
||||
m = re.match(r"^(\d{4})-(\d{2})-(\d{2})", t)
|
||||
if m:
|
||||
return f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
|
||||
m = re.match(r"^(\d{1,2})/(\d{1,2})/(\d{4})", t) # Google Sheets CSV export: M/D/YYYY
|
||||
if m:
|
||||
return f"{m.group(3)}-{int(m.group(1)):02d}-{int(m.group(2)):02d}"
|
||||
return None
|
||||
|
||||
|
||||
def parse_attempt(v):
|
||||
v = text(v)
|
||||
if v is None:
|
||||
return None
|
||||
t = v.lower()
|
||||
if "lost count" in t:
|
||||
return None
|
||||
for w, n in F["attempt_words"].items():
|
||||
if t.startswith(w):
|
||||
return n
|
||||
m = re.search(r"\d+", t)
|
||||
return int(m.group()) if m else None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- one row -> one run
|
||||
|
||||
def code_row(r, C):
|
||||
"""Code one export row. r: mapping header -> cell; C: from find_columns()."""
|
||||
g = lambda k: text(r[C[k]]) if k in C else None # noqa: E731
|
||||
flags = []
|
||||
ts = r[C["timestamp"]] if "timestamp" in C else None
|
||||
d = parse_timestamp(ts)
|
||||
source, date = ("Form", d) if d else ((text(ts) or "Unknown import"), None)
|
||||
|
||||
wings = [w for w in ("First", "Second", "Third", "Fourth") if g("wing_" + w)]
|
||||
wing = wings[0] if len(wings) == 1 else ("Multiple" if wings else "Unknown")
|
||||
section = g("wing_" + wings[0]) if len(wings) == 1 else None
|
||||
if len(wings) > 1:
|
||||
flags.append("multiple wings")
|
||||
|
||||
if "first_dir" in C:
|
||||
# Redesigned form: one direction question and one colour question. The older per-direction colour
|
||||
# columns are still in the sheet for some rows and are used only to fill gaps and to cross-check.
|
||||
direction = A["direction"].get(g("first_dir"))
|
||||
first = g("first_colour")
|
||||
legacy = [d for d in ("West", "Center") if g("leg_" + d)]
|
||||
if not direction and len(legacy) == 1:
|
||||
direction = legacy[0]
|
||||
flags.append("direction taken from the older sighting column")
|
||||
leg = g("leg_" + direction) if direction else None
|
||||
if not first and leg:
|
||||
first = leg
|
||||
elif first and leg and first != leg and first != "Unsure" and leg != "Unsure":
|
||||
flags.append("first sighting differs between the old and new columns")
|
||||
if not direction:
|
||||
direction = "Unknown"
|
||||
flags.append("direction missing")
|
||||
else:
|
||||
dirs = [d for d in ("East", "West", "Center") if g("dir_" + d)]
|
||||
if len(dirs) == 1:
|
||||
direction, first = dirs[0], g("dir_" + dirs[0])
|
||||
else:
|
||||
direction, first = ("Multiple" if dirs else "Unknown"), None
|
||||
flags.append("direction " + ("multiple" if dirs else "missing"))
|
||||
|
||||
company_raw = g("company") or "Unknown"
|
||||
suffix = F["company_left_suffix"]
|
||||
left_squad = company_raw.endswith(suffix)
|
||||
company = company_raw[: -len(suffix)] if left_squad else company_raw
|
||||
if not left_squad and company in ("1 Squadmate", "2 Squadmates") and not (date and date >= F["left_squadmates_asked_from"]):
|
||||
left_squad = None # the form did not offer "left them" yet: unknown, not "stayed together"
|
||||
alias = F.get("first_sighting_alias") or {}
|
||||
if first in alias:
|
||||
flags.append(f"first sighting answered as {first!r}")
|
||||
first = alias[first]
|
||||
pools = F["sighting_pool"]
|
||||
pool = list(pools.get(company, {}).get(direction, [])) + (list(pools["Alone"].get(direction, [])) if left_squad else [])
|
||||
if first and first != "Unsure" and pool and first not in pool:
|
||||
flags.append(f"first sighting {first} not in documented pool for {company}/{direction}")
|
||||
|
||||
cadet_raw = g("cadet")
|
||||
combat = combat2 = None
|
||||
if "cadet_actions" in C:
|
||||
act = g("cadet_actions")
|
||||
a = A["cadet_actions"].get(act, ["Unknown", None, None, "Unknown"])
|
||||
if act and act not in A["cadet_actions"]:
|
||||
flags.append("unrecognised cadet answer")
|
||||
scream, combat, combat2 = a[0], a[1], a[2]
|
||||
out = A["cadet_outcome"].get(cadet_raw)
|
||||
if a[3] is not None:
|
||||
resolution = a[3]
|
||||
if out == "DIED in combat" and not resolution.startswith("DIED"):
|
||||
flags.append("died in combat on a path without combat")
|
||||
else:
|
||||
resolution = out or "Unknown"
|
||||
else:
|
||||
scream, resolution = A["cadet_legacy"].get(cadet_raw, ["Unknown", "Unknown"])
|
||||
if cadet_raw and cadet_raw not in A["cadet_legacy"]:
|
||||
flags.append("unrecognised cadet answer")
|
||||
combat = A["combat_legacy"].get(g("combat")) if g("combat") else None
|
||||
|
||||
post_raw = g("post")
|
||||
if scream == "Investigate & help" and not resolution.startswith("DIED"):
|
||||
post = post_raw if post_raw in ("Forest", "West", "East") else "Unknown"
|
||||
elif scream in ("Ignore the scream", "Investigate & leave"):
|
||||
post = "(no fork)" # the form allowed an answer here; the game has no fork on this branch
|
||||
else:
|
||||
post = None
|
||||
|
||||
hour = A["hour"].get(g("hour"))
|
||||
died_early = scream.startswith("DIED") or resolution.startswith("DIED")
|
||||
if died_early:
|
||||
hour = None
|
||||
elif hour is None:
|
||||
hour = "Unknown"
|
||||
|
||||
enc = {c: g("enc_" + c) for c in COLORS}
|
||||
rare = A["rare"].get(g("rare"))
|
||||
bonded = [c for c in COLORS if enc[c] == ENC["bonded"]]
|
||||
burned = [c for c in COLORS if enc[c] == ENC["burned"]]
|
||||
|
||||
# Which encounter branch did the run take after the hour?
|
||||
branch = None
|
||||
if hour == "Tree":
|
||||
if enc["Blue"] or enc["Black"] or rare == "Shadow":
|
||||
branch = "Shadow (rare)"
|
||||
elif enc["Red"] == ENC["passed"] and enc["Green"]:
|
||||
# only climbing down quickly puts you above the Red: passing on it and meeting the Greens means the
|
||||
# rider let it pass
|
||||
branch = "Descend quickly: let the Red pass"
|
||||
elif enc["Green"]:
|
||||
branch = "Descend slowly: two Greens"
|
||||
if enc["Red"]:
|
||||
flags.append("tree run with both the Red and the two Greens attempted")
|
||||
elif enc["Red"]:
|
||||
branch = "Descend quickly: Red"
|
||||
elif enc["Orange"] or enc["Brown"]:
|
||||
branch = "Descend slowly: two Greens"
|
||||
flags.append("tree run with Orange/Brown but no Green recorded")
|
||||
elif hour == "Valley":
|
||||
if enc["Green"]:
|
||||
flags.append("Green recorded on a Valley run (no Greens in the valley)")
|
||||
meadow = [c for c in ("Orange", "Brown", "Blue") if enc[c]] or rare == "Blue (valley)"
|
||||
if meadow or enc["Red"] == ENC["passed"]:
|
||||
branch = "Passed the Red: meadow"
|
||||
elif enc["Red"]:
|
||||
branch = "Valley Red"
|
||||
|
||||
# How the run ended
|
||||
outcome, outcome_color = None, None
|
||||
if died_early:
|
||||
outcome = "Died before the hour"
|
||||
elif len(bonded) == 1:
|
||||
outcome, outcome_color = "Bonded", bonded[0]
|
||||
elif len(bonded) > 1:
|
||||
outcome = "Ambiguous"
|
||||
flags.append("more than one dragon marked Bonded")
|
||||
elif burned:
|
||||
order = F["burn_order"]["valley" if hour == "Valley" else "tree"]
|
||||
outcome, outcome_color = "Incinerated", [c for c in order if c in burned][-1]
|
||||
if len(burned) > 1:
|
||||
flags.append("more than one dragon marked Incinerated")
|
||||
elif hour == "DIED before the hour":
|
||||
outcome = "Died before the hour"
|
||||
else:
|
||||
outcome = "No outcome recorded"
|
||||
|
||||
rescued = enc["Green"] == ENC["rescued"] and outcome == "Bonded" and outcome_color in ("Orange", "Brown")
|
||||
attempt = parse_attempt(g("attempt"))
|
||||
if (rare == "Blue (valley)" and hour == "Tree") or (rare == "Shadow" and hour == "Valley"):
|
||||
flags.append("rare-dragon answer does not match the path")
|
||||
goal_raw = g("goal_v2")
|
||||
goal = g("goal") or (A["goal_v2"].get(goal_raw, goal_raw) if goal_raw and goal_raw != "Unknown" else None) or "(not asked/blank)"
|
||||
return {
|
||||
"source": source, "date": date, "wing": wing, "section": section,
|
||||
"squad": A["squad_any_case"].get((g("squad") or "").strip().lower(), g("squad") or "Unknown"), "attempt": attempt,
|
||||
# "I've lost count" means many attempts, so it is a later attempt, not missing
|
||||
"attempt_group": "First attempt" if attempt == 1 else ("Later attempt" if (attempt and attempt > 1) or "lost count" in (g("attempt") or "").lower() else "Unknown"),
|
||||
"goal": goal, "company": company, "left_squadmates": left_squad,
|
||||
"direction": direction, "first_sighting": first or "Unknown",
|
||||
"sighting_action": (A["sighting_action"].get(g("sighting_action")) or g("sighting_action") or "Unknown") if "sighting_action" in C else "(not asked)",
|
||||
"scream": scream, "resolution": resolution, "combat_first": combat or "(not recorded)",
|
||||
"combat_second": combat2 or "(not recorded)",
|
||||
"post_path": post, "hour": hour, "branch": branch,
|
||||
**{"enc_" + c: enc[c] for c in COLORS},
|
||||
"outcome": outcome, "outcome_color": outcome_color, "rescued": rescued,
|
||||
"saw_blue_meadow": bool(hour == "Valley" and (enc["Blue"] or rare == "Blue (valley)")),
|
||||
"saw_shadow": bool(hour == "Tree" and (enc["Blue"] or enc["Black"] or rare == "Shadow")),
|
||||
"rare_reported": (rare or "Unknown") if "rare" in C else "(not asked)",
|
||||
"malek_intent": (g("malek") or "Unknown") if "malek" in C else "(not asked)",
|
||||
"tail": g("tail"),
|
||||
"data_flags": "; ".join(flags),
|
||||
"death_note": redact(g("death_note")), "bond_note": redact(g("bond_note")),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- files
|
||||
|
||||
def read_export(path):
|
||||
"""An export as a DataFrame: the bundled JSON (data/public/form_responses.json), a CSV or the .xlsx."""
|
||||
p = str(path).lower()
|
||||
if p.endswith(".json"):
|
||||
with open(path, encoding="utf-8") as f:
|
||||
d = json.load(f)
|
||||
return pd.DataFrame(d["rows"], columns=d["headers"])
|
||||
if p.endswith(".csv"):
|
||||
return pd.read_csv(path)
|
||||
xl = pd.ExcelFile(path)
|
||||
sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0])
|
||||
return pd.read_excel(xl, sheet)
|
||||
|
||||
|
||||
def code_frame(raw):
|
||||
"""Code every row of an export. Returns (runs DataFrame with run_id, missing question keys)."""
|
||||
C, missing = find_columns(raw.columns)
|
||||
df = pd.DataFrame([code_row(r, C) for _, r in raw.iterrows()])
|
||||
df.insert(0, "run_id", range(1, len(df) + 1))
|
||||
return df, missing
|
||||
|
||||
|
||||
def load(path):
|
||||
return code_frame(read_export(path))
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = sys.argv[1:] if argv is None else argv
|
||||
src = argv[0] if argv else ROOT / "data/public/form_responses.json"
|
||||
out = argv[1] if len(argv) > 1 else None
|
||||
df, missing = load(src)
|
||||
if missing:
|
||||
print("WARNING: columns not found:", missing)
|
||||
# a whole-number column with gaps would otherwise be written as 1.0, 2.0, ... (data/dictionary.yaml: integer)
|
||||
df["attempt"] = df["attempt"].astype("Int64")
|
||||
if out:
|
||||
df.to_csv(out, index=False)
|
||||
print(f"{len(df)} runs" + (f" -> {out}" if out else ""))
|
||||
print(df.outcome.value_counts().to_string())
|
||||
print("rows with flags:", (df["data_flags"] != "").sum())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user