"""A Google Sheets export (.xlsx or .csv) -> the public source file: the sheet's own questions and answers, with the source's privacy rules applied. No recoding happens here (that is core/atlas/coding.py, at build time).""" import re from datetime import datetime import pandas as pd from ..common import read_yaml EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}") def load_sheet(path, sheet_contains="form responses"): if str(path).lower().endswith(".csv"): return pd.read_csv(path) xl = pd.ExcelFile(path) sheet = next((n for n in xl.sheet_names if sheet_contains in n.lower()), xl.sheet_names[0]) return pd.read_excel(xl, sheet) def cell(v, redact=True): if v is None or v is pd.NaT or (not isinstance(v, str) and pd.isna(v)): return None if isinstance(v, (datetime, pd.Timestamp)): return pd.Timestamp(v).strftime("%Y-%m-%d %H:%M:%S") if isinstance(v, float) and v.is_integer(): return int(v) if isinstance(v, str) and redact: return EMAIL_RE.sub("[email removed]", v) # respondents sometimes leave an email in free text return v def extra_results(manual): """Dragons read from result pages linked on Reddit (collection/manual/reddit_results.yaml).""" return [{"source": "dragonkind.com result page linked on Reddit", "url": f"https://dragonkind.com/results/{r['id']}", "name": r["name"], "colour": r["colour"], "tail": r["tail"], "sex": r["sex"], "age": r["age"], "lineage": r["lineage"]} for r in read_yaml(manual)["results"]] def transform(raw_path, spec, ctx): t, privacy = spec["transform"], spec.get("privacy") or {} raw = load_sheet(raw_path, t.get("sheet_contains", "form responses")) drop = [d.lower() for d in privacy.get("drop_columns_matching", [])] raw = raw[[c for c in raw.columns if not any(d in str(c).lower() for d in drop)]] redact = privacy.get("redact_emails", True) rows = [] for _, r in raw.iterrows(): row = {} for k, v in r.items(): c = cell(v, redact) if c is not None: row[str(k)] = c rows.append(row) out = {"source": spec["label"], "exported": ctx.get("changed"), # every header, even one blank in every row, so the coding rules see every question "headers": [str(c) for c in raw.columns], "rows": rows} if t.get("extra"): out["extra"] = extra_results(t["extra"]) return out def describe(old, new): """One report line (plus details) comparing the previous public file with the new one.""" if old is None: return [f"first export: {len(new['rows'])} rows"] key = lambda r: repr(sorted(r.items())) # noqa: E731 o, n = {key(r) for r in old["rows"]}, {key(r) for r in new["rows"]} lines = [f"{len(old['rows'])} → {len(new['rows'])} rows ({len(n - o)} new or edited, {len(o - n)} removed or edited)"] new_q = [h for h in new["headers"] if h not in old["headers"]] gone_q = [h for h in old["headers"] if h not in new["headers"]] if new_q: lines.append(f" - New questions (add them to `core/spec/form.yaml`): {', '.join(new_q)}") if gone_q: lines.append(f" - Questions no longer in the sheet: {', '.join(gone_q)}") return lines