- collection/pipeline.yaml declares every source (extract kind, transform, privacy rules, load target, run source); `python -m collection update|build` - extract keeps data/raw/<source>/current + previous only (git-ignored); data/manifest.json records the current raw checksum and change date - transforms are pure; validate checks runs against data/dictionary.yaml before load; load writes byte-stable files only when they change - data/public/runs_clean.csv: every run from every source with a new `source` column (reddit spreadsheet, reddit thread, reported, bot, reddit pdf); the sheet's own row label moves to `entry`; run_id is text - bot importer ported to Python (same output as bot.js on the fixture); labels and crosswalk in collection/specs/bot.yaml, written into graph.js - Reddit through the official API (result links to add by hand); the 12 result pages move to collection/manual/reddit_results.yaml - "Report a run": a GitHub issue form generated from collection/specs/report_form.yaml, collected as source "reported", plus a site tab pointing to it, the community spreadsheet and the megathread - analysis and the route model read runs_clean.csv (sources in measures.yaml); stats.json and policy.json unchanged apart from the factor's new name - daily workflow runs the pipeline; the raw store is kept between runs in the Actions cache, encrypted with RAW_ARCHIVE_KEY Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
76 lines
3.2 KiB
Python
76 lines
3.2 KiB
Python
"""A Google Sheets export (.xlsx or .csv) -> the public source file: the sheet's own questions and answers, with the
|
|
source's privacy rules applied. No recoding happens here (that is core/atlas/coding.py, at build time)."""
|
|
import re
|
|
from datetime import datetime
|
|
|
|
import pandas as pd
|
|
|
|
from ..common import read_yaml
|
|
|
|
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
|
|
|
|
|
def load_sheet(path, sheet_contains="form responses"):
|
|
if str(path).lower().endswith(".csv"):
|
|
return pd.read_csv(path)
|
|
xl = pd.ExcelFile(path)
|
|
sheet = next((n for n in xl.sheet_names if sheet_contains in n.lower()), xl.sheet_names[0])
|
|
return pd.read_excel(xl, sheet)
|
|
|
|
|
|
def cell(v, redact=True):
|
|
if v is None or v is pd.NaT or (not isinstance(v, str) and pd.isna(v)):
|
|
return None
|
|
if isinstance(v, (datetime, pd.Timestamp)):
|
|
return pd.Timestamp(v).strftime("%Y-%m-%d %H:%M:%S")
|
|
if isinstance(v, float) and v.is_integer():
|
|
return int(v)
|
|
if isinstance(v, str) and redact:
|
|
return EMAIL_RE.sub("[email removed]", v) # respondents sometimes leave an email in free text
|
|
return v
|
|
|
|
|
|
def extra_results(manual):
|
|
"""Dragons read from result pages linked on Reddit (collection/manual/reddit_results.yaml)."""
|
|
return [{"source": "dragonkind.com result page linked on Reddit", "url": f"https://dragonkind.com/results/{r['id']}",
|
|
"name": r["name"], "colour": r["colour"], "tail": r["tail"], "sex": r["sex"], "age": r["age"],
|
|
"lineage": r["lineage"]} for r in read_yaml(manual)["results"]]
|
|
|
|
|
|
def transform(raw_path, spec, ctx):
|
|
t, privacy = spec["transform"], spec.get("privacy") or {}
|
|
raw = load_sheet(raw_path, t.get("sheet_contains", "form responses"))
|
|
drop = [d.lower() for d in privacy.get("drop_columns_matching", [])]
|
|
raw = raw[[c for c in raw.columns if not any(d in str(c).lower() for d in drop)]]
|
|
redact = privacy.get("redact_emails", True)
|
|
rows = []
|
|
for _, r in raw.iterrows():
|
|
row = {}
|
|
for k, v in r.items():
|
|
c = cell(v, redact)
|
|
if c is not None:
|
|
row[str(k)] = c
|
|
rows.append(row)
|
|
out = {"source": spec["label"], "exported": ctx.get("changed"),
|
|
# every header, even one blank in every row, so the coding rules see every question
|
|
"headers": [str(c) for c in raw.columns], "rows": rows}
|
|
if t.get("extra"):
|
|
out["extra"] = extra_results(t["extra"])
|
|
return out
|
|
|
|
|
|
def describe(old, new):
|
|
"""One report line (plus details) comparing the previous public file with the new one."""
|
|
if old is None:
|
|
return [f"first export: {len(new['rows'])} rows"]
|
|
key = lambda r: repr(sorted(r.items())) # noqa: E731
|
|
o, n = {key(r) for r in old["rows"]}, {key(r) for r in new["rows"]}
|
|
lines = [f"{len(old['rows'])} → {len(new['rows'])} rows ({len(n - o)} new or edited, {len(o - n)} removed or edited)"]
|
|
new_q = [h for h in new["headers"] if h not in old["headers"]]
|
|
gone_q = [h for h in old["headers"] if h not in new["headers"]]
|
|
if new_q:
|
|
lines.append(f" - New questions (add them to `core/spec/form.yaml`): {', '.join(new_q)}")
|
|
if gone_q:
|
|
lines.append(f" - Questions no longer in the sheet: {', '.join(gone_q)}")
|
|
return lines
|