Threshing Atlas: monorepo, route graph, bot importer, daily source checks
- apps/atlas: the web page (valley map, route flow, Dragon Register) - packages/core: coding rules and statistics (engine.js) and the shared route graph (graph.js): scene graph, decision-variable graph, bot-to-form crosswalk, Beta credible intervals - packages/ingest: DragonKind bot importer to anonymised trajectories (no prose, no account identifiers); data/bot holds Ken's sample - research/python: independent reference implementation + update_sources.py - tests: JS/Python parity checks and graph/importer unit tests - CI, GitHub Pages deploy, daily check of the community sheets and route PDF - Email addresses in free-text form answers are redacted; the raw spreadsheet is not committed - MIT code licence, CC BY 4.0 data licence Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
0052d1a5fd
commit
c3d067b88c
@@ -0,0 +1,49 @@
|
||||
"""export_raw.py — convert the Google Sheets .xlsx (or .csv) export into the raw JSON the web page loads.
|
||||
|
||||
python export_raw.py ../../data/raw/form/Threshing_Routes_Responses.xlsx ../../data/public/form_responses.json
|
||||
|
||||
The JSON is a list of objects keyed by the original question text (no recoding happens here, so the page
|
||||
applies exactly the same cleaning rules to it as it would to a freshly uploaded export).
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
import re
|
||||
|
||||
import pandas as pd
|
||||
|
||||
EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
|
||||
|
||||
def load_raw(path):
|
||||
if str(path).lower().endswith(".csv"):
|
||||
return pd.read_csv(path)
|
||||
xl = pd.ExcelFile(path)
|
||||
sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0])
|
||||
return pd.read_excel(xl, sheet)
|
||||
|
||||
|
||||
def cell(v):
|
||||
if v is None or v is pd.NaT or (not isinstance(v, str) and pd.isna(v)):
|
||||
return None
|
||||
if isinstance(v, (datetime, pd.Timestamp)):
|
||||
return pd.Timestamp(v).strftime("%Y-%m-%d %H:%M:%S")
|
||||
if isinstance(v, float) and v.is_integer():
|
||||
return int(v)
|
||||
if isinstance(v, str):
|
||||
return EMAIL_RE.sub("[email removed]", v) # respondents sometimes leave an email in free text
|
||||
return v
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
src = sys.argv[1] if len(sys.argv) > 1 else "../../data/raw/form/Threshing_Routes_Responses.xlsx"
|
||||
out = sys.argv[2] if len(sys.argv) > 2 else "../../data/public/form_responses.json"
|
||||
raw = load_raw(src)
|
||||
rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()]
|
||||
# keep every header present even if a row is blank there, so column detection sees all questions
|
||||
headers = [str(c) for c in raw.columns]
|
||||
with open(out, "w", encoding="utf-8") as f:
|
||||
json.dump({"source_file": str(src).split("/")[-1], "exported": datetime.now().strftime("%Y-%m-%d"),
|
||||
"headers": headers, "rows": rows}, f, ensure_ascii=False, separators=(",", ":"))
|
||||
print(f"{len(rows)} rows -> {out}")
|
||||
Reference in New Issue
Block a user