"""export_raw.py — convert the Google Sheets .xlsx (or .csv) export into the raw JSON the web page loads. python export_raw.py ../../data/raw/form/Threshing_Routes_Responses.xlsx ../../data/public/form_responses.json The JSON is a list of objects keyed by the original question text (no recoding happens here, so the page applies exactly the same cleaning rules to it as it would to a freshly uploaded export). """ import json import sys from datetime import datetime import re import pandas as pd EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}") def load_raw(path): if str(path).lower().endswith(".csv"): return pd.read_csv(path) xl = pd.ExcelFile(path) sheet = next((n for n in xl.sheet_names if "form responses" in n.lower()), xl.sheet_names[0]) return pd.read_excel(xl, sheet) def cell(v): if v is None or v is pd.NaT or (not isinstance(v, str) and pd.isna(v)): return None if isinstance(v, (datetime, pd.Timestamp)): return pd.Timestamp(v).strftime("%Y-%m-%d %H:%M:%S") if isinstance(v, float) and v.is_integer(): return int(v) if isinstance(v, str): return EMAIL_RE.sub("[email removed]", v) # respondents sometimes leave an email in free text return v if __name__ == "__main__": src = sys.argv[1] if len(sys.argv) > 1 else "../../data/raw/form/Threshing_Routes_Responses.xlsx" out = sys.argv[2] if len(sys.argv) > 2 else "../../data/public/form_responses.json" raw = load_raw(src) rows = [{str(k): cell(v) for k, v in r.items() if cell(v) is not None} for _, r in raw.iterrows()] # keep every header present even if a row is blank there, so column detection sees all questions headers = [str(c) for c in raw.columns] with open(out, "w", encoding="utf-8") as f: json.dump({"source_file": str(src).split("/")[-1], "exported": datetime.now().strftime("%Y-%m-%d"), "headers": headers, "rows": rows}, f, ensure_ascii=False, separators=(",", ":")) print(f"{len(rows)} rows -> {out}")