"""Read data/public/runs_clean.csv (written by `python -m collection build`) as typed records. Every analysis reads the runs from this one published file. Types come from data/dictionary.yaml: integers and booleans are parsed, empty cells are None (data_flags is "" when a row has no flags). """ import csv from pathlib import Path import yaml ROOT = Path(__file__).resolve().parents[2] PATH = ROOT / "data/public/runs_clean.csv" with open(ROOT / "data/dictionary.yaml", encoding="utf-8") as _f: TYPES = {k: v["type"] for k, v in yaml.safe_load(_f)["columns"].items()} def _parse(col, v): if v == "": return "" if col == "data_flags" else None t = TYPES.get(col, "text") if t == "integer": return int(v) if t == "boolean": return v == "True" return v def impossible(r): """A run that cannot have happened as recorded: one of its flags is in core/spec/form.yaml impossible_flags.""" from . import spec rules = spec.form()["impossible_flags"] return any(rule in f for f in (r.get("data_flags") or "").split("; ") for rule in rules) def read(path=PATH, sources=None, drop=("run_id",)): """Typed records, optionally only from some sources (the `source` column).""" with open(path, encoding="utf-8", newline="") as f: rows = [{k: _parse(k, v) for k, v in r.items() if k not in drop} for r in csv.DictReader(f)] return [r for r in rows if sources is None or r["source"] in sources]