"""Validate: a table is checked against data/dictionary.yaml before it is loaded. A failure stops the load.""" import re CHECKS = { "integer": lambda v: isinstance(v, int) and not isinstance(v, bool) or bool(re.fullmatch(r"-?\d+", str(v))), "boolean": lambda v: isinstance(v, bool) or v in ("True", "False"), "date": lambda v: bool(re.fullmatch(r"\d{4}-\d{2}-\d{2}", str(v))), "text": lambda v: isinstance(v, str), } class Invalid(Exception): pass def problems(rows, dictionary, header=None): """Every way the rows break the dictionary, as readable lines (empty when the table is valid).""" cols = dictionary["columns"] out = [] if header is not None and list(header) != list(cols): out.append(f"columns differ from the dictionary: {list(header)} vs {list(cols)}") ids = [r.get("run_id") for r in rows] if len(set(ids)) != len(ids): out.append("run_id is not unique") for name, col in cols.items(): vals = [r.get(name) for r in rows] present = [v for v in vals if v is not None and v != ""] bad_type = sorted({str(v) for v in present if not CHECKS[col["type"]](v)}) if bad_type: out.append(f"{name}: not {col['type']}: {bad_type[:5]}") if "values" in col: allowed = {str(v) for v in col["values"]} bad = sorted({str(v) for v in present if str(v) not in allowed}) if bad: out.append(f"{name}: values not in the dictionary: {bad[:10]}") if col.get("required") and len(present) != len(vals): out.append(f"{name}: required, but {len(vals) - len(present)} rows are empty") return out def check(rows, dictionary, what="table"): errs = problems(rows, dictionary) if errs: raise Invalid(f"{what} does not match data/dictionary.yaml:\n " + "\n ".join(errs))