"""The data dictionary describes every published column and value, and data/croissant.json matches it.""" import csv import json import re import sys import unittest from pathlib import Path import yaml ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT)) from collection.etl import metadata as publish_metadata # noqa: E402 with open(ROOT / "data" / "dictionary.yaml", encoding="utf-8") as f: DICT = yaml.safe_load(f) with open(ROOT / "data" / "public" / "runs_clean.csv", encoding="utf-8", newline="") as f: reader = csv.DictReader(f) HEADER = reader.fieldnames ROWS = list(reader) CHECKS = { "integer": lambda v: re.fullmatch(r"-?\d+", v), "boolean": lambda v: v in ("True", "False"), "date": lambda v: re.fullmatch(r"\d{4}-\d{2}-\d{2}", v), "text": lambda v: True, } class Dictionary(unittest.TestCase): def test_every_column_described_in_order(self): self.assertEqual(HEADER, list(DICT["columns"])) def test_every_column_has_type_and_description(self): for name, col in DICT["columns"].items(): self.assertIn(col.get("type"), CHECKS, name) self.assertTrue(col.get("description"), name) def test_values_have_the_declared_type(self): for name, col in DICT["columns"].items(): bad = {r[name] for r in ROWS if r[name] and not CHECKS[col["type"]](r[name])} self.assertFalse(bad, f"{name}: not {col['type']}: {sorted(bad)[:5]}") def test_every_value_is_listed(self): for name, col in DICT["columns"].items(): if "values" not in col: continue allowed = col["values"] # YAML reads bare Yes/No as booleans: a listed value must be written as text self.assertTrue(all(isinstance(v, str) for v in allowed), f"{name}: quote every value") bad = {r[name] for r in ROWS if r[name] and r[name] not in allowed} self.assertFalse(bad, f"{name}: values not in the dictionary: {sorted(bad)}") def test_every_published_file_exists(self): for key, f in DICT["files"].items(): self.assertTrue((ROOT / f["path"]).exists(), key) class Croissant(unittest.TestCase): def test_croissant_is_up_to_date(self): expected = publish_metadata.render(publish_metadata.build(DICT)) actual = (ROOT / "data" / "croissant.json").read_text(encoding="utf-8") self.assertEqual(actual, expected, "run python -m collection build") def test_fields_and_checksums(self): doc = json.loads((ROOT / "data" / "croissant.json").read_text(encoding="utf-8")) names = [f["name"] for f in doc["recordSet"][0]["field"]] self.assertEqual(names, HEADER) for obj in doc["distribution"]: path = ROOT / DICT["files"][obj["@id"]]["path"] self.assertEqual(obj["sha256"], publish_metadata.sha256(path), obj["@id"]) # the route PDF is not redistributed, so it is not a source of the published data self.assertNotIn("pdf", json.dumps(doc["isBasedOn"]).lower()) if __name__ == "__main__": unittest.main()