"""Write data/croissant.json (Croissant 1.0 JSON-LD) from data/dictionary.yaml. Croissant is the MLCommons metadata format for datasets: it says what each file is, its checksum, and what each column means, so tools (and people) can load and cite the data without reading this repo's code. python3 collection/publish_metadata.py # write data/croissant.json python3 collection/publish_metadata.py --check # exit 1 if data/croissant.json is out of date """ import hashlib import json import sys from pathlib import Path import yaml ROOT = Path(__file__).resolve().parents[1] DICTIONARY = ROOT / "data" / "dictionary.yaml" SOURCES = ROOT / "data" / "sources.json" OUT = ROOT / "data" / "croissant.json" CONTEXT = { "@language": "en", "@vocab": "https://schema.org/", "citeAs": "cr:citeAs", "column": "cr:column", "conformsTo": "dct:conformsTo", "cr": "http://mlcommons.org/croissant/", "data": {"@id": "cr:data", "@type": "@json"}, "dataType": {"@id": "cr:dataType", "@type": "@vocab"}, "dct": "http://purl.org/dc/terms/", "examples": {"@id": "cr:examples", "@type": "@json"}, "extract": "cr:extract", "field": "cr:field", "fileObject": "cr:fileObject", "fileProperty": "cr:fileProperty", "format": "cr:format", "includes": "cr:includes", "isLiveDataset": "cr:isLiveDataset", "jsonPath": "cr:jsonPath", "key": "cr:key", "md5": "cr:md5", "parentField": "cr:parentField", "path": "cr:path", "recordSet": "cr:recordSet", "references": "cr:references", "regex": "cr:regex", "repeated": "cr:repeated", "replace": "cr:replace", "sc": "https://schema.org/", "separator": "cr:separator", "source": "cr:source", "subField": "cr:subField", "transform": "cr:transform", } # Croissant has no boolean-as-text type; the CSV writes True/False, which sc:Boolean parses. DATA_TYPES = {"text": "sc:Text", "integer": "sc:Integer", "boolean": "sc:Boolean", "date": "sc:Date"} def sha256(path): h = hashlib.sha256() with open(path, "rb") as f: for block in iter(lambda: f.read(1 << 20), b""): h.update(block) return h.hexdigest() def load_dictionary(): with open(DICTIONARY, encoding="utf-8") as f: return yaml.safe_load(f) def field_description(spec): text = " ".join(spec["description"].split()) if spec.get("values"): text += " Values: " + "; ".join(str(v) for v in spec["values"]) + "." return text def build(dictionary=None): d = dictionary or load_dictionary() ds = d["dataset"] with open(SOURCES, encoding="utf-8") as f: sources = json.load(f) distribution = [] for key, spec in d["files"].items(): path = ROOT / spec["path"] entry = { "@type": "cr:FileObject", "@id": key, "name": key, "description": " ".join(spec["description"].split()), "contentUrl": spec.get("url") or f"{ds['repository']}/blob/main/{spec['path']}", "encodingFormat": spec["format"], "sha256": sha256(path), "contentSize": f"{path.stat().st_size} B", } distribution.append(entry) fields = [ { "@type": "cr:Field", "@id": f"runs/{name}", "name": name, "description": field_description(spec), "dataType": DATA_TYPES[spec["type"]], "source": {"fileObject": {"@id": "runs_clean"}, "extract": {"column": name}}, } for name, spec in d["columns"].items() ] return { "@context": CONTEXT, "@type": "sc:Dataset", "conformsTo": "http://mlcommons.org/croissant/1.0", "name": ds["name"], "description": " ".join(ds["description"].split()), "url": ds["url"], "sameAs": ds["repository"], "license": ds["license"], "creator": {"@type": "sc:Person", "name": ds["creator"], "url": f"https://github.com/{ds['creator']}"}, "keywords": ds["keywords"], "citeAs": ds["citation"], "isLiveDataset": True, "isBasedOn": [ {"@type": "sc:CreativeWork", "name": s["name"], "url": s["view"]} for key, s in sources.items() if key != "pdf" # the route PDF is only fingerprinted, never redistributed ], "distribution": distribution, "recordSet": [ { "@type": "cr:RecordSet", "@id": "runs", "name": "runs", "description": "One coded threshing run per row (data/derived/runs_clean.csv).", "key": {"@id": "runs/run_id"}, "field": fields, } ], } def render(doc): return json.dumps(doc, indent=2, ensure_ascii=False) + "\n" def main(argv): text = render(build()) if "--check" in argv: if not OUT.exists() or OUT.read_text(encoding="utf-8") != text: print("data/croissant.json is out of date: run python3 collection/publish_metadata.py", file=sys.stderr) return 1 print("data/croissant.json is up to date") return 0 OUT.write_text(text, encoding="utf-8") print(f"wrote {OUT.relative_to(ROOT)}") return 0 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))