"""Write data/croissant.json (Croissant 1.0 JSON-LD) from data/dictionary.yaml. Croissant is the MLCommons metadata format for datasets: it says what each file is, its checksum, and what each column means, so tools (and people) can load and cite the data without reading this repo's code. Written by `python -m collection build` (a test checks it is up to date). """ import json from . import forge from .common import ROOT, pipeline, read_yaml, sha256_file OUT = "data/croissant.json" CONTEXT = { "@language": "en", "@vocab": "https://schema.org/", "citeAs": "cr:citeAs", "column": "cr:column", "conformsTo": "dct:conformsTo", "cr": "http://mlcommons.org/croissant/", "data": {"@id": "cr:data", "@type": "@json"}, "dataType": {"@id": "cr:dataType", "@type": "@vocab"}, "dct": "http://purl.org/dc/terms/", "examples": {"@id": "cr:examples", "@type": "@json"}, "extract": "cr:extract", "field": "cr:field", "fileObject": "cr:fileObject", "fileProperty": "cr:fileProperty", "format": "cr:format", "includes": "cr:includes", "isLiveDataset": "cr:isLiveDataset", "jsonPath": "cr:jsonPath", "key": "cr:key", "md5": "cr:md5", "parentField": "cr:parentField", "path": "cr:path", "recordSet": "cr:recordSet", "references": "cr:references", "regex": "cr:regex", "repeated": "cr:repeated", "replace": "cr:replace", "sc": "https://schema.org/", "separator": "cr:separator", "source": "cr:source", "subField": "cr:subField", "transform": "cr:transform", } # Croissant has no boolean-as-text type; the CSV writes True/False, which sc:Boolean parses. DATA_TYPES = {"text": "sc:Text", "integer": "sc:Integer", "boolean": "sc:Boolean", "date": "sc:Date"} sha256 = sha256_file def load_dictionary(): return read_yaml("data/dictionary.yaml") def field_description(spec): text = " ".join(spec["description"].split()) if spec.get("values"): text += " Values: " + "; ".join(str(v) for v in spec["values"]) + "." return text def build(dictionary=None): d = dictionary or load_dictionary() ds = d["dataset"] fg = forge.config() sources = pipeline()["sources"] distribution = [] for key, spec in d["files"].items(): path = ROOT / spec["path"] entry = { "@type": "cr:FileObject", "@id": key, "name": key, "description": " ".join(spec["description"].split()), "contentUrl": forge.file_url(spec["path"], fg), "encodingFormat": spec["format"], "sha256": sha256(path), "contentSize": f"{path.stat().st_size} B", } distribution.append(entry) fields = [ { "@type": "cr:Field", "@id": f"runs/{name}", "name": name, "description": field_description(spec), "dataType": DATA_TYPES[spec["type"]], "source": {"fileObject": {"@id": "runs_clean"}, "extract": {"column": name}}, } for name, spec in d["columns"].items() ] return { "@context": CONTEXT, "@type": "sc:Dataset", "conformsTo": "http://mlcommons.org/croissant/1.0", "name": ds["name"], "description": " ".join(ds["description"].split()), "url": fg.get("site") or forge.repo_url(fg), "sameAs": forge.repo_url(fg), "license": ds["license"], "creator": {"@type": "sc:Person", "name": ds["creator"]}, "keywords": ds["keywords"], "citeAs": ds["citation"].format(repository=forge.repo_url(fg)), "isLiveDataset": True, "isBasedOn": [ {"@type": "sc:CreativeWork", "name": s["label"], "url": next(iter(s["links"].values()))} for s in sources.values() # the route PDF is only fingerprinted, never redistributed; the bot's repo is private if (s.get("privacy") or {}).get("redistribute", True) and s.get("links") ], "distribution": distribution, "recordSet": [ { "@type": "cr:RecordSet", "@id": "runs", "name": "runs", "description": "One coded threshing run per row, from every source (data/public/runs_clean.csv).", "key": {"@id": "runs/run_id"}, "field": fields, } ], } def render(doc): return json.dumps(doc, indent=2, ensure_ascii=False) + "\n"