- data/dictionary.yaml: every published file, and every column of runs_clean.csv with its type, meaning and allowed values - collection/publish_metadata.py writes data/croissant.json (Croissant 1.0) from it, with a sha256 per file; `npm run metadata`, run by the daily update - tests/core/test_dictionary.py: every column and value described, types hold, croissant.json up to date and checksums match - attempt is now written as a whole number (was 1.0, 2.0); no statistic changes - the site ships both files and the Methods text links them Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
164 lines
5.2 KiB
Python
164 lines
5.2 KiB
Python
"""Write data/croissant.json (Croissant 1.0 JSON-LD) from data/dictionary.yaml.
|
|
|
|
Croissant is the MLCommons metadata format for datasets: it says what each file is, its checksum, and what each
|
|
column means, so tools (and people) can load and cite the data without reading this repo's code.
|
|
|
|
python3 collection/publish_metadata.py # write data/croissant.json
|
|
python3 collection/publish_metadata.py --check # exit 1 if data/croissant.json is out of date
|
|
"""
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DICTIONARY = ROOT / "data" / "dictionary.yaml"
|
|
SOURCES = ROOT / "data" / "sources.json"
|
|
OUT = ROOT / "data" / "croissant.json"
|
|
|
|
CONTEXT = {
|
|
"@language": "en",
|
|
"@vocab": "https://schema.org/",
|
|
"citeAs": "cr:citeAs",
|
|
"column": "cr:column",
|
|
"conformsTo": "dct:conformsTo",
|
|
"cr": "http://mlcommons.org/croissant/",
|
|
"data": {"@id": "cr:data", "@type": "@json"},
|
|
"dataType": {"@id": "cr:dataType", "@type": "@vocab"},
|
|
"dct": "http://purl.org/dc/terms/",
|
|
"examples": {"@id": "cr:examples", "@type": "@json"},
|
|
"extract": "cr:extract",
|
|
"field": "cr:field",
|
|
"fileObject": "cr:fileObject",
|
|
"fileProperty": "cr:fileProperty",
|
|
"format": "cr:format",
|
|
"includes": "cr:includes",
|
|
"isLiveDataset": "cr:isLiveDataset",
|
|
"jsonPath": "cr:jsonPath",
|
|
"key": "cr:key",
|
|
"md5": "cr:md5",
|
|
"parentField": "cr:parentField",
|
|
"path": "cr:path",
|
|
"recordSet": "cr:recordSet",
|
|
"references": "cr:references",
|
|
"regex": "cr:regex",
|
|
"repeated": "cr:repeated",
|
|
"replace": "cr:replace",
|
|
"sc": "https://schema.org/",
|
|
"separator": "cr:separator",
|
|
"source": "cr:source",
|
|
"subField": "cr:subField",
|
|
"transform": "cr:transform",
|
|
}
|
|
|
|
# Croissant has no boolean-as-text type; the CSV writes True/False, which sc:Boolean parses.
|
|
DATA_TYPES = {"text": "sc:Text", "integer": "sc:Integer", "boolean": "sc:Boolean", "date": "sc:Date"}
|
|
|
|
|
|
def sha256(path):
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as f:
|
|
for block in iter(lambda: f.read(1 << 20), b""):
|
|
h.update(block)
|
|
return h.hexdigest()
|
|
|
|
|
|
def load_dictionary():
|
|
with open(DICTIONARY, encoding="utf-8") as f:
|
|
return yaml.safe_load(f)
|
|
|
|
|
|
def field_description(spec):
|
|
text = " ".join(spec["description"].split())
|
|
if spec.get("values"):
|
|
text += " Values: " + "; ".join(str(v) for v in spec["values"]) + "."
|
|
return text
|
|
|
|
|
|
def build(dictionary=None):
|
|
d = dictionary or load_dictionary()
|
|
ds = d["dataset"]
|
|
with open(SOURCES, encoding="utf-8") as f:
|
|
sources = json.load(f)
|
|
|
|
distribution = []
|
|
for key, spec in d["files"].items():
|
|
path = ROOT / spec["path"]
|
|
entry = {
|
|
"@type": "cr:FileObject",
|
|
"@id": key,
|
|
"name": key,
|
|
"description": " ".join(spec["description"].split()),
|
|
"contentUrl": spec.get("url") or f"{ds['repository']}/blob/main/{spec['path']}",
|
|
"encodingFormat": spec["format"],
|
|
"sha256": sha256(path),
|
|
"contentSize": f"{path.stat().st_size} B",
|
|
}
|
|
distribution.append(entry)
|
|
|
|
fields = [
|
|
{
|
|
"@type": "cr:Field",
|
|
"@id": f"runs/{name}",
|
|
"name": name,
|
|
"description": field_description(spec),
|
|
"dataType": DATA_TYPES[spec["type"]],
|
|
"source": {"fileObject": {"@id": "runs_clean"}, "extract": {"column": name}},
|
|
}
|
|
for name, spec in d["columns"].items()
|
|
]
|
|
|
|
return {
|
|
"@context": CONTEXT,
|
|
"@type": "sc:Dataset",
|
|
"conformsTo": "http://mlcommons.org/croissant/1.0",
|
|
"name": ds["name"],
|
|
"description": " ".join(ds["description"].split()),
|
|
"url": ds["url"],
|
|
"sameAs": ds["repository"],
|
|
"license": ds["license"],
|
|
"creator": {"@type": "sc:Person", "name": ds["creator"], "url": f"https://github.com/{ds['creator']}"},
|
|
"keywords": ds["keywords"],
|
|
"citeAs": ds["citation"],
|
|
"isLiveDataset": True,
|
|
"isBasedOn": [
|
|
{"@type": "sc:CreativeWork", "name": s["name"], "url": s["view"]}
|
|
for key, s in sources.items()
|
|
if key != "pdf" # the route PDF is only fingerprinted, never redistributed
|
|
],
|
|
"distribution": distribution,
|
|
"recordSet": [
|
|
{
|
|
"@type": "cr:RecordSet",
|
|
"@id": "runs",
|
|
"name": "runs",
|
|
"description": "One coded threshing run per row (data/derived/runs_clean.csv).",
|
|
"key": {"@id": "runs/run_id"},
|
|
"field": fields,
|
|
}
|
|
],
|
|
}
|
|
|
|
|
|
def render(doc):
|
|
return json.dumps(doc, indent=2, ensure_ascii=False) + "\n"
|
|
|
|
|
|
def main(argv):
|
|
text = render(build())
|
|
if "--check" in argv:
|
|
if not OUT.exists() or OUT.read_text(encoding="utf-8") != text:
|
|
print("data/croissant.json is out of date: run python3 collection/publish_metadata.py", file=sys.stderr)
|
|
return 1
|
|
print("data/croissant.json is up to date")
|
|
return 0
|
|
OUT.write_text(text, encoding="utf-8")
|
|
print(f"wrote {OUT.relative_to(ROOT)}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|