Files
ekcSchianoandClaude Opus 5.5 d606ed1651 Reddit comment runs in the route model and the odds; Black report gate wording; cite both megathreads
- planner/policy.yaml: `reddit comment` is a model source. Only runs that break a game rule (the rule_flags
  checks) are left out; partial runs are kept. Its own reporting-bias group, and out of the low-bias sample
  (riders posting a run pick the notable ones). 77 of the 82 runs are used.
- analysis/stats/measures.yaml: the site statistics use every source answered in the form's terms (form,
  "Report a run", Reddit comments).
- analysis/black/evidence.py: a shadow run is "off the gates" only when an answer contradicts a gate (3); runs
  that do not say are counted separately (14).
- croissant isBasedOn lists both megathreads (pipeline.yaml `cite`).
- Rebuilt: policy.json, stats.json, BLACK.md, black.json, croissant.json.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
2026-11-19 14:50:08 +00:00

139 lines
4.6 KiB
Python

"""Write data/croissant.json (Croissant 1.0 JSON-LD) from data/dictionary.yaml.
Croissant is the MLCommons metadata format for datasets: it says what each file is, its checksum, and what each
column means, so tools (and people) can load and cite the data without reading this repo's code.
Written by `python -m collection build` (a test checks it is up to date).
"""
import json
from . import forge
from .common import ROOT, pipeline, read_yaml, sha256_file
OUT = "data/croissant.json"
CONTEXT = {
"@language": "en",
"@vocab": "https://schema.org/",
"citeAs": "cr:citeAs",
"column": "cr:column",
"conformsTo": "dct:conformsTo",
"cr": "http://mlcommons.org/croissant/",
"data": {"@id": "cr:data", "@type": "@json"},
"dataType": {"@id": "cr:dataType", "@type": "@vocab"},
"dct": "http://purl.org/dc/terms/",
"examples": {"@id": "cr:examples", "@type": "@json"},
"extract": "cr:extract",
"field": "cr:field",
"fileObject": "cr:fileObject",
"fileProperty": "cr:fileProperty",
"format": "cr:format",
"includes": "cr:includes",
"isLiveDataset": "cr:isLiveDataset",
"jsonPath": "cr:jsonPath",
"key": "cr:key",
"md5": "cr:md5",
"parentField": "cr:parentField",
"path": "cr:path",
"recordSet": "cr:recordSet",
"references": "cr:references",
"regex": "cr:regex",
"repeated": "cr:repeated",
"replace": "cr:replace",
"sc": "https://schema.org/",
"separator": "cr:separator",
"source": "cr:source",
"subField": "cr:subField",
"transform": "cr:transform",
}
# Croissant has no boolean-as-text type; the CSV writes True/False, which sc:Boolean parses.
DATA_TYPES = {"text": "sc:Text", "integer": "sc:Integer", "boolean": "sc:Boolean", "date": "sc:Date"}
sha256 = sha256_file
def load_dictionary():
return read_yaml("data/dictionary.yaml")
def field_description(spec):
text = " ".join(spec["description"].split())
if spec.get("values"):
text += " Values: " + "; ".join(str(v) for v in spec["values"]) + "."
return text
def build(dictionary=None):
d = dictionary or load_dictionary()
ds = d["dataset"]
fg = forge.config()
sources = pipeline()["sources"]
distribution = []
for key, spec in d["files"].items():
path = ROOT / spec["path"]
entry = {
"@type": "cr:FileObject",
"@id": key,
"name": key,
"description": " ".join(spec["description"].split()),
"contentUrl": forge.file_url(spec["path"], fg),
"encodingFormat": spec["format"],
"sha256": sha256(path),
"contentSize": f"{path.stat().st_size} B",
}
distribution.append(entry)
fields = [
{
"@type": "cr:Field",
"@id": f"runs/{name}",
"name": name,
"description": field_description(spec),
"dataType": DATA_TYPES[spec["type"]],
"source": {"fileObject": {"@id": "runs_clean"}, "extract": {"column": name}},
}
for name, spec in d["columns"].items()
]
return {
"@context": CONTEXT,
"@type": "sc:Dataset",
"conformsTo": "http://mlcommons.org/croissant/1.0",
"name": ds["name"],
"description": " ".join(ds["description"].split()),
"url": fg.get("site") or forge.repo_url(fg),
"sameAs": forge.repo_url(fg),
"license": ds["license"],
"creator": {"@type": "sc:Person", "name": ds["creator"]},
"keywords": ds["keywords"],
"citeAs": ds["citation"].format(repository=forge.repo_url(fg)),
"isLiveDataset": True,
"isBasedOn": [
{"@type": "sc:CreativeWork", "url": s["links"][k],
"name": s["label"] + (f" ({k.replace('part', 'part ')})" if len(s.get("cite") or ()) > 1 else "")}
for s in sources.values()
# the route PDF is only fingerprinted, never redistributed; the bot's repo is private
if (s.get("privacy") or {}).get("redistribute", True) and s.get("links")
# `cite` (collection/pipeline.yaml) names the links the source is based on; default: the first link
for k in (s.get("cite") or [next(iter(s["links"]))])
],
"distribution": distribution,
"recordSet": [
{
"@type": "cr:RecordSet",
"@id": "runs",
"name": "runs",
"description": "One coded threshing run per row, from every source (data/public/runs_clean.csv).",
"key": {"@id": "runs/run_id"},
"field": fields,
}
],
}
def render(doc):
return json.dumps(doc, indent=2, ensure_ascii=False) + "\n"