Collection as an ETL pipeline; runs_clean.csv from every source; Report a run
- collection/pipeline.yaml declares every source (extract kind, transform, privacy rules, load target, run source); `python -m collection update|build` - extract keeps data/raw/<source>/current + previous only (git-ignored); data/manifest.json records the current raw checksum and change date - transforms are pure; validate checks runs against data/dictionary.yaml before load; load writes byte-stable files only when they change - data/public/runs_clean.csv: every run from every source with a new `source` column (reddit spreadsheet, reddit thread, reported, bot, reddit pdf); the sheet's own row label moves to `entry`; run_id is text - bot importer ported to Python (same output as bot.js on the fixture); labels and crosswalk in collection/specs/bot.yaml, written into graph.js - Reddit through the official API (result links to add by hand); the 12 result pages move to collection/manual/reddit_results.yaml - "Report a run": a GitHub issue form generated from collection/specs/report_form.yaml, collected as source "reported", plus a site tab pointing to it, the community spreadsheet and the megathread - analysis and the route model read runs_clean.csv (sources in measures.yaml); stats.json and policy.json unchanged apart from the factor's new name - daily workflow runs the pipeline; the raw store is kept between runs in the Actions cache, encrypted with RAW_ARCHIVE_KEY Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
61bc50583c
commit
3b24a11f5a
@@ -0,0 +1,133 @@
|
||||
"""Write data/croissant.json (Croissant 1.0 JSON-LD) from data/dictionary.yaml.
|
||||
|
||||
Croissant is the MLCommons metadata format for datasets: it says what each file is, its checksum, and what each
|
||||
column means, so tools (and people) can load and cite the data without reading this repo's code.
|
||||
|
||||
Written by `python -m collection build` (a test checks it is up to date).
|
||||
"""
|
||||
import json
|
||||
|
||||
from .common import ROOT, pipeline, read_yaml, sha256_file
|
||||
|
||||
OUT = "data/croissant.json"
|
||||
|
||||
CONTEXT = {
|
||||
"@language": "en",
|
||||
"@vocab": "https://schema.org/",
|
||||
"citeAs": "cr:citeAs",
|
||||
"column": "cr:column",
|
||||
"conformsTo": "dct:conformsTo",
|
||||
"cr": "http://mlcommons.org/croissant/",
|
||||
"data": {"@id": "cr:data", "@type": "@json"},
|
||||
"dataType": {"@id": "cr:dataType", "@type": "@vocab"},
|
||||
"dct": "http://purl.org/dc/terms/",
|
||||
"examples": {"@id": "cr:examples", "@type": "@json"},
|
||||
"extract": "cr:extract",
|
||||
"field": "cr:field",
|
||||
"fileObject": "cr:fileObject",
|
||||
"fileProperty": "cr:fileProperty",
|
||||
"format": "cr:format",
|
||||
"includes": "cr:includes",
|
||||
"isLiveDataset": "cr:isLiveDataset",
|
||||
"jsonPath": "cr:jsonPath",
|
||||
"key": "cr:key",
|
||||
"md5": "cr:md5",
|
||||
"parentField": "cr:parentField",
|
||||
"path": "cr:path",
|
||||
"recordSet": "cr:recordSet",
|
||||
"references": "cr:references",
|
||||
"regex": "cr:regex",
|
||||
"repeated": "cr:repeated",
|
||||
"replace": "cr:replace",
|
||||
"sc": "https://schema.org/",
|
||||
"separator": "cr:separator",
|
||||
"source": "cr:source",
|
||||
"subField": "cr:subField",
|
||||
"transform": "cr:transform",
|
||||
}
|
||||
|
||||
# Croissant has no boolean-as-text type; the CSV writes True/False, which sc:Boolean parses.
|
||||
DATA_TYPES = {"text": "sc:Text", "integer": "sc:Integer", "boolean": "sc:Boolean", "date": "sc:Date"}
|
||||
|
||||
|
||||
sha256 = sha256_file
|
||||
|
||||
|
||||
def load_dictionary():
|
||||
return read_yaml("data/dictionary.yaml")
|
||||
|
||||
|
||||
def field_description(spec):
|
||||
text = " ".join(spec["description"].split())
|
||||
if spec.get("values"):
|
||||
text += " Values: " + "; ".join(str(v) for v in spec["values"]) + "."
|
||||
return text
|
||||
|
||||
|
||||
def build(dictionary=None):
|
||||
d = dictionary or load_dictionary()
|
||||
ds = d["dataset"]
|
||||
sources = pipeline()["sources"]
|
||||
|
||||
distribution = []
|
||||
for key, spec in d["files"].items():
|
||||
path = ROOT / spec["path"]
|
||||
entry = {
|
||||
"@type": "cr:FileObject",
|
||||
"@id": key,
|
||||
"name": key,
|
||||
"description": " ".join(spec["description"].split()),
|
||||
"contentUrl": spec.get("url") or f"{ds['repository']}/blob/main/{spec['path']}",
|
||||
"encodingFormat": spec["format"],
|
||||
"sha256": sha256(path),
|
||||
"contentSize": f"{path.stat().st_size} B",
|
||||
}
|
||||
distribution.append(entry)
|
||||
|
||||
fields = [
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": f"runs/{name}",
|
||||
"name": name,
|
||||
"description": field_description(spec),
|
||||
"dataType": DATA_TYPES[spec["type"]],
|
||||
"source": {"fileObject": {"@id": "runs_clean"}, "extract": {"column": name}},
|
||||
}
|
||||
for name, spec in d["columns"].items()
|
||||
]
|
||||
|
||||
return {
|
||||
"@context": CONTEXT,
|
||||
"@type": "sc:Dataset",
|
||||
"conformsTo": "http://mlcommons.org/croissant/1.0",
|
||||
"name": ds["name"],
|
||||
"description": " ".join(ds["description"].split()),
|
||||
"url": ds["url"],
|
||||
"sameAs": ds["repository"],
|
||||
"license": ds["license"],
|
||||
"creator": {"@type": "sc:Person", "name": ds["creator"], "url": f"https://github.com/{ds['creator']}"},
|
||||
"keywords": ds["keywords"],
|
||||
"citeAs": ds["citation"],
|
||||
"isLiveDataset": True,
|
||||
"isBasedOn": [
|
||||
{"@type": "sc:CreativeWork", "name": s["label"], "url": next(iter(s["links"].values()))}
|
||||
for s in sources.values()
|
||||
# the route PDF is only fingerprinted, never redistributed; the bot's repo is private
|
||||
if (s.get("privacy") or {}).get("redistribute", True) and s.get("links")
|
||||
],
|
||||
"distribution": distribution,
|
||||
"recordSet": [
|
||||
{
|
||||
"@type": "cr:RecordSet",
|
||||
"@id": "runs",
|
||||
"name": "runs",
|
||||
"description": "One coded threshing run per row, from every source (data/public/runs_clean.csv).",
|
||||
"key": {"@id": "runs/run_id"},
|
||||
"field": fields,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def render(doc):
|
||||
return json.dumps(doc, indent=2, ensure_ascii=False) + "\n"
|
||||
Reference in New Issue
Block a user