Step 5: data dictionary and Croissant metadata
- data/dictionary.yaml: every published file, and every column of runs_clean.csv with its type, meaning and allowed values - collection/publish_metadata.py writes data/croissant.json (Croissant 1.0) from it, with a sha256 per file; `npm run metadata`, run by the daily update - tests/core/test_dictionary.py: every column and value described, types hold, croissant.json up to date and checksums match - attempt is now written as a whole number (was 1.0, 2.0); no statistic changes - the site ships both files and the Methods text links them Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
11b4a0398e
commit
63b5d40e54
@@ -30,6 +30,7 @@ jobs:
|
||||
node analysis/network/build-graph.js
|
||||
npm run policy
|
||||
npm run stats
|
||||
npm run metadata
|
||||
npm test
|
||||
# The PR is opened with ATLAS_BOT_TOKEN (a fine-grained token, see docs/REPO_SETTINGS.md) when it is set:
|
||||
# changes made with the built-in GITHUB_TOKEN do not start other workflows, so its merge would not
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
"""Write data/croissant.json (Croissant 1.0 JSON-LD) from data/dictionary.yaml.
|
||||
|
||||
Croissant is the MLCommons metadata format for datasets: it says what each file is, its checksum, and what each
|
||||
column means, so tools (and people) can load and cite the data without reading this repo's code.
|
||||
|
||||
python3 collection/publish_metadata.py # write data/croissant.json
|
||||
python3 collection/publish_metadata.py --check # exit 1 if data/croissant.json is out of date
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DICTIONARY = ROOT / "data" / "dictionary.yaml"
|
||||
SOURCES = ROOT / "data" / "sources.json"
|
||||
OUT = ROOT / "data" / "croissant.json"
|
||||
|
||||
CONTEXT = {
|
||||
"@language": "en",
|
||||
"@vocab": "https://schema.org/",
|
||||
"citeAs": "cr:citeAs",
|
||||
"column": "cr:column",
|
||||
"conformsTo": "dct:conformsTo",
|
||||
"cr": "http://mlcommons.org/croissant/",
|
||||
"data": {"@id": "cr:data", "@type": "@json"},
|
||||
"dataType": {"@id": "cr:dataType", "@type": "@vocab"},
|
||||
"dct": "http://purl.org/dc/terms/",
|
||||
"examples": {"@id": "cr:examples", "@type": "@json"},
|
||||
"extract": "cr:extract",
|
||||
"field": "cr:field",
|
||||
"fileObject": "cr:fileObject",
|
||||
"fileProperty": "cr:fileProperty",
|
||||
"format": "cr:format",
|
||||
"includes": "cr:includes",
|
||||
"isLiveDataset": "cr:isLiveDataset",
|
||||
"jsonPath": "cr:jsonPath",
|
||||
"key": "cr:key",
|
||||
"md5": "cr:md5",
|
||||
"parentField": "cr:parentField",
|
||||
"path": "cr:path",
|
||||
"recordSet": "cr:recordSet",
|
||||
"references": "cr:references",
|
||||
"regex": "cr:regex",
|
||||
"repeated": "cr:repeated",
|
||||
"replace": "cr:replace",
|
||||
"sc": "https://schema.org/",
|
||||
"separator": "cr:separator",
|
||||
"source": "cr:source",
|
||||
"subField": "cr:subField",
|
||||
"transform": "cr:transform",
|
||||
}
|
||||
|
||||
# Croissant has no boolean-as-text type; the CSV writes True/False, which sc:Boolean parses.
|
||||
DATA_TYPES = {"text": "sc:Text", "integer": "sc:Integer", "boolean": "sc:Boolean", "date": "sc:Date"}
|
||||
|
||||
|
||||
def sha256(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for block in iter(lambda: f.read(1 << 20), b""):
|
||||
h.update(block)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def load_dictionary():
|
||||
with open(DICTIONARY, encoding="utf-8") as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
|
||||
def field_description(spec):
|
||||
text = " ".join(spec["description"].split())
|
||||
if spec.get("values"):
|
||||
text += " Values: " + "; ".join(str(v) for v in spec["values"]) + "."
|
||||
return text
|
||||
|
||||
|
||||
def build(dictionary=None):
|
||||
d = dictionary or load_dictionary()
|
||||
ds = d["dataset"]
|
||||
with open(SOURCES, encoding="utf-8") as f:
|
||||
sources = json.load(f)
|
||||
|
||||
distribution = []
|
||||
for key, spec in d["files"].items():
|
||||
path = ROOT / spec["path"]
|
||||
entry = {
|
||||
"@type": "cr:FileObject",
|
||||
"@id": key,
|
||||
"name": key,
|
||||
"description": " ".join(spec["description"].split()),
|
||||
"contentUrl": spec.get("url") or f"{ds['repository']}/blob/main/{spec['path']}",
|
||||
"encodingFormat": spec["format"],
|
||||
"sha256": sha256(path),
|
||||
"contentSize": f"{path.stat().st_size} B",
|
||||
}
|
||||
distribution.append(entry)
|
||||
|
||||
fields = [
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": f"runs/{name}",
|
||||
"name": name,
|
||||
"description": field_description(spec),
|
||||
"dataType": DATA_TYPES[spec["type"]],
|
||||
"source": {"fileObject": {"@id": "runs_clean"}, "extract": {"column": name}},
|
||||
}
|
||||
for name, spec in d["columns"].items()
|
||||
]
|
||||
|
||||
return {
|
||||
"@context": CONTEXT,
|
||||
"@type": "sc:Dataset",
|
||||
"conformsTo": "http://mlcommons.org/croissant/1.0",
|
||||
"name": ds["name"],
|
||||
"description": " ".join(ds["description"].split()),
|
||||
"url": ds["url"],
|
||||
"sameAs": ds["repository"],
|
||||
"license": ds["license"],
|
||||
"creator": {"@type": "sc:Person", "name": ds["creator"], "url": f"https://github.com/{ds['creator']}"},
|
||||
"keywords": ds["keywords"],
|
||||
"citeAs": ds["citation"],
|
||||
"isLiveDataset": True,
|
||||
"isBasedOn": [
|
||||
{"@type": "sc:CreativeWork", "name": s["name"], "url": s["view"]}
|
||||
for key, s in sources.items()
|
||||
if key != "pdf" # the route PDF is only fingerprinted, never redistributed
|
||||
],
|
||||
"distribution": distribution,
|
||||
"recordSet": [
|
||||
{
|
||||
"@type": "cr:RecordSet",
|
||||
"@id": "runs",
|
||||
"name": "runs",
|
||||
"description": "One coded threshing run per row (data/derived/runs_clean.csv).",
|
||||
"key": {"@id": "runs/run_id"},
|
||||
"field": fields,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def render(doc):
|
||||
return json.dumps(doc, indent=2, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def main(argv):
|
||||
text = render(build())
|
||||
if "--check" in argv:
|
||||
if not OUT.exists() or OUT.read_text(encoding="utf-8") != text:
|
||||
print("data/croissant.json is out of date: run python3 collection/publish_metadata.py", file=sys.stderr)
|
||||
return 1
|
||||
print("data/croissant.json is up to date")
|
||||
return 0
|
||||
OUT.write_text(text, encoding="utf-8")
|
||||
print(f"wrote {OUT.relative_to(ROOT)}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -304,6 +304,8 @@ def main(argv=None):
|
||||
df, missing = load(src)
|
||||
if missing:
|
||||
print("WARNING: columns not found:", missing)
|
||||
# a whole-number column with gaps would otherwise be written as 1.0, 2.0, ... (data/dictionary.yaml: integer)
|
||||
df["attempt"] = df["attempt"].astype("Int64")
|
||||
df.to_csv(out, index=False)
|
||||
print(f"{len(df)} runs -> {out}")
|
||||
print(df.outcome.value_counts().to_string())
|
||||
|
||||
@@ -8,3 +8,11 @@
|
||||
| Reddit megathreads (r/fourthwing, Dragonkind bonding Parts 3 and 4) | qualitative; routes cited in the valley map | manual |
|
||||
| DragonKind bot exports | `bot/runs-YYYY-MM-DD.jsonl` | by pull request, see `docs/BOT_DATA_PROTOCOL.md` |
|
||||
| Discord | not yet | planned, with admin consent and anonymised usernames |
|
||||
|
||||
## What each column means
|
||||
|
||||
`data/dictionary.yaml` describes every published file and every column of `data/derived/runs_clean.csv`: its type,
|
||||
what it means and every value it can take. `data/croissant.json` says the same in the
|
||||
[Croissant](https://docs.mlcommons.org/croissant/) metadata format (with a sha256 checksum for every file), so
|
||||
dataset tools can load and cite the data. After changing the data or the dictionary, run `npm run metadata`; the
|
||||
tests fail if a column or value is not described or the metadata is out of date.
|
||||
|
||||
@@ -0,0 +1,724 @@
|
||||
{
|
||||
"@context": {
|
||||
"@language": "en",
|
||||
"@vocab": "https://schema.org/",
|
||||
"citeAs": "cr:citeAs",
|
||||
"column": "cr:column",
|
||||
"conformsTo": "dct:conformsTo",
|
||||
"cr": "http://mlcommons.org/croissant/",
|
||||
"data": {
|
||||
"@id": "cr:data",
|
||||
"@type": "@json"
|
||||
},
|
||||
"dataType": {
|
||||
"@id": "cr:dataType",
|
||||
"@type": "@vocab"
|
||||
},
|
||||
"dct": "http://purl.org/dc/terms/",
|
||||
"examples": {
|
||||
"@id": "cr:examples",
|
||||
"@type": "@json"
|
||||
},
|
||||
"extract": "cr:extract",
|
||||
"field": "cr:field",
|
||||
"fileObject": "cr:fileObject",
|
||||
"fileProperty": "cr:fileProperty",
|
||||
"format": "cr:format",
|
||||
"includes": "cr:includes",
|
||||
"isLiveDataset": "cr:isLiveDataset",
|
||||
"jsonPath": "cr:jsonPath",
|
||||
"key": "cr:key",
|
||||
"md5": "cr:md5",
|
||||
"parentField": "cr:parentField",
|
||||
"path": "cr:path",
|
||||
"recordSet": "cr:recordSet",
|
||||
"references": "cr:references",
|
||||
"regex": "cr:regex",
|
||||
"repeated": "cr:repeated",
|
||||
"replace": "cr:replace",
|
||||
"sc": "https://schema.org/",
|
||||
"separator": "cr:separator",
|
||||
"source": "cr:source",
|
||||
"subField": "cr:subField",
|
||||
"transform": "cr:transform"
|
||||
},
|
||||
"@type": "sc:Dataset",
|
||||
"conformsTo": "http://mlcommons.org/croissant/1.0",
|
||||
"name": "Threshing Atlas",
|
||||
"description": "Community-logged runs of the Dragonkind \"threshing\" (the dragon-bonding game based on Fourth Wing), coded into one record per run: every choice, what the game rolled, and how the run ended. Also the route model built from them, the statistics shown on the Threshing Atlas page, and the route graph shared with a DragonKind bot.",
|
||||
"url": "https://ekcschiano.github.io/threshing-atlas/",
|
||||
"sameAs": "https://github.com/ekcSchiano/threshing-atlas",
|
||||
"license": "https://creativecommons.org/licenses/by/4.0/",
|
||||
"creator": {
|
||||
"@type": "sc:Person",
|
||||
"name": "ekcSchiano",
|
||||
"url": "https://github.com/ekcSchiano"
|
||||
},
|
||||
"keywords": [
|
||||
"Dragonkind",
|
||||
"Fourth Wing",
|
||||
"threshing",
|
||||
"decision tree",
|
||||
"route planning",
|
||||
"community data"
|
||||
],
|
||||
"citeAs": "ekcSchiano and contributors (2026). Threshing Atlas. https://github.com/ekcSchiano/threshing-atlas",
|
||||
"isLiveDataset": true,
|
||||
"isBasedOn": [
|
||||
{
|
||||
"@type": "sc:CreativeWork",
|
||||
"name": "Threshing Routes form responses (u/ClumsyBadger)",
|
||||
"url": "https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0"
|
||||
},
|
||||
{
|
||||
"@type": "sc:CreativeWork",
|
||||
"name": "Unofficial Dragon Register responses",
|
||||
"url": "https://docs.google.com/spreadsheets/d/1KMiE9sec0l597Pzue1OhGYf07NF4howtQ_lPeqx7jXQ"
|
||||
}
|
||||
],
|
||||
"distribution": [
|
||||
{
|
||||
"@type": "cr:FileObject",
|
||||
"@id": "runs_clean",
|
||||
"name": "runs_clean",
|
||||
"description": "One coded run per row of the community form (core/atlas/coding.py, rules in core/spec/form.yaml).",
|
||||
"contentUrl": "https://ekcschiano.github.io/threshing-atlas/data/runs_clean.csv",
|
||||
"encodingFormat": "text/csv",
|
||||
"sha256": "ef34b36881b6bc0f57453e425b1c7d76155e6572463aff0d7e89b9857823f7db",
|
||||
"contentSize": "1991643 B"
|
||||
},
|
||||
{
|
||||
"@type": "cr:FileObject",
|
||||
"@id": "form_responses",
|
||||
"name": "form_responses",
|
||||
"description": "The community \"Threshing Routes\" form export as published (headers and rows), with email addresses in free-text answers redacted. The source of runs_clean.",
|
||||
"contentUrl": "https://github.com/ekcSchiano/threshing-atlas/blob/main/data/public/form_responses.json",
|
||||
"encodingFormat": "application/json",
|
||||
"sha256": "4c5791f78ce81f5e88f1fb99d933acd06d8223ff9a573e96b6fcb3a6fd9c8d22",
|
||||
"contentSize": "8293395 B"
|
||||
},
|
||||
{
|
||||
"@type": "cr:FileObject",
|
||||
"@id": "register",
|
||||
"name": "register",
|
||||
"description": "The Unofficial Dragon Register (which dragon each rider bonded), without rider names.",
|
||||
"contentUrl": "https://ekcschiano.github.io/threshing-atlas/data/register.json",
|
||||
"encodingFormat": "application/json",
|
||||
"sha256": "f889c7e18068a934eb6872efda9b43599f4870d2e42bfdb3d447405976d59df8",
|
||||
"contentSize": "111095 B"
|
||||
},
|
||||
{
|
||||
"@type": "cr:FileObject",
|
||||
"@id": "stats",
|
||||
"name": "stats",
|
||||
"description": "Every number the Atlas page shows, for six filter presets (analysis/stats/site_stats.py; what is measured is analysis/stats/measures.yaml).",
|
||||
"contentUrl": "https://ekcschiano.github.io/threshing-atlas/data/stats.json",
|
||||
"encodingFormat": "application/json",
|
||||
"sha256": "d48b1da5176d6e85f6ee093cfb4f50dfbc6f71cfa7fec8ecb49a7c32c14f6d53",
|
||||
"contentSize": "4269825 B"
|
||||
},
|
||||
{
|
||||
"@type": "cr:FileObject",
|
||||
"@id": "policy",
|
||||
"name": "policy",
|
||||
"description": "The route model: the game (core/spec/game.yaml), the count tables behind every roll, and the best decision tree for every dragon colour under two goals, with posterior intervals (planner/build_policy.py).",
|
||||
"contentUrl": "https://ekcschiano.github.io/threshing-atlas/data/policy.json",
|
||||
"encodingFormat": "application/json",
|
||||
"sha256": "26c3adad2236493b4a77b7d46454fe01c84e8f750f0f03b6e7dbe968af32ea4e",
|
||||
"contentSize": "617892 B"
|
||||
},
|
||||
{
|
||||
"@type": "cr:FileObject",
|
||||
"@id": "graph",
|
||||
"name": "graph",
|
||||
"description": "The route graph over every source (form and bot), counted per source (analysis/network/build-graph.js).",
|
||||
"contentUrl": "https://ekcschiano.github.io/threshing-atlas/data/graph.json",
|
||||
"encodingFormat": "application/json",
|
||||
"sha256": "043227ff7d858fa8e73e7447f4cd7171bb216ab8d203b036df056d8ed542e446",
|
||||
"contentSize": "48634 B"
|
||||
}
|
||||
],
|
||||
"recordSet": [
|
||||
{
|
||||
"@type": "cr:RecordSet",
|
||||
"@id": "runs",
|
||||
"name": "runs",
|
||||
"description": "One coded threshing run per row (data/derived/runs_clean.csv).",
|
||||
"key": {
|
||||
"@id": "runs/run_id"
|
||||
},
|
||||
"field": [
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/run_id",
|
||||
"name": "run_id",
|
||||
"description": "Row number in the form export (1 = first row).",
|
||||
"dataType": "sc:Integer",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "run_id"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/source",
|
||||
"name": "source",
|
||||
"description": "\"Form\" for a form submission (it has a timestamp). Bulk-pasted logs keep the label they were pasted with; they record every run of the players who kept them, so they are the least selectively reported rows. Values: Form; Data Import; BulkUpload3; Manual Upload; Historical import; Unknown import.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "source"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/date",
|
||||
"name": "date",
|
||||
"description": "Date of a form submission (YYYY-MM-DD). Empty for bulk-pasted rows.",
|
||||
"dataType": "sc:Date",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "date"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/wing",
|
||||
"name": "wing",
|
||||
"description": "The rider's wing Values: First; Second; Third; Fourth; Multiple; Unknown.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "wing"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/section",
|
||||
"name": "section",
|
||||
"description": "The rider's section within the wing (free text as answered).",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "section"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/squad",
|
||||
"name": "squad",
|
||||
"description": "The rider's squad. Values: First; Second; Third; Don't know; Unknown.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "squad"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/attempt",
|
||||
"name": "attempt",
|
||||
"description": "Which attempt this was for the account (1 = first). Empty if not given.",
|
||||
"dataType": "sc:Integer",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "attempt"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/attempt_group",
|
||||
"name": "attempt_group",
|
||||
"description": "First attempt or a later one (\"I've lost count\" counts as later). Values: First attempt; Later attempt; Unknown.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "attempt_group"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/goal",
|
||||
"name": "goal",
|
||||
"description": "Which dragon the rider was trying to bond (older form) or why they did the run (redesigned form). Values: Red; Green; Orange; Brown; Blue; Black; Blue or Black; Red (tree); Any (for fun); Daily run data; (not asked/blank).",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "goal"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/company",
|
||||
"name": "company",
|
||||
"description": "Who entered the valley with the rider. Values: Alone; 1 Squadmate; 2 Squadmates; Unknown.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "company"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/left_squadmates",
|
||||
"name": "left_squadmates",
|
||||
"description": "Whether a rider who entered with squadmates later left them. Only asked from 2026-10-27: earlier and undated squad runs are empty (unknown).",
|
||||
"dataType": "sc:Boolean",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "left_squadmates"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/direction",
|
||||
"name": "direction",
|
||||
"description": "First direction. Values: East; West; Center; Multiple; Unknown.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "direction"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/first_sighting",
|
||||
"name": "first_sighting",
|
||||
"description": "The first dragon that found the rider (a roll of the game). Values: Red; Brown; Green; Orange; Unsure; Unknown.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "first_sighting"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/sighting_action",
|
||||
"name": "sighting_action",
|
||||
"description": "How the rider reacted to the first dragon (redesigned form only). Values: Hold still; Respect / face it; Retreat / run; Hide; Weapon; Meet its gaze; Protect squadmate; Approach; Stand alone; Unknown; (not asked).",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "sighting_action"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/scream",
|
||||
"name": "scream",
|
||||
"description": "What the rider did about the scream, or that they died before it. Values: Investigate & help; Investigate & leave; Ignore the scream; DIED at first sighting; Unknown.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "scream"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/resolution",
|
||||
"name": "resolution",
|
||||
"description": "How the cadet encounter ended. Values: Killed the bullies yourself; Squadmate's revenge; Left bullies to the dragons; Tourniquet; Left injured cadet; Took their weapons; Left them to it; DIED in combat; DIED at first sighting; Unknown.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "resolution"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/combat_first",
|
||||
"name": "combat_first",
|
||||
"description": "The rider's first move in the bully fight. Values: Assess the situation; Even the numbers (smaller cadet); Put yourself in the middle (bigger cadet); (not recorded).",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "combat_first"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/combat_second",
|
||||
"name": "combat_second",
|
||||
"description": "After assessing Values: Engage the smaller cadet; Daggers or not (protect squadmate); (not recorded).",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "combat_second"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/post_path",
|
||||
"name": "post_path",
|
||||
"description": "The fork after the bully fight (only that path has one). Values: Forest; West; East; (no fork); Unknown.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "post_path"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/hour",
|
||||
"name": "hour",
|
||||
"description": "After an hour's walk Values: Tree; Valley; DIED before the hour; Unknown.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "hour"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/branch",
|
||||
"name": "branch",
|
||||
"description": "Which bonding encounter the run reached after the hour, worked out from the dragons recorded. Values: Shadow (rare); Descend quickly: Red; Descend quickly: let the Red pass; Descend slowly: two Greens; Valley Red; Passed the Red: meadow.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "branch"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/enc_Red",
|
||||
"name": "enc_Red",
|
||||
"description": "The rider's answer for this dragon in the bonding grid. Values: Bonded!; Incinerated; Changed my mind; Saved by another.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "enc_Red"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/enc_Green",
|
||||
"name": "enc_Green",
|
||||
"description": "The rider's answer for this dragon in the bonding grid. Values: Bonded!; Incinerated; Changed my mind; Saved by another.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "enc_Green"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/enc_Orange",
|
||||
"name": "enc_Orange",
|
||||
"description": "The rider's answer for this dragon in the bonding grid. Values: Bonded!; Incinerated; Changed my mind; Saved by another.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "enc_Orange"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/enc_Brown",
|
||||
"name": "enc_Brown",
|
||||
"description": "The rider's answer for this dragon in the bonding grid. Values: Bonded!; Incinerated; Changed my mind; Saved by another.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "enc_Brown"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/enc_Blue",
|
||||
"name": "enc_Blue",
|
||||
"description": "The rider's answer for this dragon in the bonding grid. Values: Bonded!; Incinerated; Changed my mind; Saved by another.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "enc_Blue"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/enc_Black",
|
||||
"name": "enc_Black",
|
||||
"description": "The rider's answer for this dragon in the bonding grid. Values: Bonded!; Incinerated; Changed my mind; Saved by another.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "enc_Black"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/outcome",
|
||||
"name": "outcome",
|
||||
"description": "How the run ended. Values: Bonded; Incinerated; Died before the hour; No outcome recorded; Ambiguous.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "outcome"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/outcome_color",
|
||||
"name": "outcome_color",
|
||||
"description": "The dragon bonded Values: Red; Green; Orange; Brown; Blue; Black.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "outcome_color"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/rescued",
|
||||
"name": "rescued",
|
||||
"description": "The Greens' encounter ended with an Orange or Brown swooping in and bonding the rider.",
|
||||
"dataType": "sc:Boolean",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "rescued"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/saw_blue_meadow",
|
||||
"name": "saw_blue_meadow",
|
||||
"description": "A Blue waited in the meadow (valley runs).",
|
||||
"dataType": "sc:Boolean",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "saw_blue_meadow"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/saw_shadow",
|
||||
"name": "saw_shadow",
|
||||
"description": "The shadow appeared at the tree (tree runs).",
|
||||
"dataType": "sc:Boolean",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "saw_shadow"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/rare_reported",
|
||||
"name": "rare_reported",
|
||||
"description": "The rider's own answer to \"did you meet a rare dragon\". Values: No; Blue (valley); Shadow; Unknown; (not asked).",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "rare_reported"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/malek_intent",
|
||||
"name": "malek_intent",
|
||||
"description": "Whether the rider would seek Malek (die on purpose) rather than bond a dragon they did not want. Values: Yes; No; Unknown; (not asked).",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "malek_intent"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/tail",
|
||||
"name": "tail",
|
||||
"description": "The tail of the dragon bonded (redesigned form). Values: Clubtail; Daggertail; Morningstartail; Scorpiontail; Swordtail.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "tail"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/data_flags",
|
||||
"name": "data_flags",
|
||||
"description": "Inconsistencies found while coding the row",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "data_flags"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/death_note",
|
||||
"name": "death_note",
|
||||
"description": "Free-text note from riders who died (email addresses removed).",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "death_note"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/bond_note",
|
||||
"name": "bond_note",
|
||||
"description": "Free-text note from riders who bonded (email addresses removed).",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
"@id": "runs_clean"
|
||||
},
|
||||
"extract": {
|
||||
"column": "bond_note"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
+3281
-3281
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,130 @@
|
||||
# Data dictionary: every published file and every column of the coded runs.
|
||||
#
|
||||
# collection/publish_metadata.py builds data/croissant.json (Croissant, the MLCommons metadata format) from this
|
||||
# file, data/sources.json and the licences. tests/core/test_dictionary.py checks that every column of the coded runs
|
||||
# is described here and that every value it takes is listed.
|
||||
#
|
||||
# Missing answers: "Unknown", "Unsure", "(not asked)" and the other values in core/spec/form.yaml missing_values carry
|
||||
# no information about the choice; every analysis treats them, and empty cells, as missing.
|
||||
version: 1
|
||||
|
||||
dataset:
|
||||
name: Threshing Atlas
|
||||
description: >-
|
||||
Community-logged runs of the Dragonkind "threshing" (the dragon-bonding game based on Fourth Wing), coded into
|
||||
one record per run: every choice, what the game rolled, and how the run ended. Also the route model built from
|
||||
them, the statistics shown on the Threshing Atlas page, and the route graph shared with a DragonKind bot.
|
||||
url: https://ekcschiano.github.io/threshing-atlas/
|
||||
repository: https://github.com/ekcSchiano/threshing-atlas
|
||||
license: https://creativecommons.org/licenses/by/4.0/
|
||||
code_license: MIT
|
||||
creator: ekcSchiano
|
||||
keywords: [Dragonkind, Fourth Wing, threshing, decision tree, route planning, community data]
|
||||
citation: "ekcSchiano and contributors (2026). Threshing Atlas. https://github.com/ekcSchiano/threshing-atlas"
|
||||
# Third-party sources keep their own terms (DATA_LICENSE.md); the route PDF is never redistributed.
|
||||
|
||||
files:
|
||||
runs_clean:
|
||||
path: data/derived/runs_clean.csv
|
||||
url: https://ekcschiano.github.io/threshing-atlas/data/runs_clean.csv
|
||||
format: text/csv
|
||||
description: One coded run per row of the community form (core/atlas/coding.py, rules in core/spec/form.yaml).
|
||||
form_responses:
|
||||
path: data/public/form_responses.json
|
||||
format: application/json
|
||||
description: >-
|
||||
The community "Threshing Routes" form export as published (headers and rows), with email addresses in
|
||||
free-text answers redacted. The source of runs_clean.
|
||||
register:
|
||||
path: data/public/register.json
|
||||
url: https://ekcschiano.github.io/threshing-atlas/data/register.json
|
||||
format: application/json
|
||||
description: The Unofficial Dragon Register (which dragon each rider bonded), without rider names.
|
||||
stats:
|
||||
path: data/public/stats.json
|
||||
url: https://ekcschiano.github.io/threshing-atlas/data/stats.json
|
||||
format: application/json
|
||||
description: >-
|
||||
Every number the Atlas page shows, for six filter presets (analysis/stats/site_stats.py; what is measured is
|
||||
analysis/stats/measures.yaml).
|
||||
policy:
|
||||
path: data/public/policy.json
|
||||
url: https://ekcschiano.github.io/threshing-atlas/data/policy.json
|
||||
format: application/json
|
||||
description: >-
|
||||
The route model: the game (core/spec/game.yaml), the count tables behind every roll, and the best decision
|
||||
tree for every dragon colour under two goals, with posterior intervals (planner/build_policy.py).
|
||||
graph:
|
||||
path: data/public/graph.json
|
||||
url: https://ekcschiano.github.io/threshing-atlas/data/graph.json
|
||||
format: application/json
|
||||
description: The route graph over every source (form and bot), counted per source (analysis/network/build-graph.js).
|
||||
|
||||
# The columns of runs_clean. type: text, integer, boolean or date. values: every value the column may take
|
||||
# (empty cells are always allowed and mean "does not apply" or "not recorded").
|
||||
columns:
|
||||
run_id: { type: integer, description: Row number in the form export (1 = first row). }
|
||||
source:
|
||||
type: text
|
||||
description: >-
|
||||
"Form" for a form submission (it has a timestamp). Bulk-pasted logs keep the label they were pasted with;
|
||||
they record every run of the players who kept them, so they are the least selectively reported rows.
|
||||
values: [Form, Data Import, BulkUpload3, Manual Upload, Historical import, Unknown import]
|
||||
date: { type: date, description: Date of a form submission (YYYY-MM-DD). Empty for bulk-pasted rows. }
|
||||
wing: { type: text, description: The rider's wing, from the form., values: [First, Second, Third, Fourth, Multiple, Unknown] }
|
||||
section: { type: text, description: The rider's section within the wing (free text as answered). }
|
||||
squad: { type: text, description: The rider's squad., values: [First, Second, Third, "Don't know", Unknown] }
|
||||
attempt: { type: integer, description: Which attempt this was for the account (1 = first). Empty if not given. }
|
||||
attempt_group: { type: text, description: First attempt or a later one ("I've lost count" counts as later)., values: [First attempt, Later attempt, Unknown] }
|
||||
goal:
|
||||
type: text
|
||||
description: Which dragon the rider was trying to bond (older form) or why they did the run (redesigned form).
|
||||
values: [Red, Green, Orange, Brown, Blue, Black, Blue or Black, Red (tree), Any (for fun), Daily run data, "(not asked/blank)"]
|
||||
company: { type: text, description: Who entered the valley with the rider., values: [Alone, 1 Squadmate, 2 Squadmates, Unknown] }
|
||||
left_squadmates:
|
||||
type: boolean
|
||||
description: >-
|
||||
Whether a rider who entered with squadmates later left them. Only asked from 2026-10-27: earlier and undated
|
||||
squad runs are empty (unknown).
|
||||
direction: { type: text, description: First direction., values: [East, West, Center, Multiple, Unknown] }
|
||||
first_sighting: { type: text, description: The first dragon that found the rider (a roll of the game)., values: [Red, Brown, Green, Orange, Unsure, Unknown] }
|
||||
sighting_action:
|
||||
type: text
|
||||
description: How the rider reacted to the first dragon (redesigned form only).
|
||||
values: [Hold still, Respect / face it, Retreat / run, Hide, Weapon, Meet its gaze, Protect squadmate, Approach, Stand alone, Unknown, "(not asked)"]
|
||||
scream:
|
||||
type: text
|
||||
description: What the rider did about the scream, or that they died before it.
|
||||
values: [Investigate & help, Investigate & leave, Ignore the scream, DIED at first sighting, Unknown]
|
||||
resolution:
|
||||
type: text
|
||||
description: How the cadet encounter ended.
|
||||
values: [Killed the bullies yourself, Squadmate's revenge, Left bullies to the dragons, Tourniquet, Left injured cadet, Took their weapons, Left them to it, DIED in combat, DIED at first sighting, Unknown]
|
||||
combat_first:
|
||||
type: text
|
||||
description: The rider's first move in the bully fight.
|
||||
values: [Assess the situation, Even the numbers (smaller cadet), Put yourself in the middle (bigger cadet), "(not recorded)"]
|
||||
combat_second: { type: text, description: After assessing, what the rider did about the smaller cadet's dagger., values: [Engage the smaller cadet, Daggers or not (protect squadmate), "(not recorded)"] }
|
||||
post_path: { type: text, description: The fork after the bully fight (only that path has one)., values: [Forest, West, East, (no fork), Unknown] }
|
||||
hour: { type: text, description: After an hour's walk, the tallest tree or the valley centre., values: [Tree, Valley, DIED before the hour, Unknown] }
|
||||
branch:
|
||||
type: text
|
||||
description: Which bonding encounter the run reached after the hour, worked out from the dragons recorded.
|
||||
values: [Shadow (rare), "Descend quickly: Red", "Descend quickly: let the Red pass", "Descend slowly: two Greens", Valley Red, "Passed the Red: meadow"]
|
||||
enc_Red: &enc { type: text, description: The rider's answer for this dragon in the bonding grid., values: [Bonded!, Incinerated, Changed my mind, Saved by another] }
|
||||
enc_Green: *enc
|
||||
enc_Orange: *enc
|
||||
enc_Brown: *enc
|
||||
enc_Blue: *enc
|
||||
enc_Black: *enc
|
||||
outcome: { type: text, description: How the run ended., values: [Bonded, Incinerated, Died before the hour, No outcome recorded, Ambiguous] }
|
||||
outcome_color: { type: text, description: The dragon bonded, or the one that incinerated the rider (the last met)., values: [Red, Green, Orange, Brown, Blue, Black] }
|
||||
rescued: { type: boolean, description: The Greens' encounter ended with an Orange or Brown swooping in and bonding the rider. }
|
||||
saw_blue_meadow: { type: boolean, description: A Blue waited in the meadow (valley runs). }
|
||||
saw_shadow: { type: boolean, description: The shadow appeared at the tree (tree runs). }
|
||||
rare_reported: { type: text, description: The rider's own answer to "did you meet a rare dragon"., values: ["No", Blue (valley), Shadow, Unknown, "(not asked)"] }
|
||||
malek_intent: { type: text, description: Whether the rider would seek Malek (die on purpose) rather than bond a dragon they did not want., values: ["Yes", "No", Unknown, "(not asked)"] }
|
||||
tail: { type: text, description: The tail of the dragon bonded (redesigned form)., values: [Clubtail, Daggertail, Morningstartail, Scorpiontail, Swordtail] }
|
||||
data_flags: { type: text, description: Inconsistencies found while coding the row, separated by "; ". Empty when none. }
|
||||
death_note: { type: text, description: Free-text note from riders who died (email addresses removed). }
|
||||
bond_note: { type: text, description: Free-text note from riders who bonded (email addresses removed). }
|
||||
+4
-1
@@ -38,4 +38,7 @@ share.
|
||||
`measures.yaml`, `data/public/stats.json`); the upload button and the click-to-select route map are gone; a
|
||||
small JS advisor (`core/solver.js`, about 250 lines) reads the game and the Python-built model from `policy.json`
|
||||
and replaces the 900-line JS solver; `engine.js` is gone.
|
||||
5. `data/dictionary.yaml` for every published column, and Croissant metadata for the datasets.
|
||||
5. **Done.** `data/dictionary.yaml` describes every published file and every column of the coded runs (type, meaning,
|
||||
allowed values); `collection/publish_metadata.py` writes `data/croissant.json` (Croissant 1.0) from it, with a
|
||||
sha256 for every file. `tests/core/test_dictionary.py` fails when a column or value is undescribed or the metadata
|
||||
is stale. It also caught `attempt` being written as `1.0`, `2.0`: now a whole number.
|
||||
|
||||
+2
-1
@@ -13,7 +13,8 @@
|
||||
"import-bot": "node collection/bot/import-bot.js",
|
||||
"check-sources": "python3 collection/sources/update_sources.py",
|
||||
"policy": "python3 planner/build_policy.py && python3 planner/policy_report.py",
|
||||
"stats": "python3 analysis/stats/site_stats.py"
|
||||
"stats": "python3 analysis/stats/site_stats.py",
|
||||
"metadata": "python3 collection/publish_metadata.py"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
|
||||
@@ -603,7 +603,7 @@ a { color: var(--accent); text-underline-offset: 2px; }
|
||||
<section id="data-sec" aria-labelledby="h-data">
|
||||
<div class="section-head"><span class="eyebrow">Where the numbers come from</span><h2 id="h-data">Data</h2></div>
|
||||
<div class="data-status" id="data-status"><span class="loading">Loading data…</span></div>
|
||||
<p class="small muted">The data refresh every day from the community sources. Every number on this page is computed in Python (<code>analysis/stats/site_stats.py</code>) from the coded runs and published as <a href="data/stats.json">data/stats.json</a>; the coded runs themselves are <a href="data/runs_clean.csv" download>data/runs_clean.csv</a>.</p>
|
||||
<p class="small muted">The data refresh every day from the community sources. Every number on this page is computed in Python (<code>analysis/stats/site_stats.py</code>) from the coded runs and published as <a href="data/stats.json">data/stats.json</a>; the coded runs themselves are <a href="data/runs_clean.csv" download>data/runs_clean.csv</a>. Every column is described in <a href="data/dictionary.yaml">data/dictionary.yaml</a>, and <a href="data/croissant.json">data/croissant.json</a> carries the same description in the Croissant metadata format, with a checksum for every file.</p>
|
||||
</section>
|
||||
<section id="share" aria-labelledby="h-share">
|
||||
<div class="section-head"><span class="eyebrow">Open by design</span><h2 id="h-share">Share & reproduce</h2>
|
||||
|
||||
@@ -26,6 +26,7 @@ copy("core/solver.js", "solver.js");
|
||||
// The numbers the page shows (all computed in Python) and the published data
|
||||
for (const f of ["stats.json", "register.json", "graph.json", "policy.json"]) copy(`data/public/${f}`, `data/${f}`);
|
||||
copy("data/derived/runs_clean.csv", "data/runs_clean.csv");
|
||||
for (const f of ["croissant.json", "dictionary.yaml"]) copy(`data/${f}`, `data/${f}`);
|
||||
// Source files shown in the page's "view the code" panel
|
||||
copy("core/atlas/coding.py", "python/coding.py");
|
||||
copy("core/spec/form.yaml", "python/form.yaml");
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
"""The data dictionary describes every published column and value, and data/croissant.json matches it."""
|
||||
import csv
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
spec = importlib.util.spec_from_file_location("publish_metadata", ROOT / "collection" / "publish_metadata.py")
|
||||
publish_metadata = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(publish_metadata)
|
||||
|
||||
with open(ROOT / "data" / "dictionary.yaml", encoding="utf-8") as f:
|
||||
DICT = yaml.safe_load(f)
|
||||
with open(ROOT / "data" / "derived" / "runs_clean.csv", encoding="utf-8", newline="") as f:
|
||||
reader = csv.DictReader(f)
|
||||
HEADER = reader.fieldnames
|
||||
ROWS = list(reader)
|
||||
|
||||
CHECKS = {
|
||||
"integer": lambda v: re.fullmatch(r"-?\d+", v),
|
||||
"boolean": lambda v: v in ("True", "False"),
|
||||
"date": lambda v: re.fullmatch(r"\d{4}-\d{2}-\d{2}", v),
|
||||
"text": lambda v: True,
|
||||
}
|
||||
|
||||
|
||||
class Dictionary(unittest.TestCase):
|
||||
def test_every_column_described_in_order(self):
|
||||
self.assertEqual(HEADER, list(DICT["columns"]))
|
||||
|
||||
def test_every_column_has_type_and_description(self):
|
||||
for name, col in DICT["columns"].items():
|
||||
self.assertIn(col.get("type"), CHECKS, name)
|
||||
self.assertTrue(col.get("description"), name)
|
||||
|
||||
def test_values_have_the_declared_type(self):
|
||||
for name, col in DICT["columns"].items():
|
||||
bad = {r[name] for r in ROWS if r[name] and not CHECKS[col["type"]](r[name])}
|
||||
self.assertFalse(bad, f"{name}: not {col['type']}: {sorted(bad)[:5]}")
|
||||
|
||||
def test_every_value_is_listed(self):
|
||||
for name, col in DICT["columns"].items():
|
||||
if "values" not in col:
|
||||
continue
|
||||
allowed = col["values"]
|
||||
# YAML reads bare Yes/No as booleans: a listed value must be written as text
|
||||
self.assertTrue(all(isinstance(v, str) for v in allowed), f"{name}: quote every value")
|
||||
bad = {r[name] for r in ROWS if r[name] and r[name] not in allowed}
|
||||
self.assertFalse(bad, f"{name}: values not in the dictionary: {sorted(bad)}")
|
||||
|
||||
def test_every_published_file_exists(self):
|
||||
for key, f in DICT["files"].items():
|
||||
self.assertTrue((ROOT / f["path"]).exists(), key)
|
||||
|
||||
|
||||
class Croissant(unittest.TestCase):
|
||||
def test_croissant_is_up_to_date(self):
|
||||
expected = publish_metadata.render(publish_metadata.build(DICT))
|
||||
actual = (ROOT / "data" / "croissant.json").read_text(encoding="utf-8")
|
||||
self.assertEqual(actual, expected, "run python3 collection/publish_metadata.py")
|
||||
|
||||
def test_fields_and_checksums(self):
|
||||
doc = json.loads((ROOT / "data" / "croissant.json").read_text(encoding="utf-8"))
|
||||
names = [f["name"] for f in doc["recordSet"][0]["field"]]
|
||||
self.assertEqual(names, HEADER)
|
||||
for obj in doc["distribution"]:
|
||||
path = ROOT / DICT["files"][obj["@id"]]["path"]
|
||||
self.assertEqual(obj["sha256"], publish_metadata.sha256(path), obj["@id"])
|
||||
# the route PDF is not redistributed, so it is not a source of the published data
|
||||
self.assertNotIn("pdf", json.dumps(doc["isBasedOn"]).lower())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user