Step 5: data dictionary and Croissant metadata

- data/dictionary.yaml: every published file, and every column of runs_clean.csv
  with its type, meaning and allowed values
- collection/publish_metadata.py writes data/croissant.json (Croissant 1.0) from
  it, with a sha256 per file; `npm run metadata`, run by the daily update
- tests/core/test_dictionary.py: every column and value described, types hold,
  croissant.json up to date and checksums match
- attempt is now written as a whole number (was 1.0, 2.0); no statistic changes
- the site ships both files and the Methods text links them

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
This commit is contained in:
ekcSchiano
2026-11-04 01:37:10 +00:00
co-authored by Claude Opus 5.5
parent 11b4a0398e
commit 63b5d40e54
12 changed files with 4395 additions and 3284 deletions
+1
View File
@@ -30,6 +30,7 @@ jobs:
node analysis/network/build-graph.js node analysis/network/build-graph.js
npm run policy npm run policy
npm run stats npm run stats
npm run metadata
npm test npm test
# The PR is opened with ATLAS_BOT_TOKEN (a fine-grained token, see docs/REPO_SETTINGS.md) when it is set: # The PR is opened with ATLAS_BOT_TOKEN (a fine-grained token, see docs/REPO_SETTINGS.md) when it is set:
# changes made with the built-in GITHUB_TOKEN do not start other workflows, so its merge would not # changes made with the built-in GITHUB_TOKEN do not start other workflows, so its merge would not
+163
View File
@@ -0,0 +1,163 @@
"""Write data/croissant.json (Croissant 1.0 JSON-LD) from data/dictionary.yaml.
Croissant is the MLCommons metadata format for datasets: it says what each file is, its checksum, and what each
column means, so tools (and people) can load and cite the data without reading this repo's code.
python3 collection/publish_metadata.py # write data/croissant.json
python3 collection/publish_metadata.py --check # exit 1 if data/croissant.json is out of date
"""
import hashlib
import json
import sys
from pathlib import Path
import yaml
ROOT = Path(__file__).resolve().parents[1]
DICTIONARY = ROOT / "data" / "dictionary.yaml"
SOURCES = ROOT / "data" / "sources.json"
OUT = ROOT / "data" / "croissant.json"
CONTEXT = {
"@language": "en",
"@vocab": "https://schema.org/",
"citeAs": "cr:citeAs",
"column": "cr:column",
"conformsTo": "dct:conformsTo",
"cr": "http://mlcommons.org/croissant/",
"data": {"@id": "cr:data", "@type": "@json"},
"dataType": {"@id": "cr:dataType", "@type": "@vocab"},
"dct": "http://purl.org/dc/terms/",
"examples": {"@id": "cr:examples", "@type": "@json"},
"extract": "cr:extract",
"field": "cr:field",
"fileObject": "cr:fileObject",
"fileProperty": "cr:fileProperty",
"format": "cr:format",
"includes": "cr:includes",
"isLiveDataset": "cr:isLiveDataset",
"jsonPath": "cr:jsonPath",
"key": "cr:key",
"md5": "cr:md5",
"parentField": "cr:parentField",
"path": "cr:path",
"recordSet": "cr:recordSet",
"references": "cr:references",
"regex": "cr:regex",
"repeated": "cr:repeated",
"replace": "cr:replace",
"sc": "https://schema.org/",
"separator": "cr:separator",
"source": "cr:source",
"subField": "cr:subField",
"transform": "cr:transform",
}
# Croissant has no boolean-as-text type; the CSV writes True/False, which sc:Boolean parses.
DATA_TYPES = {"text": "sc:Text", "integer": "sc:Integer", "boolean": "sc:Boolean", "date": "sc:Date"}
def sha256(path):
h = hashlib.sha256()
with open(path, "rb") as f:
for block in iter(lambda: f.read(1 << 20), b""):
h.update(block)
return h.hexdigest()
def load_dictionary():
with open(DICTIONARY, encoding="utf-8") as f:
return yaml.safe_load(f)
def field_description(spec):
text = " ".join(spec["description"].split())
if spec.get("values"):
text += " Values: " + "; ".join(str(v) for v in spec["values"]) + "."
return text
def build(dictionary=None):
d = dictionary or load_dictionary()
ds = d["dataset"]
with open(SOURCES, encoding="utf-8") as f:
sources = json.load(f)
distribution = []
for key, spec in d["files"].items():
path = ROOT / spec["path"]
entry = {
"@type": "cr:FileObject",
"@id": key,
"name": key,
"description": " ".join(spec["description"].split()),
"contentUrl": spec.get("url") or f"{ds['repository']}/blob/main/{spec['path']}",
"encodingFormat": spec["format"],
"sha256": sha256(path),
"contentSize": f"{path.stat().st_size} B",
}
distribution.append(entry)
fields = [
{
"@type": "cr:Field",
"@id": f"runs/{name}",
"name": name,
"description": field_description(spec),
"dataType": DATA_TYPES[spec["type"]],
"source": {"fileObject": {"@id": "runs_clean"}, "extract": {"column": name}},
}
for name, spec in d["columns"].items()
]
return {
"@context": CONTEXT,
"@type": "sc:Dataset",
"conformsTo": "http://mlcommons.org/croissant/1.0",
"name": ds["name"],
"description": " ".join(ds["description"].split()),
"url": ds["url"],
"sameAs": ds["repository"],
"license": ds["license"],
"creator": {"@type": "sc:Person", "name": ds["creator"], "url": f"https://github.com/{ds['creator']}"},
"keywords": ds["keywords"],
"citeAs": ds["citation"],
"isLiveDataset": True,
"isBasedOn": [
{"@type": "sc:CreativeWork", "name": s["name"], "url": s["view"]}
for key, s in sources.items()
if key != "pdf" # the route PDF is only fingerprinted, never redistributed
],
"distribution": distribution,
"recordSet": [
{
"@type": "cr:RecordSet",
"@id": "runs",
"name": "runs",
"description": "One coded threshing run per row (data/derived/runs_clean.csv).",
"key": {"@id": "runs/run_id"},
"field": fields,
}
],
}
def render(doc):
return json.dumps(doc, indent=2, ensure_ascii=False) + "\n"
def main(argv):
text = render(build())
if "--check" in argv:
if not OUT.exists() or OUT.read_text(encoding="utf-8") != text:
print("data/croissant.json is out of date: run python3 collection/publish_metadata.py", file=sys.stderr)
return 1
print("data/croissant.json is up to date")
return 0
OUT.write_text(text, encoding="utf-8")
print(f"wrote {OUT.relative_to(ROOT)}")
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
+2
View File
@@ -304,6 +304,8 @@ def main(argv=None):
df, missing = load(src) df, missing = load(src)
if missing: if missing:
print("WARNING: columns not found:", missing) print("WARNING: columns not found:", missing)
# a whole-number column with gaps would otherwise be written as 1.0, 2.0, ... (data/dictionary.yaml: integer)
df["attempt"] = df["attempt"].astype("Int64")
df.to_csv(out, index=False) df.to_csv(out, index=False)
print(f"{len(df)} runs -> {out}") print(f"{len(df)} runs -> {out}")
print(df.outcome.value_counts().to_string()) print(df.outcome.value_counts().to_string())
+8
View File
@@ -8,3 +8,11 @@
| Reddit megathreads (r/fourthwing, Dragonkind bonding Parts 3 and 4) | qualitative; routes cited in the valley map | manual | | Reddit megathreads (r/fourthwing, Dragonkind bonding Parts 3 and 4) | qualitative; routes cited in the valley map | manual |
| DragonKind bot exports | `bot/runs-YYYY-MM-DD.jsonl` | by pull request, see `docs/BOT_DATA_PROTOCOL.md` | | DragonKind bot exports | `bot/runs-YYYY-MM-DD.jsonl` | by pull request, see `docs/BOT_DATA_PROTOCOL.md` |
| Discord | not yet | planned, with admin consent and anonymised usernames | | Discord | not yet | planned, with admin consent and anonymised usernames |
## What each column means
`data/dictionary.yaml` describes every published file and every column of `data/derived/runs_clean.csv`: its type,
what it means and every value it can take. `data/croissant.json` says the same in the
[Croissant](https://docs.mlcommons.org/croissant/) metadata format (with a sha256 checksum for every file), so
dataset tools can load and cite the data. After changing the data or the dictionary, run `npm run metadata`; the
tests fail if a column or value is not described or the metadata is out of date.
+724
View File
@@ -0,0 +1,724 @@
{
"@context": {
"@language": "en",
"@vocab": "https://schema.org/",
"citeAs": "cr:citeAs",
"column": "cr:column",
"conformsTo": "dct:conformsTo",
"cr": "http://mlcommons.org/croissant/",
"data": {
"@id": "cr:data",
"@type": "@json"
},
"dataType": {
"@id": "cr:dataType",
"@type": "@vocab"
},
"dct": "http://purl.org/dc/terms/",
"examples": {
"@id": "cr:examples",
"@type": "@json"
},
"extract": "cr:extract",
"field": "cr:field",
"fileObject": "cr:fileObject",
"fileProperty": "cr:fileProperty",
"format": "cr:format",
"includes": "cr:includes",
"isLiveDataset": "cr:isLiveDataset",
"jsonPath": "cr:jsonPath",
"key": "cr:key",
"md5": "cr:md5",
"parentField": "cr:parentField",
"path": "cr:path",
"recordSet": "cr:recordSet",
"references": "cr:references",
"regex": "cr:regex",
"repeated": "cr:repeated",
"replace": "cr:replace",
"sc": "https://schema.org/",
"separator": "cr:separator",
"source": "cr:source",
"subField": "cr:subField",
"transform": "cr:transform"
},
"@type": "sc:Dataset",
"conformsTo": "http://mlcommons.org/croissant/1.0",
"name": "Threshing Atlas",
"description": "Community-logged runs of the Dragonkind \"threshing\" (the dragon-bonding game based on Fourth Wing), coded into one record per run: every choice, what the game rolled, and how the run ended. Also the route model built from them, the statistics shown on the Threshing Atlas page, and the route graph shared with a DragonKind bot.",
"url": "https://ekcschiano.github.io/threshing-atlas/",
"sameAs": "https://github.com/ekcSchiano/threshing-atlas",
"license": "https://creativecommons.org/licenses/by/4.0/",
"creator": {
"@type": "sc:Person",
"name": "ekcSchiano",
"url": "https://github.com/ekcSchiano"
},
"keywords": [
"Dragonkind",
"Fourth Wing",
"threshing",
"decision tree",
"route planning",
"community data"
],
"citeAs": "ekcSchiano and contributors (2026). Threshing Atlas. https://github.com/ekcSchiano/threshing-atlas",
"isLiveDataset": true,
"isBasedOn": [
{
"@type": "sc:CreativeWork",
"name": "Threshing Routes form responses (u/ClumsyBadger)",
"url": "https://docs.google.com/spreadsheets/d/1gnw1AdRg1XT4JsOfRM6QTA0jSaS3n9ZF-rPWnYewzv0"
},
{
"@type": "sc:CreativeWork",
"name": "Unofficial Dragon Register responses",
"url": "https://docs.google.com/spreadsheets/d/1KMiE9sec0l597Pzue1OhGYf07NF4howtQ_lPeqx7jXQ"
}
],
"distribution": [
{
"@type": "cr:FileObject",
"@id": "runs_clean",
"name": "runs_clean",
"description": "One coded run per row of the community form (core/atlas/coding.py, rules in core/spec/form.yaml).",
"contentUrl": "https://ekcschiano.github.io/threshing-atlas/data/runs_clean.csv",
"encodingFormat": "text/csv",
"sha256": "ef34b36881b6bc0f57453e425b1c7d76155e6572463aff0d7e89b9857823f7db",
"contentSize": "1991643 B"
},
{
"@type": "cr:FileObject",
"@id": "form_responses",
"name": "form_responses",
"description": "The community \"Threshing Routes\" form export as published (headers and rows), with email addresses in free-text answers redacted. The source of runs_clean.",
"contentUrl": "https://github.com/ekcSchiano/threshing-atlas/blob/main/data/public/form_responses.json",
"encodingFormat": "application/json",
"sha256": "4c5791f78ce81f5e88f1fb99d933acd06d8223ff9a573e96b6fcb3a6fd9c8d22",
"contentSize": "8293395 B"
},
{
"@type": "cr:FileObject",
"@id": "register",
"name": "register",
"description": "The Unofficial Dragon Register (which dragon each rider bonded), without rider names.",
"contentUrl": "https://ekcschiano.github.io/threshing-atlas/data/register.json",
"encodingFormat": "application/json",
"sha256": "f889c7e18068a934eb6872efda9b43599f4870d2e42bfdb3d447405976d59df8",
"contentSize": "111095 B"
},
{
"@type": "cr:FileObject",
"@id": "stats",
"name": "stats",
"description": "Every number the Atlas page shows, for six filter presets (analysis/stats/site_stats.py; what is measured is analysis/stats/measures.yaml).",
"contentUrl": "https://ekcschiano.github.io/threshing-atlas/data/stats.json",
"encodingFormat": "application/json",
"sha256": "d48b1da5176d6e85f6ee093cfb4f50dfbc6f71cfa7fec8ecb49a7c32c14f6d53",
"contentSize": "4269825 B"
},
{
"@type": "cr:FileObject",
"@id": "policy",
"name": "policy",
"description": "The route model: the game (core/spec/game.yaml), the count tables behind every roll, and the best decision tree for every dragon colour under two goals, with posterior intervals (planner/build_policy.py).",
"contentUrl": "https://ekcschiano.github.io/threshing-atlas/data/policy.json",
"encodingFormat": "application/json",
"sha256": "26c3adad2236493b4a77b7d46454fe01c84e8f750f0f03b6e7dbe968af32ea4e",
"contentSize": "617892 B"
},
{
"@type": "cr:FileObject",
"@id": "graph",
"name": "graph",
"description": "The route graph over every source (form and bot), counted per source (analysis/network/build-graph.js).",
"contentUrl": "https://ekcschiano.github.io/threshing-atlas/data/graph.json",
"encodingFormat": "application/json",
"sha256": "043227ff7d858fa8e73e7447f4cd7171bb216ab8d203b036df056d8ed542e446",
"contentSize": "48634 B"
}
],
"recordSet": [
{
"@type": "cr:RecordSet",
"@id": "runs",
"name": "runs",
"description": "One coded threshing run per row (data/derived/runs_clean.csv).",
"key": {
"@id": "runs/run_id"
},
"field": [
{
"@type": "cr:Field",
"@id": "runs/run_id",
"name": "run_id",
"description": "Row number in the form export (1 = first row).",
"dataType": "sc:Integer",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "run_id"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/source",
"name": "source",
"description": "\"Form\" for a form submission (it has a timestamp). Bulk-pasted logs keep the label they were pasted with; they record every run of the players who kept them, so they are the least selectively reported rows. Values: Form; Data Import; BulkUpload3; Manual Upload; Historical import; Unknown import.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "source"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/date",
"name": "date",
"description": "Date of a form submission (YYYY-MM-DD). Empty for bulk-pasted rows.",
"dataType": "sc:Date",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "date"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/wing",
"name": "wing",
"description": "The rider's wing Values: First; Second; Third; Fourth; Multiple; Unknown.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "wing"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/section",
"name": "section",
"description": "The rider's section within the wing (free text as answered).",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "section"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/squad",
"name": "squad",
"description": "The rider's squad. Values: First; Second; Third; Don't know; Unknown.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "squad"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/attempt",
"name": "attempt",
"description": "Which attempt this was for the account (1 = first). Empty if not given.",
"dataType": "sc:Integer",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "attempt"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/attempt_group",
"name": "attempt_group",
"description": "First attempt or a later one (\"I've lost count\" counts as later). Values: First attempt; Later attempt; Unknown.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "attempt_group"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/goal",
"name": "goal",
"description": "Which dragon the rider was trying to bond (older form) or why they did the run (redesigned form). Values: Red; Green; Orange; Brown; Blue; Black; Blue or Black; Red (tree); Any (for fun); Daily run data; (not asked/blank).",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "goal"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/company",
"name": "company",
"description": "Who entered the valley with the rider. Values: Alone; 1 Squadmate; 2 Squadmates; Unknown.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "company"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/left_squadmates",
"name": "left_squadmates",
"description": "Whether a rider who entered with squadmates later left them. Only asked from 2026-10-27: earlier and undated squad runs are empty (unknown).",
"dataType": "sc:Boolean",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "left_squadmates"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/direction",
"name": "direction",
"description": "First direction. Values: East; West; Center; Multiple; Unknown.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "direction"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/first_sighting",
"name": "first_sighting",
"description": "The first dragon that found the rider (a roll of the game). Values: Red; Brown; Green; Orange; Unsure; Unknown.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "first_sighting"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/sighting_action",
"name": "sighting_action",
"description": "How the rider reacted to the first dragon (redesigned form only). Values: Hold still; Respect / face it; Retreat / run; Hide; Weapon; Meet its gaze; Protect squadmate; Approach; Stand alone; Unknown; (not asked).",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "sighting_action"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/scream",
"name": "scream",
"description": "What the rider did about the scream, or that they died before it. Values: Investigate & help; Investigate & leave; Ignore the scream; DIED at first sighting; Unknown.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "scream"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/resolution",
"name": "resolution",
"description": "How the cadet encounter ended. Values: Killed the bullies yourself; Squadmate's revenge; Left bullies to the dragons; Tourniquet; Left injured cadet; Took their weapons; Left them to it; DIED in combat; DIED at first sighting; Unknown.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "resolution"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/combat_first",
"name": "combat_first",
"description": "The rider's first move in the bully fight. Values: Assess the situation; Even the numbers (smaller cadet); Put yourself in the middle (bigger cadet); (not recorded).",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "combat_first"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/combat_second",
"name": "combat_second",
"description": "After assessing Values: Engage the smaller cadet; Daggers or not (protect squadmate); (not recorded).",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "combat_second"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/post_path",
"name": "post_path",
"description": "The fork after the bully fight (only that path has one). Values: Forest; West; East; (no fork); Unknown.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "post_path"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/hour",
"name": "hour",
"description": "After an hour's walk Values: Tree; Valley; DIED before the hour; Unknown.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "hour"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/branch",
"name": "branch",
"description": "Which bonding encounter the run reached after the hour, worked out from the dragons recorded. Values: Shadow (rare); Descend quickly: Red; Descend quickly: let the Red pass; Descend slowly: two Greens; Valley Red; Passed the Red: meadow.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "branch"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/enc_Red",
"name": "enc_Red",
"description": "The rider's answer for this dragon in the bonding grid. Values: Bonded!; Incinerated; Changed my mind; Saved by another.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "enc_Red"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/enc_Green",
"name": "enc_Green",
"description": "The rider's answer for this dragon in the bonding grid. Values: Bonded!; Incinerated; Changed my mind; Saved by another.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "enc_Green"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/enc_Orange",
"name": "enc_Orange",
"description": "The rider's answer for this dragon in the bonding grid. Values: Bonded!; Incinerated; Changed my mind; Saved by another.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "enc_Orange"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/enc_Brown",
"name": "enc_Brown",
"description": "The rider's answer for this dragon in the bonding grid. Values: Bonded!; Incinerated; Changed my mind; Saved by another.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "enc_Brown"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/enc_Blue",
"name": "enc_Blue",
"description": "The rider's answer for this dragon in the bonding grid. Values: Bonded!; Incinerated; Changed my mind; Saved by another.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "enc_Blue"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/enc_Black",
"name": "enc_Black",
"description": "The rider's answer for this dragon in the bonding grid. Values: Bonded!; Incinerated; Changed my mind; Saved by another.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "enc_Black"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/outcome",
"name": "outcome",
"description": "How the run ended. Values: Bonded; Incinerated; Died before the hour; No outcome recorded; Ambiguous.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "outcome"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/outcome_color",
"name": "outcome_color",
"description": "The dragon bonded Values: Red; Green; Orange; Brown; Blue; Black.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "outcome_color"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/rescued",
"name": "rescued",
"description": "The Greens' encounter ended with an Orange or Brown swooping in and bonding the rider.",
"dataType": "sc:Boolean",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "rescued"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/saw_blue_meadow",
"name": "saw_blue_meadow",
"description": "A Blue waited in the meadow (valley runs).",
"dataType": "sc:Boolean",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "saw_blue_meadow"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/saw_shadow",
"name": "saw_shadow",
"description": "The shadow appeared at the tree (tree runs).",
"dataType": "sc:Boolean",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "saw_shadow"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/rare_reported",
"name": "rare_reported",
"description": "The rider's own answer to \"did you meet a rare dragon\". Values: No; Blue (valley); Shadow; Unknown; (not asked).",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "rare_reported"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/malek_intent",
"name": "malek_intent",
"description": "Whether the rider would seek Malek (die on purpose) rather than bond a dragon they did not want. Values: Yes; No; Unknown; (not asked).",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "malek_intent"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/tail",
"name": "tail",
"description": "The tail of the dragon bonded (redesigned form). Values: Clubtail; Daggertail; Morningstartail; Scorpiontail; Swordtail.",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "tail"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/data_flags",
"name": "data_flags",
"description": "Inconsistencies found while coding the row",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "data_flags"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/death_note",
"name": "death_note",
"description": "Free-text note from riders who died (email addresses removed).",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "death_note"
}
}
},
{
"@type": "cr:Field",
"@id": "runs/bond_note",
"name": "bond_note",
"description": "Free-text note from riders who bonded (email addresses removed).",
"dataType": "sc:Text",
"source": {
"fileObject": {
"@id": "runs_clean"
},
"extract": {
"column": "bond_note"
}
}
}
]
}
]
}
+3281 -3281
View File
File diff suppressed because it is too large Load Diff
+130
View File
@@ -0,0 +1,130 @@
# Data dictionary: every published file and every column of the coded runs.
#
# collection/publish_metadata.py builds data/croissant.json (Croissant, the MLCommons metadata format) from this
# file, data/sources.json and the licences. tests/core/test_dictionary.py checks that every column of the coded runs
# is described here and that every value it takes is listed.
#
# Missing answers: "Unknown", "Unsure", "(not asked)" and the other values in core/spec/form.yaml missing_values carry
# no information about the choice; every analysis treats them, and empty cells, as missing.
version: 1
dataset:
name: Threshing Atlas
description: >-
Community-logged runs of the Dragonkind "threshing" (the dragon-bonding game based on Fourth Wing), coded into
one record per run: every choice, what the game rolled, and how the run ended. Also the route model built from
them, the statistics shown on the Threshing Atlas page, and the route graph shared with a DragonKind bot.
url: https://ekcschiano.github.io/threshing-atlas/
repository: https://github.com/ekcSchiano/threshing-atlas
license: https://creativecommons.org/licenses/by/4.0/
code_license: MIT
creator: ekcSchiano
keywords: [Dragonkind, Fourth Wing, threshing, decision tree, route planning, community data]
citation: "ekcSchiano and contributors (2026). Threshing Atlas. https://github.com/ekcSchiano/threshing-atlas"
# Third-party sources keep their own terms (DATA_LICENSE.md); the route PDF is never redistributed.
files:
runs_clean:
path: data/derived/runs_clean.csv
url: https://ekcschiano.github.io/threshing-atlas/data/runs_clean.csv
format: text/csv
description: One coded run per row of the community form (core/atlas/coding.py, rules in core/spec/form.yaml).
form_responses:
path: data/public/form_responses.json
format: application/json
description: >-
The community "Threshing Routes" form export as published (headers and rows), with email addresses in
free-text answers redacted. The source of runs_clean.
register:
path: data/public/register.json
url: https://ekcschiano.github.io/threshing-atlas/data/register.json
format: application/json
description: The Unofficial Dragon Register (which dragon each rider bonded), without rider names.
stats:
path: data/public/stats.json
url: https://ekcschiano.github.io/threshing-atlas/data/stats.json
format: application/json
description: >-
Every number the Atlas page shows, for six filter presets (analysis/stats/site_stats.py; what is measured is
analysis/stats/measures.yaml).
policy:
path: data/public/policy.json
url: https://ekcschiano.github.io/threshing-atlas/data/policy.json
format: application/json
description: >-
The route model: the game (core/spec/game.yaml), the count tables behind every roll, and the best decision
tree for every dragon colour under two goals, with posterior intervals (planner/build_policy.py).
graph:
path: data/public/graph.json
url: https://ekcschiano.github.io/threshing-atlas/data/graph.json
format: application/json
description: The route graph over every source (form and bot), counted per source (analysis/network/build-graph.js).
# The columns of runs_clean. type: text, integer, boolean or date. values: every value the column may take
# (empty cells are always allowed and mean "does not apply" or "not recorded").
columns:
run_id: { type: integer, description: Row number in the form export (1 = first row). }
source:
type: text
description: >-
"Form" for a form submission (it has a timestamp). Bulk-pasted logs keep the label they were pasted with;
they record every run of the players who kept them, so they are the least selectively reported rows.
values: [Form, Data Import, BulkUpload3, Manual Upload, Historical import, Unknown import]
date: { type: date, description: Date of a form submission (YYYY-MM-DD). Empty for bulk-pasted rows. }
wing: { type: text, description: The rider's wing, from the form., values: [First, Second, Third, Fourth, Multiple, Unknown] }
section: { type: text, description: The rider's section within the wing (free text as answered). }
squad: { type: text, description: The rider's squad., values: [First, Second, Third, "Don't know", Unknown] }
attempt: { type: integer, description: Which attempt this was for the account (1 = first). Empty if not given. }
attempt_group: { type: text, description: First attempt or a later one ("I've lost count" counts as later)., values: [First attempt, Later attempt, Unknown] }
goal:
type: text
description: Which dragon the rider was trying to bond (older form) or why they did the run (redesigned form).
values: [Red, Green, Orange, Brown, Blue, Black, Blue or Black, Red (tree), Any (for fun), Daily run data, "(not asked/blank)"]
company: { type: text, description: Who entered the valley with the rider., values: [Alone, 1 Squadmate, 2 Squadmates, Unknown] }
left_squadmates:
type: boolean
description: >-
Whether a rider who entered with squadmates later left them. Only asked from 2026-10-27: earlier and undated
squad runs are empty (unknown).
direction: { type: text, description: First direction., values: [East, West, Center, Multiple, Unknown] }
first_sighting: { type: text, description: The first dragon that found the rider (a roll of the game)., values: [Red, Brown, Green, Orange, Unsure, Unknown] }
sighting_action:
type: text
description: How the rider reacted to the first dragon (redesigned form only).
values: [Hold still, Respect / face it, Retreat / run, Hide, Weapon, Meet its gaze, Protect squadmate, Approach, Stand alone, Unknown, "(not asked)"]
scream:
type: text
description: What the rider did about the scream, or that they died before it.
values: [Investigate & help, Investigate & leave, Ignore the scream, DIED at first sighting, Unknown]
resolution:
type: text
description: How the cadet encounter ended.
values: [Killed the bullies yourself, Squadmate's revenge, Left bullies to the dragons, Tourniquet, Left injured cadet, Took their weapons, Left them to it, DIED in combat, DIED at first sighting, Unknown]
combat_first:
type: text
description: The rider's first move in the bully fight.
values: [Assess the situation, Even the numbers (smaller cadet), Put yourself in the middle (bigger cadet), "(not recorded)"]
combat_second: { type: text, description: After assessing, what the rider did about the smaller cadet's dagger., values: [Engage the smaller cadet, Daggers or not (protect squadmate), "(not recorded)"] }
post_path: { type: text, description: The fork after the bully fight (only that path has one)., values: [Forest, West, East, (no fork), Unknown] }
hour: { type: text, description: After an hour's walk, the tallest tree or the valley centre., values: [Tree, Valley, DIED before the hour, Unknown] }
branch:
type: text
description: Which bonding encounter the run reached after the hour, worked out from the dragons recorded.
values: [Shadow (rare), "Descend quickly: Red", "Descend quickly: let the Red pass", "Descend slowly: two Greens", Valley Red, "Passed the Red: meadow"]
enc_Red: &enc { type: text, description: The rider's answer for this dragon in the bonding grid., values: [Bonded!, Incinerated, Changed my mind, Saved by another] }
enc_Green: *enc
enc_Orange: *enc
enc_Brown: *enc
enc_Blue: *enc
enc_Black: *enc
outcome: { type: text, description: How the run ended., values: [Bonded, Incinerated, Died before the hour, No outcome recorded, Ambiguous] }
outcome_color: { type: text, description: The dragon bonded, or the one that incinerated the rider (the last met)., values: [Red, Green, Orange, Brown, Blue, Black] }
rescued: { type: boolean, description: The Greens' encounter ended with an Orange or Brown swooping in and bonding the rider. }
saw_blue_meadow: { type: boolean, description: A Blue waited in the meadow (valley runs). }
saw_shadow: { type: boolean, description: The shadow appeared at the tree (tree runs). }
rare_reported: { type: text, description: The rider's own answer to "did you meet a rare dragon"., values: ["No", Blue (valley), Shadow, Unknown, "(not asked)"] }
malek_intent: { type: text, description: Whether the rider would seek Malek (die on purpose) rather than bond a dragon they did not want., values: ["Yes", "No", Unknown, "(not asked)"] }
tail: { type: text, description: The tail of the dragon bonded (redesigned form)., values: [Clubtail, Daggertail, Morningstartail, Scorpiontail, Swordtail] }
data_flags: { type: text, description: Inconsistencies found while coding the row, separated by "; ". Empty when none. }
death_note: { type: text, description: Free-text note from riders who died (email addresses removed). }
bond_note: { type: text, description: Free-text note from riders who bonded (email addresses removed). }
+4 -1
View File
@@ -38,4 +38,7 @@ share.
`measures.yaml`, `data/public/stats.json`); the upload button and the click-to-select route map are gone; a `measures.yaml`, `data/public/stats.json`); the upload button and the click-to-select route map are gone; a
small JS advisor (`core/solver.js`, about 250 lines) reads the game and the Python-built model from `policy.json` small JS advisor (`core/solver.js`, about 250 lines) reads the game and the Python-built model from `policy.json`
and replaces the 900-line JS solver; `engine.js` is gone. and replaces the 900-line JS solver; `engine.js` is gone.
5. `data/dictionary.yaml` for every published column, and Croissant metadata for the datasets. 5. **Done.** `data/dictionary.yaml` describes every published file and every column of the coded runs (type, meaning,
allowed values); `collection/publish_metadata.py` writes `data/croissant.json` (Croissant 1.0) from it, with a
sha256 for every file. `tests/core/test_dictionary.py` fails when a column or value is undescribed or the metadata
is stale. It also caught `attempt` being written as `1.0`, `2.0`: now a whole number.
+2 -1
View File
@@ -13,7 +13,8 @@
"import-bot": "node collection/bot/import-bot.js", "import-bot": "node collection/bot/import-bot.js",
"check-sources": "python3 collection/sources/update_sources.py", "check-sources": "python3 collection/sources/update_sources.py",
"policy": "python3 planner/build_policy.py && python3 planner/policy_report.py", "policy": "python3 planner/build_policy.py && python3 planner/policy_report.py",
"stats": "python3 analysis/stats/site_stats.py" "stats": "python3 analysis/stats/site_stats.py",
"metadata": "python3 collection/publish_metadata.py"
}, },
"engines": { "engines": {
"node": ">=18" "node": ">=18"
+1 -1
View File
@@ -603,7 +603,7 @@ a { color: var(--accent); text-underline-offset: 2px; }
<section id="data-sec" aria-labelledby="h-data"> <section id="data-sec" aria-labelledby="h-data">
<div class="section-head"><span class="eyebrow">Where the numbers come from</span><h2 id="h-data">Data</h2></div> <div class="section-head"><span class="eyebrow">Where the numbers come from</span><h2 id="h-data">Data</h2></div>
<div class="data-status" id="data-status"><span class="loading">Loading data…</span></div> <div class="data-status" id="data-status"><span class="loading">Loading data…</span></div>
<p class="small muted">The data refresh every day from the community sources. Every number on this page is computed in Python (<code>analysis/stats/site_stats.py</code>) from the coded runs and published as <a href="data/stats.json">data/stats.json</a>; the coded runs themselves are <a href="data/runs_clean.csv" download>data/runs_clean.csv</a>.</p> <p class="small muted">The data refresh every day from the community sources. Every number on this page is computed in Python (<code>analysis/stats/site_stats.py</code>) from the coded runs and published as <a href="data/stats.json">data/stats.json</a>; the coded runs themselves are <a href="data/runs_clean.csv" download>data/runs_clean.csv</a>. Every column is described in <a href="data/dictionary.yaml">data/dictionary.yaml</a>, and <a href="data/croissant.json">data/croissant.json</a> carries the same description in the Croissant metadata format, with a checksum for every file.</p>
</section> </section>
<section id="share" aria-labelledby="h-share"> <section id="share" aria-labelledby="h-share">
<div class="section-head"><span class="eyebrow">Open by design</span><h2 id="h-share">Share &amp; reproduce</h2> <div class="section-head"><span class="eyebrow">Open by design</span><h2 id="h-share">Share &amp; reproduce</h2>
+1
View File
@@ -26,6 +26,7 @@ copy("core/solver.js", "solver.js");
// The numbers the page shows (all computed in Python) and the published data // The numbers the page shows (all computed in Python) and the published data
for (const f of ["stats.json", "register.json", "graph.json", "policy.json"]) copy(`data/public/${f}`, `data/${f}`); for (const f of ["stats.json", "register.json", "graph.json", "policy.json"]) copy(`data/public/${f}`, `data/${f}`);
copy("data/derived/runs_clean.csv", "data/runs_clean.csv"); copy("data/derived/runs_clean.csv", "data/runs_clean.csv");
for (const f of ["croissant.json", "dictionary.yaml"]) copy(`data/${f}`, `data/${f}`);
// Source files shown in the page's "view the code" panel // Source files shown in the page's "view the code" panel
copy("core/atlas/coding.py", "python/coding.py"); copy("core/atlas/coding.py", "python/coding.py");
copy("core/spec/form.yaml", "python/form.yaml"); copy("core/spec/form.yaml", "python/form.yaml");
+78
View File
@@ -0,0 +1,78 @@
"""The data dictionary describes every published column and value, and data/croissant.json matches it."""
import csv
import importlib.util
import json
import re
import unittest
from pathlib import Path
import yaml
ROOT = Path(__file__).resolve().parents[2]
spec = importlib.util.spec_from_file_location("publish_metadata", ROOT / "collection" / "publish_metadata.py")
publish_metadata = importlib.util.module_from_spec(spec)
spec.loader.exec_module(publish_metadata)
with open(ROOT / "data" / "dictionary.yaml", encoding="utf-8") as f:
DICT = yaml.safe_load(f)
with open(ROOT / "data" / "derived" / "runs_clean.csv", encoding="utf-8", newline="") as f:
reader = csv.DictReader(f)
HEADER = reader.fieldnames
ROWS = list(reader)
CHECKS = {
"integer": lambda v: re.fullmatch(r"-?\d+", v),
"boolean": lambda v: v in ("True", "False"),
"date": lambda v: re.fullmatch(r"\d{4}-\d{2}-\d{2}", v),
"text": lambda v: True,
}
class Dictionary(unittest.TestCase):
def test_every_column_described_in_order(self):
self.assertEqual(HEADER, list(DICT["columns"]))
def test_every_column_has_type_and_description(self):
for name, col in DICT["columns"].items():
self.assertIn(col.get("type"), CHECKS, name)
self.assertTrue(col.get("description"), name)
def test_values_have_the_declared_type(self):
for name, col in DICT["columns"].items():
bad = {r[name] for r in ROWS if r[name] and not CHECKS[col["type"]](r[name])}
self.assertFalse(bad, f"{name}: not {col['type']}: {sorted(bad)[:5]}")
def test_every_value_is_listed(self):
for name, col in DICT["columns"].items():
if "values" not in col:
continue
allowed = col["values"]
# YAML reads bare Yes/No as booleans: a listed value must be written as text
self.assertTrue(all(isinstance(v, str) for v in allowed), f"{name}: quote every value")
bad = {r[name] for r in ROWS if r[name] and r[name] not in allowed}
self.assertFalse(bad, f"{name}: values not in the dictionary: {sorted(bad)}")
def test_every_published_file_exists(self):
for key, f in DICT["files"].items():
self.assertTrue((ROOT / f["path"]).exists(), key)
class Croissant(unittest.TestCase):
def test_croissant_is_up_to_date(self):
expected = publish_metadata.render(publish_metadata.build(DICT))
actual = (ROOT / "data" / "croissant.json").read_text(encoding="utf-8")
self.assertEqual(actual, expected, "run python3 collection/publish_metadata.py")
def test_fields_and_checksums(self):
doc = json.loads((ROOT / "data" / "croissant.json").read_text(encoding="utf-8"))
names = [f["name"] for f in doc["recordSet"][0]["field"]]
self.assertEqual(names, HEADER)
for obj in doc["distribution"]:
path = ROOT / DICT["files"][obj["@id"]]["path"]
self.assertEqual(obj["sha256"], publish_metadata.sha256(path), obj["@id"])
# the route PDF is not redistributed, so it is not a source of the published data
self.assertNotIn("pdf", json.dumps(doc["isBasedOn"]).lower())
if __name__ == "__main__":
unittest.main()