Merge pull request 'Receive runs from the Reddit megathread comments (source "reddit comment")' (#18) from data/reddit-comment-runs into main
CI / test (push) Successful in 46s

Reviewed-on: #18
This commit was merged in pull request #18.
This commit is contained in:
2026-11-19 03:16:59 +00:00
10 changed files with 192 additions and 42 deletions
+1
View File
@@ -31,6 +31,7 @@ it exactly.
|---|---|---|---|
| reddit spreadsheet | `reddit_spreadsheet`: u/ClumsyBadger's Threshing Routes form and spreadsheet, linked from the megathreads | the sheet's .xlsx export | `data/public/form_responses.json` (emails redacted), coded by `core/spec/form.yaml` |
| reddit thread | `reddit_thread`: dragonkind.com result pages posted in the r/fourthwing megathreads | the threads, through the official Reddit API (needs `REDDIT_CLIENT_ID` / `REDDIT_CLIENT_SECRET`) | `manual/reddit_results.yaml`, checked by hand; the extract lists result links not in it yet |
| reddit comment | `reddit_comments`: runs riders describe in the megathread comments | not extracted here: the [threshing-reddit](https://git.kenschiano.net/DragonKind/threshing-reddit) scraper reads the threads every 6 hours and opens (or updates) a pull request that changes the public file | `data/public/reddit_comments.json`: each comment read into the "Report a run" answers, no usernames or text; corrections by hand in `manual/reddit_comments.yaml` |
| reported | `reported`: the Atlas's "Report a run" form (a GitHub issue form, `specs/report_form.yaml`) | issues labelled `run report`, through the GitHub API | `data/public/reports.json` (answers only, no usernames) |
| bot | `bot`: Ken's DragonKind bot | a local copy of the bot's private repo (`DRAGONKIND_BOT_DIR`) | `data/bot/scenes.json`, `data/bot/trajectories-<date>.jsonl` (no prose, accounts hashed with `DK_ACCOUNT_SALT`; `specs/bot.yaml`) |
| reddit pdf | `reddit_pdf`: the community route PDF | the PDF | `data/derived/pdf_fingerprint.json` only: hashes, never the text |
+6 -1
View File
@@ -64,6 +64,11 @@ def reddit(spec, env):
return json.dumps(threads, ensure_ascii=False, sort_keys=True).encode("utf-8")
def delivered(spec, env):
"""A source another repository delivers by pull request (it changes the public file itself): nothing to fetch."""
raise Skip(f"delivered by pull request from {spec.get('by') or 'another repository'}")
def forge_issues(spec, env):
"""Every issue with the label (open and closed), from the repository's host (forge.yaml: Gitea or GitHub)."""
from . import forge
@@ -107,7 +112,7 @@ def pack_local(spec, src):
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
EXTRACTORS = {"http": http, "reddit": reddit, "forge_issues": forge_issues, "local_dir": local_dir}
EXTRACTORS = {"http": http, "reddit": reddit, "forge_issues": forge_issues, "local_dir": local_dir, "delivered": delivered}
# ---------------------------------------------------------------- the raw store
+56 -9
View File
@@ -4,6 +4,7 @@ One coder per source (pipeline.yaml `runs:`). Every coder returns rows keyed by
missing answers get the same "not known" values the community form's coding uses, so every source can be filtered
the same way. The `source` column says where a run came from; `entry` keeps the spreadsheet's own row label.
"""
import functools
import json
import re
from pathlib import Path
@@ -67,8 +68,13 @@ def spreadsheet(name, spec):
def reddit_results(name, spec):
"""Dragons read from result pages linked in the megathreads: bonded, colour and tail known, route unknown."""
from .common import pipeline
comments = pipeline()["sources"].get("reddit_comments")
told = {r.get("result") for r in _comment_runs(comments)} if comments else set()
rows = []
for r in read_yaml(spec["transform"]["manual"])["results"]:
if r["id"] in told:
continue # the same run, with its route, is a "reddit comment" row
row = blank_row(run_id=f"thread-{r['id']}", source=spec["run_source"], outcome="Bonded",
outcome_color=r["colour"], tail=r["tail"])
encounter_columns(row)
@@ -110,14 +116,55 @@ def register(name, spec):
def reports(name, spec):
"""Runs reported through the "Report a run" issue form (collection/specs/report_form.yaml)."""
form = report_form.spec()
fields = {f["id"]: f for f in form["fields"]}
p = path(spec["load"])
data = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {"reports": []}
return [answers_row(f"reported-{rep['number']}", spec["run_source"], rep.get("opened"), rep["answers"])
for rep in data["reports"]]
def _comment_runs(spec):
"""data/public/reddit_comments.json with the hand corrections of collection/manual/reddit_comments.yaml:
`reject` (comment ids), `fix` (id -> answers that replace the read ones; null removes one) and `add` (runs the
reader missed: {id, thread, posted, answers})."""
p = path(spec["load"])
data = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {"runs": []}
manual = read_yaml(spec["manual"]) if spec.get("manual") and path(spec["manual"]).exists() else {}
reject = {str(x) for x in manual.get("reject") or []}
fix = {str(k): v for k, v in (manual.get("fix") or {}).items()}
out = []
for run in list(data["runs"]) + list(manual.get("add") or []):
if str(run["id"]) in reject or str(run["id"]).split("-")[0] in reject:
continue
answers = dict(run["answers"])
for k, v in (fix.get(str(run["id"])) or {}).items():
if v is None:
answers.pop(k, None)
else:
answers[k] = str(v)
out.append({**run, "answers": answers, "checked": str(run["id"]) in fix or run in (manual.get("add") or [])})
return out
def reddit_comments(name, spec):
"""Runs riders described in the megathread comments, read into the "Report a run" answers (automatic: every
row is flagged, and those not checked by hand say so)."""
rows = []
for rep in data["reports"]:
vals, flags = {}, []
for fid, answer in rep["answers"].items():
for run in _comment_runs(spec):
rows.append(answers_row(f"comment-{run['id']}", spec["run_source"], run.get("posted"), run["answers"],
["read from a Reddit comment" + ("" if run["checked"] else " automatically, not checked by hand")]))
return rows
@functools.lru_cache(maxsize=1)
def _form_fields():
return {f["id"]: f for f in report_form.spec()["fields"]}
def answers_row(run_id, source, date, answers, flags=None):
"""One run from the "Report a run" answers (option labels) -> a coded row."""
fields = _form_fields()
vals, flags = {}, list(flags or [])
for fid, answer in answers.items():
f = fields.get(fid)
if not f:
continue
@@ -128,7 +175,7 @@ def reports(name, spec):
vals[f["column"]] = f["options"][answer]
else:
flags.append(f"unrecognised answer to {fid}")
row = blank_row(run_id=f"reported-{rep['number']}", source=spec["run_source"], date=rep.get("opened"), **vals)
row = blank_row(run_id=run_id, source=source, date=date, **vals)
row["attempt_group"] = attempt_group(row["attempt"])
if row["outcome"] == "Died before the hour" and row["hour"] == "Unknown":
row["hour"] = "DIED before the hour"
@@ -146,8 +193,7 @@ def reports(name, spec):
flags.append("bonded, colour not given")
encounter_columns(row)
row["data_flags"] = "; ".join(flags)
rows.append(row)
return rows
return row
def _norm(text):
@@ -262,7 +308,8 @@ def bot(name, spec):
return rows
CODERS = {"spreadsheet": spreadsheet, "reddit_results": reddit_results, "reports": reports, "bot": bot, "register": register}
CODERS = {"spreadsheet": spreadsheet, "reddit_results": reddit_results, "reddit_comments": reddit_comments,
"reports": reports, "bot": bot, "register": register}
def build_runs(pipe):
+15
View File
@@ -0,0 +1,15 @@
# Hand corrections to the runs read from the r/fourthwing megathread comments (data/public/reddit_comments.json).
# The threshing-reddit scraper reads them automatically; every new run it finds is listed, with a link to its
# comment, in the pull request that brings it in. Check them there and correct them here. Applied when the
# runs are built (`python -m collection build`), so a correction never waits for the threads to change.
#
# reject: comment ids whose "run" is not a run (a joke, a friend's run, two runs told as one). An id without
# its "-n" suffix rejects every run read from that comment.
# fix: comment id -> answers that replace the ones read (form option labels from
# collection/specs/report_form.yaml; null removes an answer). A fixed run counts as checked by hand.
# add: runs the reader missed: { id, thread, posted: YYYY-MM-DD, answers: {...} }
# No usernames: the comment id is enough to find the comment again.
version: 1
reject: []
fix: {}
add: []
+19 -1
View File
@@ -18,7 +18,7 @@ manifest: data/manifest.json # for each source: the sha256 of the current
runs:
path: data/public/runs_clean.csv
# sources in this order; within a source, the order the coder returns
order: [reddit_spreadsheet, reddit_thread, register, reported, bot, reddit_pdf]
order: [reddit_spreadsheet, reddit_thread, reddit_comments, register, reported, bot, reddit_pdf]
sources:
reddit_spreadsheet:
@@ -69,6 +69,24 @@ sources:
load: null # the public source file is the hand-checked collection/manual/reddit_results.yaml
runs: reddit_results
reddit_comments:
label: Runs riders describe in the r/fourthwing Dragonkind megathread comments
run_source: reddit comment
links:
part3: https://www.reddit.com/r/fourthwing/comments/1wvwfp9/
part4: https://www.reddit.com/r/fourthwing/comments/1wz5cnj/
scraper: https://git.kenschiano.net/DragonKind/threshing-reddit
# delivered, not extracted: the threshing-reddit scraper (its own repository) reads the threads every 6 hours and
# opens a pull request here that changes data/public/reddit_comments.json. Each comment is read into the
# "Report a run" answers (option labels of specs/report_form.yaml); only the comment id, the day and the
# answers are kept: no usernames, no comment text. `update` leaves this source alone.
extract: { kind: delivered, by: threshing-reddit }
transform: null
load: data/public/reddit_comments.json
# hand corrections (reject / fix / add a run), applied when the runs are built
manual: collection/manual/reddit_comments.yaml
runs: reddit_comments
reported:
label: Runs reported on the Atlas ("Report a run", an issue form on the Atlas's repository)
run_source: reported
+1
View File
@@ -7,6 +7,7 @@ Every source is declared in `collection/pipeline.yaml` and collected by `python
|---|---|
| Community "Threshing Routes" form and spreadsheet, u/ClumsyBadger (`reddit spreadsheet`) | `public/form_responses.json` (emails redacted) |
| Result pages posted in the r/fourthwing megathreads (`reddit thread`) | `collection/manual/reddit_results.yaml` |
| Runs described in the r/fourthwing megathread comments (`reddit comment`) | `public/reddit_comments.json` (comment id, day and answers; no usernames or comment text) |
| The Atlas's "Report a run" form (`reported`) | `public/reports.json` (answers only, no usernames) |
| DragonKind bot exports (`bot`) | `bot/scenes.json`, `bot/trajectories-<date>.jsonl` (anonymised; see `docs/BOT_DATA_PROTOCOL.md`) |
| Community route PDF, u/ClumsyBadger (`reddit pdf`) | not redistributed: `derived/pdf_fingerprint.json` only; it maps the choices and holds no runs |
+17 -2
View File
@@ -78,6 +78,11 @@
"@type": "sc:CreativeWork",
"name": "Runs posted in the r/fourthwing Dragonkind megathreads",
"url": "https://www.reddit.com/r/fourthwing/comments/1wvwfp9/"
},
{
"@type": "sc:CreativeWork",
"name": "Runs riders describe in the r/fourthwing Dragonkind megathread comments",
"url": "https://www.reddit.com/r/fourthwing/comments/1wvwfp9/"
}
],
"distribution": [
@@ -101,6 +106,16 @@
"sha256": "319f6217e0a431454114a67276f429379ec383cd6ed2666443b48b8f48c66490",
"contentSize": "12623695 B"
},
{
"@type": "cr:FileObject",
"@id": "reddit_comments",
"name": "reddit_comments",
"description": "Runs described in the r/fourthwing megathread comments, read into the \"Report a run\" answers: comment id, day and answers only, no usernames or comment text.",
"contentUrl": "https://git.kenschiano.net/DragonKind/threshing-atlas/raw/branch/main/data/public/reddit_comments.json",
"encodingFormat": "application/json",
"sha256": "c180f8b82ed1119d4ff28697fcf45d0c7f22859597cd10f5e82a2e50ca1dc51a",
"contentSize": "105 B"
},
{
"@type": "cr:FileObject",
"@id": "reports",
@@ -166,7 +181,7 @@
"@type": "cr:Field",
"@id": "runs/run_id",
"name": "run_id",
"description": "Unique id of the run: sheet-<row of the form export>, thread-<result id>, reported-<issue number> or bot-<hash>.",
"description": "Unique id of the run: sheet-<row of the form export>, thread-<result id>, comment-<Reddit comment id>, reported-<issue number> or bot-<hash>.",
"dataType": "sc:Text",
"source": {
"fileObject": {
@@ -181,7 +196,7 @@
"@type": "cr:Field",
"@id": "runs/source",
"name": "source",
"description": "Where the run came from. \"reddit spreadsheet\": the community Threshing Routes form and spreadsheet linked from the r/fourthwing megathreads. \"reddit thread\": a dragonkind.com result page posted in a megathread (bonded, route not recorded). \"reported\": the Atlas's own \"Report a run\" form. \"bot\": an attempt logged by the DragonKind bot. \"register\": a bonded dragon entered in the community's Unofficial Dragon Register (outcome only, no route; the same run may also be in the spreadsheet). \"reddit pdf\": the community route PDF (a flowchart of the routes, not a log of runs, so it adds no rows today). Values: reddit spreadsheet; reddit thread; register; reported; bot; reddit pdf.",
"description": "Where the run came from. \"reddit spreadsheet\": the community Threshing Routes form and spreadsheet linked from the r/fourthwing megathreads. \"reddit thread\": a dragonkind.com result page posted in a megathread (bonded, route not recorded). \"reddit comment\": a run a rider described in a megathread comment, read automatically into the \"Report a run\" answers by the threshing-reddit scraper, which delivers them by pull request (hand corrections in collection/manual/reddit_comments.yaml). \"reported\": the Atlas's own \"Report a run\" form. \"bot\": an attempt logged by the DragonKind bot. \"register\": a bonded dragon entered in the community's Unofficial Dragon Register (outcome only, no route; the same run may also be in the spreadsheet). \"reddit pdf\": the community route PDF (a flowchart of the routes, not a log of runs, so it adds no rows today). Values: reddit spreadsheet; reddit thread; reddit comment; register; reported; bot; reddit pdf.",
"dataType": "sc:Text",
"source": {
"fileObject": {
+11 -3
View File
@@ -35,6 +35,12 @@ files:
description: >-
The community "Threshing Routes" form export as published (headers and rows), with email addresses in
free-text answers redacted. The source of runs_clean.
reddit_comments:
path: data/public/reddit_comments.json
format: application/json
description: >-
Runs described in the r/fourthwing megathread comments, read into the "Report a run" answers: comment id, day
and answers only, no usernames or comment text.
reports:
path: data/public/reports.json
format: application/json
@@ -63,18 +69,20 @@ files:
# The columns of runs_clean. type: text, integer, boolean or date. values: every value the column may take
# (empty cells are always allowed and mean "does not apply" or "not recorded").
columns:
run_id: { type: text, required: true, description: "Unique id of the run: sheet-<row of the form export>, thread-<result id>, reported-<issue number> or bot-<hash>." }
run_id: { type: text, required: true, description: "Unique id of the run: sheet-<row of the form export>, thread-<result id>, comment-<Reddit comment id>, reported-<issue number> or bot-<hash>." }
source:
type: text
required: true
description: >-
Where the run came from. "reddit spreadsheet": the community Threshing Routes form and spreadsheet linked from
the r/fourthwing megathreads. "reddit thread": a dragonkind.com result page posted in a megathread (bonded,
route not recorded). "reported": the Atlas's own "Report a run" form. "bot": an attempt logged by the
route not recorded). "reddit comment": a run a rider described in a megathread comment, read automatically into
the "Report a run" answers by the threshing-reddit scraper, which delivers them by pull request (hand
corrections in collection/manual/reddit_comments.yaml). "reported": the Atlas's own "Report a run" form. "bot": an attempt logged by the
DragonKind bot. "register": a bonded dragon entered in the community's Unofficial Dragon Register (outcome
only, no route; the same run may also be in the spreadsheet). "reddit pdf": the community route PDF (a
flowchart of the routes, not a log of runs, so it adds no rows today).
values: [reddit spreadsheet, reddit thread, register, reported, bot, reddit pdf]
values: [reddit spreadsheet, reddit thread, reddit comment, register, reported, bot, reddit pdf]
entry:
type: text
description: >-
+1
View File
@@ -0,0 +1 @@
{"source":"Runs riders describe in the r/fourthwing Dragonkind megathread comments","runs":[],"unread":0}
+39
View File
@@ -259,6 +259,45 @@ class Reddit(unittest.TestCase):
self.assertNotIn("someone", json.dumps(out))
class RedditComments(unittest.TestCase):
"""Runs the threshing-reddit scraper delivers (data/public/reddit_comments.json), with hand corrections."""
def source(self, public, manual):
spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"]
return {**spec, "load": str(public), "manual": str(manual)}
def test_coded_with_hand_corrections(self):
public = TMP / "reddit_comments.json"
public.write_text(json.dumps({"runs": [
{"id": "c1", "thread": "abc", "posted": "2026-11-17", "result": None,
"answers": {"company": "Alone", "direction": "East (toward the roar)", "outcome": "Bonded",
"outcome_color": "Green", "branch": "Tree: climbed down slowly to the two Greens"}},
{"id": "c2-1", "thread": "abc", "posted": "2026-11-17", "result": None,
"answers": {"company": "Alone", "outcome": "Bonded", "outcome_color": "Red"}},
{"id": "c3", "thread": "abc", "posted": None, "result": None,
"answers": {"company": "Alone", "outcome": "Incinerated at a bonding encounter", "outcome_color": "Red"}}]}),
encoding="utf-8")
manual = TMP / "reddit_comments_manual.yaml"
manual.write_text("reject: [c2]\nfix: {c1: {outcome_color: Blue, direction: null}}\n"
"add: [{id: c9, thread: abc, posted: '2026-11-16', "
"answers: {company: Two squadmates, direction: West (toward the scent), outcome: Died before the hour}}]\n",
encoding="utf-8")
rows = runs.reddit_comments("reddit_comments", self.source(public, manual))
self.assertEqual([r["run_id"] for r in rows], ["comment-c1", "comment-c3", "comment-c9"])
c1, c3, c9 = rows
self.assertEqual((c1["source"], c1["outcome_color"], c1["direction"], c1["hour"], c1["enc_Blue"]),
("reddit comment", "Blue", "Unknown", "Tree", "Bonded!"))
self.assertEqual(c1["data_flags"], "read from a Reddit comment")
self.assertEqual(c3["data_flags"], "read from a Reddit comment automatically, not checked by hand")
self.assertEqual((c9["company"], c9["direction"], c9["hour"]), ("2 Squadmates", "West", "DIED before the hour"))
self.assertEqual(validate.problems(rows, runs.DICT), [])
def test_delivered_not_extracted(self):
spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"]
with self.assertRaises(extract.Skip):
extract.EXTRACTORS[spec["extract"]["kind"]](spec["extract"], {})
class Reports(unittest.TestCase):
def issue(self, number, answers, **kw):
form = report_form.spec()