diff --git a/collection/README.md b/collection/README.md index f46a43e..0b5a349 100644 --- a/collection/README.md +++ b/collection/README.md @@ -31,6 +31,7 @@ it exactly. |---|---|---|---| | reddit spreadsheet | `reddit_spreadsheet`: u/ClumsyBadger's Threshing Routes form and spreadsheet, linked from the megathreads | the sheet's .xlsx export | `data/public/form_responses.json` (emails redacted), coded by `core/spec/form.yaml` | | reddit thread | `reddit_thread`: dragonkind.com result pages posted in the r/fourthwing megathreads | the threads, through the official Reddit API (needs `REDDIT_CLIENT_ID` / `REDDIT_CLIENT_SECRET`) | `manual/reddit_results.yaml`, checked by hand; the extract lists result links not in it yet | +| reddit comment | `reddit_comments`: runs riders describe in the megathread comments | not extracted here: the [threshing-reddit](https://git.kenschiano.net/DragonKind/threshing-reddit) scraper reads the threads every 6 hours and opens (or updates) a pull request that changes the public file | `data/public/reddit_comments.json`: each comment read into the "Report a run" answers, no usernames or text; corrections by hand in `manual/reddit_comments.yaml` | | reported | `reported`: the Atlas's "Report a run" form (a GitHub issue form, `specs/report_form.yaml`) | issues labelled `run report`, through the GitHub API | `data/public/reports.json` (answers only, no usernames) | | bot | `bot`: Ken's DragonKind bot | a local copy of the bot's private repo (`DRAGONKIND_BOT_DIR`) | `data/bot/scenes.json`, `data/bot/trajectories-.jsonl` (no prose, accounts hashed with `DK_ACCOUNT_SALT`; `specs/bot.yaml`) | | reddit pdf | `reddit_pdf`: the community route PDF | the PDF | `data/derived/pdf_fingerprint.json` only: hashes, never the text | diff --git a/collection/etl/extract.py b/collection/etl/extract.py index 55e0b86..677908c 100644 --- a/collection/etl/extract.py +++ b/collection/etl/extract.py @@ -64,6 +64,11 @@ def reddit(spec, env): return json.dumps(threads, ensure_ascii=False, sort_keys=True).encode("utf-8") +def delivered(spec, env): + """A source another repository delivers by pull request (it changes the public file itself): nothing to fetch.""" + raise Skip(f"delivered by pull request from {spec.get('by') or 'another repository'}") + + def forge_issues(spec, env): """Every issue with the label (open and closed), from the repository's host (forge.yaml: Gitea or GitHub).""" from . import forge @@ -107,7 +112,7 @@ def pack_local(spec, src): return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8") -EXTRACTORS = {"http": http, "reddit": reddit, "forge_issues": forge_issues, "local_dir": local_dir} +EXTRACTORS = {"http": http, "reddit": reddit, "forge_issues": forge_issues, "local_dir": local_dir, "delivered": delivered} # ---------------------------------------------------------------- the raw store diff --git a/collection/etl/runs.py b/collection/etl/runs.py index 28629aa..da078f4 100644 --- a/collection/etl/runs.py +++ b/collection/etl/runs.py @@ -4,6 +4,7 @@ One coder per source (pipeline.yaml `runs:`). Every coder returns rows keyed by missing answers get the same "not known" values the community form's coding uses, so every source can be filtered the same way. The `source` column says where a run came from; `entry` keeps the spreadsheet's own row label. """ +import functools import json import re from pathlib import Path @@ -67,8 +68,13 @@ def spreadsheet(name, spec): def reddit_results(name, spec): """Dragons read from result pages linked in the megathreads: bonded, colour and tail known, route unknown.""" + from .common import pipeline + comments = pipeline()["sources"].get("reddit_comments") + told = {r.get("result") for r in _comment_runs(comments)} if comments else set() rows = [] for r in read_yaml(spec["transform"]["manual"])["results"]: + if r["id"] in told: + continue # the same run, with its route, is a "reddit comment" row row = blank_row(run_id=f"thread-{r['id']}", source=spec["run_source"], outcome="Bonded", outcome_color=r["colour"], tail=r["tail"]) encounter_columns(row) @@ -110,46 +116,86 @@ def register(name, spec): def reports(name, spec): """Runs reported through the "Report a run" issue form (collection/specs/report_form.yaml).""" - form = report_form.spec() - fields = {f["id"]: f for f in form["fields"]} p = path(spec["load"]) data = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {"reports": []} - rows = [] - for rep in data["reports"]: - vals, flags = {}, [] - for fid, answer in rep["answers"].items(): - f = fields.get(fid) - if not f: - continue - if f.get("type") == "integer": - m = re.search(r"\d+", answer) - vals[f["column"]] = int(m.group()) if m else None - elif answer in f["options"]: - vals[f["column"]] = f["options"][answer] + return [answers_row(f"reported-{rep['number']}", spec["run_source"], rep.get("opened"), rep["answers"]) + for rep in data["reports"]] + + +def _comment_runs(spec): + """data/public/reddit_comments.json with the hand corrections of collection/manual/reddit_comments.yaml: + `reject` (comment ids), `fix` (id -> answers that replace the read ones; null removes one) and `add` (runs the + reader missed: {id, thread, posted, answers}).""" + p = path(spec["load"]) + data = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {"runs": []} + manual = read_yaml(spec["manual"]) if spec.get("manual") and path(spec["manual"]).exists() else {} + reject = {str(x) for x in manual.get("reject") or []} + fix = {str(k): v for k, v in (manual.get("fix") or {}).items()} + out = [] + for run in list(data["runs"]) + list(manual.get("add") or []): + if str(run["id"]) in reject or str(run["id"]).split("-")[0] in reject: + continue + answers = dict(run["answers"]) + for k, v in (fix.get(str(run["id"])) or {}).items(): + if v is None: + answers.pop(k, None) else: - flags.append(f"unrecognised answer to {fid}") - row = blank_row(run_id=f"reported-{rep['number']}", source=spec["run_source"], date=rep.get("opened"), **vals) - row["attempt_group"] = attempt_group(row["attempt"]) - if row["outcome"] == "Died before the hour" and row["hour"] == "Unknown": - row["hour"] = "DIED before the hour" - branch = row["branch"] - if branch: - tree = branch in ("Shadow (rare)", "Descend quickly: Red", "Descend quickly: let the Red pass", "Descend slowly: two Greens") - if row["hour"] == "Unknown": - row["hour"] = "Tree" if tree else "Valley" - elif row["hour"] != ("Tree" if tree else "Valley"): - flags.append("encounter does not match the tree/valley answer") - row["saw_shadow"] = branch == "Shadow (rare)" if tree else None - if branch == "Passed the Red: meadow" and row["outcome_color"] == "Blue": - row["saw_blue_meadow"] = True - if row["outcome"] == "Bonded" and not row["outcome_color"]: - flags.append("bonded, colour not given") - encounter_columns(row) - row["data_flags"] = "; ".join(flags) - rows.append(row) + answers[k] = str(v) + out.append({**run, "answers": answers, "checked": str(run["id"]) in fix or run in (manual.get("add") or [])}) + return out + + +def reddit_comments(name, spec): + """Runs riders described in the megathread comments, read into the "Report a run" answers (automatic: every + row is flagged, and those not checked by hand say so).""" + rows = [] + for run in _comment_runs(spec): + rows.append(answers_row(f"comment-{run['id']}", spec["run_source"], run.get("posted"), run["answers"], + ["read from a Reddit comment" + ("" if run["checked"] else " automatically, not checked by hand")])) return rows +@functools.lru_cache(maxsize=1) +def _form_fields(): + return {f["id"]: f for f in report_form.spec()["fields"]} + + +def answers_row(run_id, source, date, answers, flags=None): + """One run from the "Report a run" answers (option labels) -> a coded row.""" + fields = _form_fields() + vals, flags = {}, list(flags or []) + for fid, answer in answers.items(): + f = fields.get(fid) + if not f: + continue + if f.get("type") == "integer": + m = re.search(r"\d+", answer) + vals[f["column"]] = int(m.group()) if m else None + elif answer in f["options"]: + vals[f["column"]] = f["options"][answer] + else: + flags.append(f"unrecognised answer to {fid}") + row = blank_row(run_id=run_id, source=source, date=date, **vals) + row["attempt_group"] = attempt_group(row["attempt"]) + if row["outcome"] == "Died before the hour" and row["hour"] == "Unknown": + row["hour"] = "DIED before the hour" + branch = row["branch"] + if branch: + tree = branch in ("Shadow (rare)", "Descend quickly: Red", "Descend quickly: let the Red pass", "Descend slowly: two Greens") + if row["hour"] == "Unknown": + row["hour"] = "Tree" if tree else "Valley" + elif row["hour"] != ("Tree" if tree else "Valley"): + flags.append("encounter does not match the tree/valley answer") + row["saw_shadow"] = branch == "Shadow (rare)" if tree else None + if branch == "Passed the Red: meadow" and row["outcome_color"] == "Blue": + row["saw_blue_meadow"] = True + if row["outcome"] == "Bonded" and not row["outcome_color"]: + flags.append("bonded, colour not given") + encounter_columns(row) + row["data_flags"] = "; ".join(flags) + return row + + def _norm(text): from .transform.bot import normalize return normalize(text) @@ -262,7 +308,8 @@ def bot(name, spec): return rows -CODERS = {"spreadsheet": spreadsheet, "reddit_results": reddit_results, "reports": reports, "bot": bot, "register": register} +CODERS = {"spreadsheet": spreadsheet, "reddit_results": reddit_results, "reddit_comments": reddit_comments, + "reports": reports, "bot": bot, "register": register} def build_runs(pipe): diff --git a/collection/manual/reddit_comments.yaml b/collection/manual/reddit_comments.yaml new file mode 100644 index 0000000..5e6fe7d --- /dev/null +++ b/collection/manual/reddit_comments.yaml @@ -0,0 +1,15 @@ +# Hand corrections to the runs read from the r/fourthwing megathread comments (data/public/reddit_comments.json). +# The threshing-reddit scraper reads them automatically; every new run it finds is listed, with a link to its +# comment, in the pull request that brings it in. Check them there and correct them here. Applied when the +# runs are built (`python -m collection build`), so a correction never waits for the threads to change. +# +# reject: comment ids whose "run" is not a run (a joke, a friend's run, two runs told as one). An id without +# its "-n" suffix rejects every run read from that comment. +# fix: comment id -> answers that replace the ones read (form option labels from +# collection/specs/report_form.yaml; null removes an answer). A fixed run counts as checked by hand. +# add: runs the reader missed: { id, thread, posted: YYYY-MM-DD, answers: {...} } +# No usernames: the comment id is enough to find the comment again. +version: 1 +reject: [] +fix: {} +add: [] diff --git a/collection/pipeline.yaml b/collection/pipeline.yaml index 627f67f..96ca1a2 100644 --- a/collection/pipeline.yaml +++ b/collection/pipeline.yaml @@ -18,7 +18,7 @@ manifest: data/manifest.json # for each source: the sha256 of the current runs: path: data/public/runs_clean.csv # sources in this order; within a source, the order the coder returns - order: [reddit_spreadsheet, reddit_thread, register, reported, bot, reddit_pdf] + order: [reddit_spreadsheet, reddit_thread, reddit_comments, register, reported, bot, reddit_pdf] sources: reddit_spreadsheet: @@ -69,6 +69,24 @@ sources: load: null # the public source file is the hand-checked collection/manual/reddit_results.yaml runs: reddit_results + reddit_comments: + label: Runs riders describe in the r/fourthwing Dragonkind megathread comments + run_source: reddit comment + links: + part3: https://www.reddit.com/r/fourthwing/comments/1wvwfp9/ + part4: https://www.reddit.com/r/fourthwing/comments/1wz5cnj/ + scraper: https://git.kenschiano.net/DragonKind/threshing-reddit + # delivered, not extracted: the threshing-reddit scraper (its own repository) reads the threads every 6 hours and + # opens a pull request here that changes data/public/reddit_comments.json. Each comment is read into the + # "Report a run" answers (option labels of specs/report_form.yaml); only the comment id, the day and the + # answers are kept: no usernames, no comment text. `update` leaves this source alone. + extract: { kind: delivered, by: threshing-reddit } + transform: null + load: data/public/reddit_comments.json + # hand corrections (reject / fix / add a run), applied when the runs are built + manual: collection/manual/reddit_comments.yaml + runs: reddit_comments + reported: label: Runs reported on the Atlas ("Report a run", an issue form on the Atlas's repository) run_source: reported diff --git a/data/SOURCES.md b/data/SOURCES.md index 1aefbfc..06ee883 100644 --- a/data/SOURCES.md +++ b/data/SOURCES.md @@ -7,6 +7,7 @@ Every source is declared in `collection/pipeline.yaml` and collected by `python |---|---| | Community "Threshing Routes" form and spreadsheet, u/ClumsyBadger (`reddit spreadsheet`) | `public/form_responses.json` (emails redacted) | | Result pages posted in the r/fourthwing megathreads (`reddit thread`) | `collection/manual/reddit_results.yaml` | +| Runs described in the r/fourthwing megathread comments (`reddit comment`) | `public/reddit_comments.json` (comment id, day and answers; no usernames or comment text) | | The Atlas's "Report a run" form (`reported`) | `public/reports.json` (answers only, no usernames) | | DragonKind bot exports (`bot`) | `bot/scenes.json`, `bot/trajectories-.jsonl` (anonymised; see `docs/BOT_DATA_PROTOCOL.md`) | | Community route PDF, u/ClumsyBadger (`reddit pdf`) | not redistributed: `derived/pdf_fingerprint.json` only; it maps the choices and holds no runs | diff --git a/data/croissant.json b/data/croissant.json index 3ffe414..a78c3bd 100644 --- a/data/croissant.json +++ b/data/croissant.json @@ -78,6 +78,11 @@ "@type": "sc:CreativeWork", "name": "Runs posted in the r/fourthwing Dragonkind megathreads", "url": "https://www.reddit.com/r/fourthwing/comments/1wvwfp9/" + }, + { + "@type": "sc:CreativeWork", + "name": "Runs riders describe in the r/fourthwing Dragonkind megathread comments", + "url": "https://www.reddit.com/r/fourthwing/comments/1wvwfp9/" } ], "distribution": [ @@ -101,6 +106,16 @@ "sha256": "319f6217e0a431454114a67276f429379ec383cd6ed2666443b48b8f48c66490", "contentSize": "12623695 B" }, + { + "@type": "cr:FileObject", + "@id": "reddit_comments", + "name": "reddit_comments", + "description": "Runs described in the r/fourthwing megathread comments, read into the \"Report a run\" answers: comment id, day and answers only, no usernames or comment text.", + "contentUrl": "https://git.kenschiano.net/DragonKind/threshing-atlas/raw/branch/main/data/public/reddit_comments.json", + "encodingFormat": "application/json", + "sha256": "c180f8b82ed1119d4ff28697fcf45d0c7f22859597cd10f5e82a2e50ca1dc51a", + "contentSize": "105 B" + }, { "@type": "cr:FileObject", "@id": "reports", @@ -166,7 +181,7 @@ "@type": "cr:Field", "@id": "runs/run_id", "name": "run_id", - "description": "Unique id of the run: sheet-, thread-, reported- or bot-.", + "description": "Unique id of the run: sheet-, thread-, comment-, reported- or bot-.", "dataType": "sc:Text", "source": { "fileObject": { @@ -181,7 +196,7 @@ "@type": "cr:Field", "@id": "runs/source", "name": "source", - "description": "Where the run came from. \"reddit spreadsheet\": the community Threshing Routes form and spreadsheet linked from the r/fourthwing megathreads. \"reddit thread\": a dragonkind.com result page posted in a megathread (bonded, route not recorded). \"reported\": the Atlas's own \"Report a run\" form. \"bot\": an attempt logged by the DragonKind bot. \"register\": a bonded dragon entered in the community's Unofficial Dragon Register (outcome only, no route; the same run may also be in the spreadsheet). \"reddit pdf\": the community route PDF (a flowchart of the routes, not a log of runs, so it adds no rows today). Values: reddit spreadsheet; reddit thread; register; reported; bot; reddit pdf.", + "description": "Where the run came from. \"reddit spreadsheet\": the community Threshing Routes form and spreadsheet linked from the r/fourthwing megathreads. \"reddit thread\": a dragonkind.com result page posted in a megathread (bonded, route not recorded). \"reddit comment\": a run a rider described in a megathread comment, read automatically into the \"Report a run\" answers by the threshing-reddit scraper, which delivers them by pull request (hand corrections in collection/manual/reddit_comments.yaml). \"reported\": the Atlas's own \"Report a run\" form. \"bot\": an attempt logged by the DragonKind bot. \"register\": a bonded dragon entered in the community's Unofficial Dragon Register (outcome only, no route; the same run may also be in the spreadsheet). \"reddit pdf\": the community route PDF (a flowchart of the routes, not a log of runs, so it adds no rows today). Values: reddit spreadsheet; reddit thread; reddit comment; register; reported; bot; reddit pdf.", "dataType": "sc:Text", "source": { "fileObject": { diff --git a/data/dictionary.yaml b/data/dictionary.yaml index 146e128..3d672cc 100644 --- a/data/dictionary.yaml +++ b/data/dictionary.yaml @@ -35,6 +35,12 @@ files: description: >- The community "Threshing Routes" form export as published (headers and rows), with email addresses in free-text answers redacted. The source of runs_clean. + reddit_comments: + path: data/public/reddit_comments.json + format: application/json + description: >- + Runs described in the r/fourthwing megathread comments, read into the "Report a run" answers: comment id, day + and answers only, no usernames or comment text. reports: path: data/public/reports.json format: application/json @@ -63,18 +69,20 @@ files: # The columns of runs_clean. type: text, integer, boolean or date. values: every value the column may take # (empty cells are always allowed and mean "does not apply" or "not recorded"). columns: - run_id: { type: text, required: true, description: "Unique id of the run: sheet-, thread-, reported- or bot-." } + run_id: { type: text, required: true, description: "Unique id of the run: sheet-, thread-, comment-, reported- or bot-." } source: type: text required: true description: >- Where the run came from. "reddit spreadsheet": the community Threshing Routes form and spreadsheet linked from the r/fourthwing megathreads. "reddit thread": a dragonkind.com result page posted in a megathread (bonded, - route not recorded). "reported": the Atlas's own "Report a run" form. "bot": an attempt logged by the + route not recorded). "reddit comment": a run a rider described in a megathread comment, read automatically into + the "Report a run" answers by the threshing-reddit scraper, which delivers them by pull request (hand + corrections in collection/manual/reddit_comments.yaml). "reported": the Atlas's own "Report a run" form. "bot": an attempt logged by the DragonKind bot. "register": a bonded dragon entered in the community's Unofficial Dragon Register (outcome only, no route; the same run may also be in the spreadsheet). "reddit pdf": the community route PDF (a flowchart of the routes, not a log of runs, so it adds no rows today). - values: [reddit spreadsheet, reddit thread, register, reported, bot, reddit pdf] + values: [reddit spreadsheet, reddit thread, reddit comment, register, reported, bot, reddit pdf] entry: type: text description: >- diff --git a/data/public/reddit_comments.json b/data/public/reddit_comments.json new file mode 100644 index 0000000..97baeb7 --- /dev/null +++ b/data/public/reddit_comments.json @@ -0,0 +1 @@ +{"source":"Runs riders describe in the r/fourthwing Dragonkind megathread comments","runs":[],"unread":0} \ No newline at end of file diff --git a/tests/collection/test_collection.py b/tests/collection/test_collection.py index d5c9fd9..c0da559 100644 --- a/tests/collection/test_collection.py +++ b/tests/collection/test_collection.py @@ -259,6 +259,45 @@ class Reddit(unittest.TestCase): self.assertNotIn("someone", json.dumps(out)) +class RedditComments(unittest.TestCase): + """Runs the threshing-reddit scraper delivers (data/public/reddit_comments.json), with hand corrections.""" + + def source(self, public, manual): + spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"] + return {**spec, "load": str(public), "manual": str(manual)} + + def test_coded_with_hand_corrections(self): + public = TMP / "reddit_comments.json" + public.write_text(json.dumps({"runs": [ + {"id": "c1", "thread": "abc", "posted": "2026-11-17", "result": None, + "answers": {"company": "Alone", "direction": "East (toward the roar)", "outcome": "Bonded", + "outcome_color": "Green", "branch": "Tree: climbed down slowly to the two Greens"}}, + {"id": "c2-1", "thread": "abc", "posted": "2026-11-17", "result": None, + "answers": {"company": "Alone", "outcome": "Bonded", "outcome_color": "Red"}}, + {"id": "c3", "thread": "abc", "posted": None, "result": None, + "answers": {"company": "Alone", "outcome": "Incinerated at a bonding encounter", "outcome_color": "Red"}}]}), + encoding="utf-8") + manual = TMP / "reddit_comments_manual.yaml" + manual.write_text("reject: [c2]\nfix: {c1: {outcome_color: Blue, direction: null}}\n" + "add: [{id: c9, thread: abc, posted: '2026-11-16', " + "answers: {company: Two squadmates, direction: West (toward the scent), outcome: Died before the hour}}]\n", + encoding="utf-8") + rows = runs.reddit_comments("reddit_comments", self.source(public, manual)) + self.assertEqual([r["run_id"] for r in rows], ["comment-c1", "comment-c3", "comment-c9"]) + c1, c3, c9 = rows + self.assertEqual((c1["source"], c1["outcome_color"], c1["direction"], c1["hour"], c1["enc_Blue"]), + ("reddit comment", "Blue", "Unknown", "Tree", "Bonded!")) + self.assertEqual(c1["data_flags"], "read from a Reddit comment") + self.assertEqual(c3["data_flags"], "read from a Reddit comment automatically, not checked by hand") + self.assertEqual((c9["company"], c9["direction"], c9["hour"]), ("2 Squadmates", "West", "DIED before the hour")) + self.assertEqual(validate.problems(rows, runs.DICT), []) + + def test_delivered_not_extracted(self): + spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"] + with self.assertRaises(extract.Skip): + extract.EXTRACTORS[spec["extract"]["kind"]](spec["extract"], {}) + + class Reports(unittest.TestCase): def issue(self, number, answers, **kw): form = report_form.spec()