Receive runs from the Reddit megathread comments (source "reddit comment")
CI / test (pull_request) Successful in 46s

The threshing-reddit scraper (DragonKind/threshing-reddit) reads the runs
riders describe in the r/fourthwing megathreads and delivers them here by
pull request as data/public/reddit_comments.json: each run is the "Report a
run" answers (option labels of collection/specs/report_form.yaml), with the
comment id and day only; no usernames, no comment text.

- pipeline.yaml: a reddit_comments source of extract kind "delivered" (the
  atlas fetches nothing; `update` skips it with a note)
- runs.py: the coder, sharing one answers_row() with the reported coder;
  every row is flagged as read from a Reddit comment (and "not checked by
  hand" until corrected); a result page a comment already carries is not
  counted twice as a "reddit thread" row
- collection/manual/reddit_comments.yaml: reject / fix / add, applied at
  build time
- dictionary, SOURCES, README, croissant: the new source

The route model and statistics do not use the source until it is named in
planner/policy.yaml / analysis/stats/measures.yaml.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-18 22:06:14 -05:00
co-authored by Claude Opus 5.5
parent f3bbde24b7
commit 197c16dac0
10 changed files with 192 additions and 42 deletions
+39
View File
@@ -259,6 +259,45 @@ class Reddit(unittest.TestCase):
self.assertNotIn("someone", json.dumps(out))
class RedditComments(unittest.TestCase):
"""Runs the threshing-reddit scraper delivers (data/public/reddit_comments.json), with hand corrections."""
def source(self, public, manual):
spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"]
return {**spec, "load": str(public), "manual": str(manual)}
def test_coded_with_hand_corrections(self):
public = TMP / "reddit_comments.json"
public.write_text(json.dumps({"runs": [
{"id": "c1", "thread": "abc", "posted": "2026-11-17", "result": None,
"answers": {"company": "Alone", "direction": "East (toward the roar)", "outcome": "Bonded",
"outcome_color": "Green", "branch": "Tree: climbed down slowly to the two Greens"}},
{"id": "c2-1", "thread": "abc", "posted": "2026-11-17", "result": None,
"answers": {"company": "Alone", "outcome": "Bonded", "outcome_color": "Red"}},
{"id": "c3", "thread": "abc", "posted": None, "result": None,
"answers": {"company": "Alone", "outcome": "Incinerated at a bonding encounter", "outcome_color": "Red"}}]}),
encoding="utf-8")
manual = TMP / "reddit_comments_manual.yaml"
manual.write_text("reject: [c2]\nfix: {c1: {outcome_color: Blue, direction: null}}\n"
"add: [{id: c9, thread: abc, posted: '2026-11-16', "
"answers: {company: Two squadmates, direction: West (toward the scent), outcome: Died before the hour}}]\n",
encoding="utf-8")
rows = runs.reddit_comments("reddit_comments", self.source(public, manual))
self.assertEqual([r["run_id"] for r in rows], ["comment-c1", "comment-c3", "comment-c9"])
c1, c3, c9 = rows
self.assertEqual((c1["source"], c1["outcome_color"], c1["direction"], c1["hour"], c1["enc_Blue"]),
("reddit comment", "Blue", "Unknown", "Tree", "Bonded!"))
self.assertEqual(c1["data_flags"], "read from a Reddit comment")
self.assertEqual(c3["data_flags"], "read from a Reddit comment automatically, not checked by hand")
self.assertEqual((c9["company"], c9["direction"], c9["hour"]), ("2 Squadmates", "West", "DIED before the hour"))
self.assertEqual(validate.problems(rows, runs.DICT), [])
def test_delivered_not_extracted(self):
spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"]
with self.assertRaises(extract.Skip):
extract.EXTRACTORS[spec["extract"]["kind"]](spec["extract"], {})
class Reports(unittest.TestCase):
def issue(self, number, answers, **kw):
form = report_form.spec()