Receive runs from the Reddit megathread comments (source "reddit comment")
CI / test (pull_request) Successful in 46s
CI / test (pull_request) Successful in 46s
The threshing-reddit scraper (DragonKind/threshing-reddit) reads the runs riders describe in the r/fourthwing megathreads and delivers them here by pull request as data/public/reddit_comments.json: each run is the "Report a run" answers (option labels of collection/specs/report_form.yaml), with the comment id and day only; no usernames, no comment text. - pipeline.yaml: a reddit_comments source of extract kind "delivered" (the atlas fetches nothing; `update` skips it with a note) - runs.py: the coder, sharing one answers_row() with the reported coder; every row is flagged as read from a Reddit comment (and "not checked by hand" until corrected); a result page a comment already carries is not counted twice as a "reddit thread" row - collection/manual/reddit_comments.yaml: reject / fix / add, applied at build time - dictionary, SOURCES, README, croissant: the new source The route model and statistics do not use the source until it is named in planner/policy.yaml / analysis/stats/measures.yaml. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -259,6 +259,45 @@ class Reddit(unittest.TestCase):
|
||||
self.assertNotIn("someone", json.dumps(out))
|
||||
|
||||
|
||||
class RedditComments(unittest.TestCase):
|
||||
"""Runs the threshing-reddit scraper delivers (data/public/reddit_comments.json), with hand corrections."""
|
||||
|
||||
def source(self, public, manual):
|
||||
spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"]
|
||||
return {**spec, "load": str(public), "manual": str(manual)}
|
||||
|
||||
def test_coded_with_hand_corrections(self):
|
||||
public = TMP / "reddit_comments.json"
|
||||
public.write_text(json.dumps({"runs": [
|
||||
{"id": "c1", "thread": "abc", "posted": "2026-11-17", "result": None,
|
||||
"answers": {"company": "Alone", "direction": "East (toward the roar)", "outcome": "Bonded",
|
||||
"outcome_color": "Green", "branch": "Tree: climbed down slowly to the two Greens"}},
|
||||
{"id": "c2-1", "thread": "abc", "posted": "2026-11-17", "result": None,
|
||||
"answers": {"company": "Alone", "outcome": "Bonded", "outcome_color": "Red"}},
|
||||
{"id": "c3", "thread": "abc", "posted": None, "result": None,
|
||||
"answers": {"company": "Alone", "outcome": "Incinerated at a bonding encounter", "outcome_color": "Red"}}]}),
|
||||
encoding="utf-8")
|
||||
manual = TMP / "reddit_comments_manual.yaml"
|
||||
manual.write_text("reject: [c2]\nfix: {c1: {outcome_color: Blue, direction: null}}\n"
|
||||
"add: [{id: c9, thread: abc, posted: '2026-11-16', "
|
||||
"answers: {company: Two squadmates, direction: West (toward the scent), outcome: Died before the hour}}]\n",
|
||||
encoding="utf-8")
|
||||
rows = runs.reddit_comments("reddit_comments", self.source(public, manual))
|
||||
self.assertEqual([r["run_id"] for r in rows], ["comment-c1", "comment-c3", "comment-c9"])
|
||||
c1, c3, c9 = rows
|
||||
self.assertEqual((c1["source"], c1["outcome_color"], c1["direction"], c1["hour"], c1["enc_Blue"]),
|
||||
("reddit comment", "Blue", "Unknown", "Tree", "Bonded!"))
|
||||
self.assertEqual(c1["data_flags"], "read from a Reddit comment")
|
||||
self.assertEqual(c3["data_flags"], "read from a Reddit comment automatically, not checked by hand")
|
||||
self.assertEqual((c9["company"], c9["direction"], c9["hour"]), ("2 Squadmates", "West", "DIED before the hour"))
|
||||
self.assertEqual(validate.problems(rows, runs.DICT), [])
|
||||
|
||||
def test_delivered_not_extracted(self):
|
||||
spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"]
|
||||
with self.assertRaises(extract.Skip):
|
||||
extract.EXTRACTORS[spec["extract"]["kind"]](spec["extract"], {})
|
||||
|
||||
|
||||
class Reports(unittest.TestCase):
|
||||
def issue(self, number, answers, **kw):
|
||||
form = report_form.spec()
|
||||
|
||||
Reference in New Issue
Block a user