Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
70d0542ce8 | ||
|
|
57839fcc87 | ||
|
|
7ef8a58a0e | ||
|
|
27ba5fbcbc | ||
|
|
197c16dac0 |
@@ -90,18 +90,15 @@ route_thresholds: [5, 10, 20, 30]
|
||||
rare_goals: [Blue, Black, Blue or Black]
|
||||
|
||||
# "Show the paths of all riders" on the valley map (site/atlas/explorer.js): one layer per source that records a
|
||||
# route, and two layers of deaths. Unlike everything above, the layers use every source of runs_clean.csv. A layer
|
||||
# takes runs by source (and, for the spreadsheet, by its row label `entry`); the death layers take every death from
|
||||
# the route-recording sources: "deliberate" = burned at an encounter the rider did not genuinely attempt (the route
|
||||
# model's rule), "failed" = every other death. Discord has no runs yet: its layer is ready for when it is imported.
|
||||
# route, each with its own switch. Unlike everything above, the layers use every source of runs_clean.csv. Within a
|
||||
# layer the runs are split by how they ended, drawn as line styles in the layer's colour: solid (did not die), dashed
|
||||
# (chose to die: burned at an encounter the rider did not genuinely attempt, the route model's rule) and dotted
|
||||
# (failed: every other death). Discord has no runs yet: its layer is ready for when it is imported.
|
||||
path_layers:
|
||||
- { key: form, label: "Google Sheet: form posts", sources: [reddit spreadsheet], entry: [Form] }
|
||||
- { key: bulk, label: "Google Sheet: bulk logs", sources: [reddit spreadsheet], entry_not: [Form] }
|
||||
- { key: sheet, label: "Google Sheet", sources: [reddit spreadsheet] }
|
||||
- { key: reported, label: "Reported on the Atlas", sources: [reported] }
|
||||
- { key: reddit, label: "Reddit megathread comments", sources: [reddit comment] }
|
||||
- { key: bot, label: "DragonKind bot", sources: [bot] }
|
||||
- { key: discord, label: "Discord", sources: [discord] }
|
||||
- { key: died_deliberate, label: "Deaths: chose to die", death: deliberate }
|
||||
- { key: died_failed, label: "Deaths: failed runs", death: failed }
|
||||
# sources that record which dragon bonded but not the route: counted on the map, not drawn
|
||||
path_unrouted: [reddit thread, register]
|
||||
|
||||
@@ -468,30 +468,27 @@ def deliberate(r):
|
||||
|
||||
|
||||
def path_layers(runs):
|
||||
"""'Show the paths of all riders' on the valley map: for every layer in measures.yaml path_layers, how many runs
|
||||
went from each place to the next (like valley()), plus where the death layers' runs ended. Every source is used
|
||||
here, not only the statistics' sources. Sources that record no route are counted only (path_unrouted)."""
|
||||
cfg = M["path_layers"]
|
||||
routed = sorted({s for layer in cfg for s in layer.get("sources", [])})
|
||||
"""'Show the paths of all riders' on the valley map: for every layer in measures.yaml path_layers (one source or
|
||||
more), how many runs went from each place to the next (like valley()), split by how the runs ended: `lived` (did
|
||||
not die), `chose` (a deliberate death) and `failed` (every other death), plus where the deaths happened. Every
|
||||
source is used here, not only the statistics' sources. Sources that record no route are counted only."""
|
||||
out = []
|
||||
for layer in cfg:
|
||||
if "death" in layer:
|
||||
sel = [r for r in runs if r["source"] in routed and r["outcome"] in DEATHS
|
||||
and deliberate(r) == (layer["death"] == "deliberate")]
|
||||
else:
|
||||
sel = [r for r in runs if r["source"] in layer["sources"]
|
||||
and (not layer.get("entry") or r.get("entry") in layer["entry"])
|
||||
and (not layer.get("entry_not") or r.get("entry") not in layer["entry_not"])]
|
||||
v = valley(sel)
|
||||
item = {"key": layer["key"], "label": layer["label"], "runs": len(sel), "edges": v["edges"]}
|
||||
if "death" in layer:
|
||||
ends = {}
|
||||
for r in sel:
|
||||
for layer in M["path_layers"]:
|
||||
sel = [r for r in runs if r["source"] in layer["sources"]]
|
||||
kinds = {"lived": [], "chose": [], "failed": []}
|
||||
for r in sel:
|
||||
kinds["lived" if r["outcome"] not in DEATHS else "chose" if deliberate(r) else "failed"].append(r)
|
||||
ends = {}
|
||||
for k in ("chose", "failed"):
|
||||
e = {}
|
||||
for r in kinds[k]:
|
||||
seq = place_path(r)
|
||||
if seq:
|
||||
ends[seq[-1]] = ends.get(seq[-1], 0) + 1
|
||||
item["ends"] = dict(sorted(ends.items(), key=lambda kv: -kv[1]))
|
||||
out.append(item)
|
||||
e[seq[-1]] = e.get(seq[-1], 0) + 1
|
||||
ends[k] = dict(sorted(e.items(), key=lambda kv: -kv[1]))
|
||||
out.append({"key": layer["key"], "label": layer["label"], "runs": len(sel),
|
||||
"counts": {k: len(v) for k, v in kinds.items()},
|
||||
"lines": {k: valley(v)["edges"] for k, v in kinds.items()}, "ends": ends})
|
||||
unrouted = {s: sum(1 for r in runs if r["source"] == s) for s in M.get("path_unrouted", [])}
|
||||
return {"layers": out, "unrouted": unrouted}
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ it exactly.
|
||||
|---|---|---|---|
|
||||
| reddit spreadsheet | `reddit_spreadsheet`: u/ClumsyBadger's Threshing Routes form and spreadsheet, linked from the megathreads | the sheet's .xlsx export | `data/public/form_responses.json` (emails redacted), coded by `core/spec/form.yaml` |
|
||||
| reddit thread | `reddit_thread`: dragonkind.com result pages posted in the r/fourthwing megathreads | the threads, through the official Reddit API (needs `REDDIT_CLIENT_ID` / `REDDIT_CLIENT_SECRET`) | `manual/reddit_results.yaml`, checked by hand; the extract lists result links not in it yet |
|
||||
| reddit comment | `reddit_comments`: runs riders describe in the megathread comments | not extracted here: the [threshing-reddit](https://git.kenschiano.net/DragonKind/threshing-reddit) scraper reads the threads every 6 hours and opens (or updates) a pull request that changes the public file | `data/public/reddit_comments.json`: each comment read into the "Report a run" answers, no usernames or text; corrections by hand in `manual/reddit_comments.yaml` |
|
||||
| reported | `reported`: the Atlas's "Report a run" form (a GitHub issue form, `specs/report_form.yaml`) | issues labelled `run report`, through the GitHub API | `data/public/reports.json` (answers only, no usernames) |
|
||||
| bot | `bot`: Ken's DragonKind bot | a local copy of the bot's private repo (`DRAGONKIND_BOT_DIR`) | `data/bot/scenes.json`, `data/bot/trajectories-<date>.jsonl` (no prose, accounts hashed with `DK_ACCOUNT_SALT`; `specs/bot.yaml`) |
|
||||
| reddit pdf | `reddit_pdf`: the community route PDF | the PDF | `data/derived/pdf_fingerprint.json` only: hashes, never the text |
|
||||
|
||||
@@ -64,6 +64,11 @@ def reddit(spec, env):
|
||||
return json.dumps(threads, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
|
||||
|
||||
def delivered(spec, env):
|
||||
"""A source another repository delivers by pull request (it changes the public file itself): nothing to fetch."""
|
||||
raise Skip(f"delivered by pull request from {spec.get('by') or 'another repository'}")
|
||||
|
||||
|
||||
def forge_issues(spec, env):
|
||||
"""Every issue with the label (open and closed), from the repository's host (forge.yaml: Gitea or GitHub)."""
|
||||
from . import forge
|
||||
@@ -107,7 +112,7 @@ def pack_local(spec, src):
|
||||
return json.dumps(files, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
|
||||
|
||||
EXTRACTORS = {"http": http, "reddit": reddit, "forge_issues": forge_issues, "local_dir": local_dir}
|
||||
EXTRACTORS = {"http": http, "reddit": reddit, "forge_issues": forge_issues, "local_dir": local_dir, "delivered": delivered}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the raw store
|
||||
|
||||
+82
-35
@@ -4,6 +4,7 @@ One coder per source (pipeline.yaml `runs:`). Every coder returns rows keyed by
|
||||
missing answers get the same "not known" values the community form's coding uses, so every source can be filtered
|
||||
the same way. The `source` column says where a run came from; `entry` keeps the spreadsheet's own row label.
|
||||
"""
|
||||
import functools
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
@@ -67,8 +68,13 @@ def spreadsheet(name, spec):
|
||||
|
||||
def reddit_results(name, spec):
|
||||
"""Dragons read from result pages linked in the megathreads: bonded, colour and tail known, route unknown."""
|
||||
from .common import pipeline
|
||||
comments = pipeline()["sources"].get("reddit_comments")
|
||||
told = {r.get("result") for r in _comment_runs(comments)} if comments else set()
|
||||
rows = []
|
||||
for r in read_yaml(spec["transform"]["manual"])["results"]:
|
||||
if r["id"] in told:
|
||||
continue # the same run, with its route, is a "reddit comment" row
|
||||
row = blank_row(run_id=f"thread-{r['id']}", source=spec["run_source"], outcome="Bonded",
|
||||
outcome_color=r["colour"], tail=r["tail"])
|
||||
encounter_columns(row)
|
||||
@@ -110,46 +116,86 @@ def register(name, spec):
|
||||
|
||||
def reports(name, spec):
|
||||
"""Runs reported through the "Report a run" issue form (collection/specs/report_form.yaml)."""
|
||||
form = report_form.spec()
|
||||
fields = {f["id"]: f for f in form["fields"]}
|
||||
p = path(spec["load"])
|
||||
data = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {"reports": []}
|
||||
rows = []
|
||||
for rep in data["reports"]:
|
||||
vals, flags = {}, []
|
||||
for fid, answer in rep["answers"].items():
|
||||
f = fields.get(fid)
|
||||
if not f:
|
||||
continue
|
||||
if f.get("type") == "integer":
|
||||
m = re.search(r"\d+", answer)
|
||||
vals[f["column"]] = int(m.group()) if m else None
|
||||
elif answer in f["options"]:
|
||||
vals[f["column"]] = f["options"][answer]
|
||||
return [answers_row(f"reported-{rep['number']}", spec["run_source"], rep.get("opened"), rep["answers"])
|
||||
for rep in data["reports"]]
|
||||
|
||||
|
||||
def _comment_runs(spec):
|
||||
"""data/public/reddit_comments.json with the hand corrections of collection/manual/reddit_comments.yaml:
|
||||
`reject` (comment ids), `fix` (id -> answers that replace the read ones; null removes one) and `add` (runs the
|
||||
reader missed: {id, thread, posted, answers})."""
|
||||
p = path(spec["load"])
|
||||
data = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {"runs": []}
|
||||
manual = read_yaml(spec["manual"]) if spec.get("manual") and path(spec["manual"]).exists() else {}
|
||||
reject = {str(x) for x in manual.get("reject") or []}
|
||||
fix = {str(k): v for k, v in (manual.get("fix") or {}).items()}
|
||||
out = []
|
||||
for run in list(data["runs"]) + list(manual.get("add") or []):
|
||||
if str(run["id"]) in reject or str(run["id"]).split("-")[0] in reject:
|
||||
continue
|
||||
answers = dict(run["answers"])
|
||||
for k, v in (fix.get(str(run["id"])) or {}).items():
|
||||
if v is None:
|
||||
answers.pop(k, None)
|
||||
else:
|
||||
flags.append(f"unrecognised answer to {fid}")
|
||||
row = blank_row(run_id=f"reported-{rep['number']}", source=spec["run_source"], date=rep.get("opened"), **vals)
|
||||
row["attempt_group"] = attempt_group(row["attempt"])
|
||||
if row["outcome"] == "Died before the hour" and row["hour"] == "Unknown":
|
||||
row["hour"] = "DIED before the hour"
|
||||
branch = row["branch"]
|
||||
if branch:
|
||||
tree = branch in ("Shadow (rare)", "Descend quickly: Red", "Descend quickly: let the Red pass", "Descend slowly: two Greens")
|
||||
if row["hour"] == "Unknown":
|
||||
row["hour"] = "Tree" if tree else "Valley"
|
||||
elif row["hour"] != ("Tree" if tree else "Valley"):
|
||||
flags.append("encounter does not match the tree/valley answer")
|
||||
row["saw_shadow"] = branch == "Shadow (rare)" if tree else None
|
||||
if branch == "Passed the Red: meadow" and row["outcome_color"] == "Blue":
|
||||
row["saw_blue_meadow"] = True
|
||||
if row["outcome"] == "Bonded" and not row["outcome_color"]:
|
||||
flags.append("bonded, colour not given")
|
||||
encounter_columns(row)
|
||||
row["data_flags"] = "; ".join(flags)
|
||||
rows.append(row)
|
||||
answers[k] = str(v)
|
||||
out.append({**run, "answers": answers, "checked": str(run["id"]) in fix or run in (manual.get("add") or [])})
|
||||
return out
|
||||
|
||||
|
||||
def reddit_comments(name, spec):
|
||||
"""Runs riders described in the megathread comments, read into the "Report a run" answers (automatic: every
|
||||
row is flagged, and those not checked by hand say so)."""
|
||||
rows = []
|
||||
for run in _comment_runs(spec):
|
||||
rows.append(answers_row(f"comment-{run['id']}", spec["run_source"], run.get("posted"), run["answers"],
|
||||
["read from a Reddit comment" + ("" if run["checked"] else " automatically, not checked by hand")]))
|
||||
return rows
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _form_fields():
|
||||
return {f["id"]: f for f in report_form.spec()["fields"]}
|
||||
|
||||
|
||||
def answers_row(run_id, source, date, answers, flags=None):
|
||||
"""One run from the "Report a run" answers (option labels) -> a coded row."""
|
||||
fields = _form_fields()
|
||||
vals, flags = {}, list(flags or [])
|
||||
for fid, answer in answers.items():
|
||||
f = fields.get(fid)
|
||||
if not f:
|
||||
continue
|
||||
if f.get("type") == "integer":
|
||||
m = re.search(r"\d+", answer)
|
||||
vals[f["column"]] = int(m.group()) if m else None
|
||||
elif answer in f["options"]:
|
||||
vals[f["column"]] = f["options"][answer]
|
||||
else:
|
||||
flags.append(f"unrecognised answer to {fid}")
|
||||
row = blank_row(run_id=run_id, source=source, date=date, **vals)
|
||||
row["attempt_group"] = attempt_group(row["attempt"])
|
||||
if row["outcome"] == "Died before the hour" and row["hour"] == "Unknown":
|
||||
row["hour"] = "DIED before the hour"
|
||||
branch = row["branch"]
|
||||
if branch:
|
||||
tree = branch in ("Shadow (rare)", "Descend quickly: Red", "Descend quickly: let the Red pass", "Descend slowly: two Greens")
|
||||
if row["hour"] == "Unknown":
|
||||
row["hour"] = "Tree" if tree else "Valley"
|
||||
elif row["hour"] != ("Tree" if tree else "Valley"):
|
||||
flags.append("encounter does not match the tree/valley answer")
|
||||
row["saw_shadow"] = branch == "Shadow (rare)" if tree else None
|
||||
if branch == "Passed the Red: meadow" and row["outcome_color"] == "Blue":
|
||||
row["saw_blue_meadow"] = True
|
||||
if row["outcome"] == "Bonded" and not row["outcome_color"]:
|
||||
flags.append("bonded, colour not given")
|
||||
encounter_columns(row)
|
||||
row["data_flags"] = "; ".join(flags)
|
||||
return row
|
||||
|
||||
|
||||
def _norm(text):
|
||||
from .transform.bot import normalize
|
||||
return normalize(text)
|
||||
@@ -262,7 +308,8 @@ def bot(name, spec):
|
||||
return rows
|
||||
|
||||
|
||||
CODERS = {"spreadsheet": spreadsheet, "reddit_results": reddit_results, "reports": reports, "bot": bot, "register": register}
|
||||
CODERS = {"spreadsheet": spreadsheet, "reddit_results": reddit_results, "reddit_comments": reddit_comments,
|
||||
"reports": reports, "bot": bot, "register": register}
|
||||
|
||||
|
||||
def build_runs(pipe):
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
# Hand corrections to the runs read from the r/fourthwing megathread comments (data/public/reddit_comments.json).
|
||||
# The threshing-reddit scraper reads them automatically; every new run it finds is listed, with a link to its
|
||||
# comment, in the pull request that brings it in. Check them there and correct them here. Applied when the
|
||||
# runs are built (`python -m collection build`), so a correction never waits for the threads to change.
|
||||
#
|
||||
# reject: comment ids whose "run" is not a run (a joke, a friend's run, two runs told as one). An id without
|
||||
# its "-n" suffix rejects every run read from that comment.
|
||||
# fix: comment id -> answers that replace the ones read (form option labels from
|
||||
# collection/specs/report_form.yaml; null removes an answer). A fixed run counts as checked by hand.
|
||||
# add: runs the reader missed: { id, thread, posted: YYYY-MM-DD, answers: {...} }
|
||||
# No usernames: the comment id is enough to find the comment again.
|
||||
version: 1
|
||||
reject: []
|
||||
fix: {}
|
||||
add: []
|
||||
@@ -18,7 +18,7 @@ manifest: data/manifest.json # for each source: the sha256 of the current
|
||||
runs:
|
||||
path: data/public/runs_clean.csv
|
||||
# sources in this order; within a source, the order the coder returns
|
||||
order: [reddit_spreadsheet, reddit_thread, register, reported, bot, reddit_pdf]
|
||||
order: [reddit_spreadsheet, reddit_thread, reddit_comments, register, reported, bot, reddit_pdf]
|
||||
|
||||
sources:
|
||||
reddit_spreadsheet:
|
||||
@@ -69,6 +69,24 @@ sources:
|
||||
load: null # the public source file is the hand-checked collection/manual/reddit_results.yaml
|
||||
runs: reddit_results
|
||||
|
||||
reddit_comments:
|
||||
label: Runs riders describe in the r/fourthwing Dragonkind megathread comments
|
||||
run_source: reddit comment
|
||||
links:
|
||||
part3: https://www.reddit.com/r/fourthwing/comments/1wvwfp9/
|
||||
part4: https://www.reddit.com/r/fourthwing/comments/1wz5cnj/
|
||||
scraper: https://git.kenschiano.net/DragonKind/threshing-reddit
|
||||
# delivered, not extracted: the threshing-reddit scraper (its own repository) reads the threads every 6 hours and
|
||||
# opens a pull request here that changes data/public/reddit_comments.json. Each comment is read into the
|
||||
# "Report a run" answers (option labels of specs/report_form.yaml); only the comment id, the day and the
|
||||
# answers are kept: no usernames, no comment text. `update` leaves this source alone.
|
||||
extract: { kind: delivered, by: threshing-reddit }
|
||||
transform: null
|
||||
load: data/public/reddit_comments.json
|
||||
# hand corrections (reject / fix / add a run), applied when the runs are built
|
||||
manual: collection/manual/reddit_comments.yaml
|
||||
runs: reddit_comments
|
||||
|
||||
reported:
|
||||
label: Runs reported on the Atlas ("Report a run", an issue form on the Atlas's repository)
|
||||
run_source: reported
|
||||
|
||||
@@ -7,6 +7,7 @@ Every source is declared in `collection/pipeline.yaml` and collected by `python
|
||||
|---|---|
|
||||
| Community "Threshing Routes" form and spreadsheet, u/ClumsyBadger (`reddit spreadsheet`) | `public/form_responses.json` (emails redacted) |
|
||||
| Result pages posted in the r/fourthwing megathreads (`reddit thread`) | `collection/manual/reddit_results.yaml` |
|
||||
| Runs described in the r/fourthwing megathread comments (`reddit comment`) | `public/reddit_comments.json` (comment id, day and answers; no usernames or comment text) |
|
||||
| The Atlas's "Report a run" form (`reported`) | `public/reports.json` (answers only, no usernames) |
|
||||
| DragonKind bot exports (`bot`) | `bot/scenes.json`, `bot/trajectories-<date>.jsonl` (anonymised; see `docs/BOT_DATA_PROTOCOL.md`) |
|
||||
| Community route PDF, u/ClumsyBadger (`reddit pdf`) | not redistributed: `derived/pdf_fingerprint.json` only; it maps the choices and holds no runs |
|
||||
|
||||
+17
-2
@@ -78,6 +78,11 @@
|
||||
"@type": "sc:CreativeWork",
|
||||
"name": "Runs posted in the r/fourthwing Dragonkind megathreads",
|
||||
"url": "https://www.reddit.com/r/fourthwing/comments/1wvwfp9/"
|
||||
},
|
||||
{
|
||||
"@type": "sc:CreativeWork",
|
||||
"name": "Runs riders describe in the r/fourthwing Dragonkind megathread comments",
|
||||
"url": "https://www.reddit.com/r/fourthwing/comments/1wvwfp9/"
|
||||
}
|
||||
],
|
||||
"distribution": [
|
||||
@@ -101,6 +106,16 @@
|
||||
"sha256": "319f6217e0a431454114a67276f429379ec383cd6ed2666443b48b8f48c66490",
|
||||
"contentSize": "12623695 B"
|
||||
},
|
||||
{
|
||||
"@type": "cr:FileObject",
|
||||
"@id": "reddit_comments",
|
||||
"name": "reddit_comments",
|
||||
"description": "Runs described in the r/fourthwing megathread comments, read into the \"Report a run\" answers: comment id, day and answers only, no usernames or comment text.",
|
||||
"contentUrl": "https://git.kenschiano.net/DragonKind/threshing-atlas/raw/branch/main/data/public/reddit_comments.json",
|
||||
"encodingFormat": "application/json",
|
||||
"sha256": "c180f8b82ed1119d4ff28697fcf45d0c7f22859597cd10f5e82a2e50ca1dc51a",
|
||||
"contentSize": "105 B"
|
||||
},
|
||||
{
|
||||
"@type": "cr:FileObject",
|
||||
"@id": "reports",
|
||||
@@ -166,7 +181,7 @@
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/run_id",
|
||||
"name": "run_id",
|
||||
"description": "Unique id of the run: sheet-<row of the form export>, thread-<result id>, reported-<issue number> or bot-<hash>.",
|
||||
"description": "Unique id of the run: sheet-<row of the form export>, thread-<result id>, comment-<Reddit comment id>, reported-<issue number> or bot-<hash>.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
@@ -181,7 +196,7 @@
|
||||
"@type": "cr:Field",
|
||||
"@id": "runs/source",
|
||||
"name": "source",
|
||||
"description": "Where the run came from. \"reddit spreadsheet\": the community Threshing Routes form and spreadsheet linked from the r/fourthwing megathreads. \"reddit thread\": a dragonkind.com result page posted in a megathread (bonded, route not recorded). \"reported\": the Atlas's own \"Report a run\" form. \"bot\": an attempt logged by the DragonKind bot. \"register\": a bonded dragon entered in the community's Unofficial Dragon Register (outcome only, no route; the same run may also be in the spreadsheet). \"reddit pdf\": the community route PDF (a flowchart of the routes, not a log of runs, so it adds no rows today). Values: reddit spreadsheet; reddit thread; register; reported; bot; reddit pdf.",
|
||||
"description": "Where the run came from. \"reddit spreadsheet\": the community Threshing Routes form and spreadsheet linked from the r/fourthwing megathreads. \"reddit thread\": a dragonkind.com result page posted in a megathread (bonded, route not recorded). \"reddit comment\": a run a rider described in a megathread comment, read automatically into the \"Report a run\" answers by the threshing-reddit scraper, which delivers them by pull request (hand corrections in collection/manual/reddit_comments.yaml). \"reported\": the Atlas's own \"Report a run\" form. \"bot\": an attempt logged by the DragonKind bot. \"register\": a bonded dragon entered in the community's Unofficial Dragon Register (outcome only, no route; the same run may also be in the spreadsheet). \"reddit pdf\": the community route PDF (a flowchart of the routes, not a log of runs, so it adds no rows today). Values: reddit spreadsheet; reddit thread; reddit comment; register; reported; bot; reddit pdf.",
|
||||
"dataType": "sc:Text",
|
||||
"source": {
|
||||
"fileObject": {
|
||||
|
||||
+11
-3
@@ -35,6 +35,12 @@ files:
|
||||
description: >-
|
||||
The community "Threshing Routes" form export as published (headers and rows), with email addresses in
|
||||
free-text answers redacted. The source of runs_clean.
|
||||
reddit_comments:
|
||||
path: data/public/reddit_comments.json
|
||||
format: application/json
|
||||
description: >-
|
||||
Runs described in the r/fourthwing megathread comments, read into the "Report a run" answers: comment id, day
|
||||
and answers only, no usernames or comment text.
|
||||
reports:
|
||||
path: data/public/reports.json
|
||||
format: application/json
|
||||
@@ -63,18 +69,20 @@ files:
|
||||
# The columns of runs_clean. type: text, integer, boolean or date. values: every value the column may take
|
||||
# (empty cells are always allowed and mean "does not apply" or "not recorded").
|
||||
columns:
|
||||
run_id: { type: text, required: true, description: "Unique id of the run: sheet-<row of the form export>, thread-<result id>, reported-<issue number> or bot-<hash>." }
|
||||
run_id: { type: text, required: true, description: "Unique id of the run: sheet-<row of the form export>, thread-<result id>, comment-<Reddit comment id>, reported-<issue number> or bot-<hash>." }
|
||||
source:
|
||||
type: text
|
||||
required: true
|
||||
description: >-
|
||||
Where the run came from. "reddit spreadsheet": the community Threshing Routes form and spreadsheet linked from
|
||||
the r/fourthwing megathreads. "reddit thread": a dragonkind.com result page posted in a megathread (bonded,
|
||||
route not recorded). "reported": the Atlas's own "Report a run" form. "bot": an attempt logged by the
|
||||
route not recorded). "reddit comment": a run a rider described in a megathread comment, read automatically into
|
||||
the "Report a run" answers by the threshing-reddit scraper, which delivers them by pull request (hand
|
||||
corrections in collection/manual/reddit_comments.yaml). "reported": the Atlas's own "Report a run" form. "bot": an attempt logged by the
|
||||
DragonKind bot. "register": a bonded dragon entered in the community's Unofficial Dragon Register (outcome
|
||||
only, no route; the same run may also be in the spreadsheet). "reddit pdf": the community route PDF (a
|
||||
flowchart of the routes, not a log of runs, so it adds no rows today).
|
||||
values: [reddit spreadsheet, reddit thread, register, reported, bot, reddit pdf]
|
||||
values: [reddit spreadsheet, reddit thread, reddit comment, register, reported, bot, reddit pdf]
|
||||
entry:
|
||||
type: text
|
||||
description: >-
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"source":"Runs riders describe in the r/fourthwing Dragonkind megathread comments","runs":[],"unread":0}
|
||||
+31
-10
File diff suppressed because one or more lines are too long
+35
-25
@@ -434,12 +434,14 @@
|
||||
}
|
||||
}
|
||||
|
||||
// "Show the paths of all riders": one ink per layer (data/stats.json paths.layers; measures.yaml path_layers)
|
||||
const LAYER_INK = {
|
||||
form: "#6b4a33", bulk: "#2f6b57", reported: "#7a4f8f", reddit: "#a8792a", bot: "#28588a", discord: "#5865f2",
|
||||
died_deliberate: "#7d8794", died_failed: "#c2410c",
|
||||
// "Show the paths of all riders": one ink per source layer (data/stats.json paths.layers; measures.yaml path_layers),
|
||||
// and one line style per way the runs ended
|
||||
const LAYER_INK = { sheet: "#6b4a33", reported: "#7a4f8f", reddit: "#a8792a", bot: "#28588a", discord: "#5865f2" };
|
||||
const KIND = {
|
||||
lived: { dash: null, cap: "round", label: "survived" },
|
||||
chose: { dash: "11 6", cap: "butt", label: "chose to die" },
|
||||
failed: { dash: "0.1 6", cap: "round", label: "died (failed run)" },
|
||||
};
|
||||
const isDeath = (k) => k.startsWith("died_");
|
||||
const layerOn = (k) => st.layers[k] !== false; // every layer starts on
|
||||
|
||||
function drawTraffic(g) {
|
||||
@@ -453,26 +455,32 @@
|
||||
});
|
||||
}
|
||||
|
||||
/** Every layer that is switched on, side by side; the death layers dashed, with a mark where their runs ended. */
|
||||
/** Every source layer that is switched on, side by side in its own ink: solid for runs that survived, dashed for
|
||||
* deliberate deaths, dotted for failed runs, and a mark where the deaths happened. */
|
||||
function drawLayers(g, layers) {
|
||||
const shown = layers.filter((l) => l.runs && layerOn(l.key));
|
||||
const max = Math.max(1, ...shown.flatMap((l) => l.edges.map((e) => e[2])));
|
||||
const shown = layers.filter((l) => l.runs && layerOn(l.key) && l.lines);
|
||||
const all = shown.flatMap((l) => Object.values(l.lines).flatMap((es) => es.map((e) => e[2])));
|
||||
const max = Math.max(1, ...all);
|
||||
shown.forEach((l, i) => {
|
||||
const off = (i - (shown.length - 1) / 2) * 6, ink = LAYER_INK[l.key] || "#6b4a33", death = isDeath(l.key);
|
||||
l.edges.forEach(([a, b, n]) => {
|
||||
if (!L[a] || !L[b]) return;
|
||||
el("path", { d: curve(L[a], L[b], 0.05, off), fill: "none", stroke: ink, "stroke-opacity": death ? 0.5 : 0.38,
|
||||
"stroke-width": 1.2 + 16 * Math.sqrt(n / max), "stroke-linecap": death ? "butt" : "round", "stroke-dasharray": death ? "10 6" : null }, g);
|
||||
});
|
||||
if (death && l.ends) {
|
||||
const emax = Math.max(1, ...Object.values(l.ends));
|
||||
Object.entries(l.ends).forEach(([p, n]) => {
|
||||
if (!L[p]) return;
|
||||
const r = 3 + 11 * Math.sqrt(n / emax), x = L[p].x + off, y = L[p].y;
|
||||
el("circle", { cx: x, cy: y, r, fill: ink, "fill-opacity": 0.35, stroke: ink, "stroke-width": 1.5 }, g);
|
||||
el("path", { d: `M${x - r * 0.5},${y - r * 0.5}L${x + r * 0.5},${y + r * 0.5}M${x + r * 0.5},${y - r * 0.5}L${x - r * 0.5},${y + r * 0.5}`, stroke: ink, "stroke-width": 1.5 }, g);
|
||||
const base = (i - (shown.length - 1) / 2) * 9, ink = LAYER_INK[l.key] || "#6b4a33";
|
||||
["lived", "chose", "failed"].forEach((k, j) => {
|
||||
const off = base + (j - 1) * 3, kd = KIND[k];
|
||||
(l.lines[k] || []).forEach(([a, b, n]) => {
|
||||
if (!L[a] || !L[b]) return;
|
||||
const w = 1.2 + 14 * Math.sqrt(n / max);
|
||||
el("path", { d: curve(L[a], L[b], 0.05, off), fill: "none", stroke: ink, "stroke-opacity": k === "lived" ? 0.36 : 0.6,
|
||||
"stroke-width": k === "failed" ? Math.max(2.2, w) : w, "stroke-linecap": kd.cap, "stroke-dasharray": kd.dash }, g);
|
||||
});
|
||||
}
|
||||
});
|
||||
// where the deaths happened: one mark per place, sized by the deaths there
|
||||
const ends = {};
|
||||
["chose", "failed"].forEach((k) => Object.entries((l.ends || {})[k] || {}).forEach(([p, n]) => (ends[p] = (ends[p] || 0) + n)));
|
||||
const emax = Math.max(1, ...Object.values(ends));
|
||||
Object.entries(ends).forEach(([p, n]) => {
|
||||
if (!L[p]) return;
|
||||
const r = 2.5 + 9 * Math.sqrt(n / emax), x = L[p].x + base, y = L[p].y;
|
||||
el("path", { d: `M${x - r * 0.6},${y - r * 0.6}L${x + r * 0.6},${y + r * 0.6}M${x + r * 0.6},${y - r * 0.6}L${x - r * 0.6},${y + r * 0.6}`, stroke: ink, "stroke-width": 2, "stroke-opacity": 0.8 }, g);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -671,12 +679,14 @@
|
||||
const UNROUTED = { "reddit thread": "Reddit result pages", register: "the Dragon Register" };
|
||||
function trafficToggle() {
|
||||
const box = `<label class="ex-toggle"><input type="checkbox" id="ex-traffic"${st.traffic ? " checked" : ""}> Show the paths of all riders</label>`;
|
||||
if (!st.traffic || !st.paths || !st.paths.layers) return box;
|
||||
if (!st.traffic || !st.paths || !st.paths.layers || !st.paths.layers.some((l) => l.lines)) return box;
|
||||
const rows = st.paths.layers.map((l) => `<label class="ex-layer${l.runs ? "" : " none"}"><input type="checkbox" data-layer="${esc(l.key)}"${l.runs && layerOn(l.key) ? " checked" : ""}${l.runs ? "" : " disabled"}>
|
||||
<span class="sw${isDeath(l.key) ? " dash" : ""}" style="--sw: ${LAYER_INK[l.key] || "#6b4a33"}"></span><span>${esc(l.label)}</span><span class="n">${l.runs ? l.runs.toLocaleString() : "none yet"}</span></label>`).join("");
|
||||
<span class="sw" style="--sw: ${LAYER_INK[l.key] || "#6b4a33"}"></span><span>${esc(l.label)}</span><span class="n">${l.runs ? l.runs.toLocaleString() : "none yet"}</span></label>`).join("");
|
||||
const key = Object.values(KIND).map((kd) => `<span class="ex-kind"><svg viewBox="0 0 34 8" aria-hidden="true"><path d="M3 4H31" stroke="currentColor" stroke-width="2.6" stroke-linecap="${kd.cap}"${kd.dash ? ` stroke-dasharray="${kd.dash === "11 6" ? "8 4" : "0.1 5"}"` : ""}/></svg>${kd.label}</span>`).join("");
|
||||
const un = Object.entries(st.paths.unrouted || {}).filter(([, n]) => n).map(([s, n]) => `${n.toLocaleString()} from ${esc(UNROUTED[s] || s)}`);
|
||||
return `${box}<div class="ex-layers" role="group" aria-label="Which riders' paths to show">${rows}
|
||||
<p class="small muted">Line width: how many runs took each step. Dashed lines are deaths; ✕ marks where those runs ended.${un.length ? ` Not drawn (they record the dragon, not the route): ${un.join(", ")}.` : ""}</p></div>`;
|
||||
<div class="ex-kinds">${key}</div>
|
||||
<p class="small muted">Each source in its own colour; line width is how many runs took each step, and ✕ marks where runs died.${un.length ? ` Not drawn (they record the dragon, not the route): ${un.join(", ")}.` : ""}</p></div>`;
|
||||
}
|
||||
function chooser() {
|
||||
return `<div class="ex-choose">${COLOURS.map((c) => `<button type="button" class="ex-dchip" data-c="${c}" aria-pressed="${st.colour === c}" style="--c: var(${DV[c]})">${mini(c, POSE_OF[c])}<span>${c}</span></button>`).join("")}</div>`;
|
||||
|
||||
@@ -342,8 +342,10 @@ footer { border-top-color: var(--line); }
|
||||
.ex-layer { display: grid; grid-template-columns: auto 22px minmax(0, 1fr) auto; gap: 8px; align-items: center; font-size: var(--step--1); color: inherit; cursor: pointer; }
|
||||
.ex-layer.none { color: var(--muted); cursor: default; }
|
||||
.ex-layer .sw { height: 0; border-top: 4px solid var(--sw); border-radius: 2px; }
|
||||
.ex-layer .sw.dash { border-top-style: dashed; }
|
||||
.ex-layer .n { color: var(--muted); font-variant-numeric: tabular-nums; }
|
||||
.ex-kinds { display: flex; flex-wrap: wrap; gap: 4px 14px; margin-top: 4px; font-size: var(--step--1); color: var(--muted); }
|
||||
.ex-kind { display: inline-flex; align-items: center; gap: 6px; }
|
||||
.ex-kind svg { width: 34px; height: 8px; }
|
||||
.ex-head { display: grid; grid-template-columns: 64px minmax(0, 1fr) auto; gap: 10px; align-items: center; }
|
||||
.ex-head svg { width: 64px; height: 52px; }
|
||||
.ex-head h2 { color: var(--c); }
|
||||
|
||||
@@ -259,6 +259,45 @@ class Reddit(unittest.TestCase):
|
||||
self.assertNotIn("someone", json.dumps(out))
|
||||
|
||||
|
||||
class RedditComments(unittest.TestCase):
|
||||
"""Runs the threshing-reddit scraper delivers (data/public/reddit_comments.json), with hand corrections."""
|
||||
|
||||
def source(self, public, manual):
|
||||
spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"]
|
||||
return {**spec, "load": str(public), "manual": str(manual)}
|
||||
|
||||
def test_coded_with_hand_corrections(self):
|
||||
public = TMP / "reddit_comments.json"
|
||||
public.write_text(json.dumps({"runs": [
|
||||
{"id": "c1", "thread": "abc", "posted": "2026-11-17", "result": None,
|
||||
"answers": {"company": "Alone", "direction": "East (toward the roar)", "outcome": "Bonded",
|
||||
"outcome_color": "Green", "branch": "Tree: climbed down slowly to the two Greens"}},
|
||||
{"id": "c2-1", "thread": "abc", "posted": "2026-11-17", "result": None,
|
||||
"answers": {"company": "Alone", "outcome": "Bonded", "outcome_color": "Red"}},
|
||||
{"id": "c3", "thread": "abc", "posted": None, "result": None,
|
||||
"answers": {"company": "Alone", "outcome": "Incinerated at a bonding encounter", "outcome_color": "Red"}}]}),
|
||||
encoding="utf-8")
|
||||
manual = TMP / "reddit_comments_manual.yaml"
|
||||
manual.write_text("reject: [c2]\nfix: {c1: {outcome_color: Blue, direction: null}}\n"
|
||||
"add: [{id: c9, thread: abc, posted: '2026-11-16', "
|
||||
"answers: {company: Two squadmates, direction: West (toward the scent), outcome: Died before the hour}}]\n",
|
||||
encoding="utf-8")
|
||||
rows = runs.reddit_comments("reddit_comments", self.source(public, manual))
|
||||
self.assertEqual([r["run_id"] for r in rows], ["comment-c1", "comment-c3", "comment-c9"])
|
||||
c1, c3, c9 = rows
|
||||
self.assertEqual((c1["source"], c1["outcome_color"], c1["direction"], c1["hour"], c1["enc_Blue"]),
|
||||
("reddit comment", "Blue", "Unknown", "Tree", "Bonded!"))
|
||||
self.assertEqual(c1["data_flags"], "read from a Reddit comment")
|
||||
self.assertEqual(c3["data_flags"], "read from a Reddit comment automatically, not checked by hand")
|
||||
self.assertEqual((c9["company"], c9["direction"], c9["hour"]), ("2 Squadmates", "West", "DIED before the hour"))
|
||||
self.assertEqual(validate.problems(rows, runs.DICT), [])
|
||||
|
||||
def test_delivered_not_extracted(self):
|
||||
spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"]
|
||||
with self.assertRaises(extract.Skip):
|
||||
extract.EXTRACTORS[spec["extract"]["kind"]](spec["extract"], {})
|
||||
|
||||
|
||||
class Reports(unittest.TestCase):
|
||||
def issue(self, number, answers, **kw):
|
||||
form = report_form.spec()
|
||||
|
||||
Reference in New Issue
Block a user