Review summary on Reddit scraper PRs; measure the form/Reddit overlap #44

Merged
kathleen merged 1 commits from atlas/reddit-review into main 2026-11-19 16:50:49 +00:00
15 changed files with 497 additions and 6 deletions
Showing only changes of commit 98be27fc1f - Show all commits
+31
View File
@@ -0,0 +1,31 @@
# A review summary on every Reddit scraper pull request (issue #23): the runs added, changed and gone, the ones that
# break a game rule, other flags, possible copies of form rows, and result pages already counted. Posted as one
# comment on the pull request, refreshed on every push. The reviewer corrects runs in
# collection/manual/reddit_comments.yaml (collection/README.md, "Reviewing Reddit runs"), then merges.
#
# Secrets: ATLAS_BOT_TOKEN (as in update-sources.yml), to post the comment.
name: Review Reddit runs
on:
pull_request:
paths:
- data/public/reddit_comments.json
- collection/manual/reddit_comments.yaml
jobs:
summary:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- run: pip install -r requirements.txt
- name: The file as it is on main
run: |
git fetch --depth 1 origin "${{ github.base_ref }}"
git show FETCH_HEAD:data/public/reddit_comments.json > "$RUNNER_TEMP/base.json" || echo '{"runs": []}' > "$RUNNER_TEMP/base.json"
- name: Build the runs and post the summary
env:
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
run: |
python -m collection build
python -m collection review-reddit --base "$RUNNER_TEMP/base.json" --out "$RUNNER_TEMP/review.md" --pr "${{ github.event.pull_request.number }}"
+112
View File
@@ -0,0 +1,112 @@
"""Is the same attempt reported twice, in the community form and in a Reddit comment? (issue #24)
python analysis/overlap/overlap.py [--check]
There are no player ids, so a copy can only be proven by a shared dragonkind.com result-page id. Neither the form
nor the register records one, and the comment-to-result-page link is already applied when the runs are built
(collection/etl/runs.py). Everything else is measured, never removed:
- a Reddit comment run is *comparable* when it is dated and gives at least MIN_ANSWERS route answers besides how it
ended (fewer answers match dozens of form rows by chance on a busy day);
- it *matches* a form row submitted within WINDOW days when every answer it gives is the form row's answer;
- the same test with the comment's date shifted by a few weeks (SHIFTS, only where the form has rows that week) is
what chance alone produces, so matched - expected is the estimate of runs reported twice.
Writes data/public/overlap.json (the page's Data & methods tab, and the scraper pull-request summary). Nothing is
dropped: identical routes are expected, and the Reddit runs already have their own reporting-bias group in the
route model (planner/policy.yaml).
"""
import argparse
import collections
import datetime as dt
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "core"))
from atlas import runs as runs_io # noqa: E402
OUT = ROOT / "data/public/overlap.json"
FIELDS = ["company", "left_squadmates", "direction", "first_sighting", "sighting_action", "scream", "resolution",
"combat_first", "combat_second", "post_path", "hour", "branch", "outcome", "outcome_color", "tail", "attempt"]
ENDING = ("outcome", "outcome_color")
MIN_ANSWERS = 4
WINDOW = 1
SHIFTS = (-28, -21, -14, 14, 21, 28)
def given(v):
return v not in (None, "") and v != "Unknown" and not (isinstance(v, str) and v.startswith("("))
def answers(r):
return [k for k in FIELDS if given(r.get(k))]
def comparable(r):
return bool(r.get("date")) and len([k for k in answers(r) if k not in ENDING]) >= MIN_ANSWERS
def days(date, shift=0, window=WINDOW):
d = dt.date.fromisoformat(date[:10]) + dt.timedelta(days=shift)
return [(d + dt.timedelta(days=i)).isoformat() for i in range(-window, window + 1)]
def form_index(runs):
by_day = collections.defaultdict(list)
for r in runs:
if r["source"] == "reddit spreadsheet" and r.get("date"):
by_day[r["date"][:10]].append(r)
return by_day
def matches(c, by_day, shift=0):
"""Form rows within the window that give every answer the comment gives."""
keys = answers(c)
return [f for d in days(c["date"], shift) for f in by_day.get(d, []) if all(f.get(k) == c[k] for k in keys)]
def measure(runs):
by_day = form_index(runs)
comments = [r for r in runs if r["source"] == "reddit comment"]
comp = [c for c in comments if comparable(c)]
found = {c["run_id"]: [f["run_id"] for f in matches(c, by_day)] for c in comp}
matched = sum(1 for v in found.values() if v)
# chance: each comment's match rate on shifted dates, over the shifts whose window has form rows at all
expected = 0.0
for c in comp:
valid = [s for s in SHIFTS if any(by_day.get(d) for d in days(c["date"], s))]
if valid:
expected += sum(1 for s in valid if matches(c, by_day, s)) / len(valid)
return {
"schema": 1,
"rule": {"min_answers": MIN_ANSWERS, "window_days": WINDOW, "baseline_shifts_days": list(SHIFTS)},
"comment_runs": len(comments),
"comparable": len(comp),
"matched": matched,
"expected_by_chance": round(expected, 1),
"likely_double": round(max(0.0, matched - expected), 1),
"matches": [{"comment": k, "form": v} for k, v in sorted(found.items()) if v],
}
def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--check", action="store_true", help="fail if data/public/overlap.json is out of date")
a = ap.parse_args(argv)
text = json.dumps(measure(runs_io.read(drop=())), indent=1, ensure_ascii=False) + "\n"
if a.check:
if not OUT.exists() or OUT.read_text(encoding="utf-8") != text:
print(f"out of date (run `python {Path(__file__).relative_to(ROOT)}`): {OUT.relative_to(ROOT)}")
return 1
return 0
OUT.write_text(text, encoding="utf-8")
m = json.loads(text)
print(f"{OUT.relative_to(ROOT)}: {m['matched']} of {m['comparable']} comparable Reddit runs match a form row "
f"(~{m['expected_by_chance']} by chance; ~{m['likely_double']} likely reported twice)")
return 0
if __name__ == "__main__":
sys.exit(main())
+19
View File
@@ -41,6 +41,25 @@ The route PDF is a flowchart of the choices, not a log of runs, so it adds no ro
fingerprint tells us when the routes it documents change. The register lists bonded dragons without the route that fingerprint tells us when the routes it documents change. The register lists bonded dragons without the route that
led there, so it adds no rows either. led there, so it adds no rows either.
## Reviewing Reddit runs
Ken's scraper opens (or updates) a pull request from `data/reddit-comments` that changes
`data/public/reddit_comments.json`. The **Review Reddit runs** workflow (`.gitea/workflows/reddit-review.yml`) posts a
summary on it, refreshed on every push (`python -m collection review-reddit --base <main's file>` locally):
- runs added, changed and gone since main;
- **runs that break a game rule** (`core/spec/form.yaml` `impossible_flags`): left out of the route model and the
statistics until corrected. Open the comment and decide;
- other flags on the new runs (kept);
- runs that may also be in the community form: the same answers within a day, with how many chance alone gives
(`analysis/overlap/overlap.py`, `data/public/overlap.json`). Measured only, never removed;
- runs whose result page is already a `reddit thread` row (counted once automatically).
Corrections go in `manual/reddit_comments.yaml`, applied at build time, so they never wait for the threads:
`reject: [id]` (not a run), `fix: {id: {question: answer}}` (misread; `null` removes an answer) and
`add: [{id, thread, posted, answers}]` (missed). A corrected run loses the "not checked by hand" note. No usernames:
the comment id is enough to find the comment again. Then merge, once CI passes.
## Raw files and privacy ## Raw files and privacy
`data/raw/` is git-ignored and never published. It holds the current raw file of each source and the one before `data/raw/` is git-ignored and never published. It holds the current raw file of each source and the one before
+12
View File
@@ -131,6 +131,18 @@ def open_issue(title, body, env=None, f=None):
return _call("POST", f"{api_base(f)}/issues", {"title": title, "body": body}, tok, f)["number"] return _call("POST", f"{api_base(f)}/issues", {"title": title, "body": body}, tok, f)["number"]
def upsert_comment(number, marker, body, env=None, f=None):
"""Post `body` on issue or pull request `number`, replacing an earlier comment that starts with `marker`
(so a summary refreshed on every push stays one comment)."""
f = f or config()
tok = token(env)
comments = _call("GET", f"{api_base(f)}/issues/{number}/comments?limit=50&per_page=100", tok=tok, f=f) or []
mine = [c for c in comments if (c.get("body") or "").startswith(marker)]
if mine:
return _call("PATCH", f"{api_base(f)}/issues/comments/{mine[-1]['id']}", {"body": body}, tok, f)["id"]
return _call("POST", f"{api_base(f)}/issues/{number}/comments", {"body": body}, tok, f)["id"]
def ensure_label(name, color, env=None, f=None): def ensure_label(name, color, env=None, f=None):
f = f or config() f = f or config()
tok = token(env) tok = token(env)
+1
View File
@@ -88,6 +88,7 @@ def rebuild():
_step("what the runs show about the Black", [py, "analysis/black/evidence.py"]) _step("what the runs show about the Black", [py, "analysis/black/evidence.py"])
_step("screen tree", [py, "analysis/tree/build_tree.py"]) _step("screen tree", [py, "analysis/tree/build_tree.py"])
_step("known answers for the bot (guide + screen tree)", [py, "analysis/known/build_known.py"]) _step("known answers for the bot (guide + screen tree)", [py, "analysis/known/build_known.py"])
_step("runs reported twice (form + Reddit)", [py, "analysis/overlap/overlap.py"])
try: try:
import scipy # noqa: F401 import scipy # noqa: F401
_step("statistics", [py, "analysis/stats/site_stats.py"]) _step("statistics", [py, "analysis/stats/site_stats.py"])
+133
View File
@@ -0,0 +1,133 @@
"""A review summary for a Reddit scraper pull request (issue #23).
python -m collection review-reddit --base BASE.json [--out summary.md] [--pr N]
Compares the pull request's data/public/reddit_comments.json with `BASE.json` (the same file on main) and reads the
runs as built from it (data/public/runs_clean.csv, so run `python -m collection build` first). The summary lists
what a reviewer needs to decide on, before merging:
- runs added, changed (answers differ) and gone;
- runs that break a game rule: left out of the route model and the statistics until corrected;
- other flags on the new or changed runs (kept, for information);
- runs that may also be in the community form (analysis/overlap/overlap.py: same answers within a day);
- runs whose result page is already a "reddit thread" row (counted once, automatically).
Corrections go in collection/manual/reddit_comments.yaml (reject / fix / add; see collection/README.md). With
`--pr N` the summary is posted on the pull request, replacing the previous one.
"""
import importlib.util
import json
import sys
from .common import ROOT, path, pipeline, read_yaml
MARKER = "<!-- atlas:reddit-review -->"
PROVENANCE = "read from a Reddit comment"
def _overlap():
spec = importlib.util.spec_from_file_location("overlap", ROOT / "analysis/overlap/overlap.py")
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def link(run_id, thread):
cid = str(run_id).split("-")[0]
return f"[{run_id}](https://www.reddit.com/r/fourthwing/comments/{thread}/_/{cid}/)"
def diff(base, head):
"""(added, changed, gone) comment-run ids between two reddit_comments.json documents."""
b = {str(r["id"]): r for r in base.get("runs", [])}
h = {str(r["id"]): r for r in head.get("runs", [])}
added = sorted(set(h) - set(b))
gone = sorted(set(b) - set(h))
changed = sorted(k for k in set(h) & set(b) if h[k].get("answers") != b[k].get("answers"))
return added, changed, gone
def summary(base, head, runs, known_results=(), overlap=None):
"""The Markdown summary. `runs`: typed records of runs_clean.csv (with run_id)."""
sys.path.insert(0, str(ROOT / "core"))
from atlas import runs as runs_io
added, changed, gone = diff(base, head)
focus = set(added) | set(changed)
thread = {str(r["id"]): r.get("thread") for r in head.get("runs", [])}
result = {str(r["id"]): r.get("result") for r in head.get("runs", [])}
rows = {r["run_id"][len("comment-"):]: r for r in runs if r["source"] == "reddit comment"}
flags = {k: [f for f in (r.get("data_flags") or "").split("; ") if f and not f.startswith(PROVENANCE)]
for k, r in rows.items()}
impossible = sorted(k for k in rows if runs_io.impossible(rows[k]))
L = [MARKER, "## Reddit comment runs: review summary", ""]
L.append(f"**{len(head.get('runs', []))} runs** in this file: **{len(added)} new**, {len(changed)} changed, "
f"{len(gone)} gone since main. {head.get('unread', 0)} comments mention a bond or a burn but could not "
"be read as a run.")
L.append("")
L += ["### Runs that break a game rule", "",
"Left out of the route model and the statistics until corrected. Check the comment: `fix` a misread "
"answer, or `reject` a run that is not one.", ""]
bad = [k for k in impossible if k in focus] + [k for k in impossible if k not in focus]
if bad:
L += ["| Run | New? | Rule |", "|---|---|---|"]
L += [f"| {link(k, thread.get(k) or '')} | {'yes' if k in focus else ''} | "
f"{'; '.join(f for f in flags[k] if runs_io.impossible({'data_flags': f}))} |" for k in bad]
else:
L.append("None.")
L.append("")
other = [k for k in sorted(focus) if k in flags and flags[k] and k not in impossible]
L += ["### Other flags on new or changed runs (kept)", ""]
L += [f"- {link(k, thread.get(k) or '')}: {'; '.join(flags[k])}" for k in other] or ["None."]
L.append("")
if overlap is not None:
twice = {m["comment"][len("comment-"):]: m["form"] for m in overlap.get("matches", [])}
hits = [k for k in sorted(focus) if k in twice]
L += ["### May also be in the community form", "",
f"Same answers as a form row submitted within {overlap['rule']['window_days']} day(s). Over all Reddit "
f"runs: {overlap['matched']} of {overlap['comparable']} comparable runs match, about "
f"{overlap['expected_by_chance']} by chance, so about {overlap['likely_double']} look reported twice. "
"Nothing is removed: identical routes are expected.", ""]
L += [f"- {link(k, thread.get(k) or '')}: {', '.join(twice[k][:5])}" for k in hits] or ["None among the new runs."]
L.append("")
counted = [k for k in sorted(focus) if result.get(k) and str(result[k]).upper() in known_results]
L += ["### Result pages already counted", "",
"Their result page is also a \"reddit thread\" row; the run is counted once, with its route.", ""]
L += [f"- {link(k, thread.get(k) or '')}: {result[k]}" for k in counted] or ["None."]
L.append("")
L += ["### How to correct", "",
"In `collection/manual/reddit_comments.yaml` (applied at build time, on this branch or later):",
"- `reject: [comment id]`: not a run (a joke, a friend's run, two runs told as one);",
"- `fix: {comment id: {question: answer}}`: answers read wrong (`null` removes one);",
"- `add: [{id, thread, posted, answers}]`: a run the reader missed.",
"", "A corrected run loses the \"not checked by hand\" note. Then merge."]
return "\n".join(L) + "\n"
def main(base_file, out=None, pr=None):
sys.path.insert(0, str(ROOT / "core"))
from atlas import runs as runs_io
spec = pipeline()["sources"]["reddit_comments"]
base = json.loads(open(base_file, encoding="utf-8").read()) if base_file else {"runs": []}
head = json.loads(path(spec["load"]).read_text(encoding="utf-8"))
runs = runs_io.read(drop=())
thread_spec = pipeline()["sources"].get("reddit_thread") or {}
manual = (thread_spec.get("transform") or {}).get("manual")
known = {str(r["id"]).upper() for r in (read_yaml(manual)["results"] if manual else [])}
text = summary(base, head, runs, known, _overlap().measure(runs))
if out:
open(out, "w", encoding="utf-8").write(text)
else:
print(text)
if pr:
from . import forge
forge.upsert_comment(int(pr), MARKER, text)
print(f"summary posted on #{pr}")
return 0
+7
View File
@@ -132,6 +132,10 @@ def main(argv=None):
k.add_argument("--out", default="exports") k.add_argument("--out", default="exports")
sub.add_parser("rebuild", help="rebuild everything made from the runs: graph, route model, plan, reports, statistics, build") sub.add_parser("rebuild", help="rebuild everything made from the runs: graph, route model, plan, reports, statistics, build")
sub.add_parser("send-pdf", help="open a pull request on the bot's repo when the route PDF changed (private copy only)") sub.add_parser("send-pdf", help="open a pull request on the bot's repo when the route PDF changed (private copy only)")
rv = sub.add_parser("review-reddit", help="a review summary for a Reddit scraper pull request (run `build` first)")
rv.add_argument("--base", help="data/public/reddit_comments.json as it is on main")
rv.add_argument("--out", help="write the summary here (default: print it)")
rv.add_argument("--pr", help="also post it on this pull request, replacing the previous summary")
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)") r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)") r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)") c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)")
@@ -155,6 +159,9 @@ def main(argv=None):
if a.cmd == "rebuild": if a.cmd == "rebuild":
from .etl.handoff import rebuild from .etl.handoff import rebuild
return rebuild() return rebuild()
if a.cmd == "review-reddit":
from .etl.review_reddit import main as review
return review(a.base, a.out, a.pr)
if a.cmd == "send-pdf": if a.cmd == "send-pdf":
from .etl.send_pdf import send from .etl.send_pdf import send
return send() return send()
+12 -2
View File
@@ -126,6 +126,16 @@
"sha256": "b169d157e731db23c99f2f8317a9718d31801b7d64336b3d414511ac13f32493", "sha256": "b169d157e731db23c99f2f8317a9718d31801b7d64336b3d414511ac13f32493",
"contentSize": "23174 B" "contentSize": "23174 B"
}, },
{
"@type": "cr:FileObject",
"@id": "overlap",
"name": "overlap",
"description": "How many Reddit comment runs may also be in the community form (the same answers within a day), against what chance alone gives (analysis/overlap/overlap.py). Measured only: no run is removed.",
"contentUrl": "https://git.kenschiano.net/DragonKind/threshing-atlas/raw/branch/main/data/public/overlap.json",
"encodingFormat": "application/json",
"sha256": "04280a52f028334451c505ddc35004d98cc1f4595b2294c3746a09e34454e8b0",
"contentSize": "1976 B"
},
{ {
"@type": "cr:FileObject", "@type": "cr:FileObject",
"@id": "reports", "@id": "reports",
@@ -153,8 +163,8 @@
"description": "Every number the Atlas page shows, for six filter presets (analysis/stats/site_stats.py; what is measured is analysis/stats/measures.yaml).", "description": "Every number the Atlas page shows, for six filter presets (analysis/stats/site_stats.py; what is measured is analysis/stats/measures.yaml).",
"contentUrl": "https://git.kenschiano.net/DragonKind/threshing-atlas/raw/branch/main/data/public/stats.json", "contentUrl": "https://git.kenschiano.net/DragonKind/threshing-atlas/raw/branch/main/data/public/stats.json",
"encodingFormat": "application/json", "encodingFormat": "application/json",
"sha256": "89a6246da2495685733e20f2de9897299bd19d469aaeadbd8883972856787ce5", "sha256": "7431173d9b667d4914d17bbc299d994d14c3104e3e89a1eb75d690c1b21ade6a",
"contentSize": "5151910 B" "contentSize": "5110214 B"
}, },
{ {
"@type": "cr:FileObject", "@type": "cr:FileObject",
+6
View File
@@ -41,6 +41,12 @@ files:
description: >- description: >-
Runs described in the r/fourthwing megathread comments, read into the "Report a run" answers: comment id, day Runs described in the r/fourthwing megathread comments, read into the "Report a run" answers: comment id, day
and answers only, no usernames or comment text. and answers only, no usernames or comment text.
overlap:
path: data/public/overlap.json
format: application/json
description: >-
How many Reddit comment runs may also be in the community form (the same answers within a day), against what
chance alone gives (analysis/overlap/overlap.py). Measured only: no run is removed.
reports: reports:
path: data/public/reports.json path: data/public/reports.json
format: application/json format: application/json
+134
View File
@@ -0,0 +1,134 @@
{
"schema": 1,
"rule": {
"min_answers": 4,
"window_days": 1,
"baseline_shifts_days": [
-28,
-21,
-14,
14,
21,
28
]
},
"comment_runs": 82,
"comparable": 36,
"matched": 9,
"expected_by_chance": 7.3,
"likely_double": 1.7,
"matches": [
{
"comment": "comment-pdjr7gm",
"form": [
"sheet-8",
"sheet-38"
]
},
{
"comment": "comment-pdq6tsw",
"form": [
"sheet-168",
"sheet-190",
"sheet-398"
]
},
{
"comment": "comment-pe0kwol",
"form": [
"sheet-224"
]
},
{
"comment": "comment-pe2l6mz",
"form": [
"sheet-5",
"sheet-25",
"sheet-107",
"sheet-292",
"sheet-393",
"sheet-471",
"sheet-483",
"sheet-514",
"sheet-520"
]
},
{
"comment": "comment-pe2n3j1",
"form": [
"sheet-195",
"sheet-276"
]
},
{
"comment": "comment-pev3uge",
"form": [
"sheet-870",
"sheet-916",
"sheet-934",
"sheet-960",
"sheet-961",
"sheet-962",
"sheet-975",
"sheet-984",
"sheet-999",
"sheet-1008",
"sheet-1009",
"sheet-1019",
"sheet-1023",
"sheet-1068",
"sheet-1069",
"sheet-1077",
"sheet-1081",
"sheet-1089",
"sheet-1097",
"sheet-1101",
"sheet-1110",
"sheet-1129",
"sheet-1194",
"sheet-1200",
"sheet-1201",
"sheet-1235",
"sheet-1238",
"sheet-1257",
"sheet-1259",
"sheet-1264",
"sheet-1271",
"sheet-1278",
"sheet-1283",
"sheet-1290",
"sheet-1292"
]
},
{
"comment": "comment-pf22tpj",
"form": [
"sheet-1031",
"sheet-1042",
"sheet-1048",
"sheet-1140",
"sheet-1142",
"sheet-1143",
"sheet-1144",
"sheet-1171",
"sheet-1261",
"sheet-1262",
"sheet-1303",
"sheet-1304",
"sheet-1364"
]
},
{
"comment": "comment-pfhmw5d",
"form": [
"sheet-1504"
]
},
{
"comment": "comment-pgzz9hc",
"form": [
"sheet-1714"
]
}
]
}
File diff suppressed because one or more lines are too long
+2 -2
View File
@@ -6,11 +6,11 @@
"scripts": { "scripts": {
"build": "node site/build.js", "build": "node site/build.js",
"dev": "node site/build.js && python3 -m http.server 8000 -d dist", "dev": "node site/build.js && python3 -m http.server 8000 -d dist",
"test:py": "python3 -m collection build --check && python3 planner/build_experiments.py --check && python3 analysis/black/evidence.py --check && python3 analysis/tree/build_tree.py --check && python3 analysis/known/build_known.py --check && PYTHONPATH=core python3 -m unittest discover -s tests/core -p 'test_*.py' && python3 -m unittest discover -s tests/collection -p 'test_*.py'", "test:py": "python3 -m collection build --check && python3 planner/build_experiments.py --check && python3 analysis/black/evidence.py --check && python3 analysis/tree/build_tree.py --check && python3 analysis/known/build_known.py --check && python3 analysis/overlap/overlap.py --check && PYTHONPATH=core python3 -m unittest discover -s tests/core -p 'test_*.py' && python3 -m unittest discover -s tests/collection -p 'test_*.py'",
"test:unit": "node --test tests/core/graph.test.js tests/core/advisor.test.js tests/core/known.test.js", "test:unit": "node --test tests/core/graph.test.js tests/core/advisor.test.js tests/core/known.test.js",
"test": "npm run test:py && npm run test:unit", "test": "npm run test:py && npm run test:unit",
"graph": "node analysis/network/build-graph.js", "graph": "node analysis/network/build-graph.js",
"policy": "python3 planner/build_policy.py && python3 planner/policy_report.py && python3 planner/build_experiments.py && python3 analysis/black/evidence.py && python3 analysis/tree/build_tree.py && python3 analysis/known/build_known.py", "policy": "python3 planner/build_policy.py && python3 planner/policy_report.py && python3 planner/build_experiments.py && python3 analysis/black/evidence.py && python3 analysis/tree/build_tree.py && python3 analysis/known/build_known.py && python3 analysis/overlap/overlap.py",
"cycle": "python3 -m collection cycle", "cycle": "python3 -m collection cycle",
"stats": "python3 analysis/stats/site_stats.py", "stats": "python3 analysis/stats/site_stats.py",
"collect": "python3 -m collection update", "collect": "python3 -m collection update",
+2
View File
@@ -84,6 +84,7 @@
// ------------------------------------------------------------ data loading // ------------------------------------------------------------ data loading
async function loadStats() { async function loadStats() {
state.stats = await (await fetch("data/stats.json")).json(); state.stats = await (await fetch("data/stats.json")).json();
try { state.overlap = await (await fetch("data/overlap.json")).json(); } catch { state.overlap = null; }
const sel = $("#f-preset"); const sel = $("#f-preset");
sel.innerHTML = Object.entries(META().presets).map(([k, l]) => `<option value="${k}">${esc(l)}</option>`).join(""); sel.innerHTML = Object.entries(META().presets).map(([k, l]) => `<option value="${k}">${esc(l)}</option>`).join("");
sel.value = state.preset; sel.value = state.preset;
@@ -554,6 +555,7 @@
</tbody></table></div><p class="small">Bulk logs come from players who recorded every run, so they are the least selective rows. Their rare-dragon rates are several times lower than the form's, which is the strongest sign of reporting bias in this data.</p></div> </tbody></table></div><p class="small">Bulk logs come from players who recorded every run, so they are the least selective rows. Their rare-dragon rates are several times lower than the form's, which is the strongest sign of reporting bias in this data.</p></div>
<div class="panel"><h3>Rows flagged for contradictions</h3><p class="small muted">${m.n_flagged} of ${r.n} rows (${pct(m.n_flagged / r.n)}). Flagged rows stay in unless you pick "Rows without data-quality flags" above.</p><div class="tbl-wrap"><table><tbody>${flags.map(([f, n]) => `<tr><td>${esc(f)}</td><td class="num">${n}</td></tr>`).join("") || `<tr><td>None</td></tr>`}</tbody></table></div></div> <div class="panel"><h3>Rows flagged for contradictions</h3><p class="small muted">${m.n_flagged} of ${r.n} rows (${pct(m.n_flagged / r.n)}). Flagged rows stay in unless you pick "Rows without data-quality flags" above.</p><div class="tbl-wrap"><table><tbody>${flags.map(([f, n]) => `<tr><td>${esc(f)}</td><td class="num">${n}</td></tr>`).join("") || `<tr><td>None</td></tr>`}</tbody></table></div></div>
<div class="panel"><h3>How runs ended</h3><div class="tbl-wrap"><table><tbody>${outs.map(([f, n]) => `<tr><td>${esc(f)}</td><td class="num">${n}</td><td class="num">${pct(n / r.n)}</td></tr>`).join("")}</tbody></table></div><p class="small muted">"No outcome recorded" rows reached the hour but the form shows no Bonded or Incinerated. They count toward reaching a branch but not toward bond rates.</p></div> <div class="panel"><h3>How runs ended</h3><div class="tbl-wrap"><table><tbody>${outs.map(([f, n]) => `<tr><td>${esc(f)}</td><td class="num">${n}</td><td class="num">${pct(n / r.n)}</td></tr>`).join("")}</tbody></table></div><p class="small muted">"No outcome recorded" rows reached the hour but the form shows no Bonded or Incinerated. They count toward reaching a branch but not toward bond rates.</p></div>
${state.overlap && state.overlap.comparable ? `<div class="panel"><h3>The same run reported twice?</h3><p class="small">A rider may log a run in the form and post it in a megathread comment too. With no player IDs, a copy can only be measured, not proven, so nothing is removed. Of ${state.overlap.comparable} Reddit runs detailed enough to compare, ${state.overlap.matched} give the same answers as a form row sent within ${state.overlap.rule.window_days} day; chance alone (the same test a few weeks off) gives about ${state.overlap.expected_by_chance}. So about ${Math.round(state.overlap.likely_double)} of ${state.overlap.comment_runs} Reddit runs look reported twice${state.overlap.likely_double < 0.1 * state.overlap.comment_runs ? ", too few to matter" : ""}.</p></div>` : ""}
<div class="panel"><h3>Independence</h3><p class="small">The form has no player ID. One player's 50 runs count as 50 independent runs, so intervals are somewhat narrower than they should be. ${pct(Object.entries(m.sources).filter(([k]) => !["Form", "Reddit comments", "DragonKind bot", "Reported on the Atlas"].includes(k)).reduce((s, [, n]) => s + n, 0) / r.n, 0)} of rows arrived as bulk imports, each probably one player's or one group's log.</p></div> <div class="panel"><h3>Independence</h3><p class="small">The form has no player ID. One player's 50 runs count as 50 independent runs, so intervals are somewhat narrower than they should be. ${pct(Object.entries(m.sources).filter(([k]) => !["Form", "Reddit comments", "DragonKind bot", "Reported on the Atlas"].includes(k)).reduce((s, [, n]) => s + n, 0) / r.n, 0)} of rows arrived as bulk imports, each probably one player's or one group's log.</p></div>
<div class="panel"><h3>What the form does not ask</h3><p class="small">The redesigned form asks how you reacted to the first dragon (answered in ${r.factors.reach_hour.rows.filter((x) => x.factor === "sighting_action" && !["Unknown", "(not asked)"].includes(x.level)).reduce((a, x) => a + x.n, 0)} rows; most riders answer "Unsure") and how the bully fight went (${r.factors.reach_hour.rows.filter((x) => x.factor === "combat_first" && x.level !== "(not recorded)").reduce((a, x) => a + x.n, 0)} rows). It still does not record most choices inside the bonding encounters. Theories that rely on those choices cannot be confirmed or refuted here.</p></div>`; <div class="panel"><h3>What the form does not ask</h3><p class="small">The redesigned form asks how you reacted to the first dragon (answered in ${r.factors.reach_hour.rows.filter((x) => x.factor === "sighting_action" && !["Unknown", "(not asked)"].includes(x.level)).reduce((a, x) => a + x.n, 0)} rows; most riders answer "Unsure") and how the bully fight went (${r.factors.reach_hour.rows.filter((x) => x.factor === "combat_first" && x.level !== "(not recorded)").reduce((a, x) => a + x.n, 0)} rows). It still does not record most choices inside the bonding encounters. Theories that rely on those choices cannot be confirmed or refuted here.</p></div>`;
} }
+1 -1
View File
@@ -27,7 +27,7 @@ for (const f of fs.readdirSync(path.join(root, "site/atlas"))) copy(`site/atlas/
copy("core/graph.js", "graph.js"); copy("core/graph.js", "graph.js");
copy("core/solver.js", "solver.js"); copy("core/solver.js", "solver.js");
// The numbers the page shows (all computed in Python) and the published data // The numbers the page shows (all computed in Python) and the published data
for (const f of ["stats.json", "register.json", "graph.json", "policy.json", "black.json", "tree.json", "experiments.json", "pdf_encounters.json"]) copy(`data/public/${f}`, `data/${f}`); for (const f of ["stats.json", "register.json", "graph.json", "policy.json", "black.json", "tree.json", "experiments.json", "pdf_encounters.json", "overlap.json"]) copy(`data/public/${f}`, `data/${f}`);
copy("data/public/runs_clean.csv", "data/runs_clean.csv"); copy("data/public/runs_clean.csv", "data/runs_clean.csv");
for (const f of ["croissant.json", "dictionary.yaml"]) copy(`data/${f}`, `data/${f}`); for (const f of ["croissant.json", "dictionary.yaml"]) copy(`data/${f}`, `data/${f}`);
// Source files shown in the page's "view the code" panel // Source files shown in the page's "view the code" panel
+24
View File
@@ -335,6 +335,30 @@ class RedditComments(unittest.TestCase):
kept = build_policy.drop_flagged(rows) kept = build_policy.drop_flagged(rows)
self.assertEqual([r["source"] for r in kept], ["reddit comment", "reddit spreadsheet"]) self.assertEqual([r["source"] for r in kept], ["reddit comment", "reddit spreadsheet"])
def test_review_summary_and_overlap(self):
sys.path.insert(0, str(ROOT / "core"))
from collection.etl import review_reddit
ov = review_reddit._overlap()
base = {"runs": [{"id": "a1", "thread": "t", "answers": {"outcome": "Bonded"}}]}
head = {"runs": [{"id": "a1", "thread": "t", "answers": {"outcome": "Incinerated at a bonding encounter"}},
{"id": "b2", "thread": "t", "answers": {}}, {"id": "c3", "thread": "t", "answers": {}, "result": "THRESH-X1"}]}
self.assertEqual(review_reddit.diff(base, head), (["b2", "c3"], ["a1"], []))
route = dict(company="Alone", direction="East", scream="Ignore the scream", hour="Tree", outcome="Bonded",
outcome_color="Red", date="2026-10-05")
runs = [{"run_id": "comment-b2", "source": "reddit comment", "data_flags": "read from a Reddit comment; "
"Black recorded on a Valley run (the Black is only reached from the tree)", **{k: None for k in ov.FIELDS}, "date": None},
{"run_id": "comment-c3", "source": "reddit comment", "data_flags": "", **{k: None for k in ov.FIELDS}, **route},
{"run_id": "sheet-1", "source": "reddit spreadsheet", "data_flags": "", **{k: None for k in ov.FIELDS}, **route}]
m = ov.measure(runs)
self.assertEqual((m["comparable"], m["matched"]), (1, 1))
self.assertEqual(m["matches"], [{"comment": "comment-c3", "form": ["sheet-1"]}])
text = review_reddit.summary(base, head, runs, {"THRESH-X1"}, m)
self.assertIn("**2 new**, 1 changed", text)
self.assertIn("Black recorded on a Valley run", text)
self.assertIn("sheet-1", text)
self.assertIn("THRESH-X1", text)
self.assertTrue(text.startswith(review_reddit.MARKER))
def test_delivered_not_extracted(self): def test_delivered_not_extracted(self):
spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"] spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"]
with self.assertRaises(extract.Skip): with self.assertRaises(extract.Skip):