Merge pull request 'Review summary on Reddit scraper PRs; measure the form/Reddit overlap' (#44) from atlas/reddit-review into main
This commit was merged in pull request #44.
This commit is contained in:
@@ -0,0 +1,31 @@
|
|||||||
|
# A review summary on every Reddit scraper pull request (issue #23): the runs added, changed and gone, the ones that
|
||||||
|
# break a game rule, other flags, possible copies of form rows, and result pages already counted. Posted as one
|
||||||
|
# comment on the pull request, refreshed on every push. The reviewer corrects runs in
|
||||||
|
# collection/manual/reddit_comments.yaml (collection/README.md, "Reviewing Reddit runs"), then merges.
|
||||||
|
#
|
||||||
|
# Secrets: ATLAS_BOT_TOKEN (as in update-sources.yml), to post the comment.
|
||||||
|
name: Review Reddit runs
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
paths:
|
||||||
|
- data/public/reddit_comments.json
|
||||||
|
- collection/manual/reddit_comments.yaml
|
||||||
|
jobs:
|
||||||
|
summary:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
- run: pip install -r requirements.txt
|
||||||
|
- name: The file as it is on main
|
||||||
|
run: |
|
||||||
|
git fetch --depth 1 origin "${{ github.base_ref }}"
|
||||||
|
git show FETCH_HEAD:data/public/reddit_comments.json > "$RUNNER_TEMP/base.json" || echo '{"runs": []}' > "$RUNNER_TEMP/base.json"
|
||||||
|
- name: Build the runs and post the summary
|
||||||
|
env:
|
||||||
|
ATLAS_BOT_TOKEN: ${{ secrets.ATLAS_BOT_TOKEN }}
|
||||||
|
run: |
|
||||||
|
python -m collection build
|
||||||
|
python -m collection review-reddit --base "$RUNNER_TEMP/base.json" --out "$RUNNER_TEMP/review.md" --pr "${{ github.event.pull_request.number }}"
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
"""Is the same attempt reported twice, in the community form and in a Reddit comment? (issue #24)
|
||||||
|
|
||||||
|
python analysis/overlap/overlap.py [--check]
|
||||||
|
|
||||||
|
There are no player ids, so a copy can only be proven by a shared dragonkind.com result-page id. Neither the form
|
||||||
|
nor the register records one, and the comment-to-result-page link is already applied when the runs are built
|
||||||
|
(collection/etl/runs.py). Everything else is measured, never removed:
|
||||||
|
|
||||||
|
- a Reddit comment run is *comparable* when it is dated and gives at least MIN_ANSWERS route answers besides how it
|
||||||
|
ended (fewer answers match dozens of form rows by chance on a busy day);
|
||||||
|
- it *matches* a form row submitted within WINDOW days when every answer it gives is the form row's answer;
|
||||||
|
- the same test with the comment's date shifted by a few weeks (SHIFTS, only where the form has rows that week) is
|
||||||
|
what chance alone produces, so matched - expected is the estimate of runs reported twice.
|
||||||
|
|
||||||
|
Writes data/public/overlap.json (the page's Data & methods tab, and the scraper pull-request summary). Nothing is
|
||||||
|
dropped: identical routes are expected, and the Reddit runs already have their own reporting-bias group in the
|
||||||
|
route model (planner/policy.yaml).
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import collections
|
||||||
|
import datetime as dt
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
sys.path.insert(0, str(ROOT / "core"))
|
||||||
|
from atlas import runs as runs_io # noqa: E402
|
||||||
|
|
||||||
|
OUT = ROOT / "data/public/overlap.json"
|
||||||
|
FIELDS = ["company", "left_squadmates", "direction", "first_sighting", "sighting_action", "scream", "resolution",
|
||||||
|
"combat_first", "combat_second", "post_path", "hour", "branch", "outcome", "outcome_color", "tail", "attempt"]
|
||||||
|
ENDING = ("outcome", "outcome_color")
|
||||||
|
MIN_ANSWERS = 4
|
||||||
|
WINDOW = 1
|
||||||
|
SHIFTS = (-28, -21, -14, 14, 21, 28)
|
||||||
|
|
||||||
|
|
||||||
|
def given(v):
|
||||||
|
return v not in (None, "") and v != "Unknown" and not (isinstance(v, str) and v.startswith("("))
|
||||||
|
|
||||||
|
|
||||||
|
def answers(r):
|
||||||
|
return [k for k in FIELDS if given(r.get(k))]
|
||||||
|
|
||||||
|
|
||||||
|
def comparable(r):
|
||||||
|
return bool(r.get("date")) and len([k for k in answers(r) if k not in ENDING]) >= MIN_ANSWERS
|
||||||
|
|
||||||
|
|
||||||
|
def days(date, shift=0, window=WINDOW):
|
||||||
|
d = dt.date.fromisoformat(date[:10]) + dt.timedelta(days=shift)
|
||||||
|
return [(d + dt.timedelta(days=i)).isoformat() for i in range(-window, window + 1)]
|
||||||
|
|
||||||
|
|
||||||
|
def form_index(runs):
|
||||||
|
by_day = collections.defaultdict(list)
|
||||||
|
for r in runs:
|
||||||
|
if r["source"] == "reddit spreadsheet" and r.get("date"):
|
||||||
|
by_day[r["date"][:10]].append(r)
|
||||||
|
return by_day
|
||||||
|
|
||||||
|
|
||||||
|
def matches(c, by_day, shift=0):
|
||||||
|
"""Form rows within the window that give every answer the comment gives."""
|
||||||
|
keys = answers(c)
|
||||||
|
return [f for d in days(c["date"], shift) for f in by_day.get(d, []) if all(f.get(k) == c[k] for k in keys)]
|
||||||
|
|
||||||
|
|
||||||
|
def measure(runs):
|
||||||
|
by_day = form_index(runs)
|
||||||
|
comments = [r for r in runs if r["source"] == "reddit comment"]
|
||||||
|
comp = [c for c in comments if comparable(c)]
|
||||||
|
found = {c["run_id"]: [f["run_id"] for f in matches(c, by_day)] for c in comp}
|
||||||
|
matched = sum(1 for v in found.values() if v)
|
||||||
|
# chance: each comment's match rate on shifted dates, over the shifts whose window has form rows at all
|
||||||
|
expected = 0.0
|
||||||
|
for c in comp:
|
||||||
|
valid = [s for s in SHIFTS if any(by_day.get(d) for d in days(c["date"], s))]
|
||||||
|
if valid:
|
||||||
|
expected += sum(1 for s in valid if matches(c, by_day, s)) / len(valid)
|
||||||
|
return {
|
||||||
|
"schema": 1,
|
||||||
|
"rule": {"min_answers": MIN_ANSWERS, "window_days": WINDOW, "baseline_shifts_days": list(SHIFTS)},
|
||||||
|
"comment_runs": len(comments),
|
||||||
|
"comparable": len(comp),
|
||||||
|
"matched": matched,
|
||||||
|
"expected_by_chance": round(expected, 1),
|
||||||
|
"likely_double": round(max(0.0, matched - expected), 1),
|
||||||
|
"matches": [{"comment": k, "form": v} for k, v in sorted(found.items()) if v],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None):
|
||||||
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||||
|
ap.add_argument("--check", action="store_true", help="fail if data/public/overlap.json is out of date")
|
||||||
|
a = ap.parse_args(argv)
|
||||||
|
text = json.dumps(measure(runs_io.read(drop=())), indent=1, ensure_ascii=False) + "\n"
|
||||||
|
if a.check:
|
||||||
|
if not OUT.exists() or OUT.read_text(encoding="utf-8") != text:
|
||||||
|
print(f"out of date (run `python {Path(__file__).relative_to(ROOT)}`): {OUT.relative_to(ROOT)}")
|
||||||
|
return 1
|
||||||
|
return 0
|
||||||
|
OUT.write_text(text, encoding="utf-8")
|
||||||
|
m = json.loads(text)
|
||||||
|
print(f"{OUT.relative_to(ROOT)}: {m['matched']} of {m['comparable']} comparable Reddit runs match a form row "
|
||||||
|
f"(~{m['expected_by_chance']} by chance; ~{m['likely_double']} likely reported twice)")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -41,6 +41,25 @@ The route PDF is a flowchart of the choices, not a log of runs, so it adds no ro
|
|||||||
fingerprint tells us when the routes it documents change. The register lists bonded dragons without the route that
|
fingerprint tells us when the routes it documents change. The register lists bonded dragons without the route that
|
||||||
led there, so it adds no rows either.
|
led there, so it adds no rows either.
|
||||||
|
|
||||||
|
## Reviewing Reddit runs
|
||||||
|
|
||||||
|
Ken's scraper opens (or updates) a pull request from `data/reddit-comments` that changes
|
||||||
|
`data/public/reddit_comments.json`. The **Review Reddit runs** workflow (`.gitea/workflows/reddit-review.yml`) posts a
|
||||||
|
summary on it, refreshed on every push (`python -m collection review-reddit --base <main's file>` locally):
|
||||||
|
|
||||||
|
- runs added, changed and gone since main;
|
||||||
|
- **runs that break a game rule** (`core/spec/form.yaml` `impossible_flags`): left out of the route model and the
|
||||||
|
statistics until corrected. Open the comment and decide;
|
||||||
|
- other flags on the new runs (kept);
|
||||||
|
- runs that may also be in the community form: the same answers within a day, with how many chance alone gives
|
||||||
|
(`analysis/overlap/overlap.py`, `data/public/overlap.json`). Measured only, never removed;
|
||||||
|
- runs whose result page is already a `reddit thread` row (counted once automatically).
|
||||||
|
|
||||||
|
Corrections go in `manual/reddit_comments.yaml`, applied at build time, so they never wait for the threads:
|
||||||
|
`reject: [id]` (not a run), `fix: {id: {question: answer}}` (misread; `null` removes an answer) and
|
||||||
|
`add: [{id, thread, posted, answers}]` (missed). A corrected run loses the "not checked by hand" note. No usernames:
|
||||||
|
the comment id is enough to find the comment again. Then merge, once CI passes.
|
||||||
|
|
||||||
## Raw files and privacy
|
## Raw files and privacy
|
||||||
|
|
||||||
`data/raw/` is git-ignored and never published. It holds the current raw file of each source and the one before
|
`data/raw/` is git-ignored and never published. It holds the current raw file of each source and the one before
|
||||||
|
|||||||
@@ -131,6 +131,18 @@ def open_issue(title, body, env=None, f=None):
|
|||||||
return _call("POST", f"{api_base(f)}/issues", {"title": title, "body": body}, tok, f)["number"]
|
return _call("POST", f"{api_base(f)}/issues", {"title": title, "body": body}, tok, f)["number"]
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_comment(number, marker, body, env=None, f=None):
|
||||||
|
"""Post `body` on issue or pull request `number`, replacing an earlier comment that starts with `marker`
|
||||||
|
(so a summary refreshed on every push stays one comment)."""
|
||||||
|
f = f or config()
|
||||||
|
tok = token(env)
|
||||||
|
comments = _call("GET", f"{api_base(f)}/issues/{number}/comments?limit=50&per_page=100", tok=tok, f=f) or []
|
||||||
|
mine = [c for c in comments if (c.get("body") or "").startswith(marker)]
|
||||||
|
if mine:
|
||||||
|
return _call("PATCH", f"{api_base(f)}/issues/comments/{mine[-1]['id']}", {"body": body}, tok, f)["id"]
|
||||||
|
return _call("POST", f"{api_base(f)}/issues/{number}/comments", {"body": body}, tok, f)["id"]
|
||||||
|
|
||||||
|
|
||||||
def ensure_label(name, color, env=None, f=None):
|
def ensure_label(name, color, env=None, f=None):
|
||||||
f = f or config()
|
f = f or config()
|
||||||
tok = token(env)
|
tok = token(env)
|
||||||
|
|||||||
@@ -88,6 +88,7 @@ def rebuild():
|
|||||||
_step("what the runs show about the Black", [py, "analysis/black/evidence.py"])
|
_step("what the runs show about the Black", [py, "analysis/black/evidence.py"])
|
||||||
_step("screen tree", [py, "analysis/tree/build_tree.py"])
|
_step("screen tree", [py, "analysis/tree/build_tree.py"])
|
||||||
_step("known answers for the bot (guide + screen tree)", [py, "analysis/known/build_known.py"])
|
_step("known answers for the bot (guide + screen tree)", [py, "analysis/known/build_known.py"])
|
||||||
|
_step("runs reported twice (form + Reddit)", [py, "analysis/overlap/overlap.py"])
|
||||||
try:
|
try:
|
||||||
import scipy # noqa: F401
|
import scipy # noqa: F401
|
||||||
_step("statistics", [py, "analysis/stats/site_stats.py"])
|
_step("statistics", [py, "analysis/stats/site_stats.py"])
|
||||||
|
|||||||
@@ -0,0 +1,133 @@
|
|||||||
|
"""A review summary for a Reddit scraper pull request (issue #23).
|
||||||
|
|
||||||
|
python -m collection review-reddit --base BASE.json [--out summary.md] [--pr N]
|
||||||
|
|
||||||
|
Compares the pull request's data/public/reddit_comments.json with `BASE.json` (the same file on main) and reads the
|
||||||
|
runs as built from it (data/public/runs_clean.csv, so run `python -m collection build` first). The summary lists
|
||||||
|
what a reviewer needs to decide on, before merging:
|
||||||
|
|
||||||
|
- runs added, changed (answers differ) and gone;
|
||||||
|
- runs that break a game rule: left out of the route model and the statistics until corrected;
|
||||||
|
- other flags on the new or changed runs (kept, for information);
|
||||||
|
- runs that may also be in the community form (analysis/overlap/overlap.py: same answers within a day);
|
||||||
|
- runs whose result page is already a "reddit thread" row (counted once, automatically).
|
||||||
|
|
||||||
|
Corrections go in collection/manual/reddit_comments.yaml (reject / fix / add; see collection/README.md). With
|
||||||
|
`--pr N` the summary is posted on the pull request, replacing the previous one.
|
||||||
|
"""
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from .common import ROOT, path, pipeline, read_yaml
|
||||||
|
|
||||||
|
MARKER = "<!-- atlas:reddit-review -->"
|
||||||
|
PROVENANCE = "read from a Reddit comment"
|
||||||
|
|
||||||
|
|
||||||
|
def _overlap():
|
||||||
|
spec = importlib.util.spec_from_file_location("overlap", ROOT / "analysis/overlap/overlap.py")
|
||||||
|
mod = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(mod)
|
||||||
|
return mod
|
||||||
|
|
||||||
|
|
||||||
|
def link(run_id, thread):
|
||||||
|
cid = str(run_id).split("-")[0]
|
||||||
|
return f"[{run_id}](https://www.reddit.com/r/fourthwing/comments/{thread}/_/{cid}/)"
|
||||||
|
|
||||||
|
|
||||||
|
def diff(base, head):
|
||||||
|
"""(added, changed, gone) comment-run ids between two reddit_comments.json documents."""
|
||||||
|
b = {str(r["id"]): r for r in base.get("runs", [])}
|
||||||
|
h = {str(r["id"]): r for r in head.get("runs", [])}
|
||||||
|
added = sorted(set(h) - set(b))
|
||||||
|
gone = sorted(set(b) - set(h))
|
||||||
|
changed = sorted(k for k in set(h) & set(b) if h[k].get("answers") != b[k].get("answers"))
|
||||||
|
return added, changed, gone
|
||||||
|
|
||||||
|
|
||||||
|
def summary(base, head, runs, known_results=(), overlap=None):
|
||||||
|
"""The Markdown summary. `runs`: typed records of runs_clean.csv (with run_id)."""
|
||||||
|
sys.path.insert(0, str(ROOT / "core"))
|
||||||
|
from atlas import runs as runs_io
|
||||||
|
|
||||||
|
added, changed, gone = diff(base, head)
|
||||||
|
focus = set(added) | set(changed)
|
||||||
|
thread = {str(r["id"]): r.get("thread") for r in head.get("runs", [])}
|
||||||
|
result = {str(r["id"]): r.get("result") for r in head.get("runs", [])}
|
||||||
|
rows = {r["run_id"][len("comment-"):]: r for r in runs if r["source"] == "reddit comment"}
|
||||||
|
flags = {k: [f for f in (r.get("data_flags") or "").split("; ") if f and not f.startswith(PROVENANCE)]
|
||||||
|
for k, r in rows.items()}
|
||||||
|
impossible = sorted(k for k in rows if runs_io.impossible(rows[k]))
|
||||||
|
|
||||||
|
L = [MARKER, "## Reddit comment runs: review summary", ""]
|
||||||
|
L.append(f"**{len(head.get('runs', []))} runs** in this file: **{len(added)} new**, {len(changed)} changed, "
|
||||||
|
f"{len(gone)} gone since main. {head.get('unread', 0)} comments mention a bond or a burn but could not "
|
||||||
|
"be read as a run.")
|
||||||
|
L.append("")
|
||||||
|
|
||||||
|
L += ["### Runs that break a game rule", "",
|
||||||
|
"Left out of the route model and the statistics until corrected. Check the comment: `fix` a misread "
|
||||||
|
"answer, or `reject` a run that is not one.", ""]
|
||||||
|
bad = [k for k in impossible if k in focus] + [k for k in impossible if k not in focus]
|
||||||
|
if bad:
|
||||||
|
L += ["| Run | New? | Rule |", "|---|---|---|"]
|
||||||
|
L += [f"| {link(k, thread.get(k) or '')} | {'yes' if k in focus else ''} | "
|
||||||
|
f"{'; '.join(f for f in flags[k] if runs_io.impossible({'data_flags': f}))} |" for k in bad]
|
||||||
|
else:
|
||||||
|
L.append("None.")
|
||||||
|
L.append("")
|
||||||
|
|
||||||
|
other = [k for k in sorted(focus) if k in flags and flags[k] and k not in impossible]
|
||||||
|
L += ["### Other flags on new or changed runs (kept)", ""]
|
||||||
|
L += [f"- {link(k, thread.get(k) or '')}: {'; '.join(flags[k])}" for k in other] or ["None."]
|
||||||
|
L.append("")
|
||||||
|
|
||||||
|
if overlap is not None:
|
||||||
|
twice = {m["comment"][len("comment-"):]: m["form"] for m in overlap.get("matches", [])}
|
||||||
|
hits = [k for k in sorted(focus) if k in twice]
|
||||||
|
L += ["### May also be in the community form", "",
|
||||||
|
f"Same answers as a form row submitted within {overlap['rule']['window_days']} day(s). Over all Reddit "
|
||||||
|
f"runs: {overlap['matched']} of {overlap['comparable']} comparable runs match, about "
|
||||||
|
f"{overlap['expected_by_chance']} by chance, so about {overlap['likely_double']} look reported twice. "
|
||||||
|
"Nothing is removed: identical routes are expected.", ""]
|
||||||
|
L += [f"- {link(k, thread.get(k) or '')}: {', '.join(twice[k][:5])}" for k in hits] or ["None among the new runs."]
|
||||||
|
L.append("")
|
||||||
|
|
||||||
|
counted = [k for k in sorted(focus) if result.get(k) and str(result[k]).upper() in known_results]
|
||||||
|
L += ["### Result pages already counted", "",
|
||||||
|
"Their result page is also a \"reddit thread\" row; the run is counted once, with its route.", ""]
|
||||||
|
L += [f"- {link(k, thread.get(k) or '')}: {result[k]}" for k in counted] or ["None."]
|
||||||
|
L.append("")
|
||||||
|
|
||||||
|
L += ["### How to correct", "",
|
||||||
|
"In `collection/manual/reddit_comments.yaml` (applied at build time, on this branch or later):",
|
||||||
|
"- `reject: [comment id]`: not a run (a joke, a friend's run, two runs told as one);",
|
||||||
|
"- `fix: {comment id: {question: answer}}`: answers read wrong (`null` removes one);",
|
||||||
|
"- `add: [{id, thread, posted, answers}]`: a run the reader missed.",
|
||||||
|
"", "A corrected run loses the \"not checked by hand\" note. Then merge."]
|
||||||
|
return "\n".join(L) + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def main(base_file, out=None, pr=None):
|
||||||
|
sys.path.insert(0, str(ROOT / "core"))
|
||||||
|
from atlas import runs as runs_io
|
||||||
|
|
||||||
|
spec = pipeline()["sources"]["reddit_comments"]
|
||||||
|
base = json.loads(open(base_file, encoding="utf-8").read()) if base_file else {"runs": []}
|
||||||
|
head = json.loads(path(spec["load"]).read_text(encoding="utf-8"))
|
||||||
|
runs = runs_io.read(drop=())
|
||||||
|
thread_spec = pipeline()["sources"].get("reddit_thread") or {}
|
||||||
|
manual = (thread_spec.get("transform") or {}).get("manual")
|
||||||
|
known = {str(r["id"]).upper() for r in (read_yaml(manual)["results"] if manual else [])}
|
||||||
|
text = summary(base, head, runs, known, _overlap().measure(runs))
|
||||||
|
if out:
|
||||||
|
open(out, "w", encoding="utf-8").write(text)
|
||||||
|
else:
|
||||||
|
print(text)
|
||||||
|
if pr:
|
||||||
|
from . import forge
|
||||||
|
forge.upsert_comment(int(pr), MARKER, text)
|
||||||
|
print(f"summary posted on #{pr}")
|
||||||
|
return 0
|
||||||
@@ -132,6 +132,10 @@ def main(argv=None):
|
|||||||
k.add_argument("--out", default="exports")
|
k.add_argument("--out", default="exports")
|
||||||
sub.add_parser("rebuild", help="rebuild everything made from the runs: graph, route model, plan, reports, statistics, build")
|
sub.add_parser("rebuild", help="rebuild everything made from the runs: graph, route model, plan, reports, statistics, build")
|
||||||
sub.add_parser("send-pdf", help="open a pull request on the bot's repo when the route PDF changed (private copy only)")
|
sub.add_parser("send-pdf", help="open a pull request on the bot's repo when the route PDF changed (private copy only)")
|
||||||
|
rv = sub.add_parser("review-reddit", help="a review summary for a Reddit scraper pull request (run `build` first)")
|
||||||
|
rv.add_argument("--base", help="data/public/reddit_comments.json as it is on main")
|
||||||
|
rv.add_argument("--out", help="write the summary here (default: print it)")
|
||||||
|
rv.add_argument("--pr", help="also post it on this pull request, replacing the previous summary")
|
||||||
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
|
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
|
||||||
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
|
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
|
||||||
c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)")
|
c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)")
|
||||||
@@ -155,6 +159,9 @@ def main(argv=None):
|
|||||||
if a.cmd == "rebuild":
|
if a.cmd == "rebuild":
|
||||||
from .etl.handoff import rebuild
|
from .etl.handoff import rebuild
|
||||||
return rebuild()
|
return rebuild()
|
||||||
|
if a.cmd == "review-reddit":
|
||||||
|
from .etl.review_reddit import main as review
|
||||||
|
return review(a.base, a.out, a.pr)
|
||||||
if a.cmd == "send-pdf":
|
if a.cmd == "send-pdf":
|
||||||
from .etl.send_pdf import send
|
from .etl.send_pdf import send
|
||||||
return send()
|
return send()
|
||||||
|
|||||||
+12
-2
@@ -126,6 +126,16 @@
|
|||||||
"sha256": "b169d157e731db23c99f2f8317a9718d31801b7d64336b3d414511ac13f32493",
|
"sha256": "b169d157e731db23c99f2f8317a9718d31801b7d64336b3d414511ac13f32493",
|
||||||
"contentSize": "23174 B"
|
"contentSize": "23174 B"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"@type": "cr:FileObject",
|
||||||
|
"@id": "overlap",
|
||||||
|
"name": "overlap",
|
||||||
|
"description": "How many Reddit comment runs may also be in the community form (the same answers within a day), against what chance alone gives (analysis/overlap/overlap.py). Measured only: no run is removed.",
|
||||||
|
"contentUrl": "https://git.kenschiano.net/DragonKind/threshing-atlas/raw/branch/main/data/public/overlap.json",
|
||||||
|
"encodingFormat": "application/json",
|
||||||
|
"sha256": "04280a52f028334451c505ddc35004d98cc1f4595b2294c3746a09e34454e8b0",
|
||||||
|
"contentSize": "1976 B"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"@type": "cr:FileObject",
|
"@type": "cr:FileObject",
|
||||||
"@id": "reports",
|
"@id": "reports",
|
||||||
@@ -153,8 +163,8 @@
|
|||||||
"description": "Every number the Atlas page shows, for six filter presets (analysis/stats/site_stats.py; what is measured is analysis/stats/measures.yaml).",
|
"description": "Every number the Atlas page shows, for six filter presets (analysis/stats/site_stats.py; what is measured is analysis/stats/measures.yaml).",
|
||||||
"contentUrl": "https://git.kenschiano.net/DragonKind/threshing-atlas/raw/branch/main/data/public/stats.json",
|
"contentUrl": "https://git.kenschiano.net/DragonKind/threshing-atlas/raw/branch/main/data/public/stats.json",
|
||||||
"encodingFormat": "application/json",
|
"encodingFormat": "application/json",
|
||||||
"sha256": "89a6246da2495685733e20f2de9897299bd19d469aaeadbd8883972856787ce5",
|
"sha256": "7431173d9b667d4914d17bbc299d994d14c3104e3e89a1eb75d690c1b21ade6a",
|
||||||
"contentSize": "5151910 B"
|
"contentSize": "5110214 B"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"@type": "cr:FileObject",
|
"@type": "cr:FileObject",
|
||||||
|
|||||||
@@ -41,6 +41,12 @@ files:
|
|||||||
description: >-
|
description: >-
|
||||||
Runs described in the r/fourthwing megathread comments, read into the "Report a run" answers: comment id, day
|
Runs described in the r/fourthwing megathread comments, read into the "Report a run" answers: comment id, day
|
||||||
and answers only, no usernames or comment text.
|
and answers only, no usernames or comment text.
|
||||||
|
overlap:
|
||||||
|
path: data/public/overlap.json
|
||||||
|
format: application/json
|
||||||
|
description: >-
|
||||||
|
How many Reddit comment runs may also be in the community form (the same answers within a day), against what
|
||||||
|
chance alone gives (analysis/overlap/overlap.py). Measured only: no run is removed.
|
||||||
reports:
|
reports:
|
||||||
path: data/public/reports.json
|
path: data/public/reports.json
|
||||||
format: application/json
|
format: application/json
|
||||||
|
|||||||
@@ -0,0 +1,134 @@
|
|||||||
|
{
|
||||||
|
"schema": 1,
|
||||||
|
"rule": {
|
||||||
|
"min_answers": 4,
|
||||||
|
"window_days": 1,
|
||||||
|
"baseline_shifts_days": [
|
||||||
|
-28,
|
||||||
|
-21,
|
||||||
|
-14,
|
||||||
|
14,
|
||||||
|
21,
|
||||||
|
28
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"comment_runs": 82,
|
||||||
|
"comparable": 36,
|
||||||
|
"matched": 9,
|
||||||
|
"expected_by_chance": 7.3,
|
||||||
|
"likely_double": 1.7,
|
||||||
|
"matches": [
|
||||||
|
{
|
||||||
|
"comment": "comment-pdjr7gm",
|
||||||
|
"form": [
|
||||||
|
"sheet-8",
|
||||||
|
"sheet-38"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"comment": "comment-pdq6tsw",
|
||||||
|
"form": [
|
||||||
|
"sheet-168",
|
||||||
|
"sheet-190",
|
||||||
|
"sheet-398"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"comment": "comment-pe0kwol",
|
||||||
|
"form": [
|
||||||
|
"sheet-224"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"comment": "comment-pe2l6mz",
|
||||||
|
"form": [
|
||||||
|
"sheet-5",
|
||||||
|
"sheet-25",
|
||||||
|
"sheet-107",
|
||||||
|
"sheet-292",
|
||||||
|
"sheet-393",
|
||||||
|
"sheet-471",
|
||||||
|
"sheet-483",
|
||||||
|
"sheet-514",
|
||||||
|
"sheet-520"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"comment": "comment-pe2n3j1",
|
||||||
|
"form": [
|
||||||
|
"sheet-195",
|
||||||
|
"sheet-276"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"comment": "comment-pev3uge",
|
||||||
|
"form": [
|
||||||
|
"sheet-870",
|
||||||
|
"sheet-916",
|
||||||
|
"sheet-934",
|
||||||
|
"sheet-960",
|
||||||
|
"sheet-961",
|
||||||
|
"sheet-962",
|
||||||
|
"sheet-975",
|
||||||
|
"sheet-984",
|
||||||
|
"sheet-999",
|
||||||
|
"sheet-1008",
|
||||||
|
"sheet-1009",
|
||||||
|
"sheet-1019",
|
||||||
|
"sheet-1023",
|
||||||
|
"sheet-1068",
|
||||||
|
"sheet-1069",
|
||||||
|
"sheet-1077",
|
||||||
|
"sheet-1081",
|
||||||
|
"sheet-1089",
|
||||||
|
"sheet-1097",
|
||||||
|
"sheet-1101",
|
||||||
|
"sheet-1110",
|
||||||
|
"sheet-1129",
|
||||||
|
"sheet-1194",
|
||||||
|
"sheet-1200",
|
||||||
|
"sheet-1201",
|
||||||
|
"sheet-1235",
|
||||||
|
"sheet-1238",
|
||||||
|
"sheet-1257",
|
||||||
|
"sheet-1259",
|
||||||
|
"sheet-1264",
|
||||||
|
"sheet-1271",
|
||||||
|
"sheet-1278",
|
||||||
|
"sheet-1283",
|
||||||
|
"sheet-1290",
|
||||||
|
"sheet-1292"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"comment": "comment-pf22tpj",
|
||||||
|
"form": [
|
||||||
|
"sheet-1031",
|
||||||
|
"sheet-1042",
|
||||||
|
"sheet-1048",
|
||||||
|
"sheet-1140",
|
||||||
|
"sheet-1142",
|
||||||
|
"sheet-1143",
|
||||||
|
"sheet-1144",
|
||||||
|
"sheet-1171",
|
||||||
|
"sheet-1261",
|
||||||
|
"sheet-1262",
|
||||||
|
"sheet-1303",
|
||||||
|
"sheet-1304",
|
||||||
|
"sheet-1364"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"comment": "comment-pfhmw5d",
|
||||||
|
"form": [
|
||||||
|
"sheet-1504"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"comment": "comment-pgzz9hc",
|
||||||
|
"form": [
|
||||||
|
"sheet-1714"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
+2
-2
@@ -6,11 +6,11 @@
|
|||||||
"scripts": {
|
"scripts": {
|
||||||
"build": "node site/build.js",
|
"build": "node site/build.js",
|
||||||
"dev": "node site/build.js && python3 -m http.server 8000 -d dist",
|
"dev": "node site/build.js && python3 -m http.server 8000 -d dist",
|
||||||
"test:py": "python3 -m collection build --check && python3 planner/build_experiments.py --check && python3 analysis/black/evidence.py --check && python3 analysis/tree/build_tree.py --check && python3 analysis/known/build_known.py --check && PYTHONPATH=core python3 -m unittest discover -s tests/core -p 'test_*.py' && python3 -m unittest discover -s tests/collection -p 'test_*.py'",
|
"test:py": "python3 -m collection build --check && python3 planner/build_experiments.py --check && python3 analysis/black/evidence.py --check && python3 analysis/tree/build_tree.py --check && python3 analysis/known/build_known.py --check && python3 analysis/overlap/overlap.py --check && PYTHONPATH=core python3 -m unittest discover -s tests/core -p 'test_*.py' && python3 -m unittest discover -s tests/collection -p 'test_*.py'",
|
||||||
"test:unit": "node --test tests/core/graph.test.js tests/core/advisor.test.js tests/core/known.test.js",
|
"test:unit": "node --test tests/core/graph.test.js tests/core/advisor.test.js tests/core/known.test.js",
|
||||||
"test": "npm run test:py && npm run test:unit",
|
"test": "npm run test:py && npm run test:unit",
|
||||||
"graph": "node analysis/network/build-graph.js",
|
"graph": "node analysis/network/build-graph.js",
|
||||||
"policy": "python3 planner/build_policy.py && python3 planner/policy_report.py && python3 planner/build_experiments.py && python3 analysis/black/evidence.py && python3 analysis/tree/build_tree.py && python3 analysis/known/build_known.py",
|
"policy": "python3 planner/build_policy.py && python3 planner/policy_report.py && python3 planner/build_experiments.py && python3 analysis/black/evidence.py && python3 analysis/tree/build_tree.py && python3 analysis/known/build_known.py && python3 analysis/overlap/overlap.py",
|
||||||
"cycle": "python3 -m collection cycle",
|
"cycle": "python3 -m collection cycle",
|
||||||
"stats": "python3 analysis/stats/site_stats.py",
|
"stats": "python3 analysis/stats/site_stats.py",
|
||||||
"collect": "python3 -m collection update",
|
"collect": "python3 -m collection update",
|
||||||
|
|||||||
@@ -84,6 +84,7 @@
|
|||||||
// ------------------------------------------------------------ data loading
|
// ------------------------------------------------------------ data loading
|
||||||
async function loadStats() {
|
async function loadStats() {
|
||||||
state.stats = await (await fetch("data/stats.json")).json();
|
state.stats = await (await fetch("data/stats.json")).json();
|
||||||
|
try { state.overlap = await (await fetch("data/overlap.json")).json(); } catch { state.overlap = null; }
|
||||||
const sel = $("#f-preset");
|
const sel = $("#f-preset");
|
||||||
sel.innerHTML = Object.entries(META().presets).map(([k, l]) => `<option value="${k}">${esc(l)}</option>`).join("");
|
sel.innerHTML = Object.entries(META().presets).map(([k, l]) => `<option value="${k}">${esc(l)}</option>`).join("");
|
||||||
sel.value = state.preset;
|
sel.value = state.preset;
|
||||||
@@ -554,6 +555,7 @@
|
|||||||
</tbody></table></div><p class="small">Bulk logs come from players who recorded every run, so they are the least selective rows. Their rare-dragon rates are several times lower than the form's, which is the strongest sign of reporting bias in this data.</p></div>
|
</tbody></table></div><p class="small">Bulk logs come from players who recorded every run, so they are the least selective rows. Their rare-dragon rates are several times lower than the form's, which is the strongest sign of reporting bias in this data.</p></div>
|
||||||
<div class="panel"><h3>Rows flagged for contradictions</h3><p class="small muted">${m.n_flagged} of ${r.n} rows (${pct(m.n_flagged / r.n)}). Flagged rows stay in unless you pick "Rows without data-quality flags" above.</p><div class="tbl-wrap"><table><tbody>${flags.map(([f, n]) => `<tr><td>${esc(f)}</td><td class="num">${n}</td></tr>`).join("") || `<tr><td>None</td></tr>`}</tbody></table></div></div>
|
<div class="panel"><h3>Rows flagged for contradictions</h3><p class="small muted">${m.n_flagged} of ${r.n} rows (${pct(m.n_flagged / r.n)}). Flagged rows stay in unless you pick "Rows without data-quality flags" above.</p><div class="tbl-wrap"><table><tbody>${flags.map(([f, n]) => `<tr><td>${esc(f)}</td><td class="num">${n}</td></tr>`).join("") || `<tr><td>None</td></tr>`}</tbody></table></div></div>
|
||||||
<div class="panel"><h3>How runs ended</h3><div class="tbl-wrap"><table><tbody>${outs.map(([f, n]) => `<tr><td>${esc(f)}</td><td class="num">${n}</td><td class="num">${pct(n / r.n)}</td></tr>`).join("")}</tbody></table></div><p class="small muted">"No outcome recorded" rows reached the hour but the form shows no Bonded or Incinerated. They count toward reaching a branch but not toward bond rates.</p></div>
|
<div class="panel"><h3>How runs ended</h3><div class="tbl-wrap"><table><tbody>${outs.map(([f, n]) => `<tr><td>${esc(f)}</td><td class="num">${n}</td><td class="num">${pct(n / r.n)}</td></tr>`).join("")}</tbody></table></div><p class="small muted">"No outcome recorded" rows reached the hour but the form shows no Bonded or Incinerated. They count toward reaching a branch but not toward bond rates.</p></div>
|
||||||
|
${state.overlap && state.overlap.comparable ? `<div class="panel"><h3>The same run reported twice?</h3><p class="small">A rider may log a run in the form and post it in a megathread comment too. With no player IDs, a copy can only be measured, not proven, so nothing is removed. Of ${state.overlap.comparable} Reddit runs detailed enough to compare, ${state.overlap.matched} give the same answers as a form row sent within ${state.overlap.rule.window_days} day; chance alone (the same test a few weeks off) gives about ${state.overlap.expected_by_chance}. So about ${Math.round(state.overlap.likely_double)} of ${state.overlap.comment_runs} Reddit runs look reported twice${state.overlap.likely_double < 0.1 * state.overlap.comment_runs ? ", too few to matter" : ""}.</p></div>` : ""}
|
||||||
<div class="panel"><h3>Independence</h3><p class="small">The form has no player ID. One player's 50 runs count as 50 independent runs, so intervals are somewhat narrower than they should be. ${pct(Object.entries(m.sources).filter(([k]) => !["Form", "Reddit comments", "DragonKind bot", "Reported on the Atlas"].includes(k)).reduce((s, [, n]) => s + n, 0) / r.n, 0)} of rows arrived as bulk imports, each probably one player's or one group's log.</p></div>
|
<div class="panel"><h3>Independence</h3><p class="small">The form has no player ID. One player's 50 runs count as 50 independent runs, so intervals are somewhat narrower than they should be. ${pct(Object.entries(m.sources).filter(([k]) => !["Form", "Reddit comments", "DragonKind bot", "Reported on the Atlas"].includes(k)).reduce((s, [, n]) => s + n, 0) / r.n, 0)} of rows arrived as bulk imports, each probably one player's or one group's log.</p></div>
|
||||||
<div class="panel"><h3>What the form does not ask</h3><p class="small">The redesigned form asks how you reacted to the first dragon (answered in ${r.factors.reach_hour.rows.filter((x) => x.factor === "sighting_action" && !["Unknown", "(not asked)"].includes(x.level)).reduce((a, x) => a + x.n, 0)} rows; most riders answer "Unsure") and how the bully fight went (${r.factors.reach_hour.rows.filter((x) => x.factor === "combat_first" && x.level !== "(not recorded)").reduce((a, x) => a + x.n, 0)} rows). It still does not record most choices inside the bonding encounters. Theories that rely on those choices cannot be confirmed or refuted here.</p></div>`;
|
<div class="panel"><h3>What the form does not ask</h3><p class="small">The redesigned form asks how you reacted to the first dragon (answered in ${r.factors.reach_hour.rows.filter((x) => x.factor === "sighting_action" && !["Unknown", "(not asked)"].includes(x.level)).reduce((a, x) => a + x.n, 0)} rows; most riders answer "Unsure") and how the bully fight went (${r.factors.reach_hour.rows.filter((x) => x.factor === "combat_first" && x.level !== "(not recorded)").reduce((a, x) => a + x.n, 0)} rows). It still does not record most choices inside the bonding encounters. Theories that rely on those choices cannot be confirmed or refuted here.</p></div>`;
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -27,7 +27,7 @@ for (const f of fs.readdirSync(path.join(root, "site/atlas"))) copy(`site/atlas/
|
|||||||
copy("core/graph.js", "graph.js");
|
copy("core/graph.js", "graph.js");
|
||||||
copy("core/solver.js", "solver.js");
|
copy("core/solver.js", "solver.js");
|
||||||
// The numbers the page shows (all computed in Python) and the published data
|
// The numbers the page shows (all computed in Python) and the published data
|
||||||
for (const f of ["stats.json", "register.json", "graph.json", "policy.json", "black.json", "tree.json", "experiments.json", "pdf_encounters.json"]) copy(`data/public/${f}`, `data/${f}`);
|
for (const f of ["stats.json", "register.json", "graph.json", "policy.json", "black.json", "tree.json", "experiments.json", "pdf_encounters.json", "overlap.json"]) copy(`data/public/${f}`, `data/${f}`);
|
||||||
copy("data/public/runs_clean.csv", "data/runs_clean.csv");
|
copy("data/public/runs_clean.csv", "data/runs_clean.csv");
|
||||||
for (const f of ["croissant.json", "dictionary.yaml"]) copy(`data/${f}`, `data/${f}`);
|
for (const f of ["croissant.json", "dictionary.yaml"]) copy(`data/${f}`, `data/${f}`);
|
||||||
// Source files shown in the page's "view the code" panel
|
// Source files shown in the page's "view the code" panel
|
||||||
|
|||||||
@@ -335,6 +335,30 @@ class RedditComments(unittest.TestCase):
|
|||||||
kept = build_policy.drop_flagged(rows)
|
kept = build_policy.drop_flagged(rows)
|
||||||
self.assertEqual([r["source"] for r in kept], ["reddit comment", "reddit spreadsheet"])
|
self.assertEqual([r["source"] for r in kept], ["reddit comment", "reddit spreadsheet"])
|
||||||
|
|
||||||
|
def test_review_summary_and_overlap(self):
|
||||||
|
sys.path.insert(0, str(ROOT / "core"))
|
||||||
|
from collection.etl import review_reddit
|
||||||
|
ov = review_reddit._overlap()
|
||||||
|
base = {"runs": [{"id": "a1", "thread": "t", "answers": {"outcome": "Bonded"}}]}
|
||||||
|
head = {"runs": [{"id": "a1", "thread": "t", "answers": {"outcome": "Incinerated at a bonding encounter"}},
|
||||||
|
{"id": "b2", "thread": "t", "answers": {}}, {"id": "c3", "thread": "t", "answers": {}, "result": "THRESH-X1"}]}
|
||||||
|
self.assertEqual(review_reddit.diff(base, head), (["b2", "c3"], ["a1"], []))
|
||||||
|
route = dict(company="Alone", direction="East", scream="Ignore the scream", hour="Tree", outcome="Bonded",
|
||||||
|
outcome_color="Red", date="2026-10-05")
|
||||||
|
runs = [{"run_id": "comment-b2", "source": "reddit comment", "data_flags": "read from a Reddit comment; "
|
||||||
|
"Black recorded on a Valley run (the Black is only reached from the tree)", **{k: None for k in ov.FIELDS}, "date": None},
|
||||||
|
{"run_id": "comment-c3", "source": "reddit comment", "data_flags": "", **{k: None for k in ov.FIELDS}, **route},
|
||||||
|
{"run_id": "sheet-1", "source": "reddit spreadsheet", "data_flags": "", **{k: None for k in ov.FIELDS}, **route}]
|
||||||
|
m = ov.measure(runs)
|
||||||
|
self.assertEqual((m["comparable"], m["matched"]), (1, 1))
|
||||||
|
self.assertEqual(m["matches"], [{"comment": "comment-c3", "form": ["sheet-1"]}])
|
||||||
|
text = review_reddit.summary(base, head, runs, {"THRESH-X1"}, m)
|
||||||
|
self.assertIn("**2 new**, 1 changed", text)
|
||||||
|
self.assertIn("Black recorded on a Valley run", text)
|
||||||
|
self.assertIn("sheet-1", text)
|
||||||
|
self.assertIn("THRESH-X1", text)
|
||||||
|
self.assertTrue(text.startswith(review_reddit.MARKER))
|
||||||
|
|
||||||
def test_delivered_not_extracted(self):
|
def test_delivered_not_extracted(self):
|
||||||
spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"]
|
spec = read_yaml("collection/pipeline.yaml")["sources"]["reddit_comments"]
|
||||||
with self.assertRaises(extract.Skip):
|
with self.assertRaises(extract.Skip):
|
||||||
|
|||||||
Reference in New Issue
Block a user