CI / test (pull_request) Successful in 40s
- collection/etl/review_reddit.py (`python -m collection review-reddit`) and .gitea/workflows/reddit-review.yml: on every scraper pull request, one comment (refreshed on each push) listing the runs added, changed and gone, the runs that break a game rule (left out of the model and stats until corrected), other flags, runs that may also be in the form, and result pages already counted; plus how to reject / fix / add. forge.upsert_comment keeps it to one comment. collection/README.md: "Reviewing Reddit runs". - analysis/overlap/overlap.py -> data/public/overlap.json: a Reddit run with at least 4 route answers "matches" a form row giving the same answers within a day; chance is the same test a few weeks off. 9 of 36 comparable runs match, ~7.3 by chance: about 2 of 82 look reported twice, too few to matter. Nothing is removed. Shown on the page's Data & methods tab; in the dataset metadata; checked in test:py; part of `rebuild`. - Neither the form nor the register records a result-page id, so the only proven copy (comment <-> result page) stays the one #18 applies. - data/public/stats.json: the statistics of #42 (impossible runs left out), which did not make it into that merge. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UCzTd31GphpMXDhQqWrCMQ
113 lines
4.7 KiB
Python
113 lines
4.7 KiB
Python
"""Is the same attempt reported twice, in the community form and in a Reddit comment? (issue #24)
|
|
|
|
python analysis/overlap/overlap.py [--check]
|
|
|
|
There are no player ids, so a copy can only be proven by a shared dragonkind.com result-page id. Neither the form
|
|
nor the register records one, and the comment-to-result-page link is already applied when the runs are built
|
|
(collection/etl/runs.py). Everything else is measured, never removed:
|
|
|
|
- a Reddit comment run is *comparable* when it is dated and gives at least MIN_ANSWERS route answers besides how it
|
|
ended (fewer answers match dozens of form rows by chance on a busy day);
|
|
- it *matches* a form row submitted within WINDOW days when every answer it gives is the form row's answer;
|
|
- the same test with the comment's date shifted by a few weeks (SHIFTS, only where the form has rows that week) is
|
|
what chance alone produces, so matched - expected is the estimate of runs reported twice.
|
|
|
|
Writes data/public/overlap.json (the page's Data & methods tab, and the scraper pull-request summary). Nothing is
|
|
dropped: identical routes are expected, and the Reddit runs already have their own reporting-bias group in the
|
|
route model (planner/policy.yaml).
|
|
"""
|
|
import argparse
|
|
import collections
|
|
import datetime as dt
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "core"))
|
|
from atlas import runs as runs_io # noqa: E402
|
|
|
|
OUT = ROOT / "data/public/overlap.json"
|
|
FIELDS = ["company", "left_squadmates", "direction", "first_sighting", "sighting_action", "scream", "resolution",
|
|
"combat_first", "combat_second", "post_path", "hour", "branch", "outcome", "outcome_color", "tail", "attempt"]
|
|
ENDING = ("outcome", "outcome_color")
|
|
MIN_ANSWERS = 4
|
|
WINDOW = 1
|
|
SHIFTS = (-28, -21, -14, 14, 21, 28)
|
|
|
|
|
|
def given(v):
|
|
return v not in (None, "") and v != "Unknown" and not (isinstance(v, str) and v.startswith("("))
|
|
|
|
|
|
def answers(r):
|
|
return [k for k in FIELDS if given(r.get(k))]
|
|
|
|
|
|
def comparable(r):
|
|
return bool(r.get("date")) and len([k for k in answers(r) if k not in ENDING]) >= MIN_ANSWERS
|
|
|
|
|
|
def days(date, shift=0, window=WINDOW):
|
|
d = dt.date.fromisoformat(date[:10]) + dt.timedelta(days=shift)
|
|
return [(d + dt.timedelta(days=i)).isoformat() for i in range(-window, window + 1)]
|
|
|
|
|
|
def form_index(runs):
|
|
by_day = collections.defaultdict(list)
|
|
for r in runs:
|
|
if r["source"] == "reddit spreadsheet" and r.get("date"):
|
|
by_day[r["date"][:10]].append(r)
|
|
return by_day
|
|
|
|
|
|
def matches(c, by_day, shift=0):
|
|
"""Form rows within the window that give every answer the comment gives."""
|
|
keys = answers(c)
|
|
return [f for d in days(c["date"], shift) for f in by_day.get(d, []) if all(f.get(k) == c[k] for k in keys)]
|
|
|
|
|
|
def measure(runs):
|
|
by_day = form_index(runs)
|
|
comments = [r for r in runs if r["source"] == "reddit comment"]
|
|
comp = [c for c in comments if comparable(c)]
|
|
found = {c["run_id"]: [f["run_id"] for f in matches(c, by_day)] for c in comp}
|
|
matched = sum(1 for v in found.values() if v)
|
|
# chance: each comment's match rate on shifted dates, over the shifts whose window has form rows at all
|
|
expected = 0.0
|
|
for c in comp:
|
|
valid = [s for s in SHIFTS if any(by_day.get(d) for d in days(c["date"], s))]
|
|
if valid:
|
|
expected += sum(1 for s in valid if matches(c, by_day, s)) / len(valid)
|
|
return {
|
|
"schema": 1,
|
|
"rule": {"min_answers": MIN_ANSWERS, "window_days": WINDOW, "baseline_shifts_days": list(SHIFTS)},
|
|
"comment_runs": len(comments),
|
|
"comparable": len(comp),
|
|
"matched": matched,
|
|
"expected_by_chance": round(expected, 1),
|
|
"likely_double": round(max(0.0, matched - expected), 1),
|
|
"matches": [{"comment": k, "form": v} for k, v in sorted(found.items()) if v],
|
|
}
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("--check", action="store_true", help="fail if data/public/overlap.json is out of date")
|
|
a = ap.parse_args(argv)
|
|
text = json.dumps(measure(runs_io.read(drop=())), indent=1, ensure_ascii=False) + "\n"
|
|
if a.check:
|
|
if not OUT.exists() or OUT.read_text(encoding="utf-8") != text:
|
|
print(f"out of date (run `python {Path(__file__).relative_to(ROOT)}`): {OUT.relative_to(ROOT)}")
|
|
return 1
|
|
return 0
|
|
OUT.write_text(text, encoding="utf-8")
|
|
m = json.loads(text)
|
|
print(f"{OUT.relative_to(ROOT)}: {m['matched']} of {m['comparable']} comparable Reddit runs match a form row "
|
|
f"(~{m['expected_by_chance']} by chance; ~{m['likely_double']} likely reported twice)")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|