One rebuild list for the refresh, the bot cycle and the daily workflows; graph.json rebuilt #2
@@ -50,10 +50,7 @@ jobs:
|
|||||||
- name: Rebuild the analysis and run tests
|
- name: Rebuild the analysis and run tests
|
||||||
if: steps.check.outputs.changed == 'true'
|
if: steps.check.outputs.changed == 'true'
|
||||||
run: |
|
run: |
|
||||||
node analysis/network/build-graph.js
|
python -m collection rebuild
|
||||||
npm run policy
|
|
||||||
npm run stats
|
|
||||||
python -m collection build
|
|
||||||
npm test
|
npm test
|
||||||
- name: Encrypt the raw store
|
- name: Encrypt the raw store
|
||||||
id: archive
|
id: archive
|
||||||
|
|||||||
@@ -50,10 +50,7 @@ jobs:
|
|||||||
- name: Rebuild the analysis and run tests
|
- name: Rebuild the analysis and run tests
|
||||||
if: steps.check.outputs.changed == 'true'
|
if: steps.check.outputs.changed == 'true'
|
||||||
run: |
|
run: |
|
||||||
node analysis/network/build-graph.js
|
python -m collection rebuild
|
||||||
npm run policy
|
|
||||||
npm run stats
|
|
||||||
python -m collection build
|
|
||||||
npm test
|
npm test
|
||||||
- name: Encrypt the raw store
|
- name: Encrypt the raw store
|
||||||
id: archive
|
id: archive
|
||||||
|
|||||||
@@ -76,7 +76,7 @@ See `docs/BOT_DATA_PROTOCOL.md` for what the bot should log.
|
|||||||
|
|
||||||
## Running everything locally
|
## Running everything locally
|
||||||
|
|
||||||
`python -m collection refresh` does what the daily workflow does: checks every source, rebuilds the runs,
|
`python -m collection refresh` does what the daily workflow does (`python -m collection rebuild` alone redoes everything built from the runs): checks every source, rebuilds the runs,
|
||||||
the graph, the route model and the statistics. Add `--bot <folder or zip>` to import bot runs, and
|
the graph, the route model and the statistics. Add `--bot <folder or zip>` to import bot runs, and
|
||||||
`python -m collection export-bot-kit` zips the files the bot uses (see `docs/BOT_INTEGRATION.md`).
|
`python -m collection export-bot-kit` zips the files the bot uses (see `docs/BOT_INTEGRATION.md`).
|
||||||
|
|
||||||
|
|||||||
+23
-22
@@ -73,6 +73,27 @@ def _step(label, cmd):
|
|||||||
subprocess.run(cmd, cwd=ROOT, check=True)
|
subprocess.run(cmd, cwd=ROOT, check=True)
|
||||||
|
|
||||||
|
|
||||||
|
def rebuild():
|
||||||
|
"""Everything built from the runs, in order. The one list the local refresh, the bot cycle and the daily
|
||||||
|
workflows all use (`python -m collection rebuild`), so a new report only needs adding here."""
|
||||||
|
py = sys.executable
|
||||||
|
if shutil.which("node"):
|
||||||
|
_step("route graph", ["node", "analysis/network/build-graph.js"])
|
||||||
|
else:
|
||||||
|
print("\n(node is not installed: data/public/graph.json is not rebuilt; everything else is)")
|
||||||
|
_step("route model (a few minutes)", [py, "planner/build_policy.py"])
|
||||||
|
_step("experiment plan", [py, "planner/build_experiments.py"])
|
||||||
|
_step("what the runs show about the Black", [py, "analysis/black/evidence.py"])
|
||||||
|
_step("screen tree", [py, "analysis/tree/build_tree.py"])
|
||||||
|
try:
|
||||||
|
import scipy # noqa: F401
|
||||||
|
_step("statistics", [py, "analysis/stats/site_stats.py"])
|
||||||
|
except ImportError:
|
||||||
|
print("\n(scipy is not installed: data/public/stats.json is not rebuilt)")
|
||||||
|
_step("runs, generated files and metadata", [py, "-m", "collection", "build"])
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def refresh(update_main, bots=()):
|
def refresh(update_main, bots=()):
|
||||||
"""update -> (graph, policy, stats, build) when something changed. Returns 0 on success."""
|
"""update -> (graph, policy, stats, build) when something changed. Returns 0 on success."""
|
||||||
py = sys.executable
|
py = sys.executable
|
||||||
@@ -81,16 +102,7 @@ def refresh(update_main, bots=()):
|
|||||||
rc = update_main(args)
|
rc = update_main(args)
|
||||||
for i, b in enumerate(bots):
|
for i, b in enumerate(bots):
|
||||||
rc |= update_main(["update", "--source", "bot", "--file", f"bot={b}", "--report", str(path(f"exports/last-report-bot{i + 1}.md"))])
|
rc |= update_main(["update", "--source", "bot", "--file", f"bot={b}", "--report", str(path(f"exports/last-report-bot{i + 1}.md"))])
|
||||||
if shutil.which("node"):
|
rebuild()
|
||||||
_step("route graph", ["node", "analysis/network/build-graph.js"])
|
|
||||||
else:
|
|
||||||
print("\n(node is not installed: data/public/graph.json is not rebuilt; everything else is)")
|
|
||||||
_step("route model (about 2 minutes)", [py, "planner/build_policy.py"])
|
|
||||||
_step("experiment plan", [py, "planner/build_experiments.py"])
|
|
||||||
_step("what the runs show about the Black", [py, "analysis/black/evidence.py"])
|
|
||||||
_step("screen tree", [py, "analysis/tree/build_tree.py"])
|
|
||||||
_step("statistics", [py, "analysis/stats/site_stats.py"])
|
|
||||||
_step("runs, generated files and metadata", [py, "-m", "collection", "build"])
|
|
||||||
print(f"\nDone. What changed: exports/last-report.md. Send the bot its files with: {py} -m collection export-bot-kit")
|
print(f"\nDone. What changed: exports/last-report.md. Send the bot its files with: {py} -m collection export-bot-kit")
|
||||||
return rc
|
return rc
|
||||||
|
|
||||||
@@ -169,18 +181,7 @@ def cycle(update_main, bots, deliver_to=None, community=False, force=False):
|
|||||||
built = all(path(src).exists() for src in KIT.values())
|
built = all(path(src).exists() for src in KIT.values())
|
||||||
changed = force or not built or _fingerprint() != before
|
changed = force or not built or _fingerprint() != before
|
||||||
if changed:
|
if changed:
|
||||||
if shutil.which("node"):
|
rebuild()
|
||||||
_step("route graph", ["node", "analysis/network/build-graph.js"])
|
|
||||||
_step("route model (a few minutes)", [py, "planner/build_policy.py"])
|
|
||||||
_step("experiment plan", [py, "planner/build_experiments.py"])
|
|
||||||
_step("what the runs show about the Black", [py, "analysis/black/evidence.py"])
|
|
||||||
_step("screen tree", [py, "analysis/tree/build_tree.py"])
|
|
||||||
try:
|
|
||||||
import scipy # noqa: F401
|
|
||||||
_step("statistics", [py, "analysis/stats/site_stats.py"])
|
|
||||||
except ImportError:
|
|
||||||
print("\n(scipy is not installed: data/public/stats.json is not rebuilt; the bot does not use it)")
|
|
||||||
_step("runs, generated files and metadata", [py, "-m", "collection", "build"])
|
|
||||||
else:
|
else:
|
||||||
print("\nNo new runs: the route model and the plan are unchanged.")
|
print("\nNo new runs: the route model and the plan are unchanged.")
|
||||||
now = _headline()
|
now = _headline()
|
||||||
|
|||||||
@@ -109,6 +109,7 @@ def main(argv=None):
|
|||||||
fg.add_argument("args", nargs=argparse.REMAINDER)
|
fg.add_argument("args", nargs=argparse.REMAINDER)
|
||||||
k = sub.add_parser("export-bot-kit", help="zip policy.json, experiments.json, solver.js and graph.js for the bot")
|
k = sub.add_parser("export-bot-kit", help="zip policy.json, experiments.json, solver.js and graph.js for the bot")
|
||||||
k.add_argument("--out", default="exports")
|
k.add_argument("--out", default="exports")
|
||||||
|
sub.add_parser("rebuild", help="rebuild everything made from the runs: graph, route model, plan, reports, statistics, build")
|
||||||
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
|
r = sub.add_parser("refresh", help="update every source, then rebuild everything (the daily workflow, locally)")
|
||||||
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
|
r.add_argument("--bot", action="append", default=[], help="a bot folder or a .zip of its data (repeatable)")
|
||||||
c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)")
|
c = sub.add_parser("cycle", help="the loop with the bot: import its runs, refit, re-plan, deliver (safe on a timer)")
|
||||||
@@ -129,6 +130,9 @@ def main(argv=None):
|
|||||||
from .etl.handoff import export_bot_kit
|
from .etl.handoff import export_bot_kit
|
||||||
print(f"wrote {export_bot_kit(a.out)}")
|
print(f"wrote {export_bot_kit(a.out)}")
|
||||||
return 0
|
return 0
|
||||||
|
if a.cmd == "rebuild":
|
||||||
|
from .etl.handoff import rebuild
|
||||||
|
return rebuild()
|
||||||
if a.cmd == "refresh":
|
if a.cmd == "refresh":
|
||||||
from .etl.handoff import refresh
|
from .etl.handoff import refresh
|
||||||
return refresh(main, a.bot)
|
return refresh(main, a.bot)
|
||||||
|
|||||||
+2
-2
@@ -148,8 +148,8 @@
|
|||||||
"description": "The route graph over every run in runs_clean.csv, counted per source (analysis/network/build-graph.js).",
|
"description": "The route graph over every run in runs_clean.csv, counted per source (analysis/network/build-graph.js).",
|
||||||
"contentUrl": "https://git.kenschiano.net/kathleen/threshing-atlas/raw/branch/main/data/public/graph.json",
|
"contentUrl": "https://git.kenschiano.net/kathleen/threshing-atlas/raw/branch/main/data/public/graph.json",
|
||||||
"encodingFormat": "application/json",
|
"encodingFormat": "application/json",
|
||||||
"sha256": "2757b6935b3dec6854303e271e606e7e5fe089b38e3770a8db2fe664bcb74b5f",
|
"sha256": "ab58d8dd04a7b7226439f11aa16443dd67d142eba2734672729f4367fe20fe91",
|
||||||
"contentSize": "73446 B"
|
"contentSize": "73218 B"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"recordSet": [
|
"recordSet": [
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
@@ -263,7 +263,7 @@ answer to `core/spec/form.yaml` (or the value to `data/dictionary.yaml`) and run
|
|||||||
|
|
||||||
## 10. Update, 2026-11-11: first attempts, the experiment plan, the automated cycle
|
## 10. Update, 2026-11-11: first attempts, the experiment plan, the automated cycle
|
||||||
|
|
||||||
Branch `first-attempt-cycle` (not merged, not pushed).
|
Branch `first-attempt-cycle` (pushed to Gitea; not merged into main yet).
|
||||||
|
|
||||||
- **The shadow is close to a first-attempt event.** Form submissions, gated tree runs: 69 shadows in 748 first
|
- **The shadow is close to a first-attempt event.** Form submissions, gated tree runs: 69 shadows in 748 first
|
||||||
attempts (9.2%), 7 in 1,995 later ones (0.35%), in every week. Complete logs have only 28 first attempts at the
|
attempts (9.2%), 7 in 1,995 later ones (0.35%), in every week. Complete logs have only 28 first attempts at the
|
||||||
|
|||||||
Reference in New Issue
Block a user