Synthesized CLI agent-friendly principles
Public
Synthesized CLI agent-friendly principles
Agent
Pi
Model
GPT-5.5
Started
Aug 21, 12:32 PM
Messages279
Downloads0
Workspace
Reviewed experimental documentation and prior results.
UnderstandSynthesized general principles from experiment data.
BuildDocumented key CLI agent-friendly guidelines.
EditYou are PI ARM B for the autoresearch experiment. You replace the old Claude Arm B because the Claude subscription ran out. Work as a child arm under an orchestrator; do not make project decisions on your own.
READ FIRST:
Important current facts:
Your immediate task:
When done, say READY and provide the report in this session.
Planning file reading approach
Inspecting runner.py structure
>pwd && git status --short && find . -maxdepth 3 -type f | sort | sed 's#^./##' | head -200/Users/priyangapkini/nilenso/ai-playground/autoresearch
D ../editing-in-progress/.gitignore
D ../editing-in-progress/README.md
D ../editing-in-progress/config.example.toml
D ../editing-in-progress/deno.json
D ../editing-in-progress/deno.lock
D ../editing-in-progress/native/LICENSE.webui
D ../editing-in-progress/native/README.md
D ../editing-in-progress/native/aarch64-apple-darwin/libwebui-2.dylib
D ../editing-in-progress/native/aarch64-unknown-linux-gnu/libwebui-2.so
D ../editing-in-progress/native/x86_64-apple-darwin/libwebui-2.dylib
D ../editing-in-progress/native/x86_64-unknown-linux-gnu/libwebui-2.so
D ../editing-in-progress/scripts/build_app.ts
D ../editing-in-progress/scripts/build_ui.ts
D ../editing-in-progress/scripts/check_webui.ts
D ../editing-in-progress/server/app/api.ts
D ../editing-in-progress/server/app/api_test.ts
D ../editing-in-progress/server/app/local_app.ts
D ../editing-in-progress/server/app/local_app_test.ts
D ../editing-in-progress/server/app/local_document.ts
D ../editing-in-progress/server/app/local_document_test.ts
D ../editing-in-progress/server/app/persistence.ts
D ../editing-in-progress/server/app/persistence_test.ts
D ../editing-in-progress/server/cli.ts
D ../editing-in-progress/server/cli_test.ts
D ../editing-in-progress/server/config_file.ts
D ../editing-in-progress/server/config_file_test.ts
D ../editing-in-progress/server/core/config.ts
D ../editing-in-progress/server/core/config_test.ts
D ../editing-in-progress/server/core/protocol.ts
D ../editing-in-progress/server/core/protocol_test.ts
D ../editing-in-progress/server/core/scram.ts
D ../editing-in-progress/server/core/scram_test.ts
D ../editing-in-progress/server/core/uuid.ts
D ../editing-in-progress/server/core/uuid_test.ts
D ../editing-in-progress/server/http/mod.ts
D ../editing-in-progress/server/http/mod_test.ts
D ../editing-in-progress/server/log.ts
D ../editing-in-progress/server/main.ts
D ../editing-in-progress/server/main_test.ts
D ../editing-in-progress/server/network/client.ts
D ../editing-in-progress/server/network/client_test.ts
D ../editing-in-progress/server/network/coordinator.ts
D ../editing-in-progress/server/network/network_e2e_test.ts
D ../editing-in-progress/server/state/room.ts
D ../editing-in-progress/server/state/room_test.ts
D ../editing-in-progress/server/state/storage.ts
D ../editing-in-progress/server/state/storage_test.ts
D ../editing-in-progress/server/window.ts
D ../editing-in-progress/server/window_test.ts
D ../editing-in-progress/ui/index.html
D ../editing-in-progress/ui/src/automerge_doc.ts
D ../editing-in-progress/ui/src/automerge_doc_test.ts
D ../editing-in-progress/ui/src/bridge.ts
D ../editing-in-progress/ui/src/bridge_test.ts
D ../editing-in-progress/ui/src/index.tsx
D ../editing-in-progress/ui/src/state.ts
D ../editing-in-progress/ui/src/state_test.ts
D ../editing-in-progress/ui/src/style.css
D ../editing-in-progress/ui/src/test_support.ts
D ../editing-in-progress/vendor/deno-webui/LICENSE
D ../editing-in-progress/vendor/deno-webui/deno.json
D ../editing-in-progress/vendor/deno-webui/deps.ts
D ../editing-in-progress/vendor/deno-webui/mod.ts
D ../editing-in-progress/vendor/deno-webui/src/lib.ts
D ../editing-in-progress/vendor/deno-webui/src/types.ts
D ../editing-in-progress/vendor/deno-webui/src/utils.ts
D ../editing-in-progress/vendor/deno-webui/src/webui.ts
?? ../.DS_Store
?? ../.claude/worktrees/
?? ../.idea/
?? ../.vscode/
?? tools/dashboard/lever-history
.env
.gitignore
.pytest_cache/.gitignore
.pytest_cache/CACHEDIR.TAG
.pytest_cache/README.md
.venv/.gitignore
.venv/.lock
.venv/bin/activate
.venv/bin/activate_this.py
.venv/bin/activate.bat
.venv/bin/activate.csh
.venv/bin/activate.fish
.venv/bin/activate.nu
.venv/bin/activate.ps1
.venv/bin/activate.xsh
.venv/bin/deactivate.bat
.venv/bin/distro
.venv/bin/dotenv
.venv/bin/hf
.venv/bin/httpx
.venv/bin/huggingface-cli
.venv/bin/idna
.venv/bin/jsonschema
.venv/bin/lite
.venv/bin/litellm
.venv/bin/litellm-proxy
.venv/bin/normalizer
.venv/bin/py.test
.venv/bin/pydoc.bat
.venv/bin/pygmentize
.venv/bin/pytest
.venv/bin/tiny-agents
.venv/bin/tqdm
.venv/CACHEDIR.TAG
.venv/pyvenv.cfg
autoresearch.code-workspace
autoresearch/__init__.py
autoresearch/__pycache__/__init__.cpython-311.pyc
autoresearch/__pycache__/__init__.cpython-314.pyc
autoresearch/__pycache__/baseline.cpython-311.pyc
autoresearch/__pycache__/baseline.cpython-314.pyc
autoresearch/__pycache__/blocked.cpython-311.pyc
autoresearch/__pycache__/config.cpython-311.pyc
autoresearch/__pycache__/config.cpython-314.pyc
autoresearch/__pycache__/credits.cpython-311.pyc
autoresearch/__pycache__/credits.cpython-314.pyc
autoresearch/__pycache__/evaluator.cpython-311.pyc
autoresearch/__pycache__/optimize.cpython-311.pyc
autoresearch/__pycache__/orproxy.cpython-311.pyc
autoresearch/__pycache__/proposer.cpython-311.pyc
autoresearch/__pycache__/questions.cpython-311.pyc
autoresearch/__pycache__/questions.cpython-314.pyc
autoresearch/__pycache__/runner.cpython-311.pyc
autoresearch/__pycache__/runner.cpython-314.pyc
autoresearch/__pycache__/score.cpython-311.pyc
autoresearch/__pycache__/smoke.cpython-311.pyc
autoresearch/__pycache__/taxonomy.cpython-311.pyc
autoresearch/__pycache__/trace.cpython-311.pyc
autoresearch/__pycache__/worktree.cpython-311.pyc
autoresearch/baseline.py
autoresearch/blocked.py
autoresearch/config.py
autoresearch/credits.py
autoresearch/evaluator.py
autoresearch/optimize.py
autoresearch/orproxy.py
autoresearch/proposer.py
autoresearch/questions.py
autoresearch/runner.py
autoresearch/score.py
autoresearch/shim/botmap
autoresearch/smoke.py
autoresearch/taxonomy.py
autoresearch/trace.py
autoresearch/worktree.py
docs/agent-failure-taxonomy.md
docs/plan.md
docs/report/case-study.html
docs/report/README.md
docs/wiki/01-glossary.html
docs/wiki/02-domain-model.html
docs/wiki/03-system-architecture.html
docs/wiki/05-repository-map.html
docs/wiki/index.html
experiments/arm-a/AUTORESEARCH-REPORT.md
experiments/arm-a/HANDOVER.md
experiments/arm-a/README.md
experiments/arm-b/HANDOVER.md
experiments/arm-c/HANDOVER.md
experiments/artifacts/analyze.py
experiments/artifacts/answer-quality-analysis.md
experiments/artifacts/findings.md
experiments/artifacts/null-proposal.json
experiments/artifacts/overture_categories.csv
experiments/artifacts/probe.log
experiments/artifacts/prompt.md
experiments/artifacts/questions.py
experiments/artifacts/runner.out
experiments/baselines/3009509.INCOMPLETE-5of30.json
experiments/baselines/3009509.json
experiments/baselines/3009509.release-2026-07-22.0.STALE.json
experiments/baselines/INCOMPLETE-WARNING.md
experiments/baselines/README-STALE.md
experiments/failure_dataset.yaml
experiments/logs/run-20260818-134132.log
experiments/logs/run-20260818-134212.log
experiments/logs/run-20260818-161413.log
experiments/logs/run-20260818-161605.log
experiments/logs/sub-20260819-144029.log
experiments/proposals.json
experiments/questions.yaml
pyproject.toml
README.md
tests/__pycache__/test_scoring.cpython-311-pytest-9.1.1.pyc
tests/test_scoring.py
TODO.md
tools/dashboard/dashboard.py
tools/dashboard/README.md
tools/dashboard/trajectory.py
uv.lockInspecting deletions outside working directory
Planning baseline record writing
Inspecting missing agenteval directory
>find autoresearch -maxdepth 2 -type d -print && find experiments/runs/baseline-noise-run1-3009509/attempts -maxdepth 2 -type f | head -20 || true && git rev-parse --show-toplevel && git branch --show-current && git diff --statautoresearch
autoresearch/.pi-subagents
autoresearch/.pi-subagents/artifacts
autoresearch/__pycache__
autoresearch/shim
experiments/runs/baseline-noise-run1-3009509/attempts/basic-category-rollup__r2/transcript.jsonl
experiments/runs/baseline-noise-run1-3009509/attempts/basic-category-rollup__r2/commands.jsonl
experiments/runs/baseline-noise-run1-3009509/attempts/basic-category-rollup__r2/claude-stderr.log
experiments/runs/baseline-noise-run1-3009509/attempts/residential-share-cambridge__r2/transcript.jsonl
experiments/runs/baseline-noise-run1-3009509/attempts/residential-share-cambridge__r2/commands.jsonl
experiments/runs/baseline-noise-run1-3009509/attempts/residential-share-cambridge__r2/claude-stderr.log
experiments/runs/baseline-noise-run1-3009509/attempts/bus-stops-with-coffee__r2/transcript.jsonl
experiments/runs/baseline-noise-run1-3009509/attempts/bus-stops-with-coffee__r2/commands.jsonl
experiments/runs/baseline-noise-run1-3009509/attempts/bus-stops-with-coffee__r2/claude-stderr.log
experiments/runs/baseline-noise-run1-3009509/attempts/brooklyn-every-building__r2/transcript.jsonl
experiments/runs/baseline-noise-run1-3009509/attempts/brooklyn-every-building__r2/commands.jsonl
experiments/runs/baseline-noise-run1-3009509/attempts/brooklyn-every-building__r2/claude-stderr.log
experiments/runs/baseline-noise-run1-3009509/attempts/reykjavik-diacritic__r1/transcript.jsonl
experiments/runs/baseline-noise-run1-3009509/attempts/reykjavik-diacritic__r1/commands.jsonl
experiments/runs/baseline-noise-run1-3009509/attempts/reykjavik-diacritic__r1/claude-stderr.log
experiments/runs/baseline-noise-run1-3009509/attempts/hardware-near-bikepaths__r1/transcript.jsonl
experiments/runs/baseline-noise-run1-3009509/attempts/hardware-near-bikepaths__r1/commands.jsonl
experiments/runs/baseline-noise-run1-3009509/attempts/hardware-near-bikepaths__r1/claude-stderr.log
experiments/runs/baseline-noise-run1-3009509/attempts/bike-parking-coverage__r1/transcript.jsonl
experiments/runs/baseline-noise-run1-3009509/attempts/bike-parking-coverage__r1/commands.jsonl
/Users/priyangapkini/nilenso/ai-playground
feat/autoresearch
editing-in-progress/.gitignore | 5 -
editing-in-progress/README.md | 164 --
editing-in-progress/config.example.toml | 25 -
editing-in-progress/deno.json | 25 -
editing-in-progress/deno.lock | 2215 --------------------
editing-in-progress/native/LICENSE.webui | 21 -
editing-in-progress/native/README.md | 25 -
.../native/aarch64-apple-darwin/libwebui-2.dylib | Bin 299904 -> 0 bytes
.../native/aarch64-unknown-linux-gnu/libwebui-2.so | Bin 321848 -> 0 bytes
.../native/x86_64-apple-darwin/libwebui-2.dylib | Bin 271744 -> 0 bytes
.../native/x86_64-unknown-linux-gnu/libwebui-2.so | Bin 328520 -> 0 bytes
editing-in-progress/scripts/build_app.ts | 101 -
editing-in-progress/scripts/build_ui.ts | 40 -
editing-in-progress/scripts/check_webui.ts | 8 -
editing-in-progress/server/app/api.ts | 157 --
editing-in-progress/server/app/api_test.ts | 91 -
editing-in-progress/server/app/local_app.ts | 306 ---
editing-in-progress/server/app/local_app_test.ts | 49 -
editing-in-progress/server/app/local_document.ts | 73 -
.../server/app/local_document_test.ts | 39 -
editing-in-progress/server/app/persistence.ts | 104 -
editing-in-progress/server/app/persistence_test.ts | 24 -
editing-in-progress/server/cli.ts | 11 -
editing-in-progress/server/cli_test.ts | 32 -
editing-in-progress/server/config_file.ts | 66 -
editing-in-progress/server/config_file_test.ts | 38 -
editing-in-progress/server/core/config.ts | 164 --
editing-in-progress/server/core/config_test.ts | 94 -
editing-in-progress/server/core/protocol.ts | 228 --
editing-in-progress/server/core/protocol_test.ts | 109 -
editing-in-progress/server/core/scram.ts | 343 ---
editing-in-progress/server/core/scram_test.ts | 161 --
editing-in-progress/server/core/uuid.ts | 36 -
editing-in-progress/server/core/uuid_test.ts | 62 -
editing-in-progress/server/http/mod.ts | 301 ---
editing-in-progress/server/http/mod_test.ts | 249 ---
editing-in-progress/server/log.ts | 39 -
editing-in-progress/server/main.ts | 211 --
editing-in-progress/server/main_test.ts | 45 -
editing-in-progress/server/network/client.ts | 599 ------
editing-in-progress/server/network/client_test.ts | 15 -
editing-in-progress/server/network/coordinator.ts | 433 ----
.../server/network/network_e2e_test.ts | 131 --
editing-in-progress/server/state/room.ts | 181 --
editing-in-progress/server/state/room_test.ts | 102 -
editing-in-progress/server/state/storage.ts | 444 ----
editing-in-progress/server/state/storage_test.ts | 221 --
editing-in-progress/server/window.ts | 89 -
editing-in-progress/server/window_test.ts | 24 -
editing-in-progress/ui/index.html | 13 -
editing-in-progress/ui/src/automerge_doc.ts | 81 -
editing-in-progress/ui/src/automerge_doc_test.ts | 66 -
editing-in-progress/ui/src/bridge.ts | 201 --
editing-in-progress/ui/src/bridge_test.ts | 47 -
editing-in-progress/ui/src/index.tsx | 481 -----
editing-in-progress/ui/src/state.ts | 28 -
editing-in-progress/ui/src/state_test.ts | 21 -
editing-in-progress/ui/src/style.css | 372 ----
editing-in-progress/ui/src/test_support.ts | 3 -
editing-in-progress/vendor/deno-webui/LICENSE | 21 -
editing-in-progress/vendor/deno-webui/deno.json | 5 -
editing-in-progress/vendor/deno-webui/deps.ts | 42 -
editing-in-progress/vendor/deno-webui/mod.ts | 30 -
editing-in-progress/vendor/deno-webui/src/lib.ts | 488 -----
editing-in-progress/vendor/deno-webui/src/types.ts | 64 -
editing-in-progress/vendor/deno-webui/src/utils.ts | 14 -
editing-in-progress/vendor/deno-webui/src/webui.ts | 1394 ------------
67 files changed, 11271 deletions(-)Inspecting runner invocation and probe commands
>find .claude -maxdepth 4 -type f -print 2>/dev/null | sort || true && find /Users/priyangapkini/workspace/ar-b/botmap -maxdepth 3 -type f \( -path '*/.claude/*' -o -path '*/botmap/data/skill.md' \) -print 2>/dev/null | sort | head -50/Users/priyangapkini/workspace/ar-b/botmap/botmap/data/skill.mdVerifying skill source setting scope
Planning final validation with dirty status
>git status --short -- autoresearch docs experiments/arm-b tests pyproject.toml README.md TODO.md | head -100(no output)Designing contract validation approach
Planning contract validation and skill isolation
Resolving JSON key conflict with cls field
READY
Read:
docs/plan.mdexperiments/arm-b/HANDOVER.mdautoresearch/runner.pyImportant current code points:
autoresearch/runner.py inside ask().--setting-sources project.score.Attempt + trace.Call + trace.Transcript.autoresearch/taxonomy.py.autoresearch/agenteval/ package exists yet.tests/test_scoring.py.Exact source edit when approved:
In autoresearch/runner.py, change:
cmd = [ "claude", "-p", question.question, "--output-format", "stream-json", "--verbose", "--model", model, "--permission-mode", "bypassPermissions", "--allowedTools", "Bash",]to include:
"--setting-sources", "project",Likely placement:
cmd = [ "claude", "-p", question.question, "--setting-sources", "project", "--output-format", "stream-json", "--verbose", "--model", model, "--permission-mode", "bypassPermissions", "--allowedTools", "Bash",]Rationale: lowest-risk behavioral change, scoped exactly to agreed Phase 0.
Validation after orchestrator approval:
uv run pytest.claude/skills/botmap/SKILL.mdclaude -p with --setting-sources projectherdr, context-lens, lavish, find-skills, mattpocock-skills:*runner.ask() or equivalent retained attemptHazard:
.claude/skills as “project” source under --setting-sources project. If not, botmap skill may disappear too.agenteval/contract.pyCreate package:
autoresearch/agenteval/ __init__.py contract.pyImplement:
@dataclass(frozen=True)class Probe: kind: str ran: str result: str conclusive: bool
@dataclass(frozen=True)class CallVerdict: outcome: str blame: str recovery: str cls: str subtype: str | None evidence: str probes: tuple[Probe, ...]
@dataclass(frozen=True)class Record2: schema: str question_id: str repeat: int calls: tuple[dict, ...] agent_side: tuple[dict, ...] tools_used: dict[str, int] botmap_calls: int answer: dictKey choices to confirm:
cls, JSON key should be "class" because class is reserved.derive_class(outcome, blame, recovery) is the only class derivation function.validate(raw) returns list[str], never raises.load(path) can raise only for file/JSON I/O, but schema problems should be discoverable through validate.write(path, record) writes JSON with "schema": "agenteval/2".Expected derivation rules:
blame == "environment" → Eblame == "agent" → Foutcome == "degenerate" → Doutcome == "error" and recovery == "unguided" → Aoutcome == "error" and recovery == "guided" → Boutcome == "empty" and recovery == "unguided" → C"clean"? Plan table only names A-F failures.Validation tests to add:
validate() accepts minimal valid agenteval/2validate() rejects stored "class" inconsistent with axesderive_class() covers A-F tableCommands:
uv run pytestuv run python -m py_compile autoresearch/agenteval/contract.pyNo model/API spend needed for contract validation.
From repo root /Users/priyangapkini/nilenso/ai-playground:
editing-in-progress/, plus untracked IDE/worktree files.git status --short -- autoresearch docs experiments/arm-b tests pyproject.toml README.md TODO.mdConcern:
autoresearch/... and tests/... after approval.BOTMAP_REPO must remain isolated: /Users/priyangapkini/workspace/ar-b/botmap.experiments/runs/ and experiments/baselines/ are gitignored local evidence; do not overwrite/move without explicit direction.botmap/evals/."class" be omitted/null, or should we introduce "clean" despite plan saying classes A-F?load() validate and raise on bad records, or simply parse while callers use validate()?setting_sources, model, agent path/provider) be included in Record2 now, or deferred as plan says it is not Phase 0?tests/test_scoring.py or create tests/test_contract.py?Decisions from Priyanga/orchestrator:\n\n1. Phase 0a validation: approved to spend the two cheap Claude probe calls after the code change.\n2. record-v2 clean calls: use JSON class: null.\n3. Fixture sources: first copy/preserve old untracked Arm A notes into autoresearch, then old notes may be used as supporting fixture evidence.\n4. C1 candidate identity is NOT decided; Priyanga does not understand the question yet. Do not act on C1 identity until discussed further.\n5. Ignored-hint detection: use option C — strict boolean plus richer details/window.\n6. Quota failures: extend contract with attempt verdict; quota can be attempt-level class E outside per-call CallVerdict.\n7. Exit-0 did-you-mean/recovery hints: keep the recovery path. Treat guidance as something the agent should be able to use in the next iteration; if the agent does not recover, record that failure path (guided signal + agent-side failure detail), do not flatten it away.\n\nParent/orchestrator will start Phase 0a + contract.py unless told otherwise. Do not edit source files yet.
ACK.
I will not edit source files yet.
Updated Arm B understanding:
record-v2 clean call class: JSON "class": null.CallVerdict.did you mean remains guided recovery signal; if ignored, record guided signal plus agent-side failure detail.autoresearch.If parent/orchestrator is implementing Phase 0a + contract.py, I will stay ready to review/validate Arm B scope rather than duplicate edits.
Contract is now landed in the shared checkout (not committed yet):
You are unblocked for Arm B Phase 2 work. You may edit ONLY these files unless you ask first:
Do not touch botmap/evals. Do not launch evals or spend model/API quota. CLI probes in tests must be mocked/static, not real botmap spend/network. Follow docs/plan.md exactly: six probes, budget object/logging shape, inconclusive recorded, class-C subtype evidence. Acceptance target: recorded --class recreation case should produce c-wrong-column with 0-vs-86 evidence; if you cannot find the exact recorded trace, build a static fixture that captures that evidence and report the missing source.
Goal for this pass: implement probe.py with a pure testable interface and tests. Defer record-v2 wiring until probe/taxonomy contracts settle unless it is trivial and isolated. Report READY or BLOCKED with tests run and decisions needed.
Inspecting contract details
>find experiments -type f | grep -E 'failure|commands|yaml|notes|arm-a' | head -100 && rg -n "recreation|c-wrong-column|--class recreation|subtype=recreation" ../experiments/artifacts/overture_categories.csv:2075:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > diving_instruction > scuba_diving_instruction,scuba_diving_instruction,Scuba Diving Instruction,sport_or_fitness_facility,sports_and_recreation > sports_and_fitness_instruction > diving_instruction > scuba_diving_instruction,scuba_diving_instruction,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2076:sports_and_recreation,2,TRUE,sports_and_recreation > sport_or_fitness_facility > fitness_studio,fitness_studio,Fitness Studio,fitness_studio,,,,TRUE,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2077:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > fitness_studio > aerial_fitness_center,aerial_fitness_center,Aerial Fitness Center,fitness_studio,sports_and_recreation > sports_and_fitness_instruction > aerial_fitness_center,aerial_fitness_center,fitness_studio,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2078:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > fitness_studio > barre_class,barre_class,Barre Class,fitness_studio,sports_and_recreation > sports_and_fitness_instruction > barre_class,barre_class,fitness_studio,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2079:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > fitness_studio > boot_camp,boot_camp,Boot Camp,fitness_studio,sports_and_recreation > sports_and_fitness_instruction > boot_camp,boot_camp,fitness_studio,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2080:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > fitness_studio > pilates_studio,pilates_studio,Pilates Studio,fitness_studio,sports_and_recreation > sports_and_fitness_instruction > pilates_studio,pilates_studio,fitness_studio,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2081:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > fitness_studio > qi_gong_studio,qi_gong_studio,Qi Gong Studio,fitness_studio,sports_and_recreation > sports_and_fitness_instruction > qi_gong_studio,qi_gong_studio,fitness_studio,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2082:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > fitness_studio > tai_chi_studio,tai_chi_studio,Tai Chi Studio,fitness_studio,sports_and_recreation > sports_and_fitness_instruction > tai_chi_studio,tai_chi_studio,fitness_studio,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2083:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > fitness_studio > yoga_studio,yoga_studio,Yoga Studio,fitness_studio,sports_and_recreation > sports_and_fitness_instruction > yoga_studio,yoga_studio,fitness_studio,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2084:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > fitness_trainer,fitness_trainer,Fitness Trainer,sport_or_fitness_facility,sports_and_recreation > sports_and_fitness_instruction > fitness_trainer,fitness_trainer,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2085:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > flyboarding_center,flyboarding_center,Flyboarding Center,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > flyboarding_center,flyboarding_center,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2086:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > flyboarding_rental,flyboarding_rental,Flyboarding Rental,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > flyboarding_rental,flyboarding_rental,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2087:sports_and_recreation,2,TRUE,sports_and_recreation > sport_or_fitness_facility > golf_course,golf_course,Golf Course,golf_course,sports_and_recreation > sports_and_recreation_venue > golf_course,golf_course,golf_course,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2088:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > golf_course > driving_range,driving_range,Driving Range,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > golf_course > driving_range,driving_range,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2089:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > golf_instructor,golf_instructor,Golf Instructor,sport_or_fitness_facility,sports_and_recreation > sports_and_fitness_instruction > golf_instructor,golf_instructor,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2090:sports_and_recreation,2,TRUE,sports_and_recreation > sport_or_fitness_facility > gym,gym,Gym,gym,sports_and_recreation > sports_and_recreation_venue > gym,gym,gym,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2091:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > gym > cycle_studio,cycle_studio,Cycle Studio,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > gym > cycle_studio,cycle_studio,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2092:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > gymnastics_center,gymnastics_center,Gymnastics Center,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > gymnastics_center,gymnastics_center,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2093:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > hang_gliding_center,hang_gliding_center,Hang Gliding Center,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > hang_gliding_center,hang_gliding_center,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2094:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > hockey_rink,hockey_rink,Hockey Rink,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > hockey_rink,hockey_rink,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2095:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > hockey_rink > ice_hockey_rink,ice_hockey_rink,Ice Hockey Rink,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > hockey_rink > ice_hockey_rink,ice_hockey_rink,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2096:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > hockey_rink > roller_hockey_rink,roller_hockey_rink,Roller Hockey Rink,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > hockey_rink > roller_hockey_rink,roller_hockey_rink,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2097:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > horse_riding,horse_riding,Horse Riding,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > horse_riding,horse_riding,equestrian_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2098:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > horse_riding > equestrian_facility,equestrian_facility,Equestrian Facility,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > horse_riding > equestrian_facility,equestrian_facility,equestrian_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2099:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > horseback_riding_service,horseback_riding_service,Horseback Riding Service,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_rental_and_service > horseback_riding_service,horseback_riding_service,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2100:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > indoor_playcenter,indoor_playcenter,Indoor Playcenter,sport_or_fitness_facility,sports_and_recreation > indoor_playcenter,indoor_playcenter,entertainment_location,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2101:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > kiteboarding,kiteboarding,Kiteboarding,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > kiteboarding,kiteboarding,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2102:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > kiteboarding_instruction,kiteboarding_instruction,Kiteboarding Instruction,sport_or_fitness_facility,sports_and_recreation > sports_and_fitness_instruction > kiteboarding_instruction,kiteboarding_instruction,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2103:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > laser_tag,laser_tag,Laser Tag,sport_or_fitness_facility,sports_and_recreation > laser_tag,laser_tag,entertainment_location,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2104:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > miniature_golf_course,miniature_golf_course,Miniature Golf Course,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > miniature_golf_course,miniature_golf_course,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2105:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > paddleboarding_center,paddleboarding_center,Paddleboarding Center,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > paddleboarding_center,paddleboarding_center,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2106:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > paddleboarding_lessons,paddleboarding_lessons,Paddleboarding Lessons,sport_or_fitness_facility,sports_and_recreation > sports_and_fitness_instruction > paddleboarding_lessons,paddleboarding_lessons,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2107:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > paintball,paintball,Paintball,sport_or_fitness_facility,sports_and_recreation > paintball,paintball,entertainment_location,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2108:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > parasailing_ride_service,parasailing_ride_service,Parasailing Ride Service,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_rental_and_service > parasailing_ride_service,parasailing_ride_service,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2109:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > pool_billiards,pool_billiards,Pool Billiards,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > pool_billiards,pool_billiards,pool_hall,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2110:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > pool_billiards > pool_hall,pool_hall,Pool Hall,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > pool_billiards > pool_hall,pool_hall,pool_hall,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2111:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > race_track,race_track,Race Track,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > race_track,race_track,race_track,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2112:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > race_track > go_kart_track,go_kart_track,Go Kart Track,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > race_track > go_kart_track,go_kart_track,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2113:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > race_track > horse_racing_track,horse_racing_track,Horse Racing Track,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > race_track > horse_racing_track,horse_racing_track,race_track,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2114:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > race_track > motor_race_track,motor_race_track,Motor Race Track,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > race_track > motor_race_track,motor_race_track,race_track,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2115:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > race_track > track_and_field_track,track_and_field_track,Track and Field Track,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > race_track > track_and_field_track,track_and_field_track,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2116:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > race_track > velodrome,velodrome,Velodrome,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > race_track > velodrome,velodrome,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2117:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > racing_experience,racing_experience,Racing Experience,sport_or_fitness_facility,sports_and_recreation > sports_and_fitness_instruction > racing_experience,racing_experience,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2118:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > rock_climbing_gym,rock_climbing_gym,Rock Climbing Gym,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > rock_climbing_gym,rock_climbing_gym,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2119:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > rock_climbing_instructor,rock_climbing_instructor,Rock Climbing Instructor,sport_or_fitness_facility,sports_and_recreation > sports_and_fitness_instruction > rock_climbing_instructor,rock_climbing_instructor,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2120:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > running_and_track,running_and_track,Running and Track,sport_or_fitness_facility,sports_and_recreation > running_and_track,running_and_track,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2121:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > running_and_track > running,running,Running,sport_or_fitness_facility,sports_and_recreation > running_and_track > running,running,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2122:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > running_and_track > track_field_event,track_field_event,Track Field Event,sport_or_fitness_facility,sports_and_recreation > running_and_track > track_field_event,track_field_event,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2123:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > self_defense_class,self_defense_class,Self Defense Class,sport_or_fitness_facility,sports_and_recreation > sports_and_fitness_instruction > self_defense_class,self_defense_class,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2124:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > shooting_range,shooting_range,Shooting Range,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > shooting_range,shooting_range,shooting_range,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2125:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > shooting_range > archery_range,archery_range,Archery Range,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > archery_range,archery_range,shooting_range,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2126:sports_and_recreation,2,TRUE,sports_and_recreation > sport_or_fitness_facility > skate_park,skate_park,Skate Park,skate_park,sports_and_recreation > sports_and_recreation_venue > skate_park,skate_park,skate_park,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2127:sports_and_recreation,2,TRUE,sports_and_recreation > sport_or_fitness_facility > skating_rink,skating_rink,Skating Rink,skating_rink,sports_and_recreation > sports_and_recreation_venue > skating_rink,skating_rink,skating_rink,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2128:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > skating_rink > ice_skating_rink,ice_skating_rink,Ice Skating Rink,skating_rink,sports_and_recreation > sports_and_recreation_venue > skating_rink > ice_skating_rink,ice_skating_rink,skating_rink,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2129:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > skating_rink > roller_skating_rink,roller_skating_rink,Roller Skating Rink,skating_rink,sports_and_recreation > sports_and_recreation_venue > skating_rink > roller_skating_rink,roller_skating_rink,skating_rink,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2130:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > ski_and_snowboard_school,ski_and_snowboard_school,Ski and Snowboard School,sport_or_fitness_facility,sports_and_recreation > sports_and_fitness_instruction > ski_and_snowboard_school,ski_and_snowboard_school,specialty_school,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2131:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > sky_diving,sky_diving,Sky Diving,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > sky_diving,sky_diving,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2132:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sky_diving > sky_diving_drop_zone,sky_diving_drop_zone,Sky Diving Drop Zone,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > sky_diving > sky_diving_drop_zone,sky_diving_drop_zone,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2133:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sky_diving > skydiving_center,skydiving_center,Skydiving Center,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > sky_diving > skydiving_center,skydiving_center,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2134:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > snow_sport,snow_sport,Snow Sport,sport_or_fitness_facility,sports_and_recreation > snow_sport,snow_sport,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2135:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > snow_sport > bobsledding_field,bobsledding_field,Bobsledding Field,sport_or_fitness_facility,sports_and_recreation > snow_sport > bobsledding_field,bobsledding_field,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2136:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > snow_sport > ski_area,ski_area,Ski Area,sport_or_fitness_facility,sports_and_recreation > snow_sport > ski_area,ski_area,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2137:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > snow_sport > ski_chairlift,ski_chairlift,Ski Chairlift,sport_or_fitness_facility,sports_and_recreation > snow_sport > ski_chairlift,ski_chairlift,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2138:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > snow_sport > ski_chalet,ski_chalet,Ski Chalet,sport_or_fitness_facility,sports_and_recreation > snow_sport > ski_chalet,ski_chalet,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2139:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > snow_sport > ski_resort_area,ski_resort_area,Ski Resort Area,sport_or_fitness_facility,sports_and_recreation > snow_sport > ski_resort_area,ski_resort_area,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2140:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > snow_sport > snowboarding_center,snowboarding_center,Snowboarding Center,sport_or_fitness_facility,sports_and_recreation > snow_sport > snowboarding_center,snowboarding_center,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2141:sports_and_recreation,2,TRUE,sports_and_recreation > sport_or_fitness_facility > sport_court,sport_court,Sport Court,sport_court,,,,TRUE,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2142:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_court > badminton_court,badminton_court,Badminton Court,sport_court,sports_and_recreation > sports_and_recreation_venue > badminton_court,badminton_court,sport_court,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2143:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_court > basketball_court,basketball_court,Basketball Court,sport_court,sports_and_recreation > sports_and_recreation_venue > basketball_court,basketball_court,sport_court,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2144:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_court > beach_volleyball_court,beach_volleyball_court,Beach Volleyball Court,sport_court,sports_and_recreation > sports_and_recreation_venue > beach_volleyball_court,beach_volleyball_court,sport_court,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2145:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_court > bocce_ball_court,bocce_ball_court,Bocce Ball Court,sport_court,sports_and_recreation > sports_and_recreation_venue > bocce_ball_court,bocce_ball_court,sport_court,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2146:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_court > handball_court,handball_court,Handball Court,sport_court,sports_and_recreation > sports_and_recreation_venue > handball_court,handball_court,sport_court,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2147:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_court > pickleball_court,pickleball_court,Pickleball Court,sport_court,sports_and_recreation > sports_and_recreation_venue > pickleball_court,pickleball_court,sport_court,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2148:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_court > racquetball_court,racquetball_court,Racquetball Court,sport_court,sports_and_recreation > sports_and_recreation_venue > racquetball_court,racquetball_court,sport_court,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2149:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_court > squash_court,squash_court,Squash Court,sport_court,sports_and_recreation > sports_and_recreation_venue > squash_court,squash_court,sport_court,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2150:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_court > street_hockey_court,street_hockey_court,Street Hockey Court,sport_court,sports_and_recreation > sports_and_recreation_venue > hockey_field > street_hockey_court,street_hockey_court,sport_court,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2151:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_court > tennis_court,tennis_court,Tennis Court,sport_court,sports_and_recreation > sports_and_recreation_venue > tennis_court,tennis_court,sport_court,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2152:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_court > volleyball_court,volleyball_court,Volleyball Court,sport_court,sports_and_recreation > sports_and_recreation_venue > volleyball_court,volleyball_court,sport_court,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2153:sports_and_recreation,2,TRUE,sports_and_recreation > sport_or_fitness_facility > sport_field,sport_field,Sport Field,sport_field,,,,TRUE,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2154:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_field > airsoft_field,airsoft_field,Airsoft Field,sport_field,sports_and_recreation > sports_and_recreation_venue > airsoft_field,airsoft_field,sport_field,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2155:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_field > american_football_field,american_football_field,American Football Field,sport_field,sports_and_recreation > sports_and_recreation_venue > american_football_field,american_football_field,sport_field,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2156:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_field > baseball_field,baseball_field,Baseball Field,sport_field,sports_and_recreation > sports_and_recreation_venue > baseball_field,baseball_field,sport_field,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2157:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_field > disc_golf_course,disc_golf_course,Disc Golf Course,sport_field,sports_and_recreation > sports_and_recreation_venue > disc_golf_course,disc_golf_course,sport_field,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2158:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_field > futsal_field,futsal_field,Futsal Field,sport_field,sports_and_recreation > sports_and_recreation_venue > futsal_field,futsal_field,sport_field,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2159:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_field > hockey_field,hockey_field,Hockey Field,sport_field,sports_and_recreation > sports_and_recreation_venue > hockey_field,hockey_field,sport_field,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2160:sports_and_recreation,4,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_field > hockey_field > field_hockey_pitch,field_hockey_pitch,Field Hockey Pitch,sport_field,sports_and_recreation > sports_and_recreation_venue > hockey_field > field_hockey_pitch,field_hockey_pitch,sport_field,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2161:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_field > lacrosse_field,lacrosse_field,Lacrosse Field,sport_field,sports_and_recreation > sports_and_recreation_venue > lacrosse_field,lacrosse_field,sport_field,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2162:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_field > rugby_pitch,rugby_pitch,Rugby Pitch,sport_field,sports_and_recreation > sports_and_recreation_venue > rugby_pitch,rugby_pitch,sport_field,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2163:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_field > soccer_field,soccer_field,Soccer Field,sport_field,sports_and_recreation > sports_and_recreation_venue > soccer_field,soccer_field,sport_field,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2164:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > sport_field > softball_field,softball_field,Softball Field,sport_field,sports_and_recreation > sports_and_recreation_venue > softball_field,softball_field,sport_field,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2165:sports_and_recreation,2,TRUE,sports_and_recreation > sport_or_fitness_facility > sports_complex,sports_complex,Sports Complex,sports_complex,,,,TRUE,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2166:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > surfing_school,surfing_school,Surfing School,sport_or_fitness_facility,sports_and_recreation > sports_and_fitness_instruction > surfing_school,surfing_school,specialty_school,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2167:sports_and_recreation,2,TRUE,sports_and_recreation > sport_or_fitness_facility > swimming_pool,swimming_pool,Swimming Pool,swimming_pool,sports_and_recreation > sports_and_recreation_venue > swimming_pool,swimming_pool,swimming_pool,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2168:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > swimming_pool > swimming_instructor,swimming_instructor,Swimming Instructor,swimming_pool,sports_and_recreation > sports_and_fitness_instruction > swimming_instructor,swimming_instructor,swimming_pool,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2169:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > trampoline_park,trampoline_park,Trampoline Park,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > trampoline_park,trampoline_park,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2170:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > tubing_provider,tubing_provider,Tubing Provider,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > tubing_provider,tubing_provider,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2171:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > water_sport,water_sport,Water Sport,sport_or_fitness_facility,sports_and_recreation > water_sport,water_sport,recreational_location,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2172:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > water_sport > boating_place,boating_place,Boating Place,sport_or_fitness_facility,sports_and_recreation > water_sport > boating_place,boating_place,recreational_location,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2173:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > water_sport > fishing,fishing,Fishing,sport_or_fitness_facility,sports_and_recreation > water_sport > fishing,fishing,recreational_location,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2174:sports_and_recreation,4,FALSE,sports_and_recreation > sport_or_fitness_facility > water_sport > fishing > fishing_area,fishing_area,Fishing Area,sport_or_fitness_facility,sports_and_recreation > water_sport > fishing > fishing_area,fishing_area,recreational_location,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2175:sports_and_recreation,4,FALSE,sports_and_recreation > sport_or_fitness_facility > water_sport > fishing > fishing_charter,fishing_charter,Fishing Charter,sport_or_fitness_facility,sports_and_recreation > water_sport > fishing > fishing_charter,fishing_charter,recreational_location,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2176:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > water_sport > rafting_kayaking_area,rafting_kayaking_area,Rafting Kayaking Area,sport_or_fitness_facility,sports_and_recreation > water_sport > rafting_kayaking_area,rafting_kayaking_area,recreational_location,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2177:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > water_sport > sailing_area,sailing_area,Sailing Area,sport_or_fitness_facility,sports_and_recreation > water_sport > sailing_area,sailing_area,recreational_location,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2178:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > water_sport > snorkeling,snorkeling,Snorkeling,sport_or_fitness_facility,sports_and_recreation > water_sport > snorkeling,snorkeling,recreational_location,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2179:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_fitness_facility > water_sport > surfing,surfing,Surfing,sport_or_fitness_facility,sports_and_recreation > water_sport > surfing,surfing,recreational_location,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2180:sports_and_recreation,4,FALSE,sports_and_recreation > sport_or_fitness_facility > water_sport > surfing > windsurfing_center,windsurfing_center,Windsurfing Center,sport_or_fitness_facility,sports_and_recreation > water_sport > surfing > windsurfing_center,windsurfing_center,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2181:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_fitness_facility > wildlife_hunting_range,wildlife_hunting_range,Wildlife Hunting Range,sport_or_fitness_facility,sports_and_recreation > sports_and_recreation_venue > wildlife_hunting_range,wildlife_hunting_range,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2182:sports_and_recreation,1,TRUE,sports_and_recreation > sport_or_recreation_club,sport_or_recreation_club,Sport or Recreation Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league,sports_club_and_league,sport_recreation_club,,TRUE,TRUE,,,TRUE
./experiments/artifacts/overture_categories.csv:2183:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > beach_volleyball_club,beach_volleyball_club,Beach Volleyball Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > beach_volleyball_club,beach_volleyball_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2184:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > curling_club,curling_club,Curling Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > curling_club,curling_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2185:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > fencing_club,fencing_club,Fencing Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > fencing_club,fencing_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2186:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > fishing_club,fishing_club,Fishing Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > fishing_club,fishing_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2187:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > football_club,football_club,Football Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > football_club,football_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2188:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > go_kart_club,go_kart_club,Go Kart Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > go_kart_club,go_kart_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2189:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > golf_club,golf_club,Golf Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > golf_club,golf_club,golf_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2190:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_recreation_club > golf_club > indoor_golf_center,indoor_golf_center,Indoor Golf Center,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > golf_club > indoor_golf_center,indoor_golf_center,sport_fitness_facility,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2191:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > gymnastics_club,gymnastics_club,Gymnastics Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > gymnastics_club,gymnastics_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2192:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > hockey_club,hockey_club,Hockey Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > hockey_club,hockey_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2193:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_recreation_club > hockey_club > field_hockey_club,field_hockey_club,Field Hockey Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > hockey_club > field_hockey_club,field_hockey_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2194:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_recreation_club > hockey_club > ice_hockey_club,ice_hockey_club,Ice Hockey Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > hockey_club > ice_hockey_club,ice_hockey_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2195:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_recreation_club > hockey_club > roller_hockey_club,roller_hockey_club,Roller Hockey Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > hockey_club > roller_hockey_club,roller_hockey_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2196:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_recreation_club > hockey_club > street_hockey_club,street_hockey_club,Street Hockey Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > hockey_club > street_hockey_club,street_hockey_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2197:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > lacrosse_club,lacrosse_club,Lacrosse Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > lacrosse_club,lacrosse_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2198:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > lawn_bowling_club,lawn_bowling_club,Lawn Bowling Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > lawn_bowling_club,lawn_bowling_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2199:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > martial_arts_club,martial_arts_club,Martial Arts Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > martial_arts_club,martial_arts_club,martial_arts_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2200:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_recreation_club > martial_arts_club > brazilian_jiu_jitsu_club,brazilian_jiu_jitsu_club,Brazilian Jiu Jitsu Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > martial_arts_club > brazilian_jiu_jitsu_club,brazilian_jiu_jitsu_club,martial_arts_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2201:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_recreation_club > martial_arts_club > chinese_martial_arts_club,chinese_martial_arts_club,Chinese Martial Arts Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > martial_arts_club > chinese_martial_arts_club,chinese_martial_arts_club,martial_arts_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2202:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_recreation_club > martial_arts_club > karate_club,karate_club,Karate Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > martial_arts_club > karate_club,karate_club,martial_arts_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2203:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_recreation_club > martial_arts_club > kickboxing_club,kickboxing_club,Kickboxing Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > martial_arts_club > kickboxing_club,kickboxing_club,martial_arts_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2204:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_recreation_club > martial_arts_club > muay_thai_club,muay_thai_club,Muay Thai Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > martial_arts_club > muay_thai_club,muay_thai_club,martial_arts_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2205:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_recreation_club > martial_arts_club > taekwondo_club,taekwondo_club,Taekwondo Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > martial_arts_club > taekwondo_club,taekwondo_club,martial_arts_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2206:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > naturist_club,naturist_club,Naturist Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > naturist_club,naturist_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2207:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > paddle_tennis_club,paddle_tennis_club,Paddle Tennis Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > paddle_tennis_club,paddle_tennis_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2208:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > pickleball_club,pickleball_club,Pickleball Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > pickleball_club,pickleball_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2209:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > rowing_club,rowing_club,Rowing Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > rowing_club,rowing_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2210:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > running_and_track_club,running_and_track_club,Running and Track Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > running_and_track_club,running_and_track_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2211:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_recreation_club > running_and_track_club > running_club,running_club,Running Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > running_and_track_club > running_club,running_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2212:sports_and_recreation,3,FALSE,sports_and_recreation > sport_or_recreation_club > running_and_track_club > track_and_field_club,track_and_field_club,Track and Field Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > running_and_track_club > track_and_field_club,track_and_field_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2213:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > sailing_club,sailing_club,Sailing Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > sailing_club,sailing_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2214:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > soccer_club,soccer_club,Soccer Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > soccer_club,soccer_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2215:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > surf_lifesaving_club,surf_lifesaving_club,Surf Lifesaving Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > surf_lifesaving_club,surf_lifesaving_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2216:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > table_tennis_club,table_tennis_club,Table Tennis Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > table_tennis_club,table_tennis_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2217:sports_and_recreation,2,FALSE,sports_and_recreation > sport_or_recreation_club > volleyball_club,volleyball_club,Volleyball Club,sport_or_recreation_club,sports_and_recreation > sports_club_and_league > volleyball_club,volleyball_club,sport_recreation_club,,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2218:sports_and_recreation,1,TRUE,sports_and_recreation > sport_team,sport_team,Sport Team,sport_team,,,,TRUE,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2219:sports_and_recreation,2,FALSE,sports_and_recreation > sport_team > amateur_sport_team,amateur_sport_team,Amateur Sport Team,sport_team,sports_and_recreation > sports_club_and_league > amateur_sports_team,amateur_sports_team,amateur_sport_team,,TRUE,TRUE,,,TRUE
./experiments/artifacts/overture_categories.csv:2220:sports_and_recreation,2,FALSE,sports_and_recreation > sport_team > esports_team,esports_team,Esports Team,sport_team,sports_and_recreation > sports_club_and_league > esports_team,esports_team,sport_team,,TRUE,,,,
./experiments/artifacts/overture_categories.csv:2221:sports_and_recreation,2,FALSE,sports_and_recreation > sport_team > professional_sport_team,professional_sport_team,Professional Sport Team,sport_team,sports_and_recreation > sports_club_and_league > professional_sports_team,professional_sports_team,pro_sport_team,,TRUE,TRUE,,,TRUE
./experiments/artifacts/overture_categories.csv:2222:sports_and_recreation,2,FALSE,sports_and_recreation > sport_team > school_sport_team,school_sport_team,School Sport Team,sport_team,sports_and_recreation > sports_club_and_league > school_sports_team,school_sports_team,school_sport_team,,TRUE,TRUE,,,TRUE
./experiments/artifacts/overture_categories.csv:2223:sports_and_recreation,1,FALSE,sports_and_recreation > sports_and_fitness_instruction,sports_and_fitness_instruction,Sports and Fitness Instruction,sport_or_fitness_facility,sports_and_recreation > sports_and_fitness_instruction,sports_and_fitness_instruction,sport_fitness_facility,,,,TRUE,sport_or_fitness_facility,TRUE
./experiments/artifacts/overture_categories.csv:2348:travel_and_transportation,2,FALSE,travel_and_transportation > vehicle_service > recreation_vehicle_service,recreation_vehicle_service,Recreation Vehicle Service,vehicle_service,,,,TRUE,TRUE,,,,TRUE
./experiments/artifacts/overture_categories.csv:2349:travel_and_transportation,3,FALSE,travel_and_transportation > vehicle_service > recreation_vehicle_service > recreation_vehicle_repair,recreation_vehicle_repair,Recreation Vehicle Repair,vehicle_service,travel_and_transportation > automotive_and_ground_transport > automotive > automotive_services_and_repair > automotive_repair > recreation_vehicle_repair,recreation_vehicle_repair,auto_repair_service,,TRUE,,,,TRUE
./experiments/proposals.json:83: "evidence": "`--class` maps verbatim to `class=VAL` (cli.py:1252), but several advertised values live in the `subtype` column. Measured in Williamsburg NY: land_use `recreation` -> 0 as class, 86 as subtype; land_use `agriculture` -> 0 either way; segment `sidewalk` -> 0 as class, 599 as subclass. `botmap landuse --help` lists 'commercial, residential, recreation, agriculture' \u2014 two of the four return nothing. `residential` and `commercial` exist as both, which is exactly why the list looks plausible. The same text is duplicated in introspection.py:36 and the Skill cheatsheet, so an agent gets the bad values from three independent places.",
./experiments/artifacts/probe.log:92: Predominant human use of an area (commercial, residential, recreation, etc.).
./experiments/artifacts/probe.log:415: commercial, residential, recreation,
./experiments/artifacts/probe.log:1417: "value": "sports_and_recreation_venue",
./experiments/artifacts/findings.md:549:| `land_use` `recreation` | **0** | 86 |
./experiments/artifacts/findings.md:559: residential, recreation, agriculture)
./experiments/artifacts/docs/automated-improvements/report-example-01.md:58:Add a `landuse` command in overturemaps/cli.py cloning the buildings command body but with type_='land_use', plus a --class shortcut (commercial, residential, recreation, agriculture, ...) like roads --class. Then `overturemaps landuse --in "Brooklyn" --class residential` fully replaces `download -t land_use`.
[Showing lines 163-320 of 320 (50.0KB limit). Full output: /var/folders/2g/85d5_sh17wdf0fm5yw5k3gs80000gn/T/pi-bash-48932b71575610d7.log]Designing mutable budget interface
Defining ProbeResult with subtype evidence
Designing probe budget and logging
Developing probe filtering and validation strategies
Identifying location ambiguity handling
Planning probe test cases and command adjustments
Refining command usage with count and subtype filters
Planning test implementation
Running tests
>uv run pytest tests/test_probe.py tests/test_contract.py.F..F..FF..... [100%]
=================================== FAILURES ===================================
________ test_vocabulary_probe_concludes_absent_value_without_cli_call _________
def test_vocabulary_probe_concludes_absent_value_without_cli_call():
original = call(["count", "-t", "place", "--where", "categories.primary=bus_stop"])
run = runner_from({})
result = probe_empty(original, run, taxonomy=taxonomy())
assert result.subtype == "c-vocabulary"
assert "bus_stop absent" in result.evidence
> assert run.calls == []
E AssertionError: assert [('count', '-...ry=bus_stop')] == []
E
E Left contains 3 more items, first extra item: ('count', '-t', 'land_use', '--where', 'categories.primary=bus_stop')
E Use -v to get more diff
tests/test_probe.py:84: AssertionError
_________ test_argv_echo_detects_repeated_flag_dropped_by_output_echo __________
def test_argv_echo_detects_repeated_flag_dropped_by_output_echo():
original = call(
["count", "-t", "road", "--class", "trunk", "--class", "primary"],
stdout=json.dumps({"count": 0, "where": [{"key": "class", "op": "=", "value": "primary"}]}),
)
run = runner_from({})
result = probe_empty(original, run)
assert result.subtype == "c-dropped-input"
assert "repeated filters" in result.evidence
assert "only 1 of 2" in result.evidence
> assert run.calls == []
E AssertionError: assert [('count', '-...e', ...), ...] == []
E
E Left contains 8 more items, first extra item: ('count', '-t', 'road', '--where', 'subtype=trunk', '--class', ...)
E Use -v to get more diff
tests/test_probe.py:128: AssertionError
__________ test_budget_exhaustion_records_skipped_inconclusive_probe ___________
def test_budget_exhaustion_records_skipped_inconclusive_probe():
original = call(["count", "-t", "land_use", "--where", "class=beach"])
budget = ProbeBudget(max_calls=0)
run = runner_from({})
result = probe_empty(original, run, budget=budget)
assert result.subtype == "c-unknown"
assert result.probes[0].result == "skipped: probe budget exhausted"
assert result.probes[0].conclusive is False
> assert budget.log == [
{
"kind": "type_sweep",
"argv": ["count", "-t", "place", "--where", "class=beach"],
"ran": False,
"reason": "budget_exhausted",
}
]
E AssertionError: assert [{'kind': 'co...t_exhausted'}] == [{'kind': 'ty...t_exhausted'}]
E
E At index 0 diff: {'kind': 'column_swap', 'argv': ['count', '-t', 'land_use', '--where', 'subtype=beach'], 'ran': False, 'reason': 'budget_exhausted'} != {'kind': 'type_sweep', 'argv': ['count', '-t', 'place', '--where', 'class=beach'], 'ran': False, 'reason': 'budget_exhausted'}
E Left contains one more item: {'kind': 'type_sweep', 'argv': ['count', '-t', 'place', '--where', 'class=beach'], 'ran': False, 'reason': 'budget_exhausted'}
E Use -v to get more diff
tests/test_probe.py:169: AssertionError
________________ test_budget_log_records_mocked_cli_probe_calls ________________
def test_budget_log_records_mocked_cli_probe_calls():
original = call(["landuse", "--class", "recreation"])
budget = ProbeBudget(max_calls=3)
run = runner_from({("landuse", "--where", "subtype=recreation"): observation(86)})
result = probe_empty(original, run, budget=budget, taxonomy=taxonomy())
assert result.subtype == "c-wrong-column"
> assert budget.used == 1
E AssertionError: assert 3 == 1
E + where 3 = ProbeBudget(max_calls=3, used=3, log=[{'kind': 'column_swap', 'argv': ['landuse', '--where', 'subtype=recreation'], 'r..._sweep', 'argv': ['count', '-t', 'place', '--where', 'class=recreation'], 'ran': False, 'reason': 'budget_exhausted'}]).used
tests/test_probe.py:187: AssertionError
=========================== short test summary info ============================
FAILED tests/test_probe.py::test_vocabulary_probe_concludes_absent_value_without_cli_call
FAILED tests/test_probe.py::test_argv_echo_detects_repeated_flag_dropped_by_output_echo
FAILED tests/test_probe.py::test_budget_exhaustion_records_skipped_inconclusive_probe
FAILED tests/test_probe.py::test_budget_log_records_mocked_cli_probe_calls - ...
4 failed, 10 passed in 0.05s
Command exited with code 1Designing lazy evaluation with generator
>uv run pytest tests/test_probe.py tests/test_contract.py.......F...... [100%]
=================================== FAILURES ===================================
__________ test_budget_exhaustion_records_skipped_inconclusive_probe ___________
def test_budget_exhaustion_records_skipped_inconclusive_probe():
original = call(["count", "-t", "land_use", "--where", "class=beach"])
budget = ProbeBudget(max_calls=0)
run = runner_from({})
result = probe_empty(original, run, budget=budget)
assert result.subtype == "c-unknown"
assert result.probes[0].result == "skipped: probe budget exhausted"
assert result.probes[0].conclusive is False
> assert budget.log == [
{
"kind": "type_sweep",
"argv": ["count", "-t", "place", "--where", "class=beach"],
"ran": False,
"reason": "budget_exhausted",
}
]
E AssertionError: assert [{'kind': 'co...t_exhausted'}] == [{'kind': 'ty...t_exhausted'}]
E
E At index 0 diff: {'kind': 'column_swap', 'argv': ['count', '-t', 'land_use', '--where', 'subtype=beach'], 'ran': False, 'reason': 'budget_exhausted'} != {'kind': 'type_sweep', 'argv': ['count', '-t', 'place', '--where', 'class=beach'], 'ran': False, 'reason': 'budget_exhausted'}
E Left contains one more item: {'kind': 'type_sweep', 'argv': ['count', '-t', 'place', '--where', 'class=beach'], 'ran': False, 'reason': 'budget_exhausted'}
E Use -v to get more diff
tests/test_probe.py:169: AssertionError
=========================== short test summary info ============================
FAILED tests/test_probe.py::test_budget_exhaustion_records_skipped_inconclusive_probe
1 failed, 13 passed in 0.03s
Command exited with code 1Refining budget exhaustion handling
Updating test for column swap order
>uv run pytest........................................................................ [ 63%]
.......................................... [100%]
114 passed in 0.16sInspecting git status
>git status --short -- autoresearch/agenteval/probe.py tests/test_probe.py autoresearch/agenteval/contract.py autoresearch/runner.py tests/test_contract.py && git diff -- autoresearch/agenteval/probe.py tests/test_probe.py | sed -n '1,260p'M autoresearch/runner.py
?? autoresearch/agenteval/contract.py
?? autoresearch/agenteval/probe.py
?? tests/test_contract.py
?? tests/test_probe.pyReviewing probe_call tuple evaluation
>uv run pytest tests/test_probe.py && uv run pytest......... [100%]
9 passed in 0.01s
........................................................................ [ 62%]
............................................ [100%]
116 passed in 0.15sSummarizing final findings and decisions
READY
Implemented Arm B Phase 2 probe pass in allowed files only:
autoresearch/agenteval/probe.pytests/test_probe.pyWhat landed:
ProbeBudget with per-question call budget and audit log.ProbeResult carrying probes, c-* subtype, evidence, and budget.c-vocabularyc-wrong-columnc-wrong-typec-truncatedc-dropped-inputc-wrong-entityc-unknown.land_use recreation as class → 0, subtype → 86 produces c-wrong-column.Tests run:
uv run pytest tests/test_probe.pyuv run pytestResult:
9 passed116 passedNote:
--class recreation command trace under retained attempts. I used static fixture evidence from experiments/proposals.json and experiments/artifacts/findings.md, per instruction, and documented that in the test.You are PI ARM B. The evaluator-first work is now in the shared autoresearch checkout. Your role now: own the prompt-lever arm using the NEW evaluator.
Context to read:
Your clean botmap repo:
Constraints:
Task:
Planning file reading strategy
Planning to read agenteval module
>find autoresearch/agenteval -maxdepth 1 -type f -print -exec wc -l {} \; && for f in autoresearch/agenteval/*.py; do echo '---' $f; sed -n '1,220p' $f; donereturn "E"
if blame == "agent":
return "F"
if outcome == "ok":
return None
if outcome == "degenerate":
return "D"
if outcome == "empty" and recovery == "unguided":
return "C"
if outcome == "error" and recovery == "guided":
return "B"
if outcome == "error" and recovery == "unguided":
return "A"
return None
def write(path: str | Path, record: Record2) -> None:
"""Write a record as stable, indented JSON."""
target = Path(path)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(json.dumps(_record_to_json(record), indent=2, sort_keys=[REDACTED] + "\n", encoding="utf-8")
def load(path: str | Path) -> Record2:
"""Load a record from JSON.
Schema problems are reported by ``validate``; this function only parses the
file into the dataclass shape.
"""
raw = json.loads(Path(path).read_text(encoding="utf-8"))
return _record_from_json(raw)
def validate(raw: dict[str, Any]) -> list[str]:
"""Return schema problems. Never raise for malformed record content."""
problems: list[str] = []
if raw.get("schema") != SCHEMA:
problems.append(f"schema must be {SCHEMA!r}")
_require_str(raw, "question_id", problems)
if not isinstance(raw.get("repeat"), int):
problems.append("repeat must be an integer")
if not isinstance(raw.get("calls"), list):
problems.append("calls must be a list")
if not isinstance(raw.get("agent_side"), list):
problems.append("agent_side must be a list")
if not isinstance(raw.get("tools_used"), dict):
problems.append("tools_used must be an object")
if not isinstance(raw.get("botmap_calls"), int):
problems.append("botmap_calls must be an integer")
if not isinstance(raw.get("answer"), dict):
problems.append("answer must be an object")
for index, call in enumerate(raw.get("calls") if isinstance(raw.get("calls"), list) else []):
if not isinstance(call, dict):
problems.append(f"calls[{index}] must be an object")
continue
_validate_verdict(call, f"calls[{index}]", problems, allow_missing=False)
if "probes" in call:
_validate_probes(call["probes"], f"calls[{index}].probes", problems)
if raw.get("attempt") is not None:
if not isinstance(raw.get("attempt"), dict):
problems.append("attempt must be an object or null")
else:
_validate_verdict(raw["attempt"], "attempt", problems, allow_missing=False)
return problems
def _validate_verdict(raw: dict[str, Any], prefix: str, problems: list[str], *, allow_missing: bool) -> None:
missing = [key for key in ("outcome", "blame", "recovery") if key not in raw]
if missing and not allow_missing:
problems.append(f"{prefix} missing verdict axes: {', '.join(missing)}")
return
if missing:
return
outcome = raw.get("outcome")
blame = raw.get("blame")
recovery = raw.get("recovery")
if outcome not in OUTCOMES:
problems.append(f"{prefix}.outcome invalid: {outcome!r}")
if blame not in BLAMES:
problems.append(f"{prefix}.blame invalid: {blame!r}")
if recovery not in RECOVERIES:
problems.append(f"{prefix}.recovery invalid: {recovery!r}")
if outcome not in OUTCOMES or blame not in BLAMES or recovery not in RECOVERIES:
return
expected = derive_class(outcome, blame, recovery)
stored = raw.get("class", raw.get("cls"))
if stored != expected:
problems.append(f"{prefix}.class must be {expected!r} for its axes, got {stored!r}")
subtype = raw.get("subtype")
if expected == "C" and subtype is not None and subtype not in C_SUBTYPES:
problems.append(f"{prefix}.subtype invalid for class C: {subtype!r}")
if expected != "C" and subtype is not None:
problems.append(f"{prefix}.subtype must be null unless class is C")
def _validate_probes(raw: Any, prefix: str, problems: list[str]) -> None:
if not isinstance(raw, list):
problems.append(f"{prefix} must be a list")
return
for index, probe in enumerate(raw):
if not isinstance(probe, dict):
problems.append(f"{prefix}[{index}] must be an object")
continue
for key in ("kind", "ran", "result"):
if not isinstance(probe.get(key), str):
problems.append(f"{prefix}[{index}].{key} must be a string")
if not isinstance(probe.get("conclusive"), bool):
problems.append(f"{prefix}[{index}].conclusive must be a boolean")
def _require_str(raw: dict[str, Any], key: str, problems: list[str]) -> None:
if not isinstance(raw.get(key), str):
problems.append(f"{key} must be a string")
def _record_to_json(record: Record2) -> dict[str, Any]:
raw = asdict(record)
raw["calls"] = [_call_to_json(call) for call in record.calls]
if record.attempt is not None:
raw["attempt"] = _verdict_to_json(record.attempt)
return raw
def _call_to_json(call: dict[str, Any]) -> dict[str, Any]:
raw = dict(call)
if "cls" in raw:
raw["class"] = raw.pop("cls")
if "probes" in raw:
raw["probes"] = [_probe_to_json(probe) for probe in raw["probes"]]
return raw
--- autoresearch/agenteval/enrich.py
"""Enrich retained record-v2 files with post-agent differential probes."""
from __future__ import annotations
from dataclasses import asdict
import json
import os
from pathlib import Path
import subprocess
from typing import Any
from autoresearch import runner as agent_runner
from autoresearch.trace import parse_calls
from .contract import write as write_record
from .probe import ProbeBudget, ProbeObservation, probe_call
from .record import build_record
from .taxonomy import classify
def enrich_run(run_dir: str | Path, botmap_repo: str | Path, *, max_probe_calls: int = 8, timeout_s: int = 120) -> dict[str, Any]:
"""Run CLI-only probes for retained attempts and rewrite record-v2 files.
The agent transcript is not touched. Probes run after the attempt, matching
the measurement design in docs/plan.md.
"""
root = Path(run_dir)
repo = Path(botmap_repo)
runner = botmap_runner(repo, timeout_s=timeout_s)
summary = {"attempts": 0, "calls_seen": 0, "calls_probed": 0, "probe_calls": 0}
for attempt_dir in sorted((root / "attempts").iterdir() if (root / "attempts").exists() else []):
commands = attempt_dir / "commands.jsonl"
transcript = attempt_dir / "transcript.jsonl"
if not commands.exists():
continue
calls = parse_calls(commands)
record = build_record(_attempt_from_parts(attempt_dir.name, calls, transcript), transcript_path=transcript)
enriched_calls = []
for call, call_record in zip(calls, record.calls, strict=True):
summary["calls_seen"] += 1
if _should_probe(call_record):
budget = ProbeBudget(max_calls=max_probe_calls)
result = probe_call(call, runner, budget=budget)
summary["calls_probed"] += 1
summary["probe_calls"] += budget.used
verdict = classify(call, result.probes)
call_record = dict(call_record)
call_record.update(
outcome=verdict.outcome,
blame=verdict.blame,
recovery=verdict.recovery,
**{"class": verdict.cls},
subtype=verdict.subtype,
evidence=verdict.evidence,
probes=[asdict(probe) for probe in result.probes],
)
enriched_calls.append(call_record)
record = type(record)(
schema=record.schema,
question_id=record.question_id,
repeat=record.repeat,
calls=tuple(enriched_calls),
agent_side=record.agent_side,
tools_used=record.tools_used,
botmap_calls=record.botmap_calls,
answer=record.answer,
attempt=record.attempt,
)
write_record(attempt_dir / "record-v2.json", record)
summary["attempts"] += 1
return summary
def botmap_runner(repo: Path, *, timeout_s: int = 120):
"""Return a Probe runner that executes botmap in the supplied checkout."""
python = agent_runner.venv_python(repo)
def run(argv: tuple[str, ...]) -> ProbeObservation:
env = dict(os.environ)
process = subprocess.run(
[python, "-m", "botmap", *argv],
cwd=repo,
env=env,
capture_output=True,
text=True,
timeout=timeout_s,
)
return ProbeObservation(exit_code=process.returncode, stdout=process.stdout, stderr=process.stderr)
return run
def _should_probe(call: dict[str, Any]) -> bool:
if call.get("class") == "C":
return True
argv = call.get("argv") or []
return "--top" in argv or any(str(item).startswith("--top=") for item in argv)
def _attempt_from_parts(name: str, calls, transcript_path: Path):
from autoresearch.score import Attempt
from autoresearch.trace import parse_transcript
question_id, _, repeat_text = name.rpartition("__r")
repeat = int(repeat_text) if repeat_text.isdigit() else 1
return Attempt(question_id=question_id, repeat=repeat, calls=calls, transcript=parse_transcript(transcript_path))
--- autoresearch/agenteval/explain.py
"""Human-readable feedback for agent-evaluation records."""
from __future__ import annotations
from dataclasses import asdict, is_dataclass
from typing import Any
from autoresearch.agenteval.contract import Record2
CLASS_NAMES = {
"A": "hard failure",
"B": "soft failure",
"C": "silent wrong",
"D": "degenerate",
"E": "environment",
"F": "agent-side",
None: "clean",
}
def explain(record: Record2 | dict[str, Any]) -> str:
"""Turn a record-v2 attempt into fix-instruction feedback."""
raw = _raw(record)
lines = [f"QUESTION {raw.get('question_id', '<unknown>')} repeat {raw.get('repeat', '<unknown>')}"]
attempt = raw.get("attempt")
if attempt:
lines.extend(_explain_verdict("ATTEMPT", attempt))
for index, call in enumerate(raw.get("calls", [])):
cls = _class(call)
if cls is None:
continue
lines.extend(_explain_verdict(f"CALL {index}", call))
for item in raw.get("agent_side", []):
lines.extend(_explain_agent_side(item))
if len(lines) == 1:
lines.append("No classified failures.")
return "\n".join(lines)
def _explain_verdict(label: str, verdict: dict[str, Any]) -> list[str]:
cls = _class(verdict)
subtype = verdict.get("subtype")
heading = f"{label}: CLASS {cls} ({CLASS_NAMES.get(cls, 'unknown')})"
if subtype:
heading += f" - {subtype}"
lines = [heading]
argv = verdict.get("argv")
if argv is not None:
lines.append(f" The agent ran: {_format_argv(argv)}")
if "exit_code" in verdict:
lines.append(f" Result: exit {verdict.get('exit_code')}, stdout={_head(verdict, 'stdout')!r}, stderr={_head(verdict, 'stderr')!r}")
if verdict.get("evidence"):
lines.append(f" Evidence: {verdict['evidence']}")
for probe in verdict.get("probes", []):
lines.append(_explain_probe(probe))
lines.append(f" Missing behaviour: {_missing_behaviour(cls, subtype, verdict)}")
return lines
def _explain_probe(probe: dict[str, Any]) -> str:
status = "conclusive" if probe.get("conclusive") else "inconclusive"
return f" Probe ({probe.get('kind')}, {status}): `{probe.get('ran', '')}` -> {probe.get('result', '')}"
def _explain_agent_side(item: dict[str, Any]) -> list[str]:
lines = [f"AGENT-SIDE: {item.get('kind', 'unknown')}"]
if item.get("detail"):
lines.append(f" Detail: {item['detail']}")
if item.get("ignored_by_next") is not None:
lines.append(f" Strict ignored-by-next: {item['ignored_by_next']}")
if item.get("suggestions"):
lines.append(f" Suggestions: {', '.join(item['suggestions'])}")
if item.get("eventually_used") is not None:
lines.append(f" Eventually used: {item['eventually_used']} at call {item.get('used_at_call')}")
return lines
def _missing_behaviour(cls: str | None, subtype: str | None, verdict: dict[str, Any]) -> str:
if cls == "A":
return "refuse with a usable next action instead of leaving the agent stuck"
if cls == "B":
return "guided recovery was present; check whether the next agent step used it"
if cls == "C":
return _class_c_behaviour(subtype)
if cls == "D":
return "emit progress or provide a cheaper first-class route for the task"
if cls == "E":
return "exclude this attempt from tool scoring; environment failed outside the CLI contract"
if cls == "F":
return "record agent non-use separately; do not charge the tool for a usable hint being ignored"
return "none"
def _class_c_behaviour(subtype: str | None) -> str:
return {
"c-vocabulary": "never return empty without saying the value is absent from the taxonomy and naming nearest real values",
"c-wrong-column": "if a value exists in another field, say which field and give the corrected filter",
"c-wrong-type": "if the data lives under another type or verb, name that route",
"c-dropped-input": "reject or echo repeated/conflicting inputs instead of silently discarding one",
"c-truncated": "say when output hit a limit and how to raise it",
"c-wrong-entity": "confirm the resolved entity, including country/region, when the qualifier is ambiguous",
"c-unknown": "say that zero is unexplained and surface safe discovery probes",
None: "never return an unexplained empty result",
}.get(subtype, "never return an unexplained empty result")
def _format_argv(argv: Any) -> str:
return " ".join(str(part) for part in argv)
def _head(verdict: dict[str, Any], stream: str) -> str:
return str(verdict.get(f"{stream}_head", verdict.get(stream, "")))[:200]
def _class(verdict: dict[str, Any]) -> str | None:
return verdict.get("class", verdict.get("cls"))
def _raw(record: Record2 | dict[str, Any]) -> dict[str, Any]:
if isinstance(record, dict):
return record
if is_dataclass(record):
raw = asdict(record)
for call in raw.get("calls", []):
if "cls" in call:
call["class"] = call.pop("cls")
if raw.get("attempt") and "cls" in raw["attempt"]:
raw["attempt"]["class"] = raw["attempt"].pop("cls")
return raw
raise TypeError(f"unsupported record type: {type(record)!r}")
--- autoresearch/agenteval/probe.py
"""Differential probes for silent agent-evaluation failures.
Probes are deliberately pure around an injected command runner. Production
wiring can point the runner at botmap later; tests pass static observations, so
this module never spends model quota and never touches the network by itself.
"""
from __future__ import annotations
from dataclasses import dataclass, field
import json
import shlex
from typing import Any, Callable, Iterable, Mapping, Sequence
from autoresearch.trace import Call
from .contract import Probe
CommandRunner = Callable[[tuple[str, ...]], "ProbeObservation"]
@dataclass(frozen=True)
class ProbeObservation:
exit_code: int = 0
stdout: str = ""
stderr: str = ""
duration_s: float = 0.0
@dataclass
class ProbeBudget:
"""Per-question probe budget and audit log."""
max_calls: int
used: int = 0
log: list[dict[str, Any]] = field(default_factory=list)
def run(self, kind: str, argv: Sequence[str], runner: CommandRunner) -> tuple[Probe, ProbeObservation | None]:
ran = shell_join(argv)
if self.used >= self.max_calls:
self.log.append({"kind": kind, "argv": list(argv), "ran": False, "reason": "budget_exhausted"})
return Probe(kind=kind, ran=ran, result="skipped: probe budget exhausted", conclusive=False), None
self.used += 1
self.log.append({"kind": kind, "argv": list(argv), "ran": True})
observation = runner(tuple(argv))
return Probe(kind=kind, ran=ran, result=describe_observation(observation), conclusive=False), observation
@dataclass(frozen=True)
class ProbeResult:
"""The probe evidence for one call.
``subtype`` is the class-C subtype when a probe conclusively explains a
silent wrong result. It stays ``c-unknown`` when probes ran but none could
explain the empty output.
"""
probes: tuple[Probe, ...]
subtype: str
evidence: str
budget: ProbeBudget
@dataclass(frozen=True)
class TaxonomySnapshot:
"""Static knowledge a scorer may already have about value locations."""
values_by_type_and_column: Mapping[str, Mapping[str, frozenset[str]]]
def types(self) -> tuple[str, ...]:
return tuple(self.values_by_type_and_column)
def has_value(self, value: str) -> bool:
return any(value in values for columns in self.values_by_type_and_column.values() for values in columns.values())
def columns_for(self, type_name: str, value: str) -> tuple[str, ...]:
columns = self.values_by_type_and_column.get(type_name, {})
return tuple(column for column, values in columns.items() if value in values)
DEFAULT_TYPES = ("place", "building", "land", "land_use", "segment", "infrastructure", "address")
VERB_TYPES = {
"places": "place",
"place": "place",
"buildings": "building",
"building": "building",
"landuse": "land_use",
"roads": "segment",
"road": "segment",
"segments": "segment",
"infrastructure": "infrastructure",
"addresses": "address",
"address": "address",
}
COLUMN_SWAPS = {
"class": ("subtype", "subclass"),
"categories.primary": ("subtype", "categories.alternate", "class"),
"subtype": ("class", "subclass"),
"subclass": ("class", "subtype"),
}
_KIND_TO_SUBTYPE = {
"vocabulary": "c-vocabulary",
"column_swap": "c-wrong-column",
"type_sweep": "c-wrong-type",
"argv_echo": "c-dropped-input",
"limit_raise": "c-truncated",
"entity_check": "c-wrong-entity",
}
def probe_empty(
call: Call,
runner: CommandRunner,
*,
budget: ProbeBudget | None = None,
taxonomy: TaxonomySnapshot | None = None,
) -> ProbeResult:
"""Run the probes relevant to an empty call.
The runner is injected to keep this module side-effect free. Every probe
records inconclusive evidence; a conclusive probe stops further probing so
the first explanation remains attributable and budget-bounded.
"""
active_budget = budget or ProbeBudget(max_calls=8)
probes: list[Probe] = []
for probe in _empty_probe_sequence(call, runner, active_budget, taxonomy):
probes.append(probe)
if probe.conclusive:
subtype = _KIND_TO_SUBTYPE[probe.kind]
return ProbeResult(tuple(probes), subtype, probe.result, active_budget)
if probe.result == "skipped: probe budget exhausted":
break
evidence = "empty result remained unexplained by differential probes"
return ProbeResult(tuple(probes), "c-unknown", evidence, active_budget)
def probe_call(
call: Call,
runner: CommandRunner,
*,
budget: ProbeBudget | None = None,
taxonomy: TaxonomySnapshot | None = None,
) -> ProbeResult:
"""Run probes that apply to this call, empty or not."""
if is_empty_result(call):
return probe_empty(call, runner, budget=budget, taxonomy=taxonomy)
active_budget = budget or ProbeBudget(max_calls=8)
probes: list[Probe] = []
for make_probe in (
lambda: _probe_limit_raise(call, runner, active_budget),
lambda: _probe_entity_check(call, runner, active_budget),
):
probe = make_probe()
if probe is not None:
probes.append(probe)
if probe.conclusive:
return ProbeResult(tuple(probes), _KIND_TO_SUBTYPE[probe.kind], probe.result, active_budget)
if probe.result == "skipped: probe budget exhausted":
break
return ProbeResult(tuple(probes), "c-unknown", "no conclusive probe fired", active_budget)
def _empty_probe_sequence(
call: Call,
runner: CommandRunner,
budget: ProbeBudget,
taxonomy: TaxonomySnapshot | None,
) -> Iterable[Probe]:
for make_probe in (
lambda: _probe_argv_echo(call),
lambda: _probe_vocabulary(call, taxonomy),
lambda: _probe_vocabulary_cli(call, runner, budget),
lambda: _probe_column_swap(call, runner, budget, taxonomy),
lambda: _probe_entity_check(call, runner, budget),
lambda: _probe_type_sweep(call, runner, budget, taxonomy),
):
probe = make_probe()
if probe is not None:
yield probe
def _probe_vocabulary(call: Call, taxonomy: TaxonomySnapshot | None) -> Probe | None:
filters = value_filters(call.argv)
if taxonomy is None or not filters:
return None
missing = sorted({value for _, value in filters if not taxonomy.has_value(value)})
if missing:
values = ", ".join(missing)
return Probe(
kind="vocabulary",
ran=f"taxonomy lookup for {values}",
result=f"{values} absent from published taxonomy at all known levels",
conclusive=True,
)
checked = ", ".join(sorted({value for _, value in filters}))
return Probe(
kind="vocabulary",
ran=f"taxonomy lookup for {checked}",
result=f"{checked} exists somewhere in the published taxonomy; vocabulary alone is not explanatory",
conclusive=False,
)
def _probe_vocabulary_cli(call: Call, runner: CommandRunner, budget: ProbeBudget) -> Probe | None:
filters = [(column, value) for column, value in value_filters(call.argv) if column == "categories.primary"]
if not filters or feature_type(call.argv) != "place":
return None
location = location_args(call.argv)
if not location:
return None
argv = ("--json", "categories", "-t", "place", *location, "--top", "5000")
probe, observation = budget.run("vocabulary", argv, runner)
--- autoresearch/agenteval/record.py
"""Build record-v2 attempts from retained runner artifacts."""
from __future__ import annotations
from collections import Counter
import json
from pathlib import Path
from typing import Any, Iterable
from autoresearch.score import Attempt
from autoresearch.trace import Call, Transcript
from .agent_side import detect_ignored_hints
from .contract import Record2
from .taxonomy import classify, classify_attempt
def build_record(attempt: Attempt, *, transcript_path: Path | None = None) -> Record2:
"""Convert an existing runner attempt into the shared record-v2 shape."""
calls = tuple(_call_record(call) for call in attempt.calls)
agent_side = tuple(detect_ignored_hints(calls))
return Record2(
schema="agenteval/2",
question_id=attempt.question_id,
repeat=attempt.repeat,
calls=calls,
agent_side=agent_side,
tools_used=parse_tools_used(transcript_path) if transcript_path else {},
botmap_calls=len(attempt.calls),
answer={"text": attempt.transcript.final_answer, "verified": None},
attempt=classify_attempt(attempt.transcript),
)
def parse_tools_used(path: Path | None) -> dict[str, int]:
"""Count tool-use events in a Claude stream-json transcript."""
if path is None or not path.exists():
return {}
counts: Counter[str] = Counter()
for event in _events(path):
message = event.get("message") or {}
for part in message.get("content") or []:
if isinstance(part, dict) and part.get("type") == "tool_use":
name = part.get("name")
if isinstance(name, str) and name:
counts[name] += 1
return dict(counts)
def _events(path: Path) -> Iterable[dict[str, Any]]:
for line in path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
try:
event = json.loads(line)
except json.JSONDecodeError:
continue
if isinstance(event, dict):
yield event
def _call_record(call: Call) -> dict[str, Any]:
verdict = classify(call)
return {
"argv": call.argv,
"exit_code": call.exit_code,
"stdout_head": _head(call.stdout),
"stderr_head": _head(call.stderr),
"duration_s": call.duration,
"outcome": verdict.outcome,
"blame": verdict.blame,
"recovery": verdict.recovery,
"class": verdict.cls,
"subtype": verdict.subtype,
"evidence": verdict.evidence,
"probes": [
{"kind": probe.kind, "ran": probe.ran, "result": probe.result, "conclusive": probe.conclusive}
for probe in verdict.probes
],
}
def _head(text: str, limit: int = 2000) -> str:
return text[:limit]
--- autoresearch/agenteval/repair.py
"""Repair helpers for already-written record-v2 artifacts.
These functions are for instrument bugs, not result editing. They preserve the
original attempts and only recompute fields derived from evaluator logic.
"""
from __future__ import annotations
from dataclasses import asdict
import json
from pathlib import Path
import re
from typing import Any
from autoresearch.trace import Call
from .contract import write as write_record
from .record import build_record
from .taxonomy import classify
_FALSE_US_ENTITY = re.compile(
r"qualifier (?P<q>[A-Z]{2}) implies country (?P=q), but resolved .* country US / region US-(?P=q)"
)
def repair_us_state_entity_false_positives(run_dir: str | Path) -> dict[str, int]:
"""Remove stale c-wrong-entity probes caused by MA/MT country-code ambiguity.
Older probe logic checked ISO country codes before US state codes. That made
`Cambridge, MA -> US-MA` look wrong because `MA` is also Morocco. This
function strips only those stale probes and reclassifies the affected calls.
"""
root = Path(run_dir)
stats = {"records_seen": 0, "calls_repaired": 0, "probes_removed": 0}
for record_path in sorted(root.glob("attempts/*/record-v2.json")):
raw = json.loads(record_path.read_text(encoding="utf-8"))
stats["records_seen"] += 1
changed = False
repaired_calls = []
for call in raw.get("calls", []):
probes = call.get("probes") or []
kept = []
removed = 0
for probe in probes:
if probe.get("kind") == "entity_check" and _FALSE_US_ENTITY.search(str(probe.get("result", ""))):
removed += 1
continue
kept.append(probe)
if removed:
stats["probes_removed"] += removed
stats["calls_repaired"] += 1
verdict = classify(_call_from_record(call), kept)
call = dict(call)
call.update(
outcome=verdict.outcome,
blame=verdict.blame,
recovery=verdict.recovery,
**{"class": verdict.cls},
subtype=verdict.subtype,
evidence=verdict.evidence,
probes=[asdict(probe) for probe in verdict.probes],
)
changed = True
repaired_calls.append(call)
if changed:
raw["calls"] = repaired_calls
# Preserve all raw metadata while using the contract writer for a
# stable class/null JSON representation.
record = build_record(_attempt_from_raw(raw), transcript_path=None)
record = type(record)(
schema=raw["schema"],
question_id=raw["question_id"],
repeat=raw["repeat"],
calls=tuple(raw["calls"]),
agent_side=tuple(raw.get("agent_side", [])),
tools_used=dict(raw.get("tools_used", {})),
botmap_calls=int(raw.get("botmap_calls", 0)),
answer=dict(raw.get("answer", {})),
attempt=record.attempt,
)
write_record(record_path, record)
return stats
def _call_from_record(call: dict[str, Any]) -> Call:
return Call(
argv=list(call.get("argv") or []),
exit_code=int(call.get("exit_code") or 0),
stdout=str(call.get("stdout", call.get("stdout_head", "")) or ""),
stderr=str(call.get("stderr", call.get("stderr_head", "")) or ""),
duration=float(call.get("duration_s") or call.get("duration") or 0.0),
)
def _attempt_from_raw(raw: dict[str, Any]):
from autoresearch.score import Attempt
from autoresearch.trace import Transcript
calls = [_call_from_record(call) for call in raw.get("calls", [])]
return Attempt(
question_id=str(raw.get("question_id", "")),
repeat=int(raw.get("repeat") or 0),
calls=calls,
transcript=Transcript(final_answer=str((raw.get("answer") or {}).get("text", "")), completed=True, status="ok"),
)
--- autoresearch/agenteval/sabotage.py
"""Sabotage fixtures for validating invisible-failure detection.
These fixtures are static and cheap. They intentionally do not run botmap or an
LLM; they encode recorded failure shapes so taxonomy/probe code can prove it
fires before any measurement run is allowed.
"""
from __future__ import annotations
from dataclasses import asdict, dataclass
from typing import Any, Callable, Iterable
from autoresearch.agenteval.contract import CallVerdict, Probe, derive_class
Classifier = Callable[[dict[str, Any]], CallVerdict | dict[str, Any]]
@dataclass(frozen=True)
class SabotageFixture:
id: str
expected_class: str | None
expected_subtype: str | None
call: dict[str, Any]
source: str
@dataclass(frozen=True)
class SabotageResult:
fixture_id: str
passed: bool
expected_class: str | None
actual_class: str | None
expected_subtype: str | None
actual_subtype: str | None
detail: str
def expected_verdict(call: dict[str, Any]) -> CallVerdict:
"""Reference classifier for the static fixtures until taxonomy.py lands."""
cls = derive_class(call["outcome"], call["blame"], call["recovery"])
return CallVerdict(
outcome=call["outcome"],
blame=call["blame"],
recovery=call["recovery"],
cls=cls,
subtype=call.get("subtype"),
evidence=call.get("evidence", ""),
probes=tuple(call.get("probes", ())),
)
def run_sabotage(classifier: Classifier, fixtures: Iterable[SabotageFixture] | None = None) -> list[SabotageResult]:
"""Run fixtures through a classifier and report mismatches."""
checked_fixtures = FIXTURES if fixtures is None else fixtures
results: list[SabotageResult] = []
for fixture in checked_fixtures:
verdict = classifier(fixture.call)
actual_class = _field(verdict, "cls", "class")
actual_subtype = _field(verdict, "subtype")
passed = actual_class == fixture.expected_class and actual_subtype == fixture.expected_subtype
results.append(
SabotageResult(
fixture_id=fixture.id,
passed=passed,
expected_class=fixture.expected_class,
actual_class=actual_class,
expected_subtype=fixture.expected_subtype,
actual_subtype=actual_subtype,
detail="ok" if passed else f"expected {fixture.expected_class}/{fixture.expected_subtype}, got {actual_class}/{actual_subtype}",
)
)
return results
def assert_sabotage_passes(classifier: Classifier, fixtures: Iterable[SabotageFixture] | None = None) -> None:
failures = [result for result in run_sabotage(classifier, fixtures) if not result.passed]
if failures:
details = "; ".join(f"{item.fixture_id}: {item.detail}" for item in failures)
raise AssertionError(f"sabotage fixtures failed: {details}")
def _call(
*,
argv: list[str],
outcome: str,
recovery: str,
evidence: str,
exit_code: int = 0,
blame: str = "tool",
subtype: str | None = None,
stdout_head: str = "",
stderr_head: str = "",
duration_s: float = 1.0,
probes: tuple[Probe, ...] = (),
) -> dict[str, Any]:
return {
"argv": argv,
"exit_code": exit_code,
"stdout_head": stdout_head,
"stderr_head": stderr_head,
"duration_s": duration_s,
"outcome": outcome,
"blame": blame,
"recovery": recovery,
"class": derive_class(outcome, blame, recovery),
"subtype": subtype,
"evidence": evidence,
"probes": [asdict(probe) for probe in probes],
}
def _field(verdict: CallVerdict | dict[str, Any], *names: str) -> Any:
for name in names:
if isinstance(verdict, dict) and name in verdict:
return verdict[name]
if hasattr(verdict, name):
return getattr(verdict, name)
return None
FIXTURES: tuple[SabotageFixture, ...] = (
SabotageFixture(
id="a-raw-traceback-unguided",
expected_class="A",
expected_subtype=None,
source="docs/plan.md class A: raw traceback/refusal left agent unguided",
call=_call(
argv=["count", "-t", "place", "--in", "Brooklyn, US-NY", "--where", "categories.primary=coffee_shop"],
exit_code=1,
outcome="error",
recovery="unguided",
evidence="unhandled tool traceback, no retry instruction",
stderr_head="Traceback (most recent call last): ValueError: unsupported filter operator",
),
),
SabotageFixture(
id="b-did-you-mean-guided",
expected_class="B",
expected_subtype=None,
source="experiments/arm-a/HANDOVER.md C1: count-zero-hint emits bus_station near-match",
call=_call(
argv=["count", "-t", "place", "--in", "Williamsburg, Brooklyn, NY", "--where", "categories.primary=bus_stop"],
exit_code=2,
outcome="error",
recovery="guided",
evidence="tool names a concrete replacement value",
stderr_head="Bad category value 'bus_stop'. Did you mean: bus_station",
),
),
SabotageFixture(
id="c-vocabulary-bus-stop",
expected_class="C",
expected_subtype="c-vocabulary",
source="baseline-noise-run1 bus-stops-with-coffee__r2 commands.jsonl; failure_dataset no-hint-on-count-and-sample",
call=_call(
argv=["count", "-t", "place", "--in", "Williamsburg, Brooklyn, NY", "--where", "categories.primary=bus_stop"],
outcome="empty",
recovery="unguided",
subtype="c-vocabulary",
evidence="bus_stop returned count 0 while taxonomy contains bus_station, not bus_stop",
stdout_head='{"count": 0}',
probes=(Probe("vocabulary", "categories -t place --top 5000 | grep bus_stop", "absent; nearest bus_station", True),),
),
),
SabotageFixture(
id="c-wrong-column-recreation",
expected_class="C",
expected_subtype="c-wrong-column",
source="docs/plan.md differential probe example: --class recreation -> 0, subtype=recreation -> 86",
call=_call(
argv=["landuse", "--in", "Cambridge, MA", "--class", "recreation"],
outcome="empty",
recovery="unguided",
subtype="c-wrong-column",
evidence="value exists as subtype, not class",
stdout_head="0 rows",
probes=(Probe("column_swap", "landuse --where subtype=recreation", "86 rows", True),),
),
),
SabotageFixture(
id="c-wrong-type-water-as-landuse",
expected_class="C",
expected_subtype="c-wrong-type",
source="experiments/failure_dataset.yaml water/landuse convenience-verb defects",
call=_call(
argv=["landuse", "--in", "Boston", "--class", "water"],
outcome="empty",
recovery="unguided",
subtype="c-wrong-type",
evidence="same intent is answerable through the water verb/type sweep",
stdout_head="0 rows",
probes=(Probe("type_sweep", "water --in Boston", "non-empty", True),),
),
),
SabotageFixture(
id="c-dropped-input-repeat-class",
expected_class="C",
expected_subtype="c-dropped-input",
source="docs/plan.md and failure_dataset class-flag-not-repeatable: repeated --class silently keeps last",
call=_call(
argv=["roads", "--in", "Malta", "--class", "trunk", "--class", "primary"],
outcome="empty",
recovery="unguided",
subtype="c-dropped-input",
evidence="repeated flag supplied two values but only last value affected the result",
stdout_head="0 rows",
probes=(Probe("argv_echo", "inspect parsed filters", "trunk dropped; primary kept", True),),
),
),
SabotageFixture(
id="c-truncated-bus-station",
expected_class="C",
expected_subtype="c-truncated",
source="docs/plan.md: categories --top 200 omitted bus_station; --top 5000 found all 963",
call=_call(
argv=["categories", "-t", "place", "--in", "Williamsburg, Brooklyn, NY", "--top", "200"],
outcome="empty",
recovery="unguided",
subtype="c-truncated",
evidence="agent inferred absent after reading a capped taxonomy listing",
--- autoresearch/agenteval/score.py
"""Class-weighted scoring for the shared agent evaluator.
The top-level shape is fixed from the Phase 5 scoring decision:
- correctness and recoverability: 60%
- token efficiency: 20%
- wall-clock time: 20%
The 60 correctness/recoverability points are split as:
- final outcome correctness: 20
- self-recovery: 20
- guidance / error quality: 12
- execution / route quality: 6
- failure severity / attribution: 2
Class E is excluded. Class F is recorded but not charged to the CLI.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Iterable
from .contract import AttemptVerdict, CallVerdict
CORRECTNESS_RECOVERABILITY_WEIGHT = 0.60
TOKEN_EFFICIENCY_WEIGHT = 0.20
WALLCLOCK_WEIGHT = 0.20
FINAL_OUTCOME_POINTS = 20.0
SELF_RECOVERY_POINTS = 20.0
GUIDANCE_POINTS = 12.0
ROUTE_POINTS = 6.0
ATTRIBUTION_POINTS = 2.0
TOTAL_CORRECTNESS_POINTS = (
FINAL_OUTCOME_POINTS
+ SELF_RECOVERY_POINTS
+ GUIDANCE_POINTS
+ ROUTE_POINTS
+ ATTRIBUTION_POINTS
)
# TODO(phase-5+): calibrate exact penalties against measured distribution.
CLASS_SEVERITY_PENALTIES = {
"A": 1.00,
"B": 0.10,
"C": 1.00,
"D": 0.45,
}
@dataclass(frozen=True)
class RecoveryStats:
recoverable_failures: int
recovered_failures: int
self_recovery_rate: float | None
extra_calls: int
extra_tokens: int
extra_wallclock_ms: int
@dataclass(frozen=True)
class ScoreBreakdown:
final_outcome: float
self_recovery: float
guidance: float
route_quality: float
attribution: float
correctness_recoverability: float
token_efficiency: float
wallclock: float
@dataclass(frozen=True)
class Score:
value: float | None
excluded: bool
charged: tuple[CallVerdict, ...]
recorded_not_charged: tuple[CallVerdict, ...]
environment: tuple[CallVerdict, ...]
breakdown: ScoreBreakdown
recovery: RecoveryStats
attempt_environment: AttemptVerdict | None = None
def score_attempt(
calls: Iterable[CallVerdict],
*,
attempt: AttemptVerdict | None = None,
agent_side: Iterable[dict] = (),
completed: bool = True,
token_efficiency: float = 1.0,
wallclock: float = 1.0,
extra_tokens: int = 0,
extra_wallclock_ms: int = 0,
) -> Score:
"""Score one attempt.
``token_efficiency`` and ``wallclock`` are normalized 0..1 values supplied by
the outer evaluator when a reference baseline exists. This module owns the
class semantics and recovery accounting.
"""
call_tuple = tuple(calls)
agent_side_tuple = tuple(agent_side)
environment = tuple(call for call in call_tuple if call.cls == "E")
recorded_not_charged = tuple(call for call in call_tuple if call.cls == "F")
charged = tuple(call for call in call_tuple if call.cls not in {"E", "F"})
recovery = _recovery_stats(charged, agent_side_tuple, extra_tokens, extra_wallclock_ms)
empty_breakdown = ScoreBreakdown(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0)
if attempt is not None and attempt.cls == "E" and not charged:
return Score(
value=None,
excluded=True,
charged=(),
recorded_not_charged=recorded_not_charged,
environment=environment,
breakdown=empty_breakdown,
recovery=recovery,
attempt_environment=attempt,
)
breakdown = _breakdown(
charged,
completed=completed,
recovery=recovery,
token_efficiency=[REDACTED]
wallclock=wallclock,
)
value = (
CORRECTNESS_RECOVERABILITY_WEIGHT * breakdown.correctness_recoverability
+ TOKEN_EFFICIENCY_WEIGHT * breakdown.token_efficiency
+ WALLCLOCK_WEIGHT * breakdown.wallclock
)
return Score(
value=max(0.0, min(1.0, value)),
excluded=False,
charged=charged,
recorded_not_charged=recorded_not_charged,
environment=environment,
breakdown=breakdown,
recovery=recovery,
attempt_environment=attempt if attempt and attempt.cls == "E" else None,
)
def _breakdown(
charged: tuple[CallVerdict, ...],
*,
completed: bool,
recovery: RecoveryStats,
token_efficiency: float,
wallclock: float,
) -> ScoreBreakdown:
final_outcome = _final_outcome_quality(charged, completed)
self_recovery = recovery.self_recovery_rate if recovery.self_recovery_rate is not None else 1.0
guidance = _guidance_quality(charged)
route_quality = _route_quality(charged)
attribution = _attribution_quality(charged)
correctness = (
FINAL_OUTCOME_POINTS * final_outcome
+ SELF_RECOVERY_POINTS * self_recovery
+ GUIDANCE_POINTS * guidance
+ ROUTE_POINTS * route_quality
+ ATTRIBUTION_POINTS * attribution
) / TOTAL_CORRECTNESS_POINTS
return ScoreBreakdown(
final_outcome=final_outcome,
self_recovery=self_recovery,
guidance=guidance,
route_quality=route_quality,
attribution=attribution,
correctness_recoverability=max(0.0, min(1.0, correctness)),
token_efficiency=[REDACTED] min(1.0, token_efficiency)),
wallclock=max(0.0, min(1.0, wallclock)),
)
def _final_outcome_quality(calls: tuple[CallVerdict, ...], completed: bool) -> float:
if not completed:
return 0.0
if any(call.cls == "C" for call in calls):
return 0.0
if any(call.cls == "A" for call in calls):
return 0.35
if any(call.cls == "D" for call in calls):
return 0.75
return 1.0
def _recovery_stats(
calls: tuple[CallVerdict, ...],
agent_side: tuple[dict, ...],
extra_tokens: int,
extra_wallclock_ms: int,
) -> RecoveryStats:
recoverable = tuple(call for call in calls if call.recovery == "guided")
ignored_hints = sum(1 for detail in agent_side if detail.get("kind") == "ignored_hint")
recovered = max(0, len(recoverable) - ignored_hints)
rate = None if not recoverable else recovered / len(recoverable)
first_failure_index = next((index for index, call in enumerate(calls) if call.cls is not None), None)
extra_calls = 0 if first_failure_index is None else max(0, len(calls) - first_failure_index - 1)
return RecoveryStats(
recoverable_failures=len(recoverable),
recovered_failures=recovered,
self_recovery_rate=rate,
extra_calls=extra_calls,
extra_tokens=[REDACTED]
extra_wallclock_ms=extra_wallclock_ms,
)
def _guidance_quality(calls: tuple[CallVerdict, ...]) -> float:
failures = tuple(call for call in calls if call.cls is not None)
if not failures:
return 1.0
values = []
for call in failures:
if call.cls == "B" or call.recovery == "guided":
--- autoresearch/agenteval/taxonomy.py
"""Three-axis classifier for record-v2 call and attempt verdicts.
The taxonomy names what happened without deciding how much it costs. The
A-F class is always derived through :func:`contract.derive_class`; clean calls
therefore carry ``cls=None`` and serialize as JSON ``"class": null``.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Protocol
from .contract import AttemptVerdict, CallVerdict, Probe, derive_class
class CallLike(Protocol):
argv: list[str]
exit_code: int
stdout: str
stderr: str
duration: float
_NETWORK_MARKERS = (
"curlcode",
"aws error network_connection",
"timeout was reached",
"connection reset by peer",
"could not connect to",
"failed to connect",
"temporary failure in name resolution",
"connection timed out",
)
_QUOTA_MARKERS = ("session limit", "usage limit", "rate limit")
_C_SUBTYPE_BY_PROBE = {
"vocabulary": "c-vocabulary",
"column_swap": "c-wrong-column",
"column-swap": "c-wrong-column",
"type_sweep": "c-wrong-type",
"type-sweep": "c-wrong-type",
"argv_echo": "c-dropped-input",
"argv-echo": "c-dropped-input",
"limit_raise": "c-truncated",
"limit-raise": "c-truncated",
"entity_check": "c-wrong-entity",
"entity-check": "c-wrong-entity",
}
@dataclass(frozen=True)
class TranscriptLike:
"""Small adapter shape for attempt-level environment failures."""
final_answer: str = ""
quota_exhausted: bool = False
def classify(call: CallLike, probes: tuple[Probe, ...] | list[Probe] = ()) -> CallVerdict:
"""Classify one botmap call using the three axes.
Probe evidence decides class-C subtypes. Without conclusive probe evidence,
an empty unguided call is still class C, subtype ``c-unknown``; it must never
silently become clean.
"""
probe_tuple = tuple(_coerce_probe(probe) for probe in probes)
text = _text(call)
if _is_network_failure(text):
return _verdict(
outcome="error",
blame="environment",
recovery="unguided",
subtype=None,
evidence="network/data access failure, not a tool defect",
probes=probe_tuple,
)
if _has_guidance(text):
return _verdict(
outcome="error",
blame="tool",
recovery="guided",
subtype=None,
evidence="tool named a recovery path for the next iteration",
probes=probe_tuple,
)
if _get(call, "exit_code", 0) != 0:
return _verdict(
outcome="error",
blame="tool",
recovery="unguided",
subtype=None,
evidence=_first_line(text) or "command failed without recovery guidance",
probes=probe_tuple,
)
conclusive_subtype = _class_c_subtype(probe_tuple)
if conclusive_subtype != "c-unknown":
return _verdict(
outcome="empty",
blame="tool",
recovery="unguided",
subtype=conclusive_subtype,
evidence=_empty_evidence(conclusive_subtype, probe_tuple),
probes=probe_tuple,
)
if _is_degenerate(call):
return _verdict(
outcome="degenerate",
blame="tool",
recovery="n/a",
subtype=None,
evidence="command eventually answered but by an unusable route",
probes=probe_tuple,
)
if _is_empty(call):
return _verdict(
outcome="empty",
blame="tool",
recovery="unguided",
subtype="c-unknown",
evidence="empty result with no conclusive probe; reason unknown",
probes=probe_tuple,
)
return _verdict(
outcome="ok",
blame="tool",
recovery="n/a",
subtype=None,
evidence="call completed with usable output",
probes=probe_tuple,
)
def classify_attempt(transcript: object) -> AttemptVerdict | None:
"""Classify attempt-level failures that may have no botmap call.
Quota exhaustion belongs here: it is class E environment evidence, but there
may be no ``Call`` to attach it to.
"""
quota = bool(getattr(transcript, "quota_exhausted", False))
answer = str(getattr(transcript, "final_answer", "") or "")
if not quota and not any(marker in answer.lower() for marker in _QUOTA_MARKERS):
return None
outcome = "error"
blame = "environment"
recovery = "unguided"
return AttemptVerdict(
outcome=outcome,
blame=blame,
recovery=recovery,
cls=derive_class(outcome, blame, recovery),
subtype=None,
evidence="model/session quota exhausted before the attempt could be measured",
)
def _verdict(
*,
outcome: str,
blame: str,
recovery: str,
subtype: str | None,
evidence: str,
probes: tuple[Probe, ...],
) -> CallVerdict:
return CallVerdict(
outcome=outcome,
blame=blame,
recovery=recovery,
cls=derive_class(outcome, blame, recovery),
subtype=subtype,
evidence=evidence,
probes=probes,
)
def _text(call: CallLike) -> str:
return f"{_get(call, 'stdout', _get(call, 'stdout_head', '')) or ''}\n{_get(call, 'stderr', _get(call, 'stderr_head', '')) or ''}"
def _get(call: object, name: str, default: object = None) -> object:
if isinstance(call, dict):
return call.get(name, default)
return getattr(call, name, default)
def _coerce_probe(probe: Probe | dict) -> Probe:
if isinstance(probe, Probe):
return probe
return Probe(
kind=str(probe.get("kind", "")),
ran=str(probe.get("ran", "")),
result=str(probe.get("result", "")),
conclusive=bool(probe.get("conclusive", False)),
)
def _first_line(text: str) -> str:
return next((line.strip() for line in text.splitlines() if line.strip()), "")
def _is_network_failure(text: str) -> bool:
low = text.lower()
return any(marker in low for marker in _NETWORK_MARKERS)
def _has_guidance(text: str) -> bool:
low = text.lower()
return any(marker in low for marker in ("did you mean:", "try `", "try:", "use --"))
def _is_empty(call: CallLike) -> bool:
stdout = str(_get(call, "stdout", _get(call, "stdout_head", "")) or "").strip().lower()
[Showing lines 366-1820 of 1820 (50.0KB limit). Full output: /var/folders/2g/85d5_sh17wdf0fm5yw5k3gs80000gn/T/pi-bash-b94a1858c0e6b9d0.log]Planning comprehensive import search
>rg -n "agenteval|from \. import .*score|from \.agenteval|import .*score|score\.feedback|score_attempt|record-v2|build_record|enrich" autoresearch tests tools experiments -g'*.py' -g'*.md' | head -200autoresearch/smoke.py:17:from . import baseline, config, questions as qmod, runner, score
autoresearch/smoke.py:164: for line in score.feedback(q, attempts).splitlines()[:14]:
experiments/orchestrator/HANDOVER.md:1:# Orchestrator handover — agenteval run
experiments/orchestrator/HANDOVER.md:12:- Phase 1: `autoresearch/agenteval/contract.py` landed with record-v2 schema, class derivation, validation, JSON class `null` for clean calls, and attempt-level verdicts for quota/environment failures.
experiments/orchestrator/HANDOVER.md:14:- Phase 3 zero-spend pass over retained baseline attempts produced `experiments/agenteval-baseline-noise-run1-distribution.json`.
experiments/orchestrator/HANDOVER.md:20:- `e84e248 Wire record-v2 artifacts for retained attempts`
experiments/orchestrator/HANDOVER.md:28:experiments/runs/agenteval-measurement-3009509/
experiments/orchestrator/HANDOVER.md:38:attempts/<question>__r<repeat>/{commands.jsonl,transcript.jsonl,claude-stderr.log,record-v2.json}
experiments/orchestrator/HANDOVER.md:45:ps -p $(cat experiments/runs/agenteval-measurement-3009509/pid) -o pid,etime,command
experiments/orchestrator/HANDOVER.md:46:cat experiments/runs/agenteval-measurement-3009509/summary.json
experiments/orchestrator/HANDOVER.md:47:tail -f experiments/runs/agenteval-measurement-3009509/run.log
experiments/orchestrator/HANDOVER.md:68:experiments/runs/agenteval-measurement-3009509-retry-incomplete/
experiments/orchestrator/HANDOVER.md:89:Post-run probe enrichment completed for both original and retry runs. Main summary files:
experiments/orchestrator/HANDOVER.md:92:experiments/runs/agenteval-measurement-3009509/agenteval-summary.json
experiments/orchestrator/HANDOVER.md:93:experiments/runs/agenteval-measurement-3009509/agenteval-summary-with-retries.json
experiments/orchestrator/HANDOVER.md:94:experiments/runs/agenteval-measurement-3009509-retry-incomplete/agenteval-summary.json
experiments/orchestrator/HANDOVER.md:143:- record-v2 clean calls: JSON `"class": null`.
tests/test_repair_records.py:3:from autoresearch.agenteval.repair import repair_us_state_entity_false_positives
tests/test_repair_records.py:10: "schema": "agenteval/2",
tests/test_repair_records.py:41: (attempt / "record-v2.json").write_text(json.dumps(record), encoding="utf-8")
tests/test_repair_records.py:45: repaired = json.loads((attempt / "record-v2.json").read_text())
tools/dashboard/dashboard.py:254: return run_progress_state("agenteval-measurement-3009509")
tools/dashboard/dashboard.py:800: retry_card = progress_card("Retry incomplete attempts", run_progress_state("agenteval-measurement-3009509-retry-incomplete"))
tests/test_analyze_records.py:3:from autoresearch.agenteval.analyze import summarize_run, summarize_with_completed_retries, write_combined_summary, write_summary
tests/test_analyze_records.py:9: (attempt / "record-v2.json").write_text(
tests/test_analyze_records.py:12: "schema": "agenteval/2",
tests/test_analyze_records.py:40: assert out == tmp_path / "agenteval-summary.json"
tests/test_analyze_records.py:64: assert out == base / "agenteval-summary-with-retries.json"
tests/test_analyze_records.py:70: attempt.joinpath("record-v2.json").write_text(
tests/test_analyze_records.py:73: "schema": "agenteval/2",
autoresearch/evaluator.py:22:from . import config, runner, score
autoresearch/evaluator.py:112: oa.log(score.feedback(example, attempts))
experiments/paired/categories-truncation-hint/result.md:26:experiments/runs/agenteval-measurement-3009509/agenteval-summary-with-retries.json
experiments/paired/categories-truncation-hint/result.md:47:experiments/runs/after-categories-truncation-hint-00bff1a/agenteval-summary.json
tests/test_record_v2.py:3:from autoresearch.agenteval.contract import validate
tests/test_record_v2.py:4:from autoresearch.agenteval.record import build_record, parse_tools_used
tests/test_record_v2.py:9:def test_build_record_from_attempt_contains_verdicts_and_agent_side():
tests/test_record_v2.py:20: record = build_record(attempt)
tests/test_explain.py:1:from autoresearch.agenteval.contract import Probe, Record2, SCHEMA
tests/test_explain.py:2:from autoresearch.agenteval.explain import explain
experiments/paired/count-wrong-column-hint/result.md:27:experiments/runs/agenteval-measurement-3009509/agenteval-summary-with-retries.json
experiments/paired/count-wrong-column-hint/result.md:48:experiments/runs/after-count-wrong-column-hint-7c794ff/agenteval-summary.json
tests/test_agenteval_score.py:1:from autoresearch.agenteval.contract import AttemptVerdict, CallVerdict, derive_class
tests/test_agenteval_score.py:2:from autoresearch.agenteval.score import score_attempt
tests/test_agenteval_score.py:29: score = score_attempt([verdict("ok", recovery="n/a")])
tests/test_agenteval_score.py:43: assert score_attempt([guided_hint]).value > score_attempt([silent_zero]).value
tests/test_agenteval_score.py:47: one_silent = score_attempt([verdict("empty", recovery="unguided", subtype="c-unknown")])
tests/test_agenteval_score.py:48: several_guided = score_attempt([verdict("error", recovery="guided") for _ in range(3)])
tests/test_agenteval_score.py:57: with_network = score_attempt([network, clean])
tests/test_agenteval_score.py:58: without_network = score_attempt([clean])
tests/test_agenteval_score.py:68: score = score_attempt([], attempt=quota)
tests/test_agenteval_score.py:79: score = score_attempt([agent_fault, clean], agent_side=[{"kind": "ignored_hint", "strict": True}])
tests/test_agenteval_score.py:81: assert score.value == score_attempt([clean]).value
tests/test_agenteval_score.py:90: score = score_attempt([guided_hint, clean], extra_tokens=[REDACTED] extra_wallclock_ms=456)
tests/test_agenteval_score.py:103: with_agent_detail = score_attempt([guided_hint], agent_side=[{
tests/test_agenteval_score.py:108: without_agent_detail = score_attempt([guided_hint])
tests/test_contract.py:1:from autoresearch.agenteval.contract import (
tests/test_contract.py:24: path = tmp_path / "record-v2.json"
tests/test_scoring.py:12:from autoresearch import baseline, config, credits, score
tests/test_scoring.py:107: assert "botmap count -t place" in score.feedback(QUESTION, [a])
tests/test_scoring.py:111: text = score.feedback(QUESTION, [a])
tests/test_scoring.py:118: assert "--category" in score.feedback(QUESTION, [a])
tests/test_scoring.py:122: assert "learned nothing" in score.feedback(QUESTION, [a])
autoresearch/agenteval/analyze.py:1:"""Offline analysis helpers for record-v2 measurement runs."""
autoresearch/agenteval/analyze.py:12: """Return class/subtype histograms for a run containing record-v2 files."""
autoresearch/agenteval/analyze.py:40: target = Path(out) if out is not None else root / "agenteval-summary.json"
autoresearch/agenteval/analyze.py:48: target = Path(out) if out is not None else root / "agenteval-summary-with-retries.json"
autoresearch/agenteval/analyze.py:113: for path in sorted(run_dir.glob("attempts/*/record-v2.json")):
tests/test_agent_side.py:1:from autoresearch.agenteval.agent_side import detect_ignored_hints
tests/test_agenteval_taxonomy.py:3:from autoresearch.agenteval.contract import Probe, derive_class
tests/test_agenteval_taxonomy.py:4:from autoresearch.agenteval.taxonomy import classify, classify_attempt
autoresearch/agenteval/score.py:87:def score_attempt(
autoresearch/agenteval/explain.py:8:from autoresearch.agenteval.contract import Record2
autoresearch/agenteval/explain.py:22: """Turn a record-v2 attempt into fix-instruction feedback."""
autoresearch/agenteval/contract.py:15:SCHEMA = "agenteval/2"
autoresearch/agenteval/record.py:1:"""Build record-v2 attempts from retained runner artifacts."""
autoresearch/agenteval/record.py:18:def build_record(attempt: Attempt, *, transcript_path: Path | None = None) -> Record2:
autoresearch/agenteval/record.py:19: """Convert an existing runner attempt into the shared record-v2 shape."""
autoresearch/agenteval/record.py:23: schema="agenteval/2",
tests/test_sabotage_integration.py:1:from autoresearch.agenteval.sabotage import FIXTURES, run_sabotage
tests/test_sabotage_integration.py:2:from autoresearch.agenteval.taxonomy import classify, classify_attempt, TranscriptLike
autoresearch/runner.py:108: from .agenteval.contract import write as write_record2
autoresearch/runner.py:109: from .agenteval.record import build_record
autoresearch/runner.py:111: write_record2(keep / "record-v2.json", build_record(attempt, transcript_path=transcript_path))
autoresearch/agenteval/taxonomy.py:1:"""Three-axis classifier for record-v2 call and attempt verdicts.
autoresearch/agenteval/agent_side.py:1:"""Agent-side failure detectors for record-v2 attempts."""
autoresearch/agenteval/enrich.py:1:"""Enrich retained record-v2 files with post-agent differential probes."""
autoresearch/agenteval/enrich.py:17:from .record import build_record
autoresearch/agenteval/enrich.py:21:def enrich_run(run_dir: str | Path, botmap_repo: str | Path, *, max_probe_calls: int = 8, timeout_s: int = 120) -> dict[str, Any]:
autoresearch/agenteval/enrich.py:22: """Run CLI-only probes for retained attempts and rewrite record-v2 files.
autoresearch/agenteval/enrich.py:38: record = build_record(_attempt_from_parts(attempt_dir.name, calls, transcript), transcript_path=transcript)
autoresearch/agenteval/enrich.py:39: enriched_calls = []
autoresearch/agenteval/enrich.py:58: enriched_calls.append(call_record)
autoresearch/agenteval/enrich.py:63: calls=tuple(enriched_calls),
autoresearch/agenteval/enrich.py:70: write_record(attempt_dir / "record-v2.json", record)
tests/test_sabotage.py:1:from autoresearch.agenteval.contract import C_SUBTYPES, validate
tests/test_sabotage.py:2:from autoresearch.agenteval.sabotage import FIXTURES, assert_sabotage_passes, expected_verdict, run_sabotage
tests/test_sabotage.py:16: "schema": "agenteval/2",
autoresearch/agenteval/sabotage.py:13:from autoresearch.agenteval.contract import CallVerdict, Probe, derive_class
autoresearch/agenteval/repair.py:1:"""Repair helpers for already-written record-v2 artifacts.
autoresearch/agenteval/repair.py:18:from .record import build_record
autoresearch/agenteval/repair.py:35: for record_path in sorted(root.glob("attempts/*/record-v2.json")):
autoresearch/agenteval/repair.py:69: record = build_record(_attempt_from_raw(raw), transcript_path=None)
tests/test_probe.py:5:from autoresearch.agenteval.probe import (
experiments/artifacts/history/report-1.md:51:Add a `SuggestiveCommand(click.Command)` subclass in botmap/cli.py that overrides `parse_args` to catch `click.NoSuchOption` and re-raise a `UsageError` enriched from a module-level table, e.g. `OPTION_REDIRECTS = {('count','--at'): 'proximity queries live on `at`: botmap at LAT,LON -t TYPE --radius M (add --count for a number)', ('count','--radius'): same, ('places','--class'): 'places filters by --category; --class is for roads/water/landuse', ('roads','--category'): 'segments use --class, not --category', ('download','-n'): 'download has no --n; use a convenience verb or `sample -n`'}`. Fall back to `difflib.get_close_matches` over the command's declared option names ('Unknown option --catagory. Did you mean --category?'). Set `cls=SuggestiveCommand` on every `@cli.command()`.
experiments/artifacts/history/report-2.md:99:Set `download_is_legitimate: false` in evals/questions.yaml:69 and rewrite the note to "`landuse --in 'Brooklyn, US-NY'` covers this; any `download -t land_use` is avoidable." In evals/score.py, stop treating the YAML flag as the sole authority: import `TYPE_TO_VERB` from `botmap.cli` and mark a download unnecessary whenever its `-t` value is a key of that map and every other flag it carried is one the target verb accepts (`--bbox`, `--in`, `--where`, `-f`, `-o`, `-n`), regardless of the YAML flag. Reserve `coverage_gaps` for types with no TYPE_TO_VERB entry (infrastructure, division, land_cover, …), which is what the field is meant to surface. Add a case to tests/test_eval_synthesize.py asserting that a `download -t land_use --bbox …` call scores as `unnecessary_download`, and re-run the 30-run suite so the land_use entry moves out of coverage_gaps.Inspecting runner changes and scoring
Investigating runner offset settings
Inspecting baseline legacy usage
>git status --short -- autoresearch/evaluator.py autoresearch/optimize.py autoresearch/agenteval tests | head -100 && git -C /Users/priyangapkini/workspace/ar-b-new/botmap rev-parse --short HEAD && git -C /Users/priyangapkini/workspace/ar-b-new/botmap status --short && test -f /Users/priyangapkini/workspace/ar-b-new/botmap/botmap/data/skill.md && echo skill-present && test -x /Users/priyangapkini/workspace/ar-b-new/botmap/.venv/bin/python && echo venv-present || true3009509
skill-present>find experiments/runs -maxdepth 1 -type d -name 'prompt-*' -o -name '*prompt*' | sort | tail -50 && ls experiments/baselines | sort | tail -20experiments/runs/prompt-3009509-1787215716
3009509.INCOMPLETE-5of30.json
3009509.json
3009509.release-2026-07-22.0.STALE.json
INCOMPLETE-WARNING.md
README-STALE.mdHandling missing virtual environment error
>echo 'autoresearch scoped status:'; git status --short -- autoresearch/evaluator.py autoresearch/optimize.py autoresearch/agenteval tests | head -120; echo 'botmap:'; git -C /Users/priyangapkini/workspace/ar-b-new/botmap rev-parse --short HEAD; git -C /Users/priyangapkini/workspace/ar-b-new/botmap status --short; test -f /Users/priyangapkini/workspace/ar-b-new/botmap/botmap/data/skill.md; echo skill=$?; test -x /Users/priyangapkini/workspace/ar-b-new/botmap/.venv/bin/python; echo venv=$?; ls -ld /Users/priyangapkini/workspace/ar-b-new/botmap/.venv 2>/dev/null || trueautoresearch scoped status:
botmap:
3009509
skill=0
venv=1Planning inspection of measurement commands
>find autoresearch -maxdepth 1 -type f -print | sort && rg -n "measurement|agenteval|retry|progress|write_summary|enrich_run|argparse" autoresearch experiments tools/dashboard -g'*.py' | head -200autoresearch/__init__.py
autoresearch/baseline.py
autoresearch/blocked.py
autoresearch/config.py
autoresearch/credits.py
autoresearch/evaluator.py
autoresearch/optimize.py
autoresearch/orproxy.py
autoresearch/proposer.py
autoresearch/questions.py
autoresearch/runner.py
autoresearch/score.py
autoresearch/smoke.py
autoresearch/taxonomy.py
autoresearch/trace.py
autoresearch/worktree.py
autoresearch/evaluator.py:82: # one invariant the whole measurement rests on, and it costs
autoresearch/evaluator.py:96: # Every try crashed or timed out. That's a broken measurement, not a
experiments/artifacts/analyze.py:13:import argparse
experiments/artifacts/analyze.py:67: p = argparse.ArgumentParser()
autoresearch/smoke.py:13:import argparse
autoresearch/smoke.py:34: ap = argparse.ArgumentParser(description=__doc__)
autoresearch/agenteval/analyze.py:1:"""Offline analysis helpers for record-v2 measurement runs."""
autoresearch/agenteval/analyze.py:19:def summarize_with_completed_retries(run_dir: str | Path, retry_dir: str | Path) -> dict[str, Any]:
autoresearch/agenteval/analyze.py:20: """Summarize a run, replacing attempts only when their retry completed.
autoresearch/agenteval/analyze.py:22: This keeps the original measurement auditable while answering the practical
autoresearch/agenteval/analyze.py:27: retry = _completed_retry_records(Path(retry_dir))
autoresearch/agenteval/analyze.py:29: combined.update(retry)
autoresearch/agenteval/analyze.py:32: summary["retry_dir"] = str(retry_dir)
autoresearch/agenteval/analyze.py:33: summary["retry_replacements"] = sorted(retry)
autoresearch/agenteval/analyze.py:37:def write_summary(run_dir: str | Path, out: str | Path | None = None) -> Path:
autoresearch/agenteval/analyze.py:40: target = Path(out) if out is not None else root / "agenteval-summary.json"
autoresearch/agenteval/analyze.py:45:def write_combined_summary(run_dir: str | Path, retry_dir: str | Path, out: str | Path | None = None) -> Path:
autoresearch/agenteval/analyze.py:46: """Write a summary with completed retry attempts layered over the run."""
autoresearch/agenteval/analyze.py:48: target = Path(out) if out is not None else root / "agenteval-summary-with-retries.json"
autoresearch/agenteval/analyze.py:49: target.write_text(json.dumps(summarize_with_completed_retries(root, retry_dir), indent=2), encoding="utf-8")
autoresearch/agenteval/analyze.py:118:def _completed_retry_records(retry_dir: Path) -> dict[str, dict[str, Any]]:
autoresearch/agenteval/analyze.py:120: progress = retry_dir / "progress.jsonl"
autoresearch/agenteval/analyze.py:121: if progress.exists():
autoresearch/agenteval/analyze.py:122: for line in progress.read_text(encoding="utf-8").splitlines():
autoresearch/agenteval/analyze.py:128: return {name: raw for name, raw in _records_by_attempt(retry_dir).items() if name in completed}
tools/dashboard/dashboard.py:18:import argparse
tools/dashboard/dashboard.py:224:def run_progress_state(run_name: str) -> dict | None:
tools/dashboard/dashboard.py:252:def measurement_state() -> dict | None:
tools/dashboard/dashboard.py:253: """Current Phase 4 measurement run progress, if present."""
tools/dashboard/dashboard.py:254: return run_progress_state("agenteval-measurement-3009509")
tools/dashboard/dashboard.py:257:def progress_card(title: str, state: dict | None) -> str:
tools/dashboard/dashboard.py:288: happening, or a measurement is most of the way through. Reporting both as
tools/dashboard/dashboard.py:330: "progress": (done, total), "history": history}
tools/dashboard/dashboard.py:362:def arm_a_progress() -> dict[str, int]:
tools/dashboard/dashboard.py:799: measurement_card = progress_card("Phase 4 measurement run", measurement_state())
tools/dashboard/dashboard.py:800: retry_card = progress_card("Retry incomplete attempts", run_progress_state("agenteval-measurement-3009509-retry-incomplete"))
tools/dashboard/dashboard.py:801: after_trunc_card = progress_card("AFTER: categories truncation hint", run_progress_state("after-categories-truncation-hint-00bff1a"))
tools/dashboard/dashboard.py:802: after_wrong_column_card = progress_card("AFTER: count wrong-column hint", run_progress_state("after-count-wrong-column-hint-7c794ff"))
tools/dashboard/dashboard.py:806: {measurement_card}
tools/dashboard/dashboard.py:807: {retry_card}
tools/dashboard/dashboard.py:1166: p = argparse.ArgumentParser()
autoresearch/optimize.py:24:import argparse
autoresearch/optimize.py:77:measurement.
autoresearch/optimize.py:187: display_progress_bar=True,
autoresearch/optimize.py:273: p = argparse.ArgumentParser(description=__doc__,
autoresearch/optimize.py:274: formatter_class=argparse.RawDescriptionHelpFormatter)
autoresearch/runner.py:3:This is the actual measurement. We set up a scratch folder, put the tool's
autoresearch/runner.py:40: a broken measurement.
autoresearch/runner.py:108: from .agenteval.contract import write as write_record2
autoresearch/runner.py:109: from .agenteval.record import build_record
autoresearch/baseline.py:88: from the repeat-pairs, which replaced a whole second measurement worth
autoresearch/config.py:3:If any of these changed mid-run, two measurements couldn't be compared, and the
autoresearch/config.py:146: release, and measurements either side of it would not be comparable. We're
autoresearch/config.py:249: Recorded on every measurement for exactly the reason `release` is. A
autoresearch/agenteval/enrich.py:21:def enrich_run(run_dir: str | Path, botmap_repo: str | Path, *, max_probe_calls: int = 8, timeout_s: int = 120) -> dict[str, Any]:
autoresearch/agenteval/enrich.py:25: the measurement design in docs/plan.md.
autoresearch/agenteval/explain.py:8:from autoresearch.agenteval.contract import Record2
autoresearch/agenteval/explain.py:91: return "emit progress or provide a cheaper first-class route for the task"
autoresearch/agenteval/record.py:23: schema="agenteval/2",
autoresearch/agenteval/sabotage.py:5:fires before any measurement run is allowed.
autoresearch/agenteval/sabotage.py:13:from autoresearch.agenteval.contract import CallVerdict, Probe, derive_class
autoresearch/agenteval/sabotage.py:132: evidence="unhandled tool traceback, no retry instruction",
autoresearch/agenteval/sabotage.py:264: evidence="right route but no progress for a long-running operation",
autoresearch/agenteval/sabotage.py:278: evidence="Claude session limit before usable measurement",
autoresearch/agenteval/contract.py:15:SCHEMA = "agenteval/2"Inspecting dashboard run directories
Inspecting dashboard ARM names
>rg -n "Arm B|arm-b|prompt|ar-b-new|prompt-" tools/dashboard/dashboard.py | head -12061: "arm-b": {
62: "title": "Arm B — prompt lever",
90: "waits_for": "arm-b",
1081: for key, root in (("arm-b", SHARED), ("arm-c", ARM_C_TREE)):Noting dashboard update prerequisite
Estimating evaluation attempt costs
Analyzing evaluator usage of new scores
Assessing evaluator integration plan
Planning environment variable setup
Exploring openrouter setup
>rg -n "ANTHROPIC|OPENROUTER_BASE|orproxy|AUTORESEARCH_AGENT_PATH|api" autoresearch README.md experiments -g'*.py' -g'*.md' | head -200autoresearch/credits.py:24:CREDITS_URL = "https://openrouter.ai/api/v1/credits"
autoresearch/credits.py:45:def fetch(api_key: str, timeout: float = 15.0) -> Balance:
autoresearch/credits.py:48: CREDITS_URL, headers={"Authorization": f"Bearer {api_key}"}
experiments/arm-b/HANDOVER.md:17:| OpenRouter provider pin | `autoresearch/orproxy.py` | falsification: `provider=openai` refused 404 ("No allowed providers"), `provider=anthropic` succeeded. Agentic loop through it: 2 turns, Bash executed |
experiments/arm-b/HANDOVER.md:48:- `orproxy.py` has handled ~15 requests. Not exercised at 60-evaluation scale.
autoresearch/optimize.py:107: proposer: str = "api") -> None:
autoresearch/optimize.py:113: checks = config.preflight(needs_api_key=[REDACTED] subscription, sha=sha)
autoresearch/optimize.py:291: p.add_argument("--proposer", choices=("api", "subscription"), default="api",
autoresearch/optimize.py:292: help="who proposes the changes: 'api' bills OPENROUTER_API_KEY, "
autoresearch/config.py:238:# Claude Code appends `/v1/messages`, so the base stops at `/api`.
autoresearch/config.py:239:OPENROUTER_BASE = "https://openrouter.ai/api"
autoresearch/config.py:253: return os.environ.get("AUTORESEARCH_AGENT_PATH", "subscription")
autoresearch/config.py:261: returning four different hosts in four calls. See orproxy.py.
autoresearch/config.py:499:def _check_funding(needs_api_key: bool) -> str:
autoresearch/config.py:505: if not needs_api_key:
autoresearch/config.py:529:def preflight(needs_api_key: bool = True, sha: str | None = None,
autoresearch/config.py:541: balance_note = _check_funding(needs_api_key)
autoresearch/orproxy.py:29:UPSTREAM = "https://openrouter.ai/api/v1/messages"
autoresearch/orproxy.py:41: api_key = ""
autoresearch/orproxy.py:60: headers["authorization"] = f"Bearer {self.api_key}"
autoresearch/orproxy.py:93: def __init__(self, api_key: str, provider: str = PINNED_PROVIDER):
autoresearch/orproxy.py:94: handler = type("Bound", (_Handler,), {"api_key": api_key, "provider": provider})Inspecting commit history for manual runs
>git log --oneline -5 -- autoresearch/agenteval autoresearch/runner.py autoresearch/evaluator.py autoresearch/optimize.py && git status --short | head -809ed2a19 Probe category vocabulary from CLI listings
89acc62 Fix entity probe state-code ambiguity
0a75991 Apply recovery-centered evaluator rubric
7b0bdc4 Summarize measurement runs with completed retries
de7777b Add post-run probe enrichment
?? ../.DS_Store
?? ../.claude/worktrees/
?? ../.idea/
?? ../.vscode/
?? tools/dashboard/lever-historyConsidering running pytest for checks
>uv run pytest........................................................................ [ 54%]
............................................................ [100%]
132 passed in 0.18sDefining evaluator wiring prerequisites
Estimating run costs and dashboard paths
Planning evaluator baseline reuse for metrics
BLOCKED for the prompt-lever run; READY with plan.
autoresearch/evaluator.py still uses legacy scoring:
from . import config, runner, scorescore.correctness(usable)score.feedback(example, attempts)FailedCommands: len(usable[0].errors)It does not yet use:
autoresearch.agenteval.score.score_attemptautoresearch.agenteval.explain.explainrecord-v2.json as the scoring sourcerunner.py does write record-v2.json when keep_dir is provided, but evaluator scoring still happens from legacy score.py.
Tests:
uv run pytest# 132 passedNo source edits made.
Prerequisites:
~/workspace/ar-b./Users/priyangapkini/workspace/ar-b-new/botmap has .venv installed; currently missing executable .venv/bin/python.Setup:
cd /Users/priyangapkini/nilenso/ai-playground/autoresearch
export BOTMAP_REPO=/Users/priyangapkini/workspace/ar-b-new/botmapexport AUTORESEARCH_AGENT_PATH=openrouterexport AUTORESEARCH_AGENT_PROVIDER=anthropicIf needed before run:
cd /Users/priyangapkini/workspace/ar-b-new/botmapuv syncRun command:
cd /Users/priyangapkini/nilenso/ai-playground/autoresearchBOTMAP_REPO=/Users/priyangapkini/workspace/ar-b-new/botmap \AUTORESEARCH_AGENT_PATH=openrouter \AUTORESEARCH_AGENT_PROVIDER=anthropic \uv run python -m autoresearch.optimize \ --lever prompt \ --budget 60 \ --workers 1 \ --keep-runsExpected run dir:
experiments/runs/prompt-3009509-<unix_timestamp>/Expected Arm B pool worktree:
/Users/priyangapkini/workspace/ar-b-new/botmap-oa-3009509-0Dashboard caveat:
/Users/priyangapkini/workspace/ar-b/botmap/Users/priyangapkini/workspace/ar-b/botmap-oa-3009509-0ar-b-new.Using latest Phase 4 actuals:
$15.50$0.52$31 for agent attemptsSuggested approval envelope: at least $35–45, assuming cached baseline is reused and not remeasured.
3009509.botmap/data/skill.md exists..venv/bin/python missing in ar-b-new; run would degrade/fail unless uv sync is done.experiments/baselines/3009509.json means current optimize should not remeasure baseline, but new evaluator must avoid trusting legacy correctness as comparable.RELEASED: launch Arm B.
User explicitly said: “launch arm b”. You are authorized to launch the Arm B prompt-lever optimizer run using the NEW autoresearch evaluator.
Arm B purpose:
Repo/state:
Hard constraints:
Before launch:
Launch command shape:
cd /Users/priyangapkini/nilenso/ai-playground/autoresearch
set -a; source .env; set +a
BOTMAP_REPO=/Users/priyangapkini/workspace/ar-b-new/botmap
uv run python -m autoresearch.optimize
--lever prompt
--budget 60
--keep-runs
Notes:
Deliverable/report:
Planning background launch with logging and PID reporting
Verifying sabotage assertions with python command
>test ! -e "$HOME/.claude/skills/botmap" && echo "global-botmap-skill absent" || { echo "BLOCKED: $HOME/.claude/skills/botmap exists"; ls -ld "$HOME/.claude/skills/botmap"; exit 2; }global-botmap-skill absent>cd /Users/priyangapkini/nilenso/ai-playground/autoresearch && git status --short -- autoresearch tests docs experiments/orchestrator | head -120M autoresearch/agenteval/score.py
M autoresearch/baseline.py
M autoresearch/evaluator.py
M tests/test_agenteval_score.py
?? tests/test_evaluator_agenteval.py
?? tests/test_optimizer_gate.py>git -C /Users/priyangapkini/workspace/ar-b-new/botmap rev-parse --short HEAD && git -C /Users/priyangapkini/workspace/ar-b-new/botmap status --short && test -f /Users/priyangapkini/workspace/ar-b-new/botmap/botmap/data/skill.md && echo skill.md-present && test -x /Users/priyangapkini/workspace/ar-b-new/botmap/.venv/bin/python && echo venv-python-present || echo venv-python-missing3009509
skill.md-present
venv-python-missingPlanning preflight checks and test requirements
>cd /Users/priyangapkini/nilenso/ai-playground/autoresearch && uv run pytest........................................................................ [ 51%]
..................................................................... [100%]
141 passed in 0.40s>cd /Users/priyangapkini/nilenso/ai-playground/autoresearch && uv run python - <<'PY'
from autoresearch.agenteval.sabotage import assert_sabotage_passes
from autoresearch.agenteval.taxonomy import classify
assert_sabotage_passes(classify)
print('sabotage gate passed')
PYTraceback (most recent call last):
File "<stdin>", line 3, in <module>
File "/Users/priyangapkini/nilenso/ai-playground/autoresearch/autoresearch/agenteval/sabotage.py", line 79, in assert_sabotage_passes
raise AssertionError(f"sabotage fixtures failed: {details}")
AssertionError: sabotage fixtures failed: c-vocabulary-bus-stop: expected C/c-vocabulary, got C/c-unknown; c-wrong-column-recreation: expected C/c-wrong-column, got C/c-unknown; c-wrong-type-water-as-landuse: expected C/c-wrong-type, got C/c-unknown; c-dropped-input-repeat-class: expected C/c-dropped-input, got C/c-unknown; c-truncated-bus-station: expected C/c-truncated, got None/None; c-wrong-entity-malta-mt: expected C/c-wrong-entity, got None/None; e-quota-attempt-level: expected E/None, got None/None; f-ignored-guided-hint: expected F/None, got None/None
Command exited with code 1Inspecting test_sabotage_integration
Running sabotage gate with integration
>cd /Users/priyangapkini/nilenso/ai-playground/autoresearch && uv run python - <<'PY'
from autoresearch.agenteval.sabotage import FIXTURES, run_sabotage
from autoresearch.agenteval.taxonomy import classify, classify_attempt, TranscriptLike
call_level = [fixture for fixture in FIXTURES if fixture.expected_class not in {"E", "F"}]
failures = [result for result in run_sabotage(lambda call: classify(call, call.get("probes", [])), call_level) if not result.passed]
if failures:
raise SystemExit(f"call-level sabotage failed: {failures}")
quota = next(fixture for fixture in FIXTURES if fixture.id == "e-quota-attempt-level")
verdict = classify_attempt(TranscriptLike(final_answer=quota.call["stderr_head"]))
if verdict is None or verdict.cls != "E":
raise SystemExit(f"attempt-level quota sabotage failed: {verdict}")
print('sabotage gate passed')
PYsabotage gate passedEnsuring botmap virtual environment setup
>cd /Users/priyangapkini/workspace/ar-b-new/botmap && uv syncUsing CPython 3.11.15
Creating virtual environment at: .venv
Resolved 39 packages in 5ms
Building botmap @ file:///Users/priyangapkini/workspace/ar-b-new/botmap
Built botmap @ file:///Users/priyangapkini/workspace/ar-b-new/botmap
Prepared 1 package in 253ms
Installed 18 packages in 43ms
+ botmap==0.1.2 (from file:///Users/priyangapkini/workspace/ar-b-new/botmap)
+ click==8.3.1
+ colorama==0.4.6
+ iniconfig==2.3.0
+ numpy==2.4.1
+ orjson==3.11.7
+ packaging==25.0
+ pluggy==1.6.0
+ py-cpuinfo==9.0.0
+ pyarrow==22.0.0
+ pyfiglet==1.0.4
+ pygments==2.20.0
+ pytest==9.0.3
+ pytest-benchmark==5.2.3
+ pytest-mock==3.15.1
+ pyyaml==6.0.3
+ shapely==2.1.2
+ tqdm==4.67.3Inspecting environment variable keys
>cd /Users/priyangapkini/nilenso/ai-playground/autoresearch && python3 - <<'PY'
from pathlib import Path
for line in Path('.env').read_text().splitlines():
s=line.strip()
if not s or s.startswith('#') or '=' not in s: continue
key=s.removeprefix('export ').split('=',1)[0].strip()
print(key)
PYcmd = [ "claude", "-p", question.question, "--output-format", "stream-json", "--verbose", "--model", model, "--permission-mode", "bypassPermissions", "--allowedTools", "Bash",]"--setting-sources", "project",cmd = [ "claude", "-p", question.question, "--setting-sources", "project", "--output-format", "stream-json", "--verbose", "--model", model, "--permission-mode", "bypassPermissions", "--allowedTools", "Bash",]uv run pytestautoresearch/agenteval/ __init__.py contract.py@dataclass(frozen=True)class Probe: kind: str ran: str result: str conclusive: bool
@dataclass(frozen=True)class CallVerdict: outcome: str blame: str recovery: str cls: str subtype: str | None evidence: str probes: tuple[Probe, ...]
@dataclass(frozen=True)class Record2: schema: str question_id: str repeat: int calls: tuple[dict, ...] agent_side: tuple[dict, ...] tools_used: dict[str, int] botmap_calls: int answer: dictuv run pytestuv run python -m py_compile autoresearch/agenteval/contract.pygit status --short -- autoresearch docs experiments/arm-b tests pyproject.toml README.md TODO.mduv run pytest tests/test_probe.pyuv run pytest9 passed116 passeduv run pytest# 132 passedcd /Users/priyangapkini/nilenso/ai-playground/autoresearch
export BOTMAP_REPO=/Users/priyangapkini/workspace/ar-b-new/botmapexport AUTORESEARCH_AGENT_PATH=openrouterexport AUTORESEARCH_AGENT_PROVIDER=anthropiccd /Users/priyangapkini/workspace/ar-b-new/botmapuv synccd /Users/priyangapkini/nilenso/ai-playground/autoresearchBOTMAP_REPO=/Users/priyangapkini/workspace/ar-b-new/botmap \AUTORESEARCH_AGENT_PATH=openrouter \AUTORESEARCH_AGENT_PROVIDER=anthropic \uv run python -m autoresearch.optimize \ --lever prompt \ --budget 60 \ --workers 1 \ --keep-runsexperiments/runs/prompt-3009509-<unix_timestamp>//Users/priyangapkini/workspace/ar-b-new/botmap-oa-3009509-0/Users/priyangapkini/workspace/ar-b/botmap/Users/priyangapkini/workspace/ar-b/botmap-oa-3009509-0