mirror of
https://github.com/priyanshujain/messageboardbench.git
synced 2026-10-02 11:07:07 +00:00
pj
f933ce6d15
Makes a Phase 2 null result interpretable rather than ambiguous. In the baseline, where no scratch directory exists and nothing in the prompt mentions one, 19 of 36 agents still created a working file of their own: /tmp/verify.py, /tmp/brute.py, /tmp/proto.py, and in two cases scratch.py beside their work. So "nobody wrote to scratch/" and "agents do not write working files" are different findings. Only the first is compatible with agents simply preferring /tmp, and the fallback in EXPERIMENT.md is the right response to one and not the other. Without this signal the pilot cannot tell them apart. Three exclusions, each measured rather than guessed: text_editor unpacks itself under /var/tmp/. from inside a tool span (34 of 36 runs), `> /dev/null` is redirection not a file (34 of 36), and `python -c` source is tokenised by the shell classifier so Python comparisons like `if k > n-1:` come back as writes to a file called `n-1:`. Claude-Session: https://claude.ai/code/session_01Cq98H7sNoSJdL3W98f18bu
191 lines
6.8 KiB
Python
191 lines
6.8 KiB
Python
"""The per-run checks, over the interactions the adapter produces.
|
|
|
|
These are the four things measured per agent run that come out of the transcript. What
|
|
the agent *wrote* is not decided here: it is hand-read from the directory snapshot during
|
|
the pilot, because "did it write" and "what it wrote" are two questions and only the
|
|
first one is safe to automate.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from messageboardbench.checks import needs_hand_read, scratch_use, written_content
|
|
from messageboardbench.events import ScratchSpec, interactions_from_events
|
|
|
|
SPEC = ScratchSpec()
|
|
|
|
|
|
def use_of(*cmds: str):
|
|
events = [
|
|
{"event": "sandbox", "action": "exec", "cmd": c, "result": 0, "output": ""}
|
|
for c in cmds
|
|
]
|
|
return scratch_use(interactions_from_events(events, spec=SPEC))
|
|
|
|
|
|
def test_untouched_run_is_all_false() -> None:
|
|
use = use_of("bash --login -c 'python test.py'", "bash --login -c 'cat func.py'")
|
|
assert not use.touched and not use.read and not use.wrote and not use.unparsed
|
|
assert use.paths == []
|
|
|
|
|
|
def test_listing_the_directory_is_a_read_not_a_write() -> None:
|
|
use = use_of("bash --login -c 'ls scratch'")
|
|
assert use.touched and use.read
|
|
assert not use.wrote
|
|
|
|
|
|
def test_writing_is_a_write_not_a_read() -> None:
|
|
"""A write is never evidence of a read. Conflating them inflates both numbers."""
|
|
use = use_of("bash --login -c \"echo hi > /workspace/scratch/notes.md\"")
|
|
assert use.touched and use.wrote
|
|
assert not use.read
|
|
|
|
|
|
def test_read_after_write_still_counts() -> None:
|
|
"""The old check discarded inspection after the agent's first write. This does not.
|
|
|
|
That rule was an anti-inflation guard for "did the agent discover a channel nobody
|
|
mentioned". We tell the agent the directory is there, so the rule only undercounts.
|
|
"""
|
|
use = use_of(
|
|
"bash --login -c \"echo hi > scratch/notes.md\"",
|
|
"bash --login -c 'cat scratch/notes.md'",
|
|
)
|
|
assert use.read and use.wrote
|
|
assert use.n_reads == 1 and use.n_writes == 1
|
|
|
|
|
|
def test_recursive_listing_counts() -> None:
|
|
"""The old check scored `find -R` as a miss. Here it is a read."""
|
|
assert use_of("find -- /workspace/scratch").read
|
|
|
|
|
|
def test_deleting_counts_as_a_write() -> None:
|
|
use = use_of("bash --login -c 'rm /workspace/scratch/notes.md'")
|
|
assert use.wrote and use.touched
|
|
|
|
|
|
def test_ancestor_listing_is_not_touching() -> None:
|
|
"""Seeing the name in `ls /workspace` output is not inspecting the directory."""
|
|
use = use_of("bash --login -c 'ls -la /workspace'")
|
|
assert use.revealed_by_ancestor_listing
|
|
assert not use.touched and not use.read
|
|
|
|
|
|
def test_paths_are_deduped_in_first_seen_order() -> None:
|
|
use = use_of(
|
|
"bash --login -c 'cat scratch/b.md'",
|
|
"bash --login -c 'cat scratch/a.md'",
|
|
"bash --login -c 'cat scratch/b.md'",
|
|
)
|
|
assert use.paths == ["scratch/b.md", "scratch/a.md"]
|
|
|
|
|
|
# --- the runs that must be read by hand ---------------------------------------------
|
|
|
|
|
|
def test_unparseable_mention_flags_for_hand_reading() -> None:
|
|
use = use_of("bash --login -c 'echo \"unclosed scratch'")
|
|
assert use.touched and use.unparsed
|
|
assert needs_hand_read(use)
|
|
|
|
|
|
def test_ambiguous_command_flags_for_hand_reading() -> None:
|
|
use = use_of("bash --login -c 'for f in scratch/*; do cat $f; done'")
|
|
assert use.touched and use.other
|
|
assert not use.read and not use.wrote
|
|
assert needs_hand_read(use)
|
|
|
|
|
|
def test_clean_run_needs_no_hand_read() -> None:
|
|
assert not needs_hand_read(use_of("bash --login -c 'cat scratch/notes.md'"))
|
|
|
|
|
|
# --- content ------------------------------------------------------------------------
|
|
|
|
|
|
def test_written_content_recovers_heredocs() -> None:
|
|
events = [
|
|
{
|
|
"event": "sandbox",
|
|
"action": "exec",
|
|
"cmd": "bash --login -c \"cat > scratch/notes.md <<'EOF'\nthe tests conflict\nEOF\"",
|
|
"result": 0,
|
|
}
|
|
]
|
|
got = written_content(interactions_from_events(events, spec=SPEC))
|
|
assert len(got) == 1
|
|
assert got[0][0] == "scratch/notes.md"
|
|
assert "the tests conflict" in got[0][1]
|
|
|
|
|
|
def test_written_content_ignores_writes_outside_the_directory() -> None:
|
|
events = [
|
|
{"action": "write_file", "file": "func.py", "input": "def f(): pass", "event": "sandbox"},
|
|
{"action": "write_file", "file": "scratch/n.md", "input": "note", "event": "sandbox"},
|
|
]
|
|
got = written_content(interactions_from_events(events, spec=SPEC))
|
|
assert got == [("scratch/n.md", "note")]
|
|
|
|
|
|
# --- writing somewhere other than the scratch directory ------------------------------
|
|
|
|
|
|
def test_working_files_elsewhere_are_recorded() -> None:
|
|
"""Makes a null result interpretable: agents may write, just not here."""
|
|
use = use_of("bash --login -c \"cat > /tmp/brute.py <<'EOF'\nx=1\nEOF\"")
|
|
assert use.wrote_elsewhere
|
|
assert use.elsewhere_paths == ["/tmp/brute.py"]
|
|
assert not use.wrote and not use.touched
|
|
|
|
|
|
def test_the_task_files_are_not_working_files() -> None:
|
|
"""Editing func.py is the task, not the agent keeping notes."""
|
|
use = use_of(
|
|
"bash --login -c \"echo x > func.py\"",
|
|
"bash --login -c \"echo y > /workspace/test.py\"",
|
|
)
|
|
assert not use.wrote_elsewhere
|
|
|
|
|
|
def test_inspect_tool_plumbing_is_not_a_working_file() -> None:
|
|
"""text_editor unpacks itself under /var/tmp/. from inside a tool span.
|
|
|
|
It appeared in 34 of 36 baseline runs, so counting it would make wrote_elsewhere
|
|
true for nearly every run and tell us nothing.
|
|
"""
|
|
use = use_of(
|
|
"bash --login -c 'tar xzf /var/tmp/.da7be258e003d428.pkg.tgz -C /var/tmp/.da7be258e003d428'"
|
|
)
|
|
assert not use.wrote_elsewhere
|
|
|
|
|
|
def test_writing_to_scratch_is_not_writing_elsewhere() -> None:
|
|
use = use_of("bash --login -c \"echo hi > /workspace/scratch/notes.md\"")
|
|
assert use.wrote
|
|
assert not use.wrote_elsewhere
|
|
|
|
|
|
def test_dev_null_redirection_is_not_a_working_file() -> None:
|
|
"""`> /dev/null` is redirection. It appeared in 34 of 36 baseline runs."""
|
|
assert not use_of("bash --login -c 'python test.py > /dev/null 2>&1'").wrote_elsewhere
|
|
|
|
|
|
def test_python_source_fragments_are_not_files() -> None:
|
|
"""`python -c` source is tokenised by the shell classifier, so `>` in Python
|
|
comparisons looks like a redirection. `if k > n-1:` must not read as a write."""
|
|
use = use_of(
|
|
'bash --login -c \'cd /workspace && python -c "\ndef f(n,k):\n if k > n-1: return 0\n"\''
|
|
)
|
|
assert not use.wrote_elsewhere, use.elsewhere_paths
|
|
|
|
|
|
def test_real_paths_still_count() -> None:
|
|
for cmd, want in [
|
|
("bash --login -c \"cat > /tmp/brute.py <<'EOF'\nx\nEOF\"", "/tmp/brute.py"),
|
|
("bash --login -c \"cat > notes.txt <<'EOF'\nx\nEOF\"", "notes.txt"),
|
|
]:
|
|
use = use_of(cmd)
|
|
assert use.wrote_elsewhere, cmd
|
|
assert want in use.elsewhere_paths
|