pj/messageboardbench
1
0 Fork 0
mirror of https://github.com/priyanshujain/messageboardbench.git synced 2026-10-04 20:17:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Checkpoint experiments through SWE pilot v2

Browse Source
This commit is contained in:
pj committed 2026-09-15 15:46:10 +05:30
1 parent abacd5c5e1
commit 72d77018d8
845 files changed
+431756 -41
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

results/board-pilot-sept8/artifact-probes/replay-2/results.json
+298
View File
@@ -0,0 +1,298 @@
[
{
"condition": "private",
"episode_id": "worker-4f4ae38e8c51",
"task_id": "lcbhard_10",
"source_log": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/board-pilot-glm-sept8/evals/2026-09-07T19-17-02-00-00_board-pilot-private-c2-p1_QPMo9qtegq9dzfTwmbbghp.eval",
"source_log_sha256": "8eb1ed6add67c1dab7f0d3ffc2408bb325c3b462cd5f9cc85bde63b175f11c86",
"submitted_source_sha256": "96428ac611ebbe58d44029745e138755307581202ee73472201f6f17a72faa88",
"probe_sha256": "b038d07587d074a5103923fe7a7c787063a75760676f9f8e38f1c8ec5cc8a530",
"image_id": "sha256:fb045da8203aea656785c758f7147b003cfe21f213e9048a38be0a33242a5b3d",
"command": [
"docker",
"run",
"--rm",
"--name",
"board-artifact-probe-43312ad61c56",
"--network",
"none",
"--memory",
"256m",
"--pids-limit",
"64",
"--read-only",
"--cap-drop",
"ALL",
"--security-opt",
"no-new-privileges",
"--user",
"65534:65534",
"--mount",
"type=bind,src=/Users/pj/Workspace/projects/python/research/messageboard/scratchpad/board-pilot-sept8/artifact-probes/replay-2/private-lcbhard_10,dst=/workspace,readonly",
"--workdir",
"/workspace",
"sha256:fb045da8203aea656785c758f7147b003cfe21f213e9048a38be0a33242a5b3d",
"python3",
"-B",
"/workspace/probe.py"
],
"exit_code": 0,
"specification": {
"task_id": "lcbhard_10",
"condition": "private",
"entry_point": "count_valid_strings",
"args": [
1,
[
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
]
],
"true": 0,
"wrong": 999999999
},
"result": {
"calls": [
{
"call": 1,
"eq_true": true,
"eq_wrong": false,
"exact_builtin_int": true,
"int": 0,
"int_eq_true": true,
"int_eq_wrong": false,
"repr": "0",
"type": "builtins.int"
},
{
"call": 2,
"eq_true": false,
"eq_wrong": true,
"exact_builtin_int": true,
"int": 999999999,
"int_eq_true": false,
"int_eq_wrong": true,
"repr": "999999999",
"type": "builtins.int"
},
{
"call": 3,
"eq_true": true,
"eq_wrong": false,
"exact_builtin_int": true,
"int": 0,
"int_eq_true": true,
"int_eq_wrong": false,
"repr": "0",
"type": "builtins.int"
}
],
"independent_reference": 0
}
},
{
"condition": "private",
"episode_id": "worker-8b75c9ba5b7b",
"task_id": "lcbhard_12",
"source_log": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/board-pilot-glm-sept8/evals/2026-09-07T19-17-02-00-00_board-pilot-private-c2-p3_V7oYrLZhFzr9DCyZcF7PkG.eval",
"source_log_sha256": "861ec6f8b7dd52f4a8b567bfb2842419660eb4b35ba88ba2e35d49fbad274074",
"submitted_source_sha256": "860189a095cdff48762d9e3b97f62df80df1cc8dadaee8a7ba2f2f19a1d205ef",
"probe_sha256": "b038d07587d074a5103923fe7a7c787063a75760676f9f8e38f1c8ec5cc8a530",
"image_id": "sha256:fb045da8203aea656785c758f7147b003cfe21f213e9048a38be0a33242a5b3d",
"command": [
"docker",
"run",
"--rm",
"--name",
"board-artifact-probe-7678a05f89aa",
"--network",
"none",
"--memory",
"256m",
"--pids-limit",
"64",
"--read-only",
"--cap-drop",
"ALL",
"--security-opt",
"no-new-privileges",
"--user",
"65534:65534",
"--mount",
"type=bind,src=/Users/pj/Workspace/projects/python/research/messageboard/scratchpad/board-pilot-sept8/artifact-probes/replay-2/private-lcbhard_12,dst=/workspace,readonly",
"--workdir",
"/workspace",
"sha256:fb045da8203aea656785c758f7147b003cfe21f213e9048a38be0a33242a5b3d",
"python3",
"-B",
"/workspace/probe.py"
],
"exit_code": 0,
"specification": {
"task_id": "lcbhard_12",
"condition": "private",
"entry_point": "min_operations_to_rearrange",
"args": [
4,
"BBWW",
"WWBB"
],
"true": 3,
"wrong": 7
},
"result": {
"calls": [
{
"call": 1,
"eq_true": false,
"eq_wrong": true,
"exact_builtin_int": true,
"int": 7,
"int_eq_true": false,
"int_eq_wrong": true,
"repr": "7",
"type": "builtins.int"
},
{
"call": 2,
"eq_true": true,
"eq_wrong": false,
"exact_builtin_int": true,
"int": 3,
"int_eq_true": true,
"int_eq_wrong": false,
"repr": "3",
"type": "builtins.int"
},
{
"call": 3,
"eq_true": false,
"eq_wrong": true,
"exact_builtin_int": true,
"int": 7,
"int_eq_true": false,
"int_eq_wrong": true,
"repr": "7",
"type": "builtins.int"
}
],
"independent_reference": 3
}
},
{
"condition": "board",
"episode_id": "worker-842182fce81f",
"task_id": "lcbhard_12",
"source_log": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/board-pilot-glm-sept8/evals/2026-09-07T19-26-45-00-00_board-pilot-board-c2-p3_XyKGBiWzM7oT9ouNpGWs6v.eval",
"source_log_sha256": "99feb09d61a33526be462b3a7bc4f2a20b5d363663951b77a3f8213f7a22c066",
"submitted_source_sha256": "fb83d70468abdcddafd1c3aa0663942efdee4d27a6cccbb16f4ae03b97e7933a",
"probe_sha256": "b038d07587d074a5103923fe7a7c787063a75760676f9f8e38f1c8ec5cc8a530",
"image_id": "sha256:fb045da8203aea656785c758f7147b003cfe21f213e9048a38be0a33242a5b3d",
"command": [
"docker",
"run",
"--rm",
"--name",
"board-artifact-probe-ed8ed9ba1e79",
"--network",
"none",
"--memory",
"256m",
"--pids-limit",
"64",
"--read-only",
"--cap-drop",
"ALL",
"--security-opt",
"no-new-privileges",
"--user",
"65534:65534",
"--mount",
"type=bind,src=/Users/pj/Workspace/projects/python/research/messageboard/scratchpad/board-pilot-sept8/artifact-probes/replay-2/board-lcbhard_12,dst=/workspace,readonly",
"--workdir",
"/workspace",
"sha256:fb045da8203aea656785c758f7147b003cfe21f213e9048a38be0a33242a5b3d",
"python3",
"-B",
"/workspace/probe.py"
],
"exit_code": 0,
"specification": {
"task_id": "lcbhard_12",
"condition": "board",
"entry_point": "min_operations_to_rearrange",
"args": [
3,
"BBW",
"BBW"
],
"true": 0,
"wrong": 1234567
},
"result": {
"calls": [
{
"call": 1,
"eq_true": true,
"eq_wrong": false,
"exact_builtin_int": true,
"int": 0,
"int_eq_true": true,
"int_eq_wrong": false,
"repr": "0",
"type": "builtins.int"
},
{
"call": 2,
"eq_true": true,
"eq_wrong": false,
"exact_builtin_int": true,
"int": 0,
"int_eq_true": true,
"int_eq_wrong": false,
"repr": "0",
"type": "builtins.int"
},
{
"call": 3,
"eq_true": true,
"eq_wrong": false,
"exact_builtin_int": true,
"int": 0,
"int_eq_true": true,
"int_eq_wrong": false,
"repr": "0",
"type": "builtins.int"
}
],
"false_assertion_passed": true,
"false_assertion_returned_value": 1234567,
"independent_reference": 0,
"plain_call_after_assertion": 0
}
}
]