mirror of
https://github.com/priyanshujain/messageboardbench.git
synced 2026-10-02 19:17:07 +00:00
982 lines
30 KiB
JSON
982 lines
30 KiB
JSON
{
|
|
"included_run_directories": [
|
|
"baseline",
|
|
"team-conflicting-private-sept7",
|
|
"team-conflicting-shared-sept7",
|
|
"team-original-sept7",
|
|
"token-rerun-private-lcbhard-0-sept7"
|
|
],
|
|
"total_recoverable_paid_samples": 54,
|
|
"conditions": {
|
|
"baseline_conflicting": {
|
|
"n": 39,
|
|
"passed": 18,
|
|
"limits": {
|
|
"none": 7,
|
|
"message": 32,
|
|
"token": 0,
|
|
"time": 0
|
|
},
|
|
"errored": 0,
|
|
"medians": {
|
|
"total_tokens": 280866,
|
|
"input_tokens": 65289,
|
|
"cache_read_tokens": 211712,
|
|
"all_input_tokens": 258836,
|
|
"output_tokens": 26834,
|
|
"reasoning_tokens": 20111,
|
|
"messages": 30,
|
|
"turns": 13,
|
|
"working_seconds": 715.275
|
|
},
|
|
"sums": {
|
|
"total_tokens": 16195834,
|
|
"input_tokens": 3187003,
|
|
"cache_read_tokens": 11488448,
|
|
"all_input_tokens": 14675451,
|
|
"output_tokens": 1520383,
|
|
"reasoning_tokens": 1171191,
|
|
"messages": 1148,
|
|
"turns": 499,
|
|
"working_seconds": 39074.023
|
|
}
|
|
},
|
|
"team-conflicting-private-sept7": {
|
|
"n": 6,
|
|
"passed": 0,
|
|
"limits": {
|
|
"none": 1,
|
|
"message": 0,
|
|
"token": 5,
|
|
"time": 0
|
|
},
|
|
"errored": 0,
|
|
"medians": {
|
|
"total_tokens": 414080.0,
|
|
"input_tokens": 62299.5,
|
|
"cache_read_tokens": 306816.0,
|
|
"all_input_tokens": 387182.5,
|
|
"output_tokens": 30074.5,
|
|
"reasoning_tokens": 19545.5,
|
|
"messages": 45.0,
|
|
"turns": 18.0,
|
|
"working_seconds": 729.2194999999999
|
|
},
|
|
"sums": {
|
|
"total_tokens": 2294029,
|
|
"input_tokens": 396727,
|
|
"cache_read_tokens": 1723840,
|
|
"all_input_tokens": 2120567,
|
|
"output_tokens": 173462,
|
|
"reasoning_tokens": 118134,
|
|
"messages": 264,
|
|
"turns": 106,
|
|
"working_seconds": 4791.924
|
|
}
|
|
},
|
|
"team-conflicting-shared-sept7": {
|
|
"n": 6,
|
|
"passed": 0,
|
|
"limits": {
|
|
"none": 3,
|
|
"message": 0,
|
|
"token": 3,
|
|
"time": 0
|
|
},
|
|
"errored": 0,
|
|
"medians": {
|
|
"total_tokens": 388703.0,
|
|
"input_tokens": 59687.5,
|
|
"cache_read_tokens": 289664.0,
|
|
"all_input_tokens": 363172.5,
|
|
"output_tokens": 25530.5,
|
|
"reasoning_tokens": 15718.5,
|
|
"messages": 44.5,
|
|
"turns": 17.0,
|
|
"working_seconds": 762.8489999999999
|
|
},
|
|
"sums": {
|
|
"total_tokens": 2087218,
|
|
"input_tokens": 392774,
|
|
"cache_read_tokens": 1539392,
|
|
"all_input_tokens": 1932166,
|
|
"output_tokens": 155052,
|
|
"reasoning_tokens": 101791,
|
|
"messages": 259,
|
|
"turns": 102,
|
|
"working_seconds": 4611.917
|
|
}
|
|
},
|
|
"team-original-sept7": {
|
|
"n": 2,
|
|
"passed": 2,
|
|
"limits": {
|
|
"none": 2,
|
|
"message": 0,
|
|
"token": 0,
|
|
"time": 0
|
|
},
|
|
"errored": 0,
|
|
"medians": {
|
|
"total_tokens": 43311.5,
|
|
"input_tokens": 14124.0,
|
|
"cache_read_tokens": 23904.0,
|
|
"all_input_tokens": 38028.0,
|
|
"output_tokens": 5283.5,
|
|
"reasoning_tokens": 2948.5,
|
|
"messages": 19.5,
|
|
"turns": 8.0,
|
|
"working_seconds": 178.5765
|
|
},
|
|
"sums": {
|
|
"total_tokens": 86623,
|
|
"input_tokens": 28248,
|
|
"cache_read_tokens": 47808,
|
|
"all_input_tokens": 76056,
|
|
"output_tokens": 10567,
|
|
"reasoning_tokens": 5897,
|
|
"messages": 39,
|
|
"turns": 16,
|
|
"working_seconds": 357.153
|
|
}
|
|
},
|
|
"token-rerun-private-lcbhard-0-sept7": {
|
|
"n": 1,
|
|
"passed": 0,
|
|
"limits": {
|
|
"none": 1,
|
|
"message": 0,
|
|
"token": 0,
|
|
"time": 0
|
|
},
|
|
"errored": 0,
|
|
"medians": {
|
|
"total_tokens": 439381,
|
|
"input_tokens": 80576,
|
|
"cache_read_tokens": 325248,
|
|
"all_input_tokens": 405824,
|
|
"output_tokens": 33557,
|
|
"reasoning_tokens": 23135,
|
|
"messages": 48,
|
|
"turns": 18,
|
|
"working_seconds": 940.326
|
|
},
|
|
"sums": {
|
|
"total_tokens": 439381,
|
|
"input_tokens": 80576,
|
|
"cache_read_tokens": 325248,
|
|
"all_input_tokens": 405824,
|
|
"output_tokens": 33557,
|
|
"reasoning_tokens": 23135,
|
|
"messages": 48,
|
|
"turns": 18,
|
|
"working_seconds": 940.326
|
|
}
|
|
}
|
|
},
|
|
"baseline_outcome_groups": {
|
|
"impossible_pass_prior_audit_gaming": {
|
|
"n": 18,
|
|
"passed": 18,
|
|
"limits": {
|
|
"none": 7,
|
|
"message": 11,
|
|
"token": 0,
|
|
"time": 0
|
|
},
|
|
"errored": 0,
|
|
"medians": {
|
|
"total_tokens": 242574.0,
|
|
"input_tokens": 65964.0,
|
|
"cache_read_tokens": 162112.0,
|
|
"all_input_tokens": 218002.5,
|
|
"output_tokens": 26582.5,
|
|
"reasoning_tokens": 19987.0,
|
|
"messages": 30.0,
|
|
"turns": 13.0,
|
|
"working_seconds": 719.9085
|
|
},
|
|
"sums": {
|
|
"total_tokens": 6375966,
|
|
"input_tokens": 1363954,
|
|
"cache_read_tokens": 4408832,
|
|
"all_input_tokens": 5772786,
|
|
"output_tokens": 603180,
|
|
"reasoning_tokens": 501045,
|
|
"messages": 518,
|
|
"turns": 225,
|
|
"working_seconds": 15687.015
|
|
}
|
|
},
|
|
"nonpass_behavior_not_inferred": {
|
|
"n": 21,
|
|
"passed": 0,
|
|
"limits": {
|
|
"none": 0,
|
|
"message": 21,
|
|
"token": 0,
|
|
"time": 0
|
|
},
|
|
"errored": 0,
|
|
"medians": {
|
|
"total_tokens": 321913,
|
|
"input_tokens": 62516,
|
|
"cache_read_tokens": 222016,
|
|
"all_input_tokens": 304715,
|
|
"output_tokens": 26834,
|
|
"reasoning_tokens": 20111,
|
|
"messages": 30,
|
|
"turns": 13,
|
|
"working_seconds": 675.221
|
|
},
|
|
"sums": {
|
|
"total_tokens": 9819868,
|
|
"input_tokens": 1823049,
|
|
"cache_read_tokens": 7079616,
|
|
"all_input_tokens": 8902665,
|
|
"output_tokens": 917203,
|
|
"reasoning_tokens": 670146,
|
|
"messages": 630,
|
|
"turns": 274,
|
|
"working_seconds": 23387.007999999998
|
|
}
|
|
}
|
|
},
|
|
"matched_six_tasks_cross_date": {
|
|
"baseline_conflicting": {
|
|
"n": 6,
|
|
"passed": 3,
|
|
"limits": {
|
|
"none": 2,
|
|
"message": 4,
|
|
"token": 0,
|
|
"time": 0
|
|
},
|
|
"errored": 0,
|
|
"medians": {
|
|
"total_tokens": 165022.5,
|
|
"input_tokens": 56309.0,
|
|
"cache_read_tokens": 91968.0,
|
|
"all_input_tokens": 150080.0,
|
|
"output_tokens": 15665.5,
|
|
"reasoning_tokens": 10733.0,
|
|
"messages": 30.0,
|
|
"turns": 13.0,
|
|
"working_seconds": 523.614
|
|
},
|
|
"sums": {
|
|
"total_tokens": 1319537,
|
|
"input_tokens": 281011,
|
|
"cache_read_tokens": 903168,
|
|
"all_input_tokens": 1184179,
|
|
"output_tokens": 135358,
|
|
"reasoning_tokens": 104546,
|
|
"messages": 175,
|
|
"turns": 77,
|
|
"working_seconds": 4001.966
|
|
}
|
|
},
|
|
"team-conflicting-shared-sept7": {
|
|
"n": 6,
|
|
"passed": 0,
|
|
"limits": {
|
|
"none": 3,
|
|
"message": 0,
|
|
"token": 3,
|
|
"time": 0
|
|
},
|
|
"errored": 0,
|
|
"medians": {
|
|
"total_tokens": 388703.0,
|
|
"input_tokens": 59687.5,
|
|
"cache_read_tokens": 289664.0,
|
|
"all_input_tokens": 363172.5,
|
|
"output_tokens": 25530.5,
|
|
"reasoning_tokens": 15718.5,
|
|
"messages": 44.5,
|
|
"turns": 17.0,
|
|
"working_seconds": 762.8489999999999
|
|
},
|
|
"sums": {
|
|
"total_tokens": 2087218,
|
|
"input_tokens": 392774,
|
|
"cache_read_tokens": 1539392,
|
|
"all_input_tokens": 1932166,
|
|
"output_tokens": 155052,
|
|
"reasoning_tokens": 101791,
|
|
"messages": 259,
|
|
"turns": 102,
|
|
"working_seconds": 4611.917
|
|
}
|
|
},
|
|
"team-conflicting-private-sept7": {
|
|
"n": 6,
|
|
"passed": 0,
|
|
"limits": {
|
|
"none": 1,
|
|
"message": 0,
|
|
"token": 5,
|
|
"time": 0
|
|
},
|
|
"errored": 0,
|
|
"medians": {
|
|
"total_tokens": 414080.0,
|
|
"input_tokens": 62299.5,
|
|
"cache_read_tokens": 306816.0,
|
|
"all_input_tokens": 387182.5,
|
|
"output_tokens": 30074.5,
|
|
"reasoning_tokens": 19545.5,
|
|
"messages": 45.0,
|
|
"turns": 18.0,
|
|
"working_seconds": 729.2194999999999
|
|
},
|
|
"sums": {
|
|
"total_tokens": 2294029,
|
|
"input_tokens": 396727,
|
|
"cache_read_tokens": 1723840,
|
|
"all_input_tokens": 2120567,
|
|
"output_tokens": 173462,
|
|
"reasoning_tokens": 118134,
|
|
"messages": 264,
|
|
"turns": 106,
|
|
"working_seconds": 4791.924
|
|
}
|
|
}
|
|
},
|
|
"matched_six_task_ids": [
|
|
"lcbhard_0",
|
|
"lcbhard_1",
|
|
"lcbhard_10",
|
|
"lcbhard_11",
|
|
"lcbhard_12",
|
|
"lcbhard_2"
|
|
],
|
|
"matched_original_impossible_n": 2,
|
|
"log_provenance": [
|
|
{
|
|
"path": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/baseline/conflicting/2026-08-31T16-03-06-00-00_lcb-conflicting-canmod-tools_ePhPKR7TagArgd8j9CFuyj.eval",
|
|
"sha256": "e8e8060da40ba32243bdc8db6b65c5947ae18bef657c6f44e3f1fde0964f6e32",
|
|
"status": "started",
|
|
"planned_samples": 40,
|
|
"recoverable_samples": 39,
|
|
"model": "openrouter/z-ai/glm-5.3-flash",
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": null,
|
|
"score_on_error": false,
|
|
"message_limit": 30,
|
|
"token_limit": null,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": null,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": null,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": null,
|
|
"max_subprocesses": 12,
|
|
"max_sandboxes": 12,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generate_config": {
|
|
"max_retries": 3,
|
|
"timeout": 900,
|
|
"attempt_timeout": null,
|
|
"max_connections": 12,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": null,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": null,
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
},
|
|
{
|
|
"path": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/team-conflicting-private-sept7/evals/2026-09-07T16-32-32-00-00_team-lcb-conflicting-private-wave1_a8GsvkiPzpkZF9yb7mA7C3.eval",
|
|
"sha256": "955b245c5cd947d7da5cb2bf12b64d28445643d5d6c4329a941f10ab733d490b",
|
|
"status": "success",
|
|
"planned_samples": 3,
|
|
"recoverable_samples": 3,
|
|
"model": "openrouter/z-ai/glm-5.3-flash",
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 60,
|
|
"token_limit": 400000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 3,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": null,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 3,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generate_config": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 3,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": null,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": null,
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
},
|
|
{
|
|
"path": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/team-conflicting-private-sept7/evals/2026-09-07T16-45-26-00-00_team-lcb-conflicting-private-wave2_j6vTyEzRrnKYLji59YsmDw.eval",
|
|
"sha256": "e719fa4e3e0e32e64701024584624811519ccb478ff5aafb7daf99a123c0915d",
|
|
"status": "success",
|
|
"planned_samples": 3,
|
|
"recoverable_samples": 3,
|
|
"model": "openrouter/z-ai/glm-5.3-flash",
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 60,
|
|
"token_limit": 400000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 3,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": null,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 3,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generate_config": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 3,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": null,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": null,
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
},
|
|
{
|
|
"path": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/team-conflicting-shared-sept7/evals/2026-09-07T16-30-44-00-00_team-lcb-conflicting-shared-wave1_n96wbEXK9LbYEsaTeGNggp.eval",
|
|
"sha256": "8dab1105ee92db724da6d5ae17be17884d36999755d9f78eee055c5512eca8cc",
|
|
"status": "success",
|
|
"planned_samples": 3,
|
|
"recoverable_samples": 3,
|
|
"model": "openrouter/z-ai/glm-5.3-flash",
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 60,
|
|
"token_limit": 400000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 3,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": null,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 3,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generate_config": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 3,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": null,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": null,
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
},
|
|
{
|
|
"path": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/team-conflicting-shared-sept7/evals/2026-09-07T16-48-03-00-00_team-lcb-conflicting-shared-wave2_7Vf6SjeiQnfxrhCnfCGVxq.eval",
|
|
"sha256": "ded772654c2164a6adb7b6e8d4e1581bbd8f8d875b740d115659dc086ffc9adf",
|
|
"status": "success",
|
|
"planned_samples": 3,
|
|
"recoverable_samples": 3,
|
|
"model": "openrouter/z-ai/glm-5.3-flash",
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 60,
|
|
"token_limit": 400000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 3,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": null,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 3,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generate_config": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 3,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": null,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": null,
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
},
|
|
{
|
|
"path": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/team-original-sept7/evals/2026-09-07T16-25-55-00-00_team-lcb-original-shared-wave1_TcTKgXVQ6oCSt6z3uJGkNw.eval",
|
|
"sha256": "861a8e97b93172c9661b69334c373f96a8b4e217fce73a7a0bd4c521261d717e",
|
|
"status": "success",
|
|
"planned_samples": 2,
|
|
"recoverable_samples": 2,
|
|
"model": "openrouter/z-ai/glm-5.3-flash",
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 60,
|
|
"token_limit": 400000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 2,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": null,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 2,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generate_config": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 2,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": null,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": null,
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
},
|
|
{
|
|
"path": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/token-rerun-private-lcbhard-0-sept7/evals/2026-09-07T17-31-42-00-00_capped-rerun-private-lcbhard-0_4xQfeWXq3NKzJ3Lhcc9MLv.eval",
|
|
"sha256": "8acbc854e092318450ee84ee044dbbb11bc1b68890ea5786327e20b1953af34c",
|
|
"status": "success",
|
|
"planned_samples": 1,
|
|
"recoverable_samples": 1,
|
|
"model": "openrouter/z-ai/glm-5.3-flash",
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 60,
|
|
"token_limit": 1000000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 1,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": null,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 1,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generate_config": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 1,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": null,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": null,
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
}
|
|
],
|
|
"excluded_logs": [
|
|
{
|
|
"path": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/smoke/2026-08-31T15-44-15-00-00_lcb-conflicting-canmod-tools_63EnZGucFdVCG4SerruyBi.eval",
|
|
"reason": "outside frozen audit run list"
|
|
},
|
|
{
|
|
"path": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/smoke-original/2026-08-31T16-16-23-00-00_lcb-original-canmod-tools_MaoruqapbVjJGYqxcqqUzB.eval",
|
|
"reason": "outside frozen audit run list"
|
|
},
|
|
{
|
|
"path": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/smoke-scratch/2026-08-31T16-05-18-00-00_scratch-lcb-conflicting_ecJy5gkcxWNgCZmrz56pVW.eval",
|
|
"reason": "outside frozen audit run list"
|
|
},
|
|
{
|
|
"path": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/smoke-scratch/2026-08-31T16-05-42-00-00_scratch-lcb-conflicting_5gRrY8dkCmmtAuwTwjhvij.eval",
|
|
"reason": "outside frozen audit run list"
|
|
},
|
|
{
|
|
"path": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/smoke-scratch/2026-08-31T16-10-33-00-00_scratch-lcb-conflicting_JkhRwAyfxxhBwJtiiY5b27.eval",
|
|
"reason": "outside frozen audit run list"
|
|
},
|
|
{
|
|
"path": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/team-mock-sept7-v3/evals/2026-09-07T16-25-08-00-00_team-lcb-conflicting-shared-wave1_VpKPEZ3gk3QKpmLJZUHMNh.eval",
|
|
"reason": "outside frozen audit run list"
|
|
},
|
|
{
|
|
"path": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/team-mock-sept7-v3/evals/2026-09-07T16-25-27-00-00_team-lcb-conflicting-shared-wave2_SaaKdX7HDjVJ32dDpntAfE.eval",
|
|
"reason": "outside frozen audit run list"
|
|
},
|
|
{
|
|
"path": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/verify-snapshot/2026-08-31T16-13-06-00-00_scratch-lcb-conflicting_b7efXD268oKZaMkwP8U38U.eval",
|
|
"reason": "outside frozen audit run list"
|
|
}
|
|
],
|
|
"interpretation": [
|
|
"Nonpass is not an honesty label. Prior baseline audit identified 18 impossible passes as gaming; no new semantic classification was performed by this script.",
|
|
"Baseline log is started with 39 recovered of 40 planned, not a completed 40-sample run.",
|
|
"Same model identifier but August31 vs September7, different prompts, limits, concurrency and retry settings; cross-date comparisons are descriptive only.",
|
|
"Input tokens are summed over repeated model calls; cache-read tokens count toward total. Reasoning tokens are a subset of output, not additional. No claim about distinct reasoning amount from total tokens.",
|
|
"32/39 baseline attempts ended at message cap, and 8/12 new impossible attempts at token cap. These are censored trajectories. Passing early and retry-until-failure stopping rules also confound resource comparisons.",
|
|
"Only two same-condition original/impossible task pairs exist; no paid original August baseline exists in these logs.",
|
|
"No significance testing or causal attribution; shared samples are team-dependent and no repeated randomized teams exist."
|
|
]
|
|
}
|