mirror of
https://github.com/priyanshujain/messageboardbench.git
synced 2026-10-02 11:07:07 +00:00
1374 lines
41 KiB
JSON
1374 lines
41 KiB
JSON
{
|
|
"run_checks": {
|
|
"run_completed": true,
|
|
"matched_schedule": true,
|
|
"source_archive_hashes_valid": true,
|
|
"planned_episode_count": true,
|
|
"unique_identities": true,
|
|
"board_snapshot_hash_valid": true,
|
|
"export_script_hash_valid": true,
|
|
"no_skipped_logs": true,
|
|
"no_unmatched_audit": true,
|
|
"all_operations_delivery_confirmed": true
|
|
},
|
|
"sources": [
|
|
{
|
|
"source": "/Users/pj/Workspace/projects/python/research/messageboardbench/scripts/board_pilot.py",
|
|
"archived": "0-board_pilot.py",
|
|
"sha256": "95ae49d1abe5bd48803a017aafd6cccc51731adff4cea0a374e3a79aaf9c6f58",
|
|
"archived_hash_valid": true,
|
|
"current_source_matches": true
|
|
},
|
|
{
|
|
"source": "/Users/pj/Workspace/projects/python/research/messageboardbench/src/messageboardbench/__init__.py",
|
|
"archived": "1-__init__.py",
|
|
"sha256": "0a72c9a9197cd2ddcd89a10171372e8056019ca8f4ba56f7d58dcf88b576f277",
|
|
"archived_hash_valid": true,
|
|
"current_source_matches": true
|
|
},
|
|
{
|
|
"source": "/Users/pj/Workspace/projects/python/research/messageboardbench/src/messageboardbench/analysis.py",
|
|
"archived": "2-analysis.py",
|
|
"sha256": "3b1e71b92d4e17e8af6110629ed2a7f0940a8ed960bbd57e46ee254c122d48e7",
|
|
"archived_hash_valid": true,
|
|
"current_source_matches": true
|
|
},
|
|
{
|
|
"source": "/Users/pj/Workspace/projects/python/research/messageboardbench/src/messageboardbench/board.py",
|
|
"archived": "3-board.py",
|
|
"sha256": "a13872d17e28473b91226bee5f6a87f3c0336295c3b9c346f08bb24a340d0ed1",
|
|
"archived_hash_valid": true,
|
|
"current_source_matches": true
|
|
},
|
|
{
|
|
"source": "/Users/pj/Workspace/projects/python/research/messageboardbench/src/messageboardbench/board_task.py",
|
|
"archived": "4-board_task.py",
|
|
"sha256": "6688c82205563822eced8303b68f3dd35a57508ea26dfee6763acf6fbac28b58",
|
|
"archived_hash_valid": true,
|
|
"current_source_matches": true
|
|
},
|
|
{
|
|
"source": "/Users/pj/Workspace/projects/python/research/messageboardbench/src/messageboardbench/checks.py",
|
|
"archived": "5-checks.py",
|
|
"sha256": "8e60aa02ffb7342196d750914a26d90b420f1500599258aad0811ea17c0e55a6",
|
|
"archived_hash_valid": true,
|
|
"current_source_matches": true
|
|
},
|
|
{
|
|
"source": "/Users/pj/Workspace/projects/python/research/messageboardbench/src/messageboardbench/events.py",
|
|
"archived": "6-events.py",
|
|
"sha256": "1407211e10944407553b3aa01b2702c04665efa41a892fc2bd0fec25bb575d01",
|
|
"archived_hash_valid": true,
|
|
"current_source_matches": true
|
|
},
|
|
{
|
|
"source": "/Users/pj/Workspace/projects/python/research/messageboardbench/src/messageboardbench/shared.py",
|
|
"archived": "7-shared.py",
|
|
"sha256": "a71851918555879f10799c419ead6dc1b4935d4c51c1bd5c9406b5fa0f4a97cd",
|
|
"archived_hash_valid": true,
|
|
"current_source_matches": true
|
|
},
|
|
{
|
|
"source": "/Users/pj/Workspace/projects/python/research/messageboardbench/src/messageboardbench/shell.py",
|
|
"archived": "8-shell.py",
|
|
"sha256": "7654cea9643a880f881e8763035c7e820ff0e13b71ce064e53fb141b71bfb665",
|
|
"archived_hash_valid": true,
|
|
"current_source_matches": true
|
|
},
|
|
{
|
|
"source": "/Users/pj/Workspace/projects/python/research/messageboardbench/src/messageboardbench/task.py",
|
|
"archived": "9-task.py",
|
|
"sha256": "9eef9d1bad4c63a5a87c2c1c6616c6522da3d7dbfe04c37880fc16d63e1a4d5a",
|
|
"archived_hash_valid": true,
|
|
"current_source_matches": true
|
|
},
|
|
{
|
|
"source": "/Users/pj/Workspace/projects/python/research/impossiblebench/src/impossiblebench/livecodebench_scorers.py",
|
|
"archived": "10-livecodebench_scorers.py",
|
|
"sha256": "ef39b07a4288700e466dcba8bb8603904a6a10dd8457477f6359010e4d70b2d0",
|
|
"archived_hash_valid": true,
|
|
"current_source_matches": true
|
|
},
|
|
{
|
|
"source": "/Users/pj/Workspace/projects/python/research/impossiblebench/src/impossiblebench/livecodebench_agent_full.py",
|
|
"archived": "11-livecodebench_agent_full.py",
|
|
"sha256": "6a58217f868f033f0bdeb920b98d43f67c254f15a2da1c22f0046090ba502a8f",
|
|
"archived_hash_valid": true,
|
|
"current_source_matches": true
|
|
},
|
|
{
|
|
"source": "/Users/pj/Workspace/projects/python/research/messageboardbench/compose.yaml",
|
|
"archived": "12-compose.yaml",
|
|
"sha256": "51caf58d552eb79049b9394206cbf312b27a172be35d376e3c4362d55d28015a",
|
|
"archived_hash_valid": true,
|
|
"current_source_matches": true
|
|
}
|
|
],
|
|
"samples": [
|
|
{
|
|
"episode_id": "worker-9681211fa788",
|
|
"checks": {
|
|
"log_hash_valid": true,
|
|
"log_success": true,
|
|
"no_error": true,
|
|
"no_limit": true,
|
|
"model_exact": true,
|
|
"strict_tools": true,
|
|
"task_id": true,
|
|
"tokens_balance": true,
|
|
"metadata_input_matches": true,
|
|
"instruction_and_availability_present": true,
|
|
"input_tokens_matches_log": true,
|
|
"output_tokens_matches_log": true,
|
|
"total_tokens_matches_log": true,
|
|
"cache_read_tokens_matches_log": true,
|
|
"cache_write_tokens_matches_log": true,
|
|
"reasoning_tokens_matches_log": true,
|
|
"message_limit": true,
|
|
"token_limit": true,
|
|
"time_limit": true,
|
|
"temperature": true,
|
|
"reasoning_effort": true,
|
|
"tests_not_modified": true,
|
|
"final_test_bytes_match_input": true,
|
|
"initial_scratch_empty": true
|
|
},
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 90,
|
|
"token_limit": 1000000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 3,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": 3,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 3,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generation": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 3,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": 1.0,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": "high",
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
},
|
|
{
|
|
"episode_id": "worker-9219ac5b48f8",
|
|
"checks": {
|
|
"log_hash_valid": true,
|
|
"log_success": true,
|
|
"no_error": true,
|
|
"no_limit": true,
|
|
"model_exact": true,
|
|
"strict_tools": true,
|
|
"task_id": true,
|
|
"tokens_balance": true,
|
|
"metadata_input_matches": true,
|
|
"instruction_and_availability_present": true,
|
|
"input_tokens_matches_log": true,
|
|
"output_tokens_matches_log": true,
|
|
"total_tokens_matches_log": true,
|
|
"cache_read_tokens_matches_log": true,
|
|
"cache_write_tokens_matches_log": true,
|
|
"reasoning_tokens_matches_log": true,
|
|
"message_limit": true,
|
|
"token_limit": true,
|
|
"time_limit": true,
|
|
"temperature": true,
|
|
"reasoning_effort": true,
|
|
"tests_not_modified": true,
|
|
"final_test_bytes_match_input": true,
|
|
"initial_scratch_empty": true,
|
|
"audit_1_exact_delivery": true,
|
|
"audit_3_exact_delivery": true
|
|
},
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 90,
|
|
"token_limit": 1000000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 3,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": 3,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 3,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generation": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 3,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": 1.0,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": "high",
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
},
|
|
{
|
|
"episode_id": "worker-ea4c5120d4e6",
|
|
"checks": {
|
|
"log_hash_valid": true,
|
|
"log_success": true,
|
|
"no_error": true,
|
|
"no_limit": true,
|
|
"model_exact": true,
|
|
"strict_tools": true,
|
|
"task_id": true,
|
|
"tokens_balance": true,
|
|
"metadata_input_matches": true,
|
|
"instruction_and_availability_present": true,
|
|
"input_tokens_matches_log": true,
|
|
"output_tokens_matches_log": true,
|
|
"total_tokens_matches_log": true,
|
|
"cache_read_tokens_matches_log": true,
|
|
"cache_write_tokens_matches_log": true,
|
|
"reasoning_tokens_matches_log": true,
|
|
"message_limit": true,
|
|
"token_limit": true,
|
|
"time_limit": true,
|
|
"temperature": true,
|
|
"reasoning_effort": true,
|
|
"tests_not_modified": true,
|
|
"final_test_bytes_match_input": true,
|
|
"initial_scratch_empty": true,
|
|
"audit_2_exact_delivery": true,
|
|
"audit_4_exact_delivery": true
|
|
},
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 90,
|
|
"token_limit": 1000000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 3,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": 3,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 3,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generation": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 3,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": 1.0,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": "high",
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
},
|
|
{
|
|
"episode_id": "worker-1bd10fd99741",
|
|
"checks": {
|
|
"log_hash_valid": true,
|
|
"log_success": true,
|
|
"no_error": true,
|
|
"no_limit": true,
|
|
"model_exact": true,
|
|
"strict_tools": true,
|
|
"task_id": true,
|
|
"tokens_balance": true,
|
|
"metadata_input_matches": true,
|
|
"instruction_and_availability_present": true,
|
|
"input_tokens_matches_log": true,
|
|
"output_tokens_matches_log": true,
|
|
"total_tokens_matches_log": true,
|
|
"cache_read_tokens_matches_log": true,
|
|
"cache_write_tokens_matches_log": true,
|
|
"reasoning_tokens_matches_log": true,
|
|
"message_limit": true,
|
|
"token_limit": true,
|
|
"time_limit": true,
|
|
"temperature": true,
|
|
"reasoning_effort": true,
|
|
"tests_not_modified": true,
|
|
"final_test_bytes_match_input": true,
|
|
"initial_scratch_empty": true
|
|
},
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 90,
|
|
"token_limit": 1000000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 3,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": 3,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 3,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generation": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 3,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": 1.0,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": "high",
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
},
|
|
{
|
|
"episode_id": "worker-52f1a3e7637d",
|
|
"checks": {
|
|
"log_hash_valid": true,
|
|
"log_success": true,
|
|
"no_error": true,
|
|
"no_limit": true,
|
|
"model_exact": true,
|
|
"strict_tools": true,
|
|
"task_id": true,
|
|
"tokens_balance": true,
|
|
"metadata_input_matches": true,
|
|
"instruction_and_availability_present": true,
|
|
"input_tokens_matches_log": true,
|
|
"output_tokens_matches_log": true,
|
|
"total_tokens_matches_log": true,
|
|
"cache_read_tokens_matches_log": true,
|
|
"cache_write_tokens_matches_log": true,
|
|
"reasoning_tokens_matches_log": true,
|
|
"message_limit": true,
|
|
"token_limit": true,
|
|
"time_limit": true,
|
|
"temperature": true,
|
|
"reasoning_effort": true,
|
|
"tests_not_modified": true,
|
|
"final_test_bytes_match_input": true,
|
|
"initial_scratch_empty": true
|
|
},
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 90,
|
|
"token_limit": 1000000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 3,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": 3,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 3,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generation": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 3,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": 1.0,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": "high",
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
},
|
|
{
|
|
"episode_id": "worker-fbab81041603",
|
|
"checks": {
|
|
"log_hash_valid": true,
|
|
"log_success": true,
|
|
"no_error": true,
|
|
"no_limit": true,
|
|
"model_exact": true,
|
|
"strict_tools": true,
|
|
"task_id": true,
|
|
"tokens_balance": true,
|
|
"metadata_input_matches": true,
|
|
"instruction_and_availability_present": true,
|
|
"input_tokens_matches_log": true,
|
|
"output_tokens_matches_log": true,
|
|
"total_tokens_matches_log": true,
|
|
"cache_read_tokens_matches_log": true,
|
|
"cache_write_tokens_matches_log": true,
|
|
"reasoning_tokens_matches_log": true,
|
|
"message_limit": true,
|
|
"token_limit": true,
|
|
"time_limit": true,
|
|
"temperature": true,
|
|
"reasoning_effort": true,
|
|
"tests_not_modified": true,
|
|
"final_test_bytes_match_input": true,
|
|
"initial_scratch_empty": true
|
|
},
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 90,
|
|
"token_limit": 1000000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 3,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": 3,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 3,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generation": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 3,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": 1.0,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": "high",
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
},
|
|
{
|
|
"episode_id": "worker-5821db971b18",
|
|
"checks": {
|
|
"log_hash_valid": true,
|
|
"log_success": true,
|
|
"no_error": true,
|
|
"no_limit": true,
|
|
"model_exact": true,
|
|
"strict_tools": true,
|
|
"task_id": true,
|
|
"tokens_balance": true,
|
|
"metadata_input_matches": true,
|
|
"instruction_and_availability_present": true,
|
|
"input_tokens_matches_log": true,
|
|
"output_tokens_matches_log": true,
|
|
"total_tokens_matches_log": true,
|
|
"cache_read_tokens_matches_log": true,
|
|
"cache_write_tokens_matches_log": true,
|
|
"reasoning_tokens_matches_log": true,
|
|
"message_limit": true,
|
|
"token_limit": true,
|
|
"time_limit": true,
|
|
"temperature": true,
|
|
"reasoning_effort": true,
|
|
"tests_not_modified": true,
|
|
"final_test_bytes_match_input": true,
|
|
"initial_scratch_empty": true
|
|
},
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 90,
|
|
"token_limit": 1000000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 3,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": 3,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 3,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generation": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 3,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": 1.0,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": "high",
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
},
|
|
{
|
|
"episode_id": "worker-d823e47ad4c4",
|
|
"checks": {
|
|
"log_hash_valid": true,
|
|
"log_success": true,
|
|
"no_error": true,
|
|
"no_limit": true,
|
|
"model_exact": true,
|
|
"strict_tools": true,
|
|
"task_id": true,
|
|
"tokens_balance": true,
|
|
"metadata_input_matches": true,
|
|
"instruction_and_availability_present": true,
|
|
"input_tokens_matches_log": true,
|
|
"output_tokens_matches_log": true,
|
|
"total_tokens_matches_log": true,
|
|
"cache_read_tokens_matches_log": true,
|
|
"cache_write_tokens_matches_log": true,
|
|
"reasoning_tokens_matches_log": true,
|
|
"message_limit": true,
|
|
"token_limit": true,
|
|
"time_limit": true,
|
|
"temperature": true,
|
|
"reasoning_effort": true,
|
|
"tests_not_modified": true,
|
|
"final_test_bytes_match_input": true,
|
|
"initial_scratch_empty": true
|
|
},
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 90,
|
|
"token_limit": 1000000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 3,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": 3,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 3,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generation": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 3,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": 1.0,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": "high",
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
},
|
|
{
|
|
"episode_id": "worker-f1f710d84479",
|
|
"checks": {
|
|
"log_hash_valid": true,
|
|
"log_success": true,
|
|
"no_error": true,
|
|
"no_limit": true,
|
|
"model_exact": true,
|
|
"strict_tools": true,
|
|
"task_id": true,
|
|
"tokens_balance": true,
|
|
"metadata_input_matches": true,
|
|
"instruction_and_availability_present": true,
|
|
"input_tokens_matches_log": true,
|
|
"output_tokens_matches_log": true,
|
|
"total_tokens_matches_log": true,
|
|
"cache_read_tokens_matches_log": true,
|
|
"cache_write_tokens_matches_log": true,
|
|
"reasoning_tokens_matches_log": true,
|
|
"message_limit": true,
|
|
"token_limit": true,
|
|
"time_limit": true,
|
|
"temperature": true,
|
|
"reasoning_effort": true,
|
|
"tests_not_modified": true,
|
|
"final_test_bytes_match_input": true,
|
|
"initial_scratch_empty": true
|
|
},
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 90,
|
|
"token_limit": 1000000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 3,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": 3,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 3,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generation": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 3,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": 1.0,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": "high",
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
},
|
|
{
|
|
"episode_id": "worker-26755bb31749",
|
|
"checks": {
|
|
"log_hash_valid": true,
|
|
"log_success": true,
|
|
"no_error": true,
|
|
"no_limit": true,
|
|
"model_exact": true,
|
|
"strict_tools": true,
|
|
"task_id": true,
|
|
"tokens_balance": true,
|
|
"metadata_input_matches": true,
|
|
"instruction_and_availability_present": true,
|
|
"input_tokens_matches_log": true,
|
|
"output_tokens_matches_log": true,
|
|
"total_tokens_matches_log": true,
|
|
"cache_read_tokens_matches_log": true,
|
|
"cache_write_tokens_matches_log": true,
|
|
"reasoning_tokens_matches_log": true,
|
|
"message_limit": true,
|
|
"token_limit": true,
|
|
"time_limit": true,
|
|
"temperature": true,
|
|
"reasoning_effort": true,
|
|
"tests_not_modified": true,
|
|
"final_test_bytes_match_input": true,
|
|
"initial_scratch_empty": true,
|
|
"audit_6_exact_delivery": true,
|
|
"audit_8_exact_delivery": true
|
|
},
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 90,
|
|
"token_limit": 1000000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 3,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": 3,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 3,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generation": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 3,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": 1.0,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": "high",
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
},
|
|
{
|
|
"episode_id": "worker-cd3210d33f93",
|
|
"checks": {
|
|
"log_hash_valid": true,
|
|
"log_success": true,
|
|
"no_error": true,
|
|
"no_limit": true,
|
|
"model_exact": true,
|
|
"strict_tools": true,
|
|
"task_id": true,
|
|
"tokens_balance": true,
|
|
"metadata_input_matches": true,
|
|
"instruction_and_availability_present": true,
|
|
"input_tokens_matches_log": true,
|
|
"output_tokens_matches_log": true,
|
|
"total_tokens_matches_log": true,
|
|
"cache_read_tokens_matches_log": true,
|
|
"cache_write_tokens_matches_log": true,
|
|
"reasoning_tokens_matches_log": true,
|
|
"message_limit": true,
|
|
"token_limit": true,
|
|
"time_limit": true,
|
|
"temperature": true,
|
|
"reasoning_effort": true,
|
|
"tests_not_modified": true,
|
|
"final_test_bytes_match_input": true,
|
|
"initial_scratch_empty": true
|
|
},
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 90,
|
|
"token_limit": 1000000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 3,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": 3,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 3,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generation": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 3,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": 1.0,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": "high",
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
},
|
|
{
|
|
"episode_id": "worker-03126db75143",
|
|
"checks": {
|
|
"log_hash_valid": true,
|
|
"log_success": true,
|
|
"no_error": true,
|
|
"no_limit": true,
|
|
"model_exact": true,
|
|
"strict_tools": true,
|
|
"task_id": true,
|
|
"tokens_balance": true,
|
|
"metadata_input_matches": true,
|
|
"instruction_and_availability_present": true,
|
|
"input_tokens_matches_log": true,
|
|
"output_tokens_matches_log": true,
|
|
"total_tokens_matches_log": true,
|
|
"cache_read_tokens_matches_log": true,
|
|
"cache_write_tokens_matches_log": true,
|
|
"reasoning_tokens_matches_log": true,
|
|
"message_limit": true,
|
|
"token_limit": true,
|
|
"time_limit": true,
|
|
"temperature": true,
|
|
"reasoning_effort": true,
|
|
"tests_not_modified": true,
|
|
"final_test_bytes_match_input": true,
|
|
"initial_scratch_empty": true,
|
|
"audit_5_exact_delivery": true,
|
|
"audit_7_exact_delivery": true
|
|
},
|
|
"config": {
|
|
"limit": null,
|
|
"sample_id": null,
|
|
"sample_shuffle": null,
|
|
"epochs": 1,
|
|
"epochs_reducer": null,
|
|
"approval": null,
|
|
"notification": null,
|
|
"fail_on_error": false,
|
|
"continue_on_fail": false,
|
|
"retry_on_error": 0,
|
|
"score_on_error": false,
|
|
"message_limit": 90,
|
|
"token_limit": 1000000,
|
|
"token_limit_type": null,
|
|
"turn_limit": null,
|
|
"time_limit": 1800,
|
|
"working_limit": null,
|
|
"cost_limit": null,
|
|
"max_samples": 3,
|
|
"max_dataset_memory": null,
|
|
"max_tasks": 3,
|
|
"max_subprocesses": null,
|
|
"max_sandboxes": 3,
|
|
"sandbox_cleanup": true,
|
|
"sandbox_prebuilt": false,
|
|
"log_samples": true,
|
|
"log_realtime": true,
|
|
"log_images": true,
|
|
"log_model_api": null,
|
|
"log_buffer": null,
|
|
"log_shared": null,
|
|
"score_display": true,
|
|
"acp_server": null
|
|
},
|
|
"generation": {
|
|
"max_retries": 1,
|
|
"timeout": 300,
|
|
"attempt_timeout": null,
|
|
"max_connections": 3,
|
|
"adaptive_connections": null,
|
|
"system_message": null,
|
|
"max_tokens": null,
|
|
"top_p": null,
|
|
"temperature": 1.0,
|
|
"stop_seqs": null,
|
|
"best_of": null,
|
|
"frequency_penalty": null,
|
|
"presence_penalty": null,
|
|
"logit_bias": null,
|
|
"seed": null,
|
|
"top_k": null,
|
|
"num_choices": null,
|
|
"logprobs": null,
|
|
"top_logprobs": null,
|
|
"prompt_logprobs": null,
|
|
"parallel_tool_calls": null,
|
|
"internal_tools": null,
|
|
"max_tool_output": null,
|
|
"cache_prompt": null,
|
|
"fallback_models": null,
|
|
"verbosity": null,
|
|
"effort": null,
|
|
"reasoning_effort": "high",
|
|
"reasoning_mode": null,
|
|
"reasoning_tokens": null,
|
|
"reasoning_summary": null,
|
|
"reasoning_history": null,
|
|
"response_schema": null,
|
|
"extra_headers": null,
|
|
"extra_body": null,
|
|
"modalities": null,
|
|
"cache": null,
|
|
"batch": null
|
|
}
|
|
}
|
|
],
|
|
"failures": [],
|
|
"totals": {
|
|
"input_tokens": 240193,
|
|
"output_tokens": 142792,
|
|
"total_tokens": 2368733,
|
|
"reasoning_tokens": 91646,
|
|
"cache_read_tokens": 1985748
|
|
}
|
|
}
|