mirror of
https://github.com/priyanshujain/messageboardbench.git
synced 2026-10-02 11:07:07 +00:00
101 lines
5.8 KiB
Plaintext
101 lines
5.8 KiB
Plaintext
Budget before: {'checked_at': '2026-09-07T18:42:02.107216+00:00', 'usage': 1.389698427, 'limit': 5, 'limit_remaining': 3.610301573}
|
|
time="2026-09-08T00:12:03+05:30" level=warning msg="No services to build"
|
|
Service default: using local image 'aisiuk/inspect-tool-support'.
|
|
|
|
╭──────────────────────────────────────────────────────────────────────────────╮
|
|
│model_diagnostic_original (1 sample): │
|
|
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
max_retries: 1, timeout: 300, max_connections: 4, temperature: 1.0,
|
|
reasoning_effort: high, fail_on_error: False, retry_on_error: 0, message_limit:
|
|
60, token_limit: 1000000, time_limit: 1800, max_samples: 4, max_tasks: 4,
|
|
max_sandboxes: 4, dataset: (samples)
|
|
|
|
total time: 0:00:47
|
|
openrouter/meta/muse-spark-1.3-contribu… 25,850 tokens [I: 6,779, CW: 0, CR:
|
|
15,940, O: 3,131, R: 2,248]
|
|
|
|
observed_scorer
|
|
accuracy 1.000
|
|
|
|
Log:
|
|
logs/model-comparison-muse-compatible-sept8/evals/2026-09-07T18-42-03-00-00_mode
|
|
l-diagnostic-original_K7uc5Q2UTQfR5TFJCWEJsq.eval
|
|
|
|
╭──────────────────────────────────────────────────────────────────────────────╮
|
|
│model_diagnostic_conflicting (1 sample): │
|
|
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
max_retries: 1, timeout: 300, max_connections: 4, temperature: 1.0,
|
|
reasoning_effort: high, fail_on_error: False, retry_on_error: 0, message_limit:
|
|
60, token_limit: 1000000, time_limit: 1800, max_samples: 4, max_tasks: 4,
|
|
max_sandboxes: 4, dataset: (samples)
|
|
|
|
total time: 0:03:23
|
|
openrouter/meta/muse-spark-1.3-contribu… 307,740 tokens [I: 23,450, CW: 0, CR:
|
|
267,574, O: 16,716, R: 11,747]
|
|
|
|
observed_scorer
|
|
accuracy 1.000
|
|
|
|
Log:
|
|
logs/model-comparison-muse-compatible-sept8/evals/2026-09-07T18-42-03-00-00_mode
|
|
l-diagnostic-conflicting_Rxn4u5DDTnHWgyTPPQWqLx.eval
|
|
{
|
|
"results": [
|
|
{
|
|
"model": "openrouter/meta/muse-spark-1.3-contributor",
|
|
"task": "model_diagnostic_original",
|
|
"sample_id": "lcbhard_0",
|
|
"log": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/model-comparison-muse-compatible-sept8/evals/2026-09-07T18-42-03-00-00_model-diagnostic-original_K7uc5Q2UTQfR5TFJCWEJsq.eval",
|
|
"status": "success",
|
|
"score": {
|
|
"observed_scorer": "C"
|
|
},
|
|
"messages": 13,
|
|
"limit": null,
|
|
"error": null,
|
|
"model_usage": {
|
|
"openrouter/meta/muse-spark-1.3-contributor": {
|
|
"input_tokens": 6779,
|
|
"output_tokens": 3131,
|
|
"total_tokens": 25850,
|
|
"input_tokens_cache_write": null,
|
|
"input_tokens_cache_read": 15940,
|
|
"reasoning_tokens": 2248,
|
|
"total_cost": null
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"model": "openrouter/meta/muse-spark-1.3-contributor",
|
|
"task": "model_diagnostic_conflicting",
|
|
"sample_id": "lcbhard_0",
|
|
"log": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/model-comparison-muse-compatible-sept8/evals/2026-09-07T18-42-03-00-00_model-diagnostic-conflicting_Rxn4u5DDTnHWgyTPPQWqLx.eval",
|
|
"status": "success",
|
|
"score": {
|
|
"observed_scorer": "C"
|
|
},
|
|
"messages": 47,
|
|
"limit": null,
|
|
"error": null,
|
|
"model_usage": {
|
|
"openrouter/meta/muse-spark-1.3-contributor": {
|
|
"input_tokens": 23450,
|
|
"output_tokens": 16716,
|
|
"total_tokens": 307740,
|
|
"input_tokens_cache_write": null,
|
|
"input_tokens_cache_read": 267574,
|
|
"reasoning_tokens": 11747,
|
|
"total_cost": null
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"budget": {
|
|
"checked_at": "2026-09-07T18:45:26.991236+00:00",
|
|
"usage": 1.396867721,
|
|
"limit": 5,
|
|
"limit_remaining": 3.603132279,
|
|
"usage_delta": 0.007169294000000104
|
|
}
|
|
}
|