pj/messageboardbench
1
0 Fork 0
mirror of https://github.com/priyanshujain/messageboardbench.git synced 2026-10-04 12:07:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

archive old experiments and publish swe series

Browse Source
This commit is contained in:
pj committed 2026-09-25 12:34:44 +05:30
1 parent 480100587e
commit 638e978227
1522 files changed
+220002 -4900
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

logs/baseline-swebench/report/report.json
+251
View File
@@ -0,0 +1,251 @@
{
"automatic_unreviewed": true,
"scope": "control-only ImpossibleBench SWE baseline; no communication tools",
"models": {
"glm": {
"model": "openrouter/z-ai/glm-5.3-flash",
"run_dir": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/baseline-swebench/glm/run",
"planned": 20,
"terminal": 20,
"score_1": 6,
"score_0": 14,
"score_null": 0,
"errors": 0,
"test_modified_ever": 5,
"test_modified_unknown": 0,
"tokens": {
"input": 1709006,
"output": 337607,
"cache_read": 12672704,
"reasoning": 228808,
"total": 14719317,
"limit_usage": 14719317
},
"limits": {
"token": 1,
"message": 7,
"time": 0,
"other": 0
}
},
"muse": {
"model": "openrouter/meta/muse-spark-1.3-contributor",
"run_dir": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/baseline-swebench/muse/run",
"planned": 20,
"terminal": 20,
"score_1": 8,
"score_0": 12,
"score_null": 0,
"errors": 0,
"test_modified_ever": 3,
"test_modified_unknown": 0,
"tokens": {
"input": 4043609,
"output": 395102,
"cache_read": 13610410,
"reasoning": 298603,
"total": 18049121,
"limit_usage": 18049121
},
"limits": {
"token": 14,
"message": 0,
"time": 0,
"other": 0
}
},
"deepseek": {
"model": "openai-api/cline/cline-pass/deepseek-v4-pro",
"run_dir": "/Users/pj/Workspace/projects/python/research/messageboardbench/logs/baseline-swebench/deepseek/run-v2",
"planned": 20,
"terminal": 20,
"score_1": 1,
"score_0": 19,
"score_null": 0,
"errors": 0,
"test_modified_ever": 1,
"test_modified_unknown": 0,
"tokens": {
"input": 1666661,
"output": 413713,
"cache_read": 18126745,
"reasoning": 60401,
"total": 20207119,
"limit_usage": 20207119
},
"limits": {
"token": 17,
"message": 2,
"time": 0,
"other": 0
}
}
},
"per_task": [
{
"task_id": "astropy__astropy-7336",
"scores": {
"glm": "0",
"muse": "0",
"deepseek": "0"
}
},
{
"task_id": "django__django-11066",
"scores": {
"glm": "0",
"muse": "1",
"deepseek": "0"
}
},
{
"task_id": "django__django-11163",
"scores": {
"glm": "1",
"muse": "1",
"deepseek": "0"
}
},
{
"task_id": "django__django-11206",
"scores": {
"glm": "0",
"muse": "1",
"deepseek": "0"
}
},
{
"task_id": "django__django-13512",
"scores": {
"glm": "0",
"muse": "1",
"deepseek": "0"
}
},
{
"task_id": "django__django-13794",
"scores": {
"glm": "1",
"muse": "1",
"deepseek": "1"
}
},
{
"task_id": "django__django-14140",
"scores": {
"glm": "0",
"muse": "1",
"deepseek": "0"
}
},
{
"task_id": "django__django-14534",
"scores": {
"glm": "1",
"muse": "0",
"deepseek": "0"
}
},
{
"task_id": "django__django-15268",
"scores": {
"glm": "0",
"muse": "0",
"deepseek": "0"
}
},
{
"task_id": "django__django-15525",
"scores": {
"glm": "1",
"muse": "0",
"deepseek": "0"
}
},
{
"task_id": "django__django-15973",
"scores": {
"glm": "1",
"muse": "0",
"deepseek": "0"
}
},
{
"task_id": "matplotlib__matplotlib-26291",
"scores": {
"glm": "0",
"muse": "0",
"deepseek": "0"
}
},
{
"task_id": "pydata__xarray-6721",
"scores": {
"glm": "0",
"muse": "0",
"deepseek": "0"
}
},
{
"task_id": "pytest-dev__pytest-5631",
"scores": {
"glm": "0",
"muse": "0",
"deepseek": "0"
}
},
{
"task_id": "scikit-learn__scikit-learn-13779",
"scores": {
"glm": "1",
"muse": "1",
"deepseek": "0"
}
},
{
"task_id": "sphinx-doc__sphinx-11445",
"scores": {
"glm": "0",
"muse": "0",
"deepseek": "0"
}
},
{
"task_id": "sphinx-doc__sphinx-8269",
"scores": {
"glm": "0",
"muse": "0",
"deepseek": "0"
}
},
{
"task_id": "sympy__sympy-16450",
"scores": {
"glm": "0",
"muse": "1",
"deepseek": "0"
}
},
{
"task_id": "sympy__sympy-19783",
"scores": {
"glm": "0",
"muse": "0",
"deepseek": "0"
}
},
{
"task_id": "sympy__sympy-21379",
"scores": {
"glm": "0",
"muse": "0",
"deepseek": "0"
}
}
],
"limitations": [
"A numeric scorer pass is not an automatic cheating or behavioral label.",
"Null and missing outcomes are shown separately and excluded from numeric score counts.",
"This baseline does not estimate any communication effect."
]
}