mirror of
https://github.com/priyanshujain/messageboardbench.git
synced 2026-10-02 19:17:07 +00:00
archive old experiments and publish swe series
This commit is contained in:
1 parent
480100587e
commit
638e978227
1522 files changed
+220002
-4900
No files matched your search
@@ -37,6 +37,11 @@ def main():
|
||||
if feedback_operations_path.is_file() else [])
|
||||
before = json.loads((args.run / 'budget-before.json').read_text())
|
||||
after = json.loads((args.run / 'budget-after.json').read_text())
|
||||
if before.get('provider') == 'clinepass':
|
||||
account_limitation = ('ClinePass account quota is unavailable from the '
|
||||
'documented API; use episode usage and the Cline dashboard.')
|
||||
else:
|
||||
account_limitation = 'Account usage changes may include billing delay or other account activity.'
|
||||
result = {
|
||||
'all': summarize(rows),
|
||||
'by_condition': {c: summarize([r for r in rows if r['condition'] == c])
|
||||
@@ -56,7 +61,7 @@ def main():
|
||||
'Reasoning tokens are a subset of output, not an additional cost.',
|
||||
'Input is uncached; cached input is reported separately and contributes to total.',
|
||||
'Working seconds are summed episode time, not experiment wall time.',
|
||||
'Account usage changes may include billing delay or other account activity.',
|
||||
account_limitation,
|
||||
'Small dependent development-task samples; descriptive comparisons only.',
|
||||
],
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user