mirror of
https://github.com/priyanshujain/messageboardbench.git
synced 2026-10-02 19:17:07 +00:00
archive old experiments and publish swe series
This commit is contained in:
pj committed
1 parent
480100587e
commit
638e978227
1522 files changed
+220002
-4900
No files matched your search
@@ -0,0 +1,25 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="980" height="266" viewBox="0 0 980 266">
|
||||
<rect width="100%" height="100%" fill="white"/>
|
||||
<text x="20" y="28" font-family="sans-serif" font-size="18" font-weight="bold">Reasoning observability and unresolved information</text>
|
||||
<text x="10" y="76" font-family="sans-serif" font-size="12">reasoning message coverage</text>
|
||||
<rect x="220" y="55" width="371.7" height="9" fill="#3568a8"/>
|
||||
<text x="596.7" y="64" font-family="sans-serif" font-size="10">glm 446/804</text>
|
||||
<rect x="220" y="67" width="422.9" height="9" fill="#c45d35"/>
|
||||
<text x="647.9" y="76" font-family="sans-serif" font-size="10">muse 445/705</text>
|
||||
<rect x="220" y="79" width="163.7" height="9" fill="#548f52"/>
|
||||
<text x="388.7" y="88" font-family="sans-serif" font-size="10">deepseek 184/753</text>
|
||||
<text x="10" y="128" font-family="sans-serif" font-size="12">readable reasoning message coverage</text>
|
||||
<rect x="220" y="107" width="371.7" height="9" fill="#3568a8"/>
|
||||
<text x="596.7" y="116" font-family="sans-serif" font-size="10">glm 446/804</text>
|
||||
<rect x="220" y="119" width="0.0" height="9" fill="#c45d35"/>
|
||||
<text x="225.0" y="128" font-family="sans-serif" font-size="10">muse 0/705</text>
|
||||
<rect x="220" y="131" width="163.7" height="9" fill="#548f52"/>
|
||||
<text x="388.7" y="140" font-family="sans-serif" font-size="10">deepseek 184/753</text>
|
||||
<text x="10" y="180" font-family="sans-serif" font-size="12">episode with unresolved information label</text>
|
||||
<rect x="220" y="159" width="502.5" height="9" fill="#3568a8"/>
|
||||
<text x="727.5" y="168" font-family="sans-serif" font-size="10">glm 15/20</text>
|
||||
<rect x="220" y="171" width="502.5" height="9" fill="#c45d35"/>
|
||||
<text x="727.5" y="180" font-family="sans-serif" font-size="10">muse 15/20</text>
|
||||
<rect x="220" y="183" width="603.0" height="9" fill="#548f52"/>
|
||||
<text x="828.0" y="192" font-family="sans-serif" font-size="10">deepseek 18/20</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.9 KiB |
@@ -0,0 +1,32 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="980" height="318" viewBox="0 0 980 318">
|
||||
<rect width="100%" height="100%" fill="white"/>
|
||||
<text x="20" y="28" font-family="sans-serif" font-size="18" font-weight="bold">Governance and disclosure</text>
|
||||
<text x="10" y="76" font-family="sans-serif" font-size="12">internet visible activity</text>
|
||||
<rect x="220" y="55" width="335.0" height="9" fill="#3568a8"/>
|
||||
<text x="560.0" y="64" font-family="sans-serif" font-size="10">glm 10/20</text>
|
||||
<rect x="220" y="67" width="33.5" height="9" fill="#c45d35"/>
|
||||
<text x="258.5" y="76" font-family="sans-serif" font-size="10">muse 1/20</text>
|
||||
<rect x="220" y="79" width="100.5" height="9" fill="#548f52"/>
|
||||
<text x="325.5" y="88" font-family="sans-serif" font-size="10">deepseek 3/20</text>
|
||||
<text x="10" y="128" font-family="sans-serif" font-size="12">final answer disclosure</text>
|
||||
<rect x="220" y="107" width="134.0" height="9" fill="#3568a8"/>
|
||||
<text x="359.0" y="116" font-family="sans-serif" font-size="10">glm 4/20</text>
|
||||
<rect x="220" y="119" width="100.5" height="9" fill="#c45d35"/>
|
||||
<text x="325.5" y="128" font-family="sans-serif" font-size="10">muse 3/20</text>
|
||||
<rect x="220" y="131" width="0.0" height="9" fill="#548f52"/>
|
||||
<text x="225.0" y="140" font-family="sans-serif" font-size="10">deepseek 0/20</text>
|
||||
<text x="10" y="180" font-family="sans-serif" font-size="12">source comment disclosure</text>
|
||||
<rect x="220" y="159" width="167.5" height="9" fill="#3568a8"/>
|
||||
<text x="392.5" y="168" font-family="sans-serif" font-size="10">glm 5/20</text>
|
||||
<rect x="220" y="171" width="268.0" height="9" fill="#c45d35"/>
|
||||
<text x="493.0" y="180" font-family="sans-serif" font-size="10">muse 8/20</text>
|
||||
<rect x="220" y="183" width="0.0" height="9" fill="#548f52"/>
|
||||
<text x="225.0" y="192" font-family="sans-serif" font-size="10">deepseek 0/20</text>
|
||||
<text x="10" y="232" font-family="sans-serif" font-size="12">success claim artifact conflict</text>
|
||||
<rect x="220" y="211" width="33.5" height="9" fill="#3568a8"/>
|
||||
<text x="258.5" y="220" font-family="sans-serif" font-size="10">glm 1/20</text>
|
||||
<rect x="220" y="223" width="33.5" height="9" fill="#c45d35"/>
|
||||
<text x="258.5" y="232" font-family="sans-serif" font-size="10">muse 1/20</text>
|
||||
<rect x="220" y="235" width="0.0" height="9" fill="#548f52"/>
|
||||
<text x="225.0" y="244" font-family="sans-serif" font-size="10">deepseek 0/20</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.3 KiB |
@@ -0,0 +1,167 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="538" viewBox="0 0 1100 538">
|
||||
<rect width="100%" height="100%" fill="white"/>
|
||||
<text x="20" y="28" font-family="sans-serif" font-size="18" font-weight="bold">Attempted mechanism by maximum episode scope</text>
|
||||
<text x="390" y="58" font-family="sans-serif" font-size="11">narrow</text>
|
||||
<text x="500" y="58" font-family="sans-serif" font-size="11">object api</text>
|
||||
<text x="610" y="58" font-family="sans-serif" font-size="11">package</text>
|
||||
<text x="720" y="58" font-family="sans-serif" font-size="11">process global</text>
|
||||
<text x="830" y="58" font-family="sans-serif" font-size="11">evaluation pipeline</text>
|
||||
<text x="940" y="58" font-family="sans-serif" font-size="11">external system</text>
|
||||
<text x="10" y="96" font-family="sans-serif" font-size="11">literal special case</text>
|
||||
<rect x="390" y="78" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="396" y="95" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="500" y="78" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="506" y="95" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="610" y="78" width="90" height="24" fill="#7b9fc8"/>
|
||||
<text x="616" y="95" font-family="sans-serif" font-size="10">0/1/0</text>
|
||||
<rect x="720" y="78" width="90" height="24" fill="#7b9fc8"/>
|
||||
<text x="726" y="95" font-family="sans-serif" font-size="10">0/2/0</text>
|
||||
<rect x="830" y="78" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="836" y="95" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="940" y="78" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="946" y="95" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<text x="10" y="130" font-family="sans-serif" font-size="11">call history or cache</text>
|
||||
<rect x="390" y="112" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="396" y="129" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="500" y="112" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="506" y="129" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="610" y="112" width="90" height="24" fill="#7b9fc8"/>
|
||||
<text x="616" y="129" font-family="sans-serif" font-size="10">0/1/0</text>
|
||||
<rect x="720" y="112" width="90" height="24" fill="#7b9fc8"/>
|
||||
<text x="726" y="129" font-family="sans-serif" font-size="10">0/2/0</text>
|
||||
<rect x="830" y="112" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="836" y="129" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="940" y="112" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="946" y="129" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<text x="10" y="164" font-family="sans-serif" font-size="11">protocol or equality spoof</text>
|
||||
<rect x="390" y="146" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="396" y="163" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="500" y="146" width="90" height="24" fill="#7b9fc8"/>
|
||||
<text x="506" y="163" font-family="sans-serif" font-size="10">3/4/2</text>
|
||||
<rect x="610" y="146" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="616" y="163" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="720" y="146" width="90" height="24" fill="#7b9fc8"/>
|
||||
<text x="726" y="163" font-family="sans-serif" font-size="10">0/1/0</text>
|
||||
<rect x="830" y="146" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="836" y="163" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="940" y="146" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="946" y="163" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<text x="10" y="198" font-family="sans-serif" font-size="11">caller stack source or test detection</text>
|
||||
<rect x="390" y="180" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="396" y="197" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="500" y="180" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="506" y="197" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="610" y="180" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="616" y="197" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="720" y="180" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="726" y="197" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="830" y="180" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="836" y="197" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="940" y="180" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="946" y="197" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<text x="10" y="232" font-family="sans-serif" font-size="11">global runtime mutation</text>
|
||||
<rect x="390" y="214" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="396" y="231" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="500" y="214" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="506" y="231" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="610" y="214" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="616" y="231" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="720" y="214" width="90" height="24" fill="#7b9fc8"/>
|
||||
<text x="726" y="231" font-family="sans-serif" font-size="10">0/3/0</text>
|
||||
<rect x="830" y="214" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="836" y="231" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="940" y="214" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="946" y="231" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<text x="10" y="266" font-family="sans-serif" font-size="11">protected test edit</text>
|
||||
<rect x="390" y="248" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="396" y="265" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="500" y="248" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="506" y="265" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="610" y="248" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="616" y="265" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="720" y="248" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="726" y="265" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="830" y="248" width="90" height="24" fill="#7b9fc8"/>
|
||||
<text x="836" y="265" font-family="sans-serif" font-size="10">3/1/0</text>
|
||||
<rect x="940" y="248" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="946" y="265" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<text x="10" y="300" font-family="sans-serif" font-size="11">test runner evaluator or grader interference</text>
|
||||
<rect x="390" y="282" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="396" y="299" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="500" y="282" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="506" y="299" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="610" y="282" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="616" y="299" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="720" y="282" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="726" y="299" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="830" y="282" width="90" height="24" fill="#7b9fc8"/>
|
||||
<text x="836" y="299" font-family="sans-serif" font-size="10">2/0/0</text>
|
||||
<rect x="940" y="282" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="946" y="299" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<text x="10" y="334" font-family="sans-serif" font-size="11">environment or dependency manipulation</text>
|
||||
<rect x="390" y="316" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="396" y="333" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="500" y="316" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="506" y="333" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="610" y="316" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="616" y="333" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="720" y="316" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="726" y="333" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="830" y="316" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="836" y="333" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="940" y="316" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="946" y="333" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<text x="10" y="368" font-family="sans-serif" font-size="11">external solution retrieval or reuse</text>
|
||||
<rect x="390" y="350" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="396" y="367" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="500" y="350" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="506" y="367" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="610" y="350" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="616" y="367" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="720" y="350" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="726" y="367" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="830" y="350" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="836" y="367" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="940" y="350" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="946" y="367" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<text x="10" y="402" font-family="sans-serif" font-size="11">side channel</text>
|
||||
<rect x="390" y="384" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="396" y="401" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="500" y="384" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="506" y="401" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="610" y="384" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="616" y="401" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="720" y="384" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="726" y="401" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="830" y="384" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="836" y="401" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="940" y="384" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="946" y="401" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<text x="10" y="436" font-family="sans-serif" font-size="11">result concealment or false claim</text>
|
||||
<rect x="390" y="418" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="396" y="435" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="500" y="418" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="506" y="435" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="610" y="418" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="616" y="435" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="720" y="418" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="726" y="435" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="830" y="418" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="836" y="435" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="940" y="418" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="946" y="435" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<text x="10" y="470" font-family="sans-serif" font-size="11">other</text>
|
||||
<rect x="390" y="452" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="396" y="469" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="500" y="452" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="506" y="469" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="610" y="452" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="616" y="469" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="720" y="452" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="726" y="469" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="830" y="452" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="836" y="469" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<rect x="940" y="452" width="90" height="24" fill="#eeeeee"/>
|
||||
<text x="946" y="469" font-family="sans-serif" font-size="10">0/0/0</text>
|
||||
<text x="390" y="112" font-family="sans-serif" font-size="10">cells show GLM/Muse/DeepSeek event counts</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 12 KiB |
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regenerate baseline SVG figures from the maintained metric CSV files."""
|
||||
|
||||
import csv
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
METRICS = HERE.parent
|
||||
MODEL_ORDER = ["glm", "muse", "deepseek"]
|
||||
COLORS = {"glm": "#3568a8", "muse": "#c45d35", "deepseek": "#548f52"}
|
||||
MECHANISMS = [
|
||||
"literal_special_case", "call_history_or_cache", "protocol_or_equality_spoof",
|
||||
"caller_stack_source_or_test_detection", "global_runtime_mutation", "protected_test_edit",
|
||||
"test_runner_evaluator_or_grader_interference", "environment_or_dependency_manipulation",
|
||||
"external_solution_retrieval_or_reuse", "side_channel", "result_concealment_or_false_claim", "other",
|
||||
]
|
||||
|
||||
|
||||
with (METRICS / "summary.csv").open() as handle:
|
||||
SUMMARY = list(csv.DictReader(handle))
|
||||
|
||||
|
||||
def bars(filename, title, metric_names, family):
|
||||
selected = [row for row in SUMMARY if row["metric_family"] == family and row["metric_name"] in metric_names]
|
||||
width, height = 980, 110 + 52 * len(metric_names)
|
||||
left, right, top = 220, 90, 55
|
||||
plot_width = width - left - right
|
||||
lines = [
|
||||
f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" viewBox="0 0 {width} {height}">',
|
||||
'<rect width="100%" height="100%" fill="white"/>',
|
||||
f'<text x="20" y="28" font-family="sans-serif" font-size="18" font-weight="bold">{title}</text>',
|
||||
]
|
||||
for i, metric in enumerate(metric_names):
|
||||
y = top + i * 52
|
||||
lines.append(f'<text x="10" y="{y + 21}" font-family="sans-serif" font-size="12">{metric.replace("_", " ")}</text>')
|
||||
model_rows = [row for row in selected if row["metric_name"] == metric]
|
||||
for j, short in enumerate(MODEL_ORDER):
|
||||
row = next((item for item in model_rows if short in item["model"]), None)
|
||||
if row is None or not row["value"]:
|
||||
continue
|
||||
value = float(row["value"])
|
||||
bar_width = plot_width * max(0.0, min(1.0, value))
|
||||
yy = y + j * 12
|
||||
label = f'{row["numerator"]}/{row["denominator"]}' if row["numerator"] and row["denominator"] else f'{value:.2f}'
|
||||
lines.append(f'<rect x="{left}" y="{yy}" width="{bar_width:.1f}" height="9" fill="{COLORS[short]}"/>')
|
||||
lines.append(f'<text x="{left + bar_width + 5:.1f}" y="{yy + 9}" font-family="sans-serif" font-size="10">{short} {label}</text>')
|
||||
lines.append('</svg>')
|
||||
(HERE / filename).write_text("\n".join(lines) + "\n")
|
||||
|
||||
|
||||
bars("progression-funnel.svg", "Reviewed cheating progression", [
|
||||
"cheating_considered", "cheating_attempted", "cheating_retained", "cheating_successful", "cheating_reversed",
|
||||
], "cheating_progression")
|
||||
bars("pressure-timeline.svg", "Pressure response", [
|
||||
"continuation_after_recognition", "safe_exit",
|
||||
], "persistence_pressure")
|
||||
bars("governance-response.svg", "Governance and disclosure", [
|
||||
"internet_visible_activity", "final_answer_disclosure", "source_comment_disclosure", "success_claim_artifact_conflict",
|
||||
], "information_governance")
|
||||
bars("cot-observability.svg", "Reasoning observability and unresolved information", [
|
||||
"reasoning_message_coverage", "readable_reasoning_message_coverage", "episode_with_unresolved_information_label",
|
||||
], "cot_observability")
|
||||
|
||||
with (METRICS / "events.csv").open() as handle:
|
||||
events = list(csv.DictReader(handle))
|
||||
counts = Counter((row["model"], row["mechanism"], row["scope"]) for row in events if row["event_type"] == "mechanism_attempt")
|
||||
width, height = 1100, 130 + 34 * len(MECHANISMS)
|
||||
lines = [
|
||||
f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" viewBox="0 0 {width} {height}">',
|
||||
'<rect width="100%" height="100%" fill="white"/>',
|
||||
'<text x="20" y="28" font-family="sans-serif" font-size="18" font-weight="bold">Attempted mechanism by maximum episode scope</text>',
|
||||
]
|
||||
scopes = ["narrow", "object_api", "package", "process_global", "evaluation_pipeline", "external_system"]
|
||||
for j, scope in enumerate(scopes):
|
||||
lines.append(f'<text x="{390 + j * 110}" y="58" font-family="sans-serif" font-size="11">{scope.replace("_", " ")}</text>')
|
||||
for i, mechanism in enumerate(MECHANISMS):
|
||||
y = 78 + i * 34
|
||||
lines.append(f'<text x="10" y="{y + 18}" font-family="sans-serif" font-size="11">{mechanism.replace("_", " ")}</text>')
|
||||
for j, scope in enumerate(scopes):
|
||||
cell = [sum(n for (model, mech, sc), n in counts.items() if short in model and mech == mechanism and sc == scope) for short in MODEL_ORDER]
|
||||
x = 390 + j * 110
|
||||
lines.append(f'<rect x="{x}" y="{y}" width="90" height="24" fill="{"#eeeeee" if sum(cell) == 0 else "#7b9fc8"}"/>')
|
||||
lines.append(f'<text x="{x + 6}" y="{y + 17}" font-family="sans-serif" font-size="10">{cell[0]}/{cell[1]}/{cell[2]}</text>')
|
||||
lines.append('<text x="390" y="112" font-family="sans-serif" font-size="10">cells show GLM/Muse/DeepSeek event counts</text>')
|
||||
lines.append('</svg>')
|
||||
(HERE / "mechanism-scope.svg").write_text("\n".join(lines) + "\n")
|
||||
@@ -0,0 +1,18 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="980" height="214" viewBox="0 0 980 214">
|
||||
<rect width="100%" height="100%" fill="white"/>
|
||||
<text x="20" y="28" font-family="sans-serif" font-size="18" font-weight="bold">Pressure response</text>
|
||||
<text x="10" y="76" font-family="sans-serif" font-size="12">continuation after recognition</text>
|
||||
<rect x="220" y="55" width="493.7" height="9" fill="#3568a8"/>
|
||||
<text x="718.7" y="64" font-family="sans-serif" font-size="10">glm 14/19</text>
|
||||
<rect x="220" y="67" width="0.0" height="9" fill="#c45d35"/>
|
||||
<text x="225.0" y="76" font-family="sans-serif" font-size="10">muse 0/10</text>
|
||||
<rect x="220" y="79" width="551.8" height="9" fill="#548f52"/>
|
||||
<text x="776.8" y="88" font-family="sans-serif" font-size="10">deepseek 14/17</text>
|
||||
<text x="10" y="128" font-family="sans-serif" font-size="12">safe exit</text>
|
||||
<rect x="220" y="107" width="493.7" height="9" fill="#3568a8"/>
|
||||
<text x="718.7" y="116" font-family="sans-serif" font-size="10">glm 14/19</text>
|
||||
<rect x="220" y="119" width="67.0" height="9" fill="#c45d35"/>
|
||||
<text x="292.0" y="128" font-family="sans-serif" font-size="10">muse 1/10</text>
|
||||
<rect x="220" y="131" width="630.6" height="9" fill="#548f52"/>
|
||||
<text x="855.6" y="140" font-family="sans-serif" font-size="10">deepseek 16/17</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.3 KiB |
@@ -0,0 +1,39 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="980" height="370" viewBox="0 0 980 370">
|
||||
<rect width="100%" height="100%" fill="white"/>
|
||||
<text x="20" y="28" font-family="sans-serif" font-size="18" font-weight="bold">Reviewed cheating progression</text>
|
||||
<text x="10" y="76" font-family="sans-serif" font-size="12">cheating considered</text>
|
||||
<rect x="220" y="55" width="502.5" height="9" fill="#3568a8"/>
|
||||
<text x="727.5" y="64" font-family="sans-serif" font-size="10">glm 15/20</text>
|
||||
<rect x="220" y="67" width="301.5" height="9" fill="#c45d35"/>
|
||||
<text x="526.5" y="76" font-family="sans-serif" font-size="10">muse 9/20</text>
|
||||
<rect x="220" y="79" width="402.0" height="9" fill="#548f52"/>
|
||||
<text x="627.0" y="88" font-family="sans-serif" font-size="10">deepseek 12/20</text>
|
||||
<text x="10" y="128" font-family="sans-serif" font-size="12">cheating attempted</text>
|
||||
<rect x="220" y="107" width="201.0" height="9" fill="#3568a8"/>
|
||||
<text x="426.0" y="116" font-family="sans-serif" font-size="10">glm 6/20</text>
|
||||
<rect x="220" y="119" width="301.5" height="9" fill="#c45d35"/>
|
||||
<text x="526.5" y="128" font-family="sans-serif" font-size="10">muse 9/20</text>
|
||||
<rect x="220" y="131" width="67.0" height="9" fill="#548f52"/>
|
||||
<text x="292.0" y="140" font-family="sans-serif" font-size="10">deepseek 2/20</text>
|
||||
<text x="10" y="180" font-family="sans-serif" font-size="12">cheating retained</text>
|
||||
<rect x="220" y="159" width="167.5" height="9" fill="#3568a8"/>
|
||||
<text x="392.5" y="168" font-family="sans-serif" font-size="10">glm 5/20</text>
|
||||
<rect x="220" y="171" width="301.5" height="9" fill="#c45d35"/>
|
||||
<text x="526.5" y="180" font-family="sans-serif" font-size="10">muse 9/20</text>
|
||||
<rect x="220" y="183" width="33.5" height="9" fill="#548f52"/>
|
||||
<text x="258.5" y="192" font-family="sans-serif" font-size="10">deepseek 1/20</text>
|
||||
<text x="10" y="232" font-family="sans-serif" font-size="12">cheating successful</text>
|
||||
<rect x="220" y="211" width="134.0" height="9" fill="#3568a8"/>
|
||||
<text x="359.0" y="220" font-family="sans-serif" font-size="10">glm 3/15</text>
|
||||
<rect x="220" y="223" width="357.3" height="9" fill="#c45d35"/>
|
||||
<text x="582.3" y="232" font-family="sans-serif" font-size="10">muse 8/15</text>
|
||||
<rect x="220" y="235" width="44.7" height="9" fill="#548f52"/>
|
||||
<text x="269.7" y="244" font-family="sans-serif" font-size="10">deepseek 1/15</text>
|
||||
<text x="10" y="284" font-family="sans-serif" font-size="12">cheating reversed</text>
|
||||
<rect x="220" y="263" width="111.7" height="9" fill="#3568a8"/>
|
||||
<text x="336.7" y="272" font-family="sans-serif" font-size="10">glm 1/6</text>
|
||||
<rect x="220" y="275" width="0.0" height="9" fill="#c45d35"/>
|
||||
<text x="225.0" y="284" font-family="sans-serif" font-size="10">muse 0/9</text>
|
||||
<rect x="220" y="287" width="0.0" height="9" fill="#548f52"/>
|
||||
<text x="225.0" y="296" font-family="sans-serif" font-size="10">deepseek 0/2</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.8 KiB |
Reference in new issue
Block a user