test(confusion-matrix): reject malformed inputs and keep missing data out of the cells

cmd/internal-tools/confusion-matrix/fixtures_test.go

@@ -0,0 +1,304 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// fixtureRun is one campaign of one implementation at one seed, in the shape
+// implementation-sweep and campaign write together.
+type fixtureRun struct {
+	Seed     int64
+	ExitCode int
+	TimedOut bool
+	Violated []string
+	// Surfaces is the locatableSurfaces reading the trace records. A nil map
+	// with NoTrace false still writes a reading of every surface false.
+	Surfaces map[string]bool
+	NoTrace  bool
+}
+
+type fixtureReview struct {
+	Overall string
+	Clauses map[string]string
+	Minutes int
+}
+
+type fixtureImplementation struct {
+	Name        string
+	Model       string
+	FailedStage string
+	Runs        []fixtureRun
+	Review      *fixtureReview
+	RawReview   string
+	Adjudicated map[string]string
+}
+
+type fixture struct {
+	Sweep      string
+	Reviews    string
+	Assignment string
+	Mapping    string
+}
+
+const defaultMapping = `
+| property | clauses | surfaces |
+| --- | --- | --- |
+| sentOnlyAfterConfirmation | R5 R15 | stateWords |
+| serverHoldsEachMessageOnce | R15 | none |
+| unsentReachesZero | R14 | unsentCount |
+
+| surface | never observed | note |
+| --- | --- | --- |
+| composer | unlocatable | R1 obliges one |
+| stateWords | unlocatable | R4 obliges one on every composed message |
+| unsentCount | inconclusive | R6 hides the count at zero |
+`
+
+func writeFixture(t *testing.T, implementations []fixtureImplementation, mappingBody string) fixture {
+	t.Helper()
+	root := t.TempDir()
+	built := fixture{
+		Sweep:      filepath.Join(root, "sweep"),
+		Reviews:    filepath.Join(root, "reviews"),
+		Assignment: filepath.Join(root, "assignment.md"),
+		Mapping:    filepath.Join(root, "property-clauses.md"),
+	}
+	mustMkdir(t, built.Sweep)
+	mustMkdir(t, built.Reviews)
+	mustWrite(t, built.Mapping, mappingBody)
+
+	var planned []map[string]any
+	var assignmentRows []string
+	assignmentRows = append(assignmentRows, "| implementation | model |", "| --- | --- |")
+	var records []string
+	for _, implementation := range implementations {
+		planned = append(planned, map[string]any{"name": implementation.Name})
+		if implementation.Model != "" {
+			assignmentRows = append(assignmentRows,
+				fmt.Sprintf("| %s | %s |", implementation.Name, implementation.Model))
+		}
+		records = append(records, writeImplementation(t, built, implementation))
+		writeReviewFiles(t, built.Reviews, implementation)
+	}
+	mustWrite(t, built.Assignment, strings.Join(assignmentRows, "\n")+"\n")
+	mustWriteJSON(t, filepath.Join(built.Sweep, sweepManifestFileName), map[string]any{
+		"generator":       "seeded",
+		"platform":        "web",
+		"spec_path":       "paper/experiments/e4/spec/spec.ts",
+		"max_steps":       400,
+		"seeds":           []int{1, 2},
+		"host":            "anton",
+		"implementations": planned,
+	})
+	mustWrite(t, filepath.Join(built.Sweep, sweepRecordsFileName), strings.Join(records, "\n")+"\n")
+	return built
+}
+
+func writeImplementation(t *testing.T, built fixture, implementation fixtureImplementation) string {
+	t.Helper()
+	record := map[string]any{"implementation": implementation.Name}
+	if implementation.FailedStage != "" {
+		record["failed_stage"] = implementation.FailedStage
+		record["error"] = "bun run build exited 1"
+		return encode(t, record)
+	}
+	var runs []map[string]any
+	for _, run := range implementation.Runs {
+		seedText := strconv.FormatInt(run.Seed, 10)
+		campaignDirectory := filepath.Join(built.Sweep, implementation.Name, "seed-"+seedText)
+		mustMkdir(t, campaignDirectory)
+		mustWriteJSON(t, filepath.Join(campaignDirectory, campaignManifestFileName), map[string]any{
+			"arm":       implementation.Name,
+			"generator": "seeded",
+			"platform":  "web",
+			"max_steps": 400,
+			"seeds":     []int64{run.Seed},
+		})
+		runDirectory := filepath.Join("seed-"+seedText, "20260820T110000Z")
+		campaignRun := map[string]any{
+			"seed":          run.Seed,
+			"exit_code":     run.ExitCode,
+			"steps":         400,
+			"actions":       380,
+			"run_directory": runDirectory,
+		}
+		if run.TimedOut {
+			campaignRun["timed_out"] = true
+		}
+		if len(run.Violated) > 0 {
+			campaignRun["violated_properties"] = run.Violated
+		}
+		mustWrite(t, filepath.Join(campaignDirectory, campaignRecordsFileName), encode(t, campaignRun)+"\n")
+		if !run.NoTrace {
+			writeTrace(t, filepath.Join(campaignDirectory, runDirectory), run.Surfaces)
+		}
+		runs = append(runs, map[string]any{
+			"seed":               run.Seed,
+			"exit_code":          run.ExitCode,
+			"campaign_directory": campaignDirectory,
+		})
+	}
+	record["runs"] = runs
+	return encode(t, record)
+}
+
+// writeTrace writes the one line the verifier emits on the first snapshot, when
+// every extractor is reported as a change from null.
+func writeTrace(t *testing.T, directory string, surfaces map[string]bool) {
+	t.Helper()
+	mustMkdir(t, directory)
+	reading := map[string]bool{}
+	for _, surface := range []string{"appRoot", "composer", "submit", "stateWords",
+		"unsentCount", "pendingIndicator", "offlineIndicator", "retryControl"} {
+		reading[surface] = surfaces[surface]
+	}
+	line := map[string]any{
+		"step": 1,
+		"extractor_changes": map[string]any{
+			surfacesExtractor: map[string]any{"prev": nil, "curr": reading},
+		},
+	}
+	mustWrite(t, filepath.Join(directory, traceFileName), encode(t, line)+"\n")
+}
+
+func writeReviewFiles(t *testing.T, directory string, implementation fixtureImplementation) {
+	t.Helper()
+	if implementation.RawReview != "" {
+		mustWrite(t, filepath.Join(directory, implementation.Name+".md"), implementation.RawReview)
+		return
+	}
+	if implementation.Review == nil {
+		return
+	}
+	mustWrite(t, filepath.Join(directory, implementation.Name+".md"), renderReview(*implementation.Review))
+	if len(implementation.Adjudicated) == 0 {
+		return
+	}
+	rows := []string{"| clause | verdict | why |", "| --- | --- | --- |"}
+	for _, clause := range allClauses() {
+		label, resolved := implementation.Adjudicated[clause]
+		if !resolved {
+			continue
+		}
+		rows = append(rows, fmt.Sprintf("| %s | %s | joint reread |", clause, label))
+	}
+	mustWrite(t, filepath.Join(directory, implementation.Name+"-adjudication.md"),
+		strings.Join(rows, "\n")+"\n")
+}
+
+func renderReview(review fixtureReview) string {
+	minutes := review.Minutes
+	if minutes == 0 {
+		minutes = 45
+	}
+	lines := []string{
+		"# review",
+		"",
+		"reviewer: Jane",
+		"date: 2026-09-02",
+		fmt.Sprintf("minutes: %d", minutes),
+		"",
+		"| clause | verdict | justification | steps |",
+		"| --- | --- | --- | --- |",
+	}
+	for _, clause := range allClauses() {
+		label := review.Clauses[clause]
+		if label == "" {
+			label = clauseMeets
+		}
+		lines = append(lines, fmt.Sprintf("| %s | %s | seen by hand | offline, compose, online |", clause, label))
+	}
+	lines = append(lines, "", "overall: "+review.Overall, "")
+	return strings.Join(lines, "\n")
+}
+
+func runTool(t *testing.T, built fixture) (result, string) {
+	t.Helper()
+	var stdout, stderr bytes.Buffer
+	jsonPath := filepath.Join(t.TempDir(), "matrix.json")
+	err := run([]string{
+		"--sweep", built.Sweep,
+		"--reviews", built.Reviews,
+		"--assignment", built.Assignment,
+		"--property-clauses", built.Mapping,
+		"--json", jsonPath,
+	}, &stdout, &stderr)
+	if err != nil {
+		t.Fatalf("run: %v\nstderr: %s", err, stderr.String())
+	}
+	body, err := os.ReadFile(jsonPath)
+	if err != nil {
+		t.Fatalf("read emitted summary: %v", err)
+	}
+	var emitted result
+	if err := json.Unmarshal(body, &emitted); err != nil {
+		t.Fatalf("emitted summary is not valid JSON: %v", err)
+	}
+	return emitted, stdout.String()
+}
+
+func cleanRun(seed int64, violated ...string) fixtureRun {
+	return fixtureRun{
+		Seed:     seed,
+		Violated: violated,
+		Surfaces: map[string]bool{"appRoot": true, "composer": true, "submit": true, "stateWords": true},
+	}
+}
+
+func mustMkdir(t *testing.T, path string) {
+	t.Helper()
+	if err := os.MkdirAll(path, 0o755); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func mustWrite(t *testing.T, path, body string) {
+	t.Helper()
+	mustMkdir(t, filepath.Dir(path))
+	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func mustWriteJSON(t *testing.T, path string, value any) {
+	t.Helper()
+	mustWrite(t, path, encode(t, value)+"\n")
+}
+
+func encode(t *testing.T, value any) string {
+	t.Helper()
+	body, err := json.Marshal(value)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(body)
+}
+
+func outcomeFor(t *testing.T, emitted result, name string) implementationOutcome {
+	t.Helper()
+	for _, row := range emitted.Outcomes {
+		if row.Implementation == name {
+			return row
+		}
+	}
+	t.Fatalf("%s carries no cell; excluded as %v", name, emitted.Excluded)
+	return implementationOutcome{}
+}
+
+func exclusionFor(t *testing.T, emitted result, name string) exclusion {
+	t.Helper()
+	for _, entry := range emitted.Excluded {
+		if entry.Implementation == name {
+			return entry
+		}
+	}
+	t.Fatalf("%s was not excluded; it scored %v", name, emitted.Outcomes)
+	return exclusion{}
+}

cmd/internal-tools/confusion-matrix/parse_test.go

@@ -0,0 +1,222 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func runExpectingError(t *testing.T, built fixture) string {
+	t.Helper()
+	var stdout, stderr bytes.Buffer
+	err := run([]string{
+		"--sweep", built.Sweep,
+		"--reviews", built.Reviews,
+		"--assignment", built.Assignment,
+		"--property-clauses", built.Mapping,
+	}, &stdout, &stderr)
+	if err == nil {
+		t.Fatalf("the tool reported success on input it must refuse:\n%s", stdout.String())
+	}
+	return err.Error()
+}
+
+func scoredFixture(name, model string) fixtureImplementation {
+	return fixtureImplementation{
+		Name: name, Model: model,
+		Runs:   []fixtureRun{cleanRun(1)},
+		Review: &fixtureReview{Overall: overallNotDefective},
+	}
+}
+
+func TestMappingRefusesInputTheMatrixCannotBeScoredFrom(t *testing.T) {
+	tests := []struct {
+		name    string
+		mapping string
+		want    string
+	}{
+		{
+			name: "a property reads a surface the surface table never declares",
+			mapping: "| property | clauses | surfaces |\n| p | R1 | badgeRow |\n" +
+				"| surface | never observed | note |\n| composer | unlocatable | |\n",
+			want: "the surface table does not declare",
+		},
+		{
+			name:    "no surface is declared at all",
+			mapping: "| property | clauses | surfaces |\n| p | R1 | none |\n",
+			want:    "declares no surfaces",
+		},
+		{
+			name: "a property names a clause outside the requirement",
+			mapping: "| property | clauses | surfaces |\n| p | R21 | none |\n" +
+				"| surface | never observed | note |\n| composer | unlocatable | |\n",
+			want: "which is not one of R1 to R20",
+		},
+		{
+			name: "a surface says something other than what a miss means",
+			mapping: "| property | clauses | surfaces |\n| p | R1 | none |\n" +
+				"| surface | never observed | note |\n| composer | maybe | |\n",
+			want: "want unlocatable or inconclusive",
+		},
+		{
+			name: "one property is mapped twice",
+			mapping: "| property | clauses | surfaces |\n| p | R1 | none |\n| p | R2 | none |\n" +
+				"| surface | never observed | note |\n| composer | unlocatable | |\n",
+			want: "is mapped twice",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			built := writeFixture(t, []fixtureImplementation{scoredFixture("impl-01", "Opus 5")}, test.mapping)
+			if got := runExpectingError(t, built); !strings.Contains(got, test.want) {
+				t.Fatalf("error %q does not name the problem %q", got, test.want)
+			}
+		})
+	}
+}
+
+func TestAssignmentRefusesAModelTheSampleWasNotDrawnFrom(t *testing.T) {
+	built := writeFixture(t, []fixtureImplementation{scoredFixture("impl-01", "Opus 5")}, defaultMapping)
+	mustWrite(t, built.Assignment, "| implementation | model |\n| impl-01 | Opus 4 |\n")
+	if got := runExpectingError(t, built); !strings.Contains(got, "which is none of Sonnet 5, Opus 5, Fable 5") {
+		t.Fatalf("error %q does not refuse the unknown model", got)
+	}
+}
+
+func TestSweepRecordingAnImplementationTwiceIsRefused(t *testing.T) {
+	built := writeFixture(t, []fixtureImplementation{scoredFixture("impl-01", "Opus 5")}, defaultMapping)
+	path := filepath.Join(built.Sweep, sweepRecordsFileName)
+	body, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	mustWrite(t, path, string(body)+string(body))
+	if got := runExpectingError(t, built); !strings.Contains(got, "a second time") {
+		t.Fatalf("error %q does not refuse the repeated implementation", got)
+	}
+}
+
+func TestMalformedVerdictFormsAreExcludedWithTheirReason(t *testing.T) {
+	tests := []struct {
+		name string
+		body string
+		want string
+	}{
+		{
+			name: "a clause carries a word that is not a verdict",
+			body: reviewWithRow("| R7 | probably fine | |"),
+			want: "want meets, violates or cannot tell",
+		},
+		{
+			name: "the form closes with no overall verdict",
+			body: strings.Replace(renderReview(fixtureReview{Overall: overallNotDefective}),
+				"overall: not defective", "", 1),
+			want: "no overall verdict",
+		},
+		{
+			name: "the overall verdict is neither answer",
+			body: strings.Replace(renderReview(fixtureReview{Overall: overallNotDefective}),
+				"overall: not defective", "overall: mostly ok", 1),
+			want: "is neither defective nor not defective",
+		},
+		{
+			name: "a clause is filed twice with two labels",
+			body: renderReview(fixtureReview{Overall: overallNotDefective}) +
+				"\n| R3 | violates | filed again |\n",
+			want: "is filed twice",
+		},
+		{
+			name: "a clause row is missing",
+			body: strings.Replace(renderReview(fixtureReview{Overall: overallNotDefective}),
+				"| R12 | meets | seen by hand | offline, compose, online |\n", "", 1),
+			want: "no row for clause R12",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			built := writeFixture(t, []fixtureImplementation{{
+				Name: "impl-01", Model: "Opus 5",
+				Runs:      []fixtureRun{cleanRun(1)},
+				RawReview: test.body,
+			}}, defaultMapping)
+			emitted, stdout := runTool(t, built)
+			entry := exclusionFor(t, emitted, "impl-01")
+			if entry.Reason != missingMalformed {
+				t.Fatalf("impl-01 excluded as %q, want %q", entry.Reason, missingMalformed)
+			}
+			if !strings.Contains(entry.Detail, test.want) {
+				t.Errorf("exclusion detail %q does not name %q", entry.Detail, test.want)
+			}
+			if !strings.Contains(stdout, missingMalformed) {
+				t.Errorf("the report never prints the malformed form:\n%s", stdout)
+			}
+			if emitted.Implementations.Scored != 0 {
+				t.Errorf("a malformed form scored %d implementation(s), want none", emitted.Implementations.Scored)
+			}
+		})
+	}
+}
+
+func TestAnUnfilledMappingStillReportsTheImplementationMatrix(t *testing.T) {
+	unfilled := "| property | clauses | surfaces |\n| TODO | TODO | TODO |\n" +
+		"| surface | never observed | note |\n| stateWords | unlocatable | R4 obliges one |\n"
+	emitted, stdout := runTool(t, writeFixture(t, []fixtureImplementation{{
+		Name: "impl-01", Model: "Opus 5",
+		Runs:   []fixtureRun{cleanRun(1, "serverHoldsEachMessageOnce")},
+		Review: &fixtureReview{Overall: overallDefective, Clauses: map[string]string{"R15": clauseViolates}},
+	}}, unfilled))
+
+	if emitted.Implementations.TruePositive != 1 {
+		t.Fatalf("implementation matrix = %+v, want one true positive", emitted.Implementations)
+	}
+	if emitted.Clauses.TruePositive != 0 || emitted.Clauses.FalseNegative != 1 {
+		t.Errorf("clause matrix = %+v, want every clause uncovered", emitted.Clauses)
+	}
+	if emitted.Coverage.MappingTodoRows != 1 {
+		t.Errorf("coverage reports %d TODO row(s), want 1", emitted.Coverage.MappingTodoRows)
+	}
+	if !strings.Contains(stdout, "maps no property to a clause") {
+		t.Errorf("the report never says the mapping is unfilled:\n%s", stdout)
+	}
+}
+
+func TestAnImplementationOutsideTheAssignmentIsMissingData(t *testing.T) {
+	emitted, _ := runTool(t, writeFixture(t, []fixtureImplementation{
+		scoredFixture("impl-01", "Opus 5"),
+		{
+			Name: "impl-02",
+			Runs: []fixtureRun{cleanRun(1)}, Review: &fixtureReview{Overall: overallNotDefective},
+		},
+	}, defaultMapping))
+
+	if entry := exclusionFor(t, emitted, "impl-02"); entry.Reason != missingNoModel {
+		t.Fatalf("impl-02 excluded as %q, want %q", entry.Reason, missingNoModel)
+	}
+}
+
+func TestSurfacesArePooledAcrossTheSeedsOneImplementationWasSweptAt(t *testing.T) {
+	emitted, _ := runTool(t, writeFixture(t, []fixtureImplementation{{
+		Name: "impl-01", Model: "Fable 5",
+		Runs: []fixtureRun{
+			{Seed: 1, Surfaces: map[string]bool{"composer": true}},
+			{Seed: 2, Surfaces: map[string]bool{"stateWords": true}},
+		},
+		Review: &fixtureReview{Overall: overallNotDefective},
+	}}, defaultMapping))
+
+	row := outcomeFor(t, emitted, "impl-01")
+	if len(row.UnlocatableSurfaces) != 0 {
+		t.Fatalf("impl-01 reports %v unlocatable, want none: each surface was located on one seed or the other",
+			row.UnlocatableSurfaces)
+	}
+	if emitted.Clauses.UnevaluatedSurfaceMissed != 0 {
+		t.Errorf("clause matrix reports %d unevaluated pair(s), want none", emitted.Clauses.UnevaluatedSurfaceMissed)
+	}
+}
+
+func reviewWithRow(row string) string {
+	body := renderReview(fixtureReview{Overall: overallNotDefective})
+	return strings.Replace(body, "| R7 | meets | seen by hand | offline, compose, online |", row, 1)
+}