feat(confusion-matrix): score the checker against a blind reviewer

Cross-tabulates the properties that fired against the human verdict, one
cell per implementation, over a sweep whose implementations all passed
their own generated tests. An implementation that failed to build, has no
usable run, or carries no filed verdict is listed as missing data rather
than counted as a clean cell.

Landing the package in one commit because the intermediate splits would
not link.

cmd/internal-tools/confusion-matrix/assignment.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+)
+
+// The three models the sample is drawn from, in the capability order
+// model-implementations.md fixed before any implementation was generated.
+var capabilityOrder = []string{"Sonnet 5", "Opus 5", "Fable 5"}
+
+var (
+	implementationName = regexp.MustCompile(`^impl-\d+$`)
+	nonAlphanumeric    = regexp.MustCompile(`[^a-z0-9]`)
+)
+
+func loadAssignment(path string) (map[string]string, error) {
+	body, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read assignment: %w", err)
+	}
+	assignments := map[string]string{}
+	for _, row := range parseTableRows(string(body)) {
+		name := row.cell(0)
+		if !implementationName.MatchString(name) {
+			continue
+		}
+		model, ok := canonicalModel(row.cell(1))
+		if !ok {
+			return nil, fmt.Errorf("%s line %d: %s is assigned model %q, which is none of %s",
+				path, row.Line, name, row.cell(1), strings.Join(capabilityOrder, ", "))
+		}
+		if existing, seen := assignments[name]; seen && existing != model {
+			return nil, fmt.Errorf("%s line %d: %s is assigned to both %s and %s",
+				path, row.Line, name, existing, model)
+		}
+		assignments[name] = model
+	}
+	if len(assignments) == 0 {
+		return nil, fmt.Errorf("%s maps no implementation to a model", path)
+	}
+	return assignments, nil
+}
+
+func canonicalModel(value string) (string, bool) {
+	key := nonAlphanumeric.ReplaceAllString(strings.ToLower(value), "")
+	for _, model := range capabilityOrder {
+		if key == nonAlphanumeric.ReplaceAllString(strings.ToLower(model), "") {
+			return model, true
+		}
+	}
+	return "", false
+}

cmd/internal-tools/confusion-matrix/checker.go

@@ -0,0 +1,329 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"maps"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+const (
+	sweepManifestFileName    = "sweep.json"
+	sweepRecordsFileName     = "implementations.jsonl"
+	campaignManifestFileName = "campaign.json"
+	campaignRecordsFileName  = "runs.jsonl"
+	traceFileName            = "trace.jsonl"
+	surfacesExtractor        = "locatableSurfaces"
+	maxRecordBytes           = 4 * 1024 * 1024
+	maxTraceLineBytes        = 16 * 1024 * 1024
+)
+
+// Run exclusion reasons, kept in the vocabulary analyze already uses so the two
+// tools describe the same run the same way.
+const (
+	reasonLaunchError = "launch error"
+	reasonTimedOut    = "timed out"
+	reasonNonzeroExit = "nonzero exit"
+	reasonTraceError  = "unreadable trace"
+)
+
+type sweepManifest struct {
+	SpecPath        string `json:"spec_path"`
+	Implementations []struct {
+		Name string `json:"name"`
+	} `json:"implementations"`
+}
+
+type sweepRunRecord struct {
+	Seed              int64  `json:"seed"`
+	ExitCode          int    `json:"exit_code"`
+	LaunchError       string `json:"launch_error"`
+	CampaignDirectory string `json:"campaign_directory"`
+}
+
+type sweepImplementationRecord struct {
+	Name        string           `json:"implementation"`
+	FailedStage string           `json:"failed_stage"`
+	Error       string           `json:"error"`
+	Runs        []sweepRunRecord `json:"runs"`
+}
+
+type campaignRunRecord struct {
+	Seed               int64    `json:"seed"`
+	ExitCode           int      `json:"exit_code"`
+	LaunchError        string   `json:"launch_error"`
+	TimedOut           bool     `json:"timed_out"`
+	TraceError         string   `json:"trace_error"`
+	RunDirectory       string   `json:"run_directory"`
+	ViolatedProperties []string `json:"violated_properties"`
+}
+
+// checkerVerdict is one implementation's whole checker side, pooled across the
+// seeds it was swept at.
+type checkerVerdict struct {
+	Implementation   string
+	FailedStage      string
+	FailedError      string
+	RunsRecorded     int
+	RunsUsable       int
+	ExcludedByReason map[string]int
+	FiredProperties  []string
+	// SurfacesObserved holds every locatable surface seen true on at least one
+	// step of at least one usable run. A surface missing from it was never
+	// located across the whole sweep of this implementation.
+	SurfacesObserved map[string]bool
+	SurfacesKnown    bool
+	TraceErrors      []string
+}
+
+func (v checkerVerdict) fired() bool { return len(v.FiredProperties) > 0 }
+
+type checkerSide struct {
+	Directory string
+	SpecPath  string
+	Planned   []string
+	Verdicts  []checkerVerdict
+}
+
+func loadChecker(directory string) (checkerSide, error) {
+	body, err := os.ReadFile(filepath.Join(directory, sweepManifestFileName))
+	if err != nil {
+		return checkerSide{}, fmt.Errorf("read %s: %w", sweepManifestFileName, err)
+	}
+	var declared sweepManifest
+	if err := json.Unmarshal(body, &declared); err != nil {
+		return checkerSide{}, fmt.Errorf("parse %s in %s: %w", sweepManifestFileName, directory, err)
+	}
+	side := checkerSide{Directory: directory, SpecPath: declared.SpecPath}
+	for _, planned := range declared.Implementations {
+		side.Planned = append(side.Planned, planned.Name)
+	}
+
+	records, err := readSweepRecords(filepath.Join(directory, sweepRecordsFileName))
+	if err != nil {
+		return checkerSide{}, err
+	}
+	for _, record := range records {
+		verdict, err := readImplementation(directory, record)
+		if err != nil {
+			return checkerSide{}, err
+		}
+		side.Verdicts = append(side.Verdicts, verdict)
+	}
+	slices.SortFunc(side.Verdicts, func(a, b checkerVerdict) int {
+		return strings.Compare(a.Implementation, b.Implementation)
+	})
+	return side, nil
+}
+
+func readSweepRecords(path string) ([]sweepImplementationRecord, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("read %s: %w", sweepRecordsFileName, err)
+	}
+	defer file.Close()
+
+	var records []sweepImplementationRecord
+	scanner := bufio.NewScanner(file)
+	scanner.Buffer(make([]byte, 0, 64*1024), maxRecordBytes)
+	lineNumber := 0
+	seen := map[string]bool{}
+	for scanner.Scan() {
+		lineNumber++
+		raw := strings.TrimSpace(scanner.Text())
+		if raw == "" {
+			continue
+		}
+		var record sweepImplementationRecord
+		if err := json.Unmarshal([]byte(raw), &record); err != nil {
+			return nil, fmt.Errorf("%s line %d: %w", sweepRecordsFileName, lineNumber, err)
+		}
+		if record.Name == "" {
+			return nil, fmt.Errorf("%s line %d names no implementation", sweepRecordsFileName, lineNumber)
+		}
+		if seen[record.Name] {
+			return nil, fmt.Errorf("%s line %d records %s a second time: its runs would be pooled twice",
+				sweepRecordsFileName, lineNumber, record.Name)
+		}
+		seen[record.Name] = true
+		records = append(records, record)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("read %s: %w", sweepRecordsFileName, err)
+	}
+	return records, nil
+}
+
+func readImplementation(sweepDirectory string, record sweepImplementationRecord) (checkerVerdict, error) {
+	verdict := checkerVerdict{
+		Implementation:   record.Name,
+		FailedStage:      record.FailedStage,
+		FailedError:      record.Error,
+		ExcludedByReason: map[string]int{},
+		SurfacesObserved: map[string]bool{},
+	}
+	fired := map[string]bool{}
+	for _, run := range record.Runs {
+		verdict.RunsRecorded++
+		if run.LaunchError != "" {
+			verdict.ExcludedByReason[reasonLaunchError]++
+			continue
+		}
+		directory := resolveCampaignDirectory(sweepDirectory, record.Name, run)
+		campaignRuns, err := readCampaignRuns(directory)
+		if err != nil {
+			return checkerVerdict{}, err
+		}
+		for _, campaignRun := range campaignRuns {
+			if reason := excludedBecause(campaignRun); reason != "" {
+				verdict.ExcludedByReason[reason]++
+				continue
+			}
+			verdict.RunsUsable++
+			for _, property := range campaignRun.ViolatedProperties {
+				fired[property] = true
+			}
+			observed, err := readObservedSurfaces(filepath.Join(directory, campaignRun.RunDirectory))
+			if err != nil {
+				verdict.TraceErrors = append(verdict.TraceErrors, err.Error())
+				continue
+			}
+			verdict.SurfacesKnown = true
+			for surface, seen := range observed {
+				if seen {
+					verdict.SurfacesObserved[surface] = true
+				}
+			}
+		}
+	}
+	if len(fired) > 0 {
+		verdict.FiredProperties = slices.Sorted(maps.Keys(fired))
+	}
+	if len(verdict.ExcludedByReason) == 0 {
+		verdict.ExcludedByReason = nil
+	}
+	return verdict, nil
+}
+
+// resolveCampaignDirectory prefers the path the sweep recorded and falls back to
+// the layout it names, so a sweep directory read on another machine than the one
+// that wrote its absolute paths still resolves.
+func resolveCampaignDirectory(sweepDirectory, name string, run sweepRunRecord) string {
+	if run.CampaignDirectory != "" {
+		if _, err := os.Stat(run.CampaignDirectory); err == nil {
+			return run.CampaignDirectory
+		}
+	}
+	return filepath.Join(sweepDirectory, name, "seed-"+strconv.FormatInt(run.Seed, 10))
+}
+
+func readCampaignRuns(directory string) ([]campaignRunRecord, error) {
+	if _, err := os.Stat(filepath.Join(directory, campaignManifestFileName)); err != nil {
+		return nil, fmt.Errorf("read %s in %s: %w", campaignManifestFileName, directory, err)
+	}
+	file, err := os.Open(filepath.Join(directory, campaignRecordsFileName))
+	if err != nil {
+		return nil, fmt.Errorf("read %s: %w", campaignRecordsFileName, err)
+	}
+	defer file.Close()
+
+	var records []campaignRunRecord
+	scanner := bufio.NewScanner(file)
+	scanner.Buffer(make([]byte, 0, 64*1024), maxRecordBytes)
+	lineNumber := 0
+	for scanner.Scan() {
+		lineNumber++
+		raw := strings.TrimSpace(scanner.Text())
+		if raw == "" {
+			continue
+		}
+		var record campaignRunRecord
+		if err := json.Unmarshal([]byte(raw), &record); err != nil {
+			return nil, fmt.Errorf("%s line %d in %s: %w", campaignRecordsFileName, lineNumber, directory, err)
+		}
+		records = append(records, record)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("read %s in %s: %w", campaignRecordsFileName, directory, err)
+	}
+	return records, nil
+}
+
+func excludedBecause(record campaignRunRecord) string {
+	switch {
+	case record.LaunchError != "":
+		return reasonLaunchError
+	case record.TimedOut:
+		return reasonTimedOut
+	case record.ExitCode != 0:
+		return reasonNonzeroExit
+	case record.TraceError != "":
+		return reasonTraceError
+	default:
+		return ""
+	}
+}
+
+type traceLine struct {
+	ExtractorChanges map[string]struct {
+		Curr json.RawMessage `json:"curr"`
+	} `json:"extractor_changes"`
+}
+
+// readObservedSurfaces replays one run's locatableSurfaces readings. The
+// verifier emits every extractor as a change on the first snapshot and only on
+// a difference afterwards, so a surface true on any recorded change was located
+// at least once, and one absent from every change was never located at all.
+func readObservedSurfaces(runDirectory string) (map[string]bool, error) {
+	path := filepath.Join(runDirectory, traceFileName)
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("read %s: %w", path, err)
+	}
+	defer file.Close()
+
+	observed := map[string]bool{}
+	found := false
+	scanner := bufio.NewScanner(file)
+	scanner.Buffer(make([]byte, 0, 64*1024), maxTraceLineBytes)
+	lineNumber := 0
+	for scanner.Scan() {
+		lineNumber++
+		raw := strings.TrimSpace(scanner.Text())
+		if raw == "" {
+			continue
+		}
+		var line traceLine
+		if err := json.Unmarshal([]byte(raw), &line); err != nil {
+			return nil, fmt.Errorf("%s line %d: %w", path, lineNumber, err)
+		}
+		change, present := line.ExtractorChanges[surfacesExtractor]
+		if !present || len(change.Curr) == 0 {
+			continue
+		}
+		var reading map[string]bool
+		if err := json.Unmarshal(change.Curr, &reading); err != nil {
+			return nil, fmt.Errorf("%s line %d: %s is not an object of booleans: %w",
+				path, lineNumber, surfacesExtractor, err)
+		}
+		found = true
+		for surface, located := range reading {
+			if located {
+				observed[surface] = true
+			}
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("read %s: %w", path, err)
+	}
+	if !found {
+		return nil, fmt.Errorf("%s records no %s reading: this run cannot say whether a surface was located",
+			path, surfacesExtractor)
+	}
+	return observed, nil
+}

cmd/internal-tools/confusion-matrix/main.go

@@ -0,0 +1,111 @@
+// Command confusion-matrix cross-tabulates e4's checker verdicts against the
+// blind human review, which is the measure model-implementations.md
+// pre-registers: an implementation whose own suite passed, scored on whether a
+// property fired and on whether the reviewer found a defect.
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"time"
+)
+
+const usage = `confusion-matrix cross-tabulates the e4 checker against the blind human review.
+
+Usage:
+  confusion-matrix --sweep <dir> --reviews <dir> --assignment <path> --property-clauses <path> [--json <path>]
+
+--sweep is the directory implementation-sweep wrote: sweep.json, implementations.jsonl
+and the per-implementation campaign directories under it.
+
+--reviews holds one impl-NN.md per implementation in the shape review-protocol.md
+fixes, plus any impl-NN-adjudication.md whose resolved labels replace the first
+rater's for the clauses it names.
+
+--assignment is implementations/assignment.md, the blinded implementation-to-model
+mapping, opened only after the last verdict is filed.
+
+--property-clauses declares which requirement clauses each property covers and which
+locatable surfaces it reads. Without it a fired property cannot be scored against a
+clause and a portability miss cannot be told from a clean run.
+`
+
+func run(arguments []string, stdout, stderr io.Writer) error {
+	flagSet := flag.NewFlagSet("confusion-matrix", flag.ContinueOnError)
+	flagSet.SetOutput(stderr)
+	flagSet.Usage = func() {
+		fmt.Fprint(stderr, usage)
+		flagSet.PrintDefaults()
+	}
+	var sweepDirectory string
+	var reviewsDirectory string
+	var assignmentPath string
+	var mappingPath string
+	var jsonPath string
+	flagSet.StringVar(&sweepDirectory, "sweep", "", "directory implementation-sweep wrote")
+	flagSet.StringVar(&reviewsDirectory, "reviews", "", "directory holding impl-NN.md verdict forms")
+	flagSet.StringVar(&assignmentPath, "assignment", "", "implementations/assignment.md, the implementation-to-model mapping")
+	flagSet.StringVar(&mappingPath, "property-clauses", "", "the declared property-to-clause and surface mapping")
+	flagSet.StringVar(&jsonPath, "json", "", "write the machine-readable summary here, or - for stdout")
+	if err := flagSet.Parse(arguments); err != nil {
+		return err
+	}
+	for name, value := range map[string]string{
+		"--sweep":            sweepDirectory,
+		"--reviews":          reviewsDirectory,
+		"--assignment":       assignmentPath,
+		"--property-clauses": mappingPath,
+	} {
+		if value == "" {
+			return fmt.Errorf("%s is required", name)
+		}
+	}
+
+	mapping, err := loadMapping(mappingPath)
+	if err != nil {
+		return err
+	}
+	assignments, err := loadAssignment(assignmentPath)
+	if err != nil {
+		return err
+	}
+	checker, err := loadChecker(sweepDirectory)
+	if err != nil {
+		return err
+	}
+	reviews, err := loadReviews(reviewsDirectory)
+	if err != nil {
+		return err
+	}
+
+	result := crossTabulate(checker, reviews, assignments, mapping, time.Now().UTC())
+	writeReport(result, stdout)
+
+	if jsonPath == "" {
+		return nil
+	}
+	body, err := json.MarshalIndent(result, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal summary: %w", err)
+	}
+	body = append(body, '\n')
+	if jsonPath == "-" {
+		_, err = stdout.Write(body)
+		return err
+	}
+	return os.WriteFile(jsonPath, body, 0o644)
+}
+
+func main() {
+	if err := run(os.Args[1:], os.Stdout, os.Stderr); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return
+		}
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+}

cmd/internal-tools/confusion-matrix/mapping.go

@@ -0,0 +1,232 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"slices"
+	"strings"
+)
+
+// clauseCount is R1 to R20, the twenty clauses requirement.md numbers and the
+// twenty rows review-protocol.md requires on every verdict form.
+const clauseCount = 20
+
+const (
+	surfaceUnlocatable  = "unlocatable"
+	surfaceInconclusive = "inconclusive"
+)
+
+const todoMarker = "todo"
+
+// propertyMapping is one row of the property table: the clauses a property is
+// the oracle for, and the locatable surfaces it reads.
+type propertyMapping struct {
+	Property string
+	Clauses  []string
+	Surfaces []string
+}
+
+// surfaceMapping says what a surface never observed across an implementation's
+// whole sweep means. Only a surface the requirement obliges every
+// implementation to show at all times can be read as unlocatable; a surface
+// that is legitimately absent when nothing is in that state is inconclusive
+// and can never mark a property unevaluated.
+type surfaceMapping struct {
+	Surface       string
+	NeverObserved string
+	Note          string
+}
+
+type mapping struct {
+	Path               string
+	Properties         []propertyMapping
+	Surfaces           []surfaceMapping
+	PropertyTodoRows   int
+	byProperty         map[string]propertyMapping
+	coveringProperty   map[string][]string
+	unlocatableSurface map[string]bool
+}
+
+var clausePattern = regexp.MustCompile(`^[Rr]([0-9]{1,2})$`)
+
+func canonicalClause(value string) (string, bool) {
+	match := clausePattern.FindStringSubmatch(strings.TrimSpace(value))
+	if match == nil {
+		return "", false
+	}
+	number := match[1]
+	trimmed := strings.TrimLeft(number, "0")
+	if trimmed == "" {
+		return "", false
+	}
+	clause := "R" + trimmed
+	if !slices.Contains(allClauses(), clause) {
+		return "", false
+	}
+	return clause, true
+}
+
+func allClauses() []string {
+	clauses := make([]string, 0, clauseCount)
+	for index := 1; index <= clauseCount; index++ {
+		clauses = append(clauses, fmt.Sprintf("R%d", index))
+	}
+	return clauses
+}
+
+func loadMapping(path string) (mapping, error) {
+	body, err := os.ReadFile(path)
+	if err != nil {
+		return mapping{}, fmt.Errorf("read property-clause mapping: %w", err)
+	}
+	result := mapping{
+		Path:               path,
+		byProperty:         map[string]propertyMapping{},
+		coveringProperty:   map[string][]string{},
+		unlocatableSurface: map[string]bool{},
+	}
+	section := ""
+	for _, row := range parseTableRows(string(body)) {
+		head := strings.ToLower(row.cell(0))
+		switch head {
+		case "property":
+			section = "property"
+			continue
+		case "surface":
+			section = "surface"
+			continue
+		}
+		switch section {
+		case "property":
+			if err := result.addProperty(row); err != nil {
+				return mapping{}, fmt.Errorf("%s line %d: %w", path, row.Line, err)
+			}
+		case "surface":
+			if err := result.addSurface(row); err != nil {
+				return mapping{}, fmt.Errorf("%s line %d: %w", path, row.Line, err)
+			}
+		default:
+			return mapping{}, fmt.Errorf("%s line %d: table row before any header naming property or surface", path, row.Line)
+		}
+	}
+	if len(result.Surfaces) == 0 {
+		return mapping{}, fmt.Errorf("%s declares no surfaces: a portability miss cannot be told from a clean run without them", path)
+	}
+	for _, property := range result.Properties {
+		for _, surface := range property.Surfaces {
+			if _, declared := result.surfaceByName(surface); !declared {
+				return mapping{}, fmt.Errorf("%s: property %q reads surface %q, which the surface table does not declare",
+					path, property.Property, surface)
+			}
+		}
+	}
+	return result, nil
+}
+
+func (m *mapping) addProperty(row tableRow) error {
+	name := row.cell(0)
+	if name == "" {
+		return nil
+	}
+	if strings.EqualFold(name, todoMarker) {
+		m.PropertyTodoRows++
+		return nil
+	}
+	if _, seen := m.byProperty[name]; seen {
+		return fmt.Errorf("property %q is mapped twice", name)
+	}
+	entry := propertyMapping{Property: name}
+	for _, item := range splitList(row.cell(1)) {
+		if strings.EqualFold(item, todoMarker) || item == "-" || strings.EqualFold(item, "none") {
+			continue
+		}
+		clause, ok := canonicalClause(item)
+		if !ok {
+			return fmt.Errorf("property %q names clause %q, which is not one of R1 to R%d", name, item, clauseCount)
+		}
+		if slices.Contains(entry.Clauses, clause) {
+			continue
+		}
+		entry.Clauses = append(entry.Clauses, clause)
+	}
+	for _, item := range splitList(row.cell(2)) {
+		if strings.EqualFold(item, todoMarker) || item == "-" || strings.EqualFold(item, "none") {
+			continue
+		}
+		if !slices.Contains(entry.Surfaces, item) {
+			entry.Surfaces = append(entry.Surfaces, item)
+		}
+	}
+	m.Properties = append(m.Properties, entry)
+	m.byProperty[name] = entry
+	for _, clause := range entry.Clauses {
+		m.coveringProperty[clause] = append(m.coveringProperty[clause], name)
+	}
+	return nil
+}
+
+func (m *mapping) addSurface(row tableRow) error {
+	name := row.cell(0)
+	if name == "" || strings.EqualFold(name, todoMarker) {
+		return nil
+	}
+	meaning := strings.ToLower(row.cell(1))
+	if meaning != surfaceUnlocatable && meaning != surfaceInconclusive {
+		return fmt.Errorf("surface %q says %q for never observed, want %s or %s",
+			name, row.cell(1), surfaceUnlocatable, surfaceInconclusive)
+	}
+	if _, seen := m.surfaceByName(name); seen {
+		return fmt.Errorf("surface %q is declared twice", name)
+	}
+	m.Surfaces = append(m.Surfaces, surfaceMapping{Surface: name, NeverObserved: meaning, Note: row.cell(2)})
+	if meaning == surfaceUnlocatable {
+		m.unlocatableSurface[name] = true
+	}
+	return nil
+}
+
+func (m mapping) surfaceByName(name string) (surfaceMapping, bool) {
+	for _, surface := range m.Surfaces {
+		if surface.Surface == name {
+			return surface, true
+		}
+	}
+	return surfaceMapping{}, false
+}
+
+// unevaluable reports whether a property could not be evaluated against this
+// implementation because a surface it reads was never located. A surface the
+// mapping calls inconclusive never makes a property unevaluable, however many
+// steps failed to observe it.
+func (m mapping) unevaluable(property string, observed map[string]bool) bool {
+	entry, known := m.byProperty[property]
+	if !known {
+		return false
+	}
+	for _, surface := range entry.Surfaces {
+		if m.unlocatableSurface[surface] && !observed[surface] {
+			return true
+		}
+	}
+	return false
+}
+
+func (m mapping) covering(clause string) []string {
+	return m.coveringProperty[clause]
+}
+
+func (m mapping) knows(property string) bool {
+	_, known := m.byProperty[property]
+	return known
+}
+
+func (m mapping) unlocatableSurfaces() []string {
+	var names []string
+	for _, surface := range m.Surfaces {
+		if surface.NeverObserved == surfaceUnlocatable {
+			names = append(names, surface.Surface)
+		}
+	}
+	return names
+}

cmd/internal-tools/confusion-matrix/matrix.go

@@ -0,0 +1,510 @@
+package main
+
+import (
+	"fmt"
+	"maps"
+	"slices"
+	"strings"
+	"time"
+)
+
+// The four cells of the pre-registered measure, named as
+// model-implementations.md describes them.
+const (
+	cellTruePositive  = "checker fired, review confirmed"
+	cellFalsePositive = "checker fired, review found nothing"
+	cellFalseNegative = "checker silent, review found a defect"
+	cellTrueNegative  = "checker silent, review found nothing"
+)
+
+// Reasons an implementation is missing data rather than a cell of the matrix.
+const (
+	missingSweepStage    = "sweep stopped before any run"
+	missingNoUsableRun   = "no usable run"
+	missingNoSweepRecord = "no sweep record"
+	missingNoReview      = "no verdict filed"
+	missingMalformed     = "malformed verdict form"
+	missingSurfaces      = "surface locatability unknown"
+	missingNoModel       = "not in the assignment mapping"
+)
+
+type matrix struct {
+	Unit          string   `json:"unit"`
+	Scored        int      `json:"scored"`
+	TruePositive  int      `json:"true_positive"`
+	FalsePositive int      `json:"false_positive"`
+	FalseNegative int      `json:"false_negative"`
+	TrueNegative  int      `json:"true_negative"`
+	Precision     *float64 `json:"precision"`
+	Recall        *float64 `json:"recall"`
+}
+
+func (m *matrix) add(checkerPositive, humanPositive bool) {
+	m.Scored++
+	switch {
+	case checkerPositive && humanPositive:
+		m.TruePositive++
+	case checkerPositive:
+		m.FalsePositive++
+	case humanPositive:
+		m.FalseNegative++
+	default:
+		m.TrueNegative++
+	}
+}
+
+func (m *matrix) finish() {
+	m.Precision = ratio(m.TruePositive, m.TruePositive+m.FalsePositive)
+	m.Recall = ratio(m.TruePositive, m.TruePositive+m.FalseNegative)
+}
+
+func ratio(numerator, denominator int) *float64 {
+	if denominator == 0 {
+		return nil
+	}
+	value := float64(numerator) / float64(denominator)
+	return &value
+}
+
+// clauseMatrix scores one implementation-clause pair. Its three side buckets
+// hold the pairs that are not evidence either way: a clause the reviewer could
+// not judge, a clause whose every covering property was unevaluable because a
+// surface was never located, and, tracked but still scored, a clause no
+// property covers at all.
+type clauseMatrix struct {
+	matrix
+	CannotTell               int `json:"cannot_tell"`
+	UnevaluatedSurfaceMissed int `json:"unevaluated_surface_missed"`
+	UncoveredScored          int `json:"uncovered_clause_pairs_scored"`
+	UncoveredFalseNegative   int `json:"uncovered_clause_false_negatives"`
+}
+
+type implementationOutcome struct {
+	Implementation        string   `json:"implementation"`
+	Model                 string   `json:"model"`
+	Cell                  string   `json:"cell"`
+	FiredProperties       []string `json:"fired_properties,omitempty"`
+	ViolatedClauses       []string `json:"violated_clauses,omitempty"`
+	CannotTellClauses     []string `json:"cannot_tell_clauses,omitempty"`
+	UnlocatableSurfaces   []string `json:"unlocatable_surfaces,omitempty"`
+	UnevaluatedProperties []string `json:"unevaluated_properties,omitempty"`
+	// DefectOnlyOnUnevaluatedClauses marks a false negative the checker was
+	// never in a position to catch: every clause the reviewer faulted is
+	// covered only by properties a missing surface left unevaluable. It is a
+	// portability miss reported beside the matrix, never inside it.
+	DefectOnlyOnUnevaluatedClauses bool `json:"defect_only_on_unevaluated_clauses,omitempty"`
+	RunsUsable                     int  `json:"runs_usable"`
+	ReviewMinutes                  int  `json:"review_minutes,omitempty"`
+}
+
+type exclusion struct {
+	Implementation string `json:"implementation"`
+	Model          string `json:"model,omitempty"`
+	Reason         string `json:"reason"`
+	Detail         string `json:"detail,omitempty"`
+}
+
+type modelBreakdown struct {
+	Model                          string       `json:"model"`
+	Implementations                matrix       `json:"implementation_matrix"`
+	Clauses                        clauseMatrix `json:"clause_matrix"`
+	Excluded                       int          `json:"excluded"`
+	DefectOnlyOnUnevaluatedClauses int          `json:"defect_only_on_unevaluated_clauses"`
+}
+
+type portability struct {
+	Scored                     int            `json:"implementations_scored"`
+	WithUnlocatableSurface     int            `json:"implementations_with_an_unlocatable_surface"`
+	BySurface                  map[string]int `json:"implementations_by_unlocatable_surface,omitempty"`
+	SurfacesReadAsInconclusive []string       `json:"surfaces_a_miss_cannot_be_read_from,omitempty"`
+}
+
+type coverage struct {
+	MappedProperties         int      `json:"mapped_properties"`
+	MappingTodoRows          int      `json:"mapping_todo_rows"`
+	ClausesCovered           []string `json:"clauses_covered,omitempty"`
+	ClausesUncovered         []string `json:"clauses_no_property_covers,omitempty"`
+	FiredPropertiesNotMapped []string `json:"fired_properties_not_in_the_mapping,omitempty"`
+}
+
+type result struct {
+	GeneratedAt      time.Time               `json:"generated_at"`
+	SweepDirectory   string                  `json:"sweep_directory"`
+	ReviewsDirectory string                  `json:"reviews_directory"`
+	MappingPath      string                  `json:"property_clause_mapping"`
+	SpecPath         string                  `json:"spec_path,omitempty"`
+	Implementations  matrix                  `json:"implementation_matrix"`
+	Clauses          clauseMatrix            `json:"clause_matrix"`
+	ByModel          []modelBreakdown        `json:"by_model"`
+	Outcomes         []implementationOutcome `json:"outcomes"`
+	Excluded         []exclusion             `json:"excluded,omitempty"`
+	Portability      portability             `json:"portability"`
+	ReviewMinutes    int                     `json:"review_minutes_over_scored_implementations"`
+	Coverage         coverage                `json:"clause_coverage"`
+	Notes            []string                `json:"notes,omitempty"`
+}
+
+func crossTabulate(
+	checker checkerSide,
+	reviews reviewSide,
+	assignments map[string]string,
+	declared mapping,
+	now time.Time,
+) result {
+	outcome := result{
+		GeneratedAt:      now,
+		SweepDirectory:   checker.Directory,
+		ReviewsDirectory: reviews.Directory,
+		MappingPath:      declared.Path,
+		SpecPath:         checker.SpecPath,
+		Implementations:  matrix{Unit: "implementation"},
+		Clauses:          clauseMatrix{matrix: matrix{Unit: "implementation-clause pair"}},
+		Portability:      portability{BySurface: map[string]int{}},
+	}
+
+	reviewByName := map[string]reviewVerdict{}
+	for _, verdict := range reviews.Verdicts {
+		reviewByName[verdict.Implementation] = verdict
+	}
+	malformedByName := map[string]malformedReview{}
+	for _, entry := range reviews.Malformed {
+		malformedByName[entry.Implementation] = entry
+	}
+	checkerByName := map[string]checkerVerdict{}
+	for _, verdict := range checker.Verdicts {
+		checkerByName[verdict.Implementation] = verdict
+	}
+
+	byModel := map[string]*modelBreakdown{}
+	modelOf := func(name string) string { return assignments[name] }
+	breakdown := func(model string) *modelBreakdown {
+		current, seen := byModel[model]
+		if !seen {
+			current = &modelBreakdown{
+				Model:           model,
+				Implementations: matrix{Unit: "implementation"},
+				Clauses:         clauseMatrix{matrix: matrix{Unit: "implementation-clause pair"}},
+			}
+			byModel[model] = current
+		}
+		return current
+	}
+
+	unmappedFired := map[string]bool{}
+	for _, name := range implementationNames(checker, reviews, assignments) {
+		model := modelOf(name)
+		verdict, swept := checkerByName[name]
+		review, reviewed := reviewByName[name]
+
+		if reason, detail := missingData(name, model, verdict, swept, reviewed, malformedByName); reason != "" {
+			outcome.Excluded = append(outcome.Excluded, exclusion{
+				Implementation: name, Model: model, Reason: reason, Detail: detail,
+			})
+			if model != "" {
+				breakdown(model).Excluded++
+			}
+			continue
+		}
+
+		for _, property := range verdict.FiredProperties {
+			if !declared.knows(property) {
+				unmappedFired[property] = true
+			}
+		}
+		row := scoreImplementation(verdict, review, declared)
+		row.Model = model
+		outcome.Outcomes = append(outcome.Outcomes, row)
+
+		modelRow := breakdown(model)
+		outcome.Implementations.add(verdict.fired(), review.defective())
+		modelRow.Implementations.add(verdict.fired(), review.defective())
+		scoreClauses(&outcome.Clauses, &modelRow.Clauses, verdict, review, declared)
+
+		if row.DefectOnlyOnUnevaluatedClauses {
+			modelRow.DefectOnlyOnUnevaluatedClauses++
+		}
+		outcome.Portability.Scored++
+		outcome.ReviewMinutes += review.Minutes
+		if len(row.UnlocatableSurfaces) > 0 {
+			outcome.Portability.WithUnlocatableSurface++
+			for _, surface := range row.UnlocatableSurfaces {
+				outcome.Portability.BySurface[surface]++
+			}
+		}
+	}
+
+	outcome.Implementations.finish()
+	outcome.Clauses.finish()
+	for _, model := range capabilityOrder {
+		current, seen := byModel[model]
+		if !seen {
+			continue
+		}
+		current.Implementations.finish()
+		current.Clauses.finish()
+		outcome.ByModel = append(outcome.ByModel, *current)
+	}
+
+	outcome.Coverage = describeCoverage(declared, unmappedFired)
+	outcome.Portability.SurfacesReadAsInconclusive = inconclusiveSurfaces(declared)
+	outcome.Notes = buildNotes(outcome, declared, reviews)
+	return outcome
+}
+
+func implementationNames(checker checkerSide, reviews reviewSide, assignments map[string]string) []string {
+	names := map[string]bool{}
+	for _, planned := range checker.Planned {
+		names[planned] = true
+	}
+	for _, verdict := range checker.Verdicts {
+		names[verdict.Implementation] = true
+	}
+	for _, verdict := range reviews.Verdicts {
+		names[verdict.Implementation] = true
+	}
+	for _, entry := range reviews.Malformed {
+		names[entry.Implementation] = true
+	}
+	for name := range assignments {
+		names[name] = true
+	}
+	return slices.Sorted(maps.Keys(names))
+}
+
+// missingData names why an implementation carries no cell. A build that never
+// finished, a run that never produced a usable campaign and a verdict that was
+// never filed are all absent evidence: scoring any of them as a clean run would
+// read the gap as agreement.
+func missingData(
+	name string,
+	model string,
+	verdict checkerVerdict,
+	swept bool,
+	reviewed bool,
+	malformed map[string]malformedReview,
+) (string, string) {
+	if model == "" {
+		return missingNoModel, ""
+	}
+	if !swept {
+		return missingNoSweepRecord, ""
+	}
+	if verdict.FailedStage != "" {
+		return missingSweepStage, fmt.Sprintf("%s: %s", verdict.FailedStage, verdict.FailedError)
+	}
+	if verdict.RunsUsable == 0 {
+		return missingNoUsableRun, excludedSummary(verdict)
+	}
+	if entry, broken := malformed[name]; broken {
+		return missingMalformed, entry.Reason
+	}
+	if !reviewed {
+		return missingNoReview, ""
+	}
+	if !verdict.SurfacesKnown {
+		return missingSurfaces, strings.Join(verdict.TraceErrors, "; ")
+	}
+	return "", ""
+}
+
+func excludedSummary(verdict checkerVerdict) string {
+	if len(verdict.ExcludedByReason) == 0 {
+		return ""
+	}
+	var parts []string
+	for _, reason := range slices.Sorted(maps.Keys(verdict.ExcludedByReason)) {
+		parts = append(parts, fmt.Sprintf("%s=%d", reason, verdict.ExcludedByReason[reason]))
+	}
+	return strings.Join(parts, ", ")
+}
+
+func scoreImplementation(verdict checkerVerdict, review reviewVerdict, declared mapping) implementationOutcome {
+	row := implementationOutcome{
+		Implementation:  verdict.Implementation,
+		Cell:            cellOf(verdict.fired(), review.defective()),
+		FiredProperties: verdict.FiredProperties,
+		ViolatedClauses: review.violatedClauses(),
+		RunsUsable:      verdict.RunsUsable,
+		ReviewMinutes:   review.Minutes,
+	}
+	for _, clause := range allClauses() {
+		if review.Clauses[clause] == clauseCannotTell {
+			row.CannotTellClauses = append(row.CannotTellClauses, clause)
+		}
+	}
+	for _, surface := range declared.unlocatableSurfaces() {
+		if !verdict.SurfacesObserved[surface] {
+			row.UnlocatableSurfaces = append(row.UnlocatableSurfaces, surface)
+		}
+	}
+	for _, property := range declared.Properties {
+		if declared.unevaluable(property.Property, verdict.SurfacesObserved) {
+			row.UnevaluatedProperties = append(row.UnevaluatedProperties, property.Property)
+		}
+	}
+	row.DefectOnlyOnUnevaluatedClauses = attributableToUnevaluated(verdict, review, declared)
+	return row
+}
+
+// attributableToUnevaluated reports a false negative the checker could not have
+// caught: it fired nothing, the reviewer faulted at least one clause, and every
+// clause the reviewer faulted is covered only by properties a missing surface
+// left unevaluable.
+func attributableToUnevaluated(verdict checkerVerdict, review reviewVerdict, declared mapping) bool {
+	if verdict.fired() || !review.defective() {
+		return false
+	}
+	violated := review.violatedClauses()
+	if len(violated) == 0 {
+		return false
+	}
+	for _, clause := range violated {
+		covering := declared.covering(clause)
+		if len(covering) == 0 {
+			return false
+		}
+		for _, property := range covering {
+			if !declared.unevaluable(property, verdict.SurfacesObserved) {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+func cellOf(checkerPositive, humanPositive bool) string {
+	switch {
+	case checkerPositive && humanPositive:
+		return cellTruePositive
+	case checkerPositive:
+		return cellFalsePositive
+	case humanPositive:
+		return cellFalseNegative
+	default:
+		return cellTrueNegative
+	}
+}
+
+func scoreClauses(overall, model *clauseMatrix, verdict checkerVerdict, review reviewVerdict, declared mapping) {
+	fired := map[string]bool{}
+	for _, property := range verdict.FiredProperties {
+		fired[property] = true
+	}
+	for _, clause := range allClauses() {
+		label := review.Clauses[clause]
+		if label == clauseCannotTell {
+			overall.CannotTell++
+			model.CannotTell++
+			continue
+		}
+		covering := declared.covering(clause)
+		checkerPositive := false
+		for _, property := range covering {
+			if fired[property] {
+				checkerPositive = true
+				break
+			}
+		}
+		if !checkerPositive && len(covering) > 0 && allUnevaluable(covering, verdict, declared) {
+			overall.UnevaluatedSurfaceMissed++
+			model.UnevaluatedSurfaceMissed++
+			continue
+		}
+		humanPositive := label == clauseViolates
+		overall.add(checkerPositive, humanPositive)
+		model.add(checkerPositive, humanPositive)
+		if len(covering) == 0 {
+			overall.UncoveredScored++
+			model.UncoveredScored++
+			if humanPositive {
+				overall.UncoveredFalseNegative++
+				model.UncoveredFalseNegative++
+			}
+		}
+	}
+}
+
+func allUnevaluable(covering []string, verdict checkerVerdict, declared mapping) bool {
+	for _, property := range covering {
+		if !declared.unevaluable(property, verdict.SurfacesObserved) {
+			return false
+		}
+	}
+	return true
+}
+
+func describeCoverage(declared mapping, unmappedFired map[string]bool) coverage {
+	result := coverage{
+		MappedProperties: len(declared.Properties),
+		MappingTodoRows:  declared.PropertyTodoRows,
+	}
+	for _, clause := range allClauses() {
+		if len(declared.covering(clause)) > 0 {
+			result.ClausesCovered = append(result.ClausesCovered, clause)
+			continue
+		}
+		result.ClausesUncovered = append(result.ClausesUncovered, clause)
+	}
+	if len(unmappedFired) > 0 {
+		result.FiredPropertiesNotMapped = slices.Sorted(maps.Keys(unmappedFired))
+	}
+	return result
+}
+
+func inconclusiveSurfaces(declared mapping) []string {
+	var names []string
+	for _, surface := range declared.Surfaces {
+		if surface.NeverObserved == surfaceInconclusive {
+			names = append(names, surface.Surface)
+		}
+	}
+	return names
+}
+
+func buildNotes(outcome result, declared mapping, reviews reviewSide) []string {
+	var notes []string
+	if len(declared.Properties) == 0 {
+		notes = append(notes, fmt.Sprintf(
+			"%s maps no property to a clause, so the clause matrix is empty and no portability miss can be detected; "+
+				"the implementation matrix below stands on its own", declared.Path))
+	}
+	if declared.PropertyTodoRows > 0 {
+		notes = append(notes, fmt.Sprintf("%s still carries %d TODO row(s) in its property table",
+			declared.Path, declared.PropertyTodoRows))
+	}
+	if len(outcome.Coverage.FiredPropertiesNotMapped) > 0 {
+		notes = append(notes, fmt.Sprintf(
+			"%d fired propert(ies) are absent from the mapping and could not be attributed to a clause: %s",
+			len(outcome.Coverage.FiredPropertiesNotMapped),
+			strings.Join(outcome.Coverage.FiredPropertiesNotMapped, ", ")))
+	}
+	for _, verdict := range reviews.Verdicts {
+		if !verdict.defective() && len(verdict.violatedClauses()) > 0 {
+			notes = append(notes, fmt.Sprintf(
+				"%s files %d violating clause(s) under an overall verdict of %s; the overall verdict is what the matrix scores",
+				verdict.Implementation, len(verdict.violatedClauses()), overallNotDefective))
+		}
+		if len(verdict.Adjudicated) > 0 {
+			notes = append(notes, fmt.Sprintf("%s uses the adjudicated label for %s",
+				verdict.Implementation, strings.Join(verdict.Adjudicated, ", ")))
+		}
+	}
+	if count := attributedFalseNegatives(outcome); count > 0 {
+		notes = append(notes, fmt.Sprintf(
+			"%d false negative(s) fault only clauses whose every property a missing surface left unevaluable: "+
+				"those are portability misses, not blind spots", count))
+	}
+	notes = append(notes, "second-rater agreement and Cohen's kappa are not computed here; "+
+		"an impl-NN-adjudication.md is read and its resolved labels replace the first rater's")
+	return notes
+}
+
+func attributedFalseNegatives(outcome result) int {
+	count := 0
+	for _, row := range outcome.Outcomes {
+		if row.DefectOnlyOnUnevaluatedClauses {
+			count++
+		}
+	}
+	return count
+}

cmd/internal-tools/confusion-matrix/report.go

@@ -0,0 +1,169 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"maps"
+	"slices"
+	"strconv"
+	"strings"
+	"text/tabwriter"
+)
+
+// labelledMatrix is one row of a printed matrix: the whole sample, or one model.
+type labelledMatrix struct {
+	Label    string
+	Value    clauseMatrix
+	Excluded int
+}
+
+func writeReport(outcome result, out io.Writer) {
+	fmt.Fprintln(out, "unit: implementation whose own suite passed, scored on whether a property fired and on the reviewer's overall verdict")
+	fmt.Fprintln(out, "an implementation with no cell is missing data and is listed separately, never counted as a clean run")
+	fmt.Fprintln(out)
+	writeTable(out, []string{"group", "scored", "true pos", "false pos", "false neg", "true neg", "precision", "recall", "no cell"},
+		func(add func(...string)) {
+			for _, row := range implementationRows(outcome) {
+				add(
+					row.Label,
+					strconv.Itoa(row.Value.Scored),
+					strconv.Itoa(row.Value.TruePositive),
+					strconv.Itoa(row.Value.FalsePositive),
+					strconv.Itoa(row.Value.FalseNegative),
+					strconv.Itoa(row.Value.TrueNegative),
+					formatRatio(row.Value.Precision),
+					formatRatio(row.Value.Recall),
+					strconv.Itoa(row.Excluded),
+				)
+			}
+		})
+
+	fmt.Fprintln(out)
+	fmt.Fprintln(out, "clause pairs are an implementation against one of R1 to R20, scored through the declared property-to-clause mapping")
+	fmt.Fprintln(out, "cannot tell is the reviewer's specification-error answer and is evidence neither way")
+	fmt.Fprintln(out, "unevaluated is a clause whose every covering property a never-located surface left unrunnable: a portability miss, not a cell")
+	writeTable(out, []string{"group", "scored", "true pos", "false pos", "false neg", "true neg",
+		"precision", "recall", "cannot tell", "unevaluated", "uncovered", "uncovered false neg"},
+		func(add func(...string)) {
+			for _, row := range clauseRows(outcome) {
+				add(
+					row.Label,
+					strconv.Itoa(row.Value.Scored),
+					strconv.Itoa(row.Value.TruePositive),
+					strconv.Itoa(row.Value.FalsePositive),
+					strconv.Itoa(row.Value.FalseNegative),
+					strconv.Itoa(row.Value.TrueNegative),
+					formatRatio(row.Value.Precision),
+					formatRatio(row.Value.Recall),
+					strconv.Itoa(row.Value.CannotTell),
+					strconv.Itoa(row.Value.UnevaluatedSurfaceMissed),
+					strconv.Itoa(row.Value.UncoveredScored),
+					strconv.Itoa(row.Value.UncoveredFalseNegative),
+				)
+			}
+		})
+
+	fmt.Fprintln(out)
+	writeTable(out, []string{"implementation", "model", "cell", "usable runs", "fired", "violated clauses", "cannot tell"},
+		func(add func(...string)) {
+			for _, row := range outcome.Outcomes {
+				add(
+					row.Implementation,
+					row.Model,
+					row.Cell,
+					strconv.Itoa(row.RunsUsable),
+					joinOrDash(row.FiredProperties),
+					joinOrDash(row.ViolatedClauses),
+					strconv.Itoa(len(row.CannotTellClauses)),
+				)
+			}
+		})
+
+	if len(outcome.Excluded) > 0 {
+		fmt.Fprintf(out, "\n%d implementation(s) carry no cell: missing data, never a clean run\n", len(outcome.Excluded))
+		writeTable(out, []string{"implementation", "model", "reason", "detail"}, func(add func(...string)) {
+			for _, entry := range outcome.Excluded {
+				add(entry.Implementation, orDash(entry.Model), entry.Reason, orDash(entry.Detail))
+			}
+		})
+	}
+
+	fmt.Fprintf(out, "\nspecification portability over the %d scored implementation(s): %d needed a locating adaptation\n",
+		outcome.Portability.Scored, outcome.Portability.WithUnlocatableSurface)
+	if len(outcome.Portability.BySurface) > 0 {
+		writeTable(out, []string{"surface never located", "implementations"}, func(add func(...string)) {
+			for _, surface := range slices.Sorted(maps.Keys(outcome.Portability.BySurface)) {
+				add(surface, strconv.Itoa(outcome.Portability.BySurface[surface]))
+			}
+		})
+	}
+	if len(outcome.Portability.SurfacesReadAsInconclusive) > 0 {
+		fmt.Fprintf(out, "a miss on %s cannot be told from that surface being legitimately absent, so neither counts against portability\n",
+			strings.Join(outcome.Portability.SurfacesReadAsInconclusive, ", "))
+	}
+
+	fmt.Fprintf(out, "\nclause coverage: %d mapped propert(ies) cover %d of %d clauses\n",
+		outcome.Coverage.MappedProperties, len(outcome.Coverage.ClausesCovered), clauseCount)
+	if len(outcome.Coverage.ClausesUncovered) > 0 {
+		fmt.Fprintf(out, "no property covers %s\n", strings.Join(outcome.Coverage.ClausesUncovered, ", "))
+	}
+	fmt.Fprintf(out, "review cost over the scored implementations: %d minutes\n", outcome.ReviewMinutes)
+
+	for _, note := range outcome.Notes {
+		fmt.Fprintf(out, "\nnote: %s\n", note)
+	}
+}
+
+func implementationRows(outcome result) []labelledMatrix {
+	rows := []labelledMatrix{{
+		Label:    "all",
+		Value:    clauseMatrix{matrix: outcome.Implementations},
+		Excluded: len(outcome.Excluded),
+	}}
+	for _, model := range outcome.ByModel {
+		rows = append(rows, labelledMatrix{
+			Label:    model.Model,
+			Value:    clauseMatrix{matrix: model.Implementations},
+			Excluded: model.Excluded,
+		})
+	}
+	return rows
+}
+
+func clauseRows(outcome result) []labelledMatrix {
+	rows := []labelledMatrix{{Label: "all", Value: outcome.Clauses}}
+	for _, model := range outcome.ByModel {
+		rows = append(rows, labelledMatrix{Label: model.Model, Value: model.Clauses})
+	}
+	return rows
+}
+
+func writeTable(out io.Writer, header []string, rows func(add func(...string))) {
+	writer := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(writer, strings.Join(header, "\t"))
+	rows(func(cells ...string) {
+		fmt.Fprintln(writer, strings.Join(cells, "\t"))
+	})
+	writer.Flush()
+}
+
+func formatRatio(value *float64) string {
+	if value == nil {
+		return "n/a"
+	}
+	return strconv.FormatFloat(*value, 'f', 3, 64)
+}
+
+func joinOrDash(values []string) string {
+	if len(values) == 0 {
+		return "-"
+	}
+	return strings.Join(values, " ")
+}
+
+func orDash(value string) string {
+	if value == "" {
+		return "-"
+	}
+	return value
+}

cmd/internal-tools/confusion-matrix/review.go

@@ -0,0 +1,237 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+const (
+	clauseMeets      = "meets"
+	clauseViolates   = "violates"
+	clauseCannotTell = "cannot tell"
+)
+
+const (
+	overallDefective    = "defective"
+	overallNotDefective = "not defective"
+)
+
+// reviewVerdict is one filed verdict form: the twenty clause rows and the
+// overall verdict review-protocol.md requires, with any adjudicated label
+// already substituted for the first rater's.
+type reviewVerdict struct {
+	Implementation string
+	Path           string
+	Reviewer       string
+	Date           string
+	Minutes        int
+	Overall        string
+	Clauses        map[string]string
+	Adjudicated    []string
+}
+
+func (v reviewVerdict) defective() bool { return v.Overall == overallDefective }
+
+func (v reviewVerdict) violatedClauses() []string {
+	var violated []string
+	for _, clause := range allClauses() {
+		if v.Clauses[clause] == clauseViolates {
+			violated = append(violated, clause)
+		}
+	}
+	return violated
+}
+
+type malformedReview struct {
+	Implementation string
+	Path           string
+	Reason         string
+}
+
+type reviewSide struct {
+	Directory string
+	Verdicts  []reviewVerdict
+	Malformed []malformedReview
+}
+
+var (
+	reviewFileName      = regexp.MustCompile(`^(impl-\d+)\.md$`)
+	adjudicationName    = regexp.MustCompile(`^(impl-\d+)-adjudication\.md$`)
+	secondRaterName     = regexp.MustCompile(`^impl-\d+-r2\.md$`)
+	keyValueLine        = regexp.MustCompile(`(?i)^\s*[-*]?\s*(reviewer|date|minutes|overall verdict|overall)\s*:\s*(.+?)\s*$`)
+	leadingWholeNumbers = regexp.MustCompile(`^\d+`)
+)
+
+func loadReviews(directory string) (reviewSide, error) {
+	entries, err := os.ReadDir(directory)
+	if err != nil {
+		return reviewSide{}, fmt.Errorf("read reviews: %w", err)
+	}
+	side := reviewSide{Directory: directory}
+	adjudications := map[string]map[string]string{}
+	var forms []struct {
+		name string
+		path string
+	}
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+		path := filepath.Join(directory, entry.Name())
+		switch {
+		case secondRaterName.MatchString(entry.Name()):
+			continue
+		case adjudicationName.MatchString(entry.Name()):
+			name := adjudicationName.FindStringSubmatch(entry.Name())[1]
+			resolved, err := readAdjudication(path)
+			if err != nil {
+				side.Malformed = append(side.Malformed, malformedReview{
+					Implementation: name, Path: path, Reason: err.Error(),
+				})
+				continue
+			}
+			adjudications[name] = resolved
+		case reviewFileName.MatchString(entry.Name()):
+			forms = append(forms, struct {
+				name string
+				path string
+			}{reviewFileName.FindStringSubmatch(entry.Name())[1], path})
+		}
+	}
+
+	for _, form := range forms {
+		verdict, err := readReview(form.name, form.path)
+		if err != nil {
+			side.Malformed = append(side.Malformed, malformedReview{
+				Implementation: form.name, Path: form.path, Reason: err.Error(),
+			})
+			continue
+		}
+		for clause, label := range adjudications[form.name] {
+			if verdict.Clauses[clause] != label {
+				verdict.Adjudicated = append(verdict.Adjudicated, clause)
+			}
+			verdict.Clauses[clause] = label
+		}
+		slices.Sort(verdict.Adjudicated)
+		side.Verdicts = append(side.Verdicts, verdict)
+	}
+	slices.SortFunc(side.Verdicts, func(a, b reviewVerdict) int {
+		return strings.Compare(a.Implementation, b.Implementation)
+	})
+	slices.SortFunc(side.Malformed, func(a, b malformedReview) int {
+		return strings.Compare(a.Path, b.Path)
+	})
+	return side, nil
+}
+
+func readReview(name, path string) (reviewVerdict, error) {
+	body, err := os.ReadFile(path)
+	if err != nil {
+		return reviewVerdict{}, err
+	}
+	verdict := reviewVerdict{Implementation: name, Path: path, Clauses: map[string]string{}}
+	for _, line := range strings.Split(string(body), "\n") {
+		match := keyValueLine.FindStringSubmatch(normalizeCell(line))
+		if match == nil {
+			continue
+		}
+		value := strings.TrimSpace(match[2])
+		switch strings.ToLower(match[1]) {
+		case "reviewer":
+			verdict.Reviewer = value
+		case "date":
+			verdict.Date = value
+		case "minutes":
+			if digits := leadingWholeNumbers.FindString(value); digits != "" {
+				verdict.Minutes, _ = strconv.Atoi(digits)
+			}
+		case "overall", "overall verdict":
+			overall, ok := canonicalOverall(value)
+			if !ok {
+				return reviewVerdict{}, fmt.Errorf("overall verdict %q is neither %s nor %s", value, overallDefective, overallNotDefective)
+			}
+			verdict.Overall = overall
+		}
+	}
+	clauses, err := readClauseRows(string(body))
+	if err != nil {
+		return reviewVerdict{}, err
+	}
+	verdict.Clauses = clauses
+	for _, clause := range allClauses() {
+		if _, filed := verdict.Clauses[clause]; !filed {
+			return reviewVerdict{}, fmt.Errorf("no row for clause %s: the form must carry all %d", clause, clauseCount)
+		}
+	}
+	if verdict.Overall == "" {
+		return reviewVerdict{}, fmt.Errorf("no overall verdict: the matrix scores the reviewer's own %s or %s", overallDefective, overallNotDefective)
+	}
+	return verdict, nil
+}
+
+func readClauseRows(body string) (map[string]string, error) {
+	clauses := map[string]string{}
+	for _, row := range parseTableRows(body) {
+		clause, ok := canonicalClause(row.cell(0))
+		if !ok {
+			continue
+		}
+		label, ok := canonicalClauseVerdict(row.cell(1))
+		if !ok {
+			return nil, fmt.Errorf("clause %s has verdict %q, want %s, %s or %s",
+				clause, row.cell(1), clauseMeets, clauseViolates, clauseCannotTell)
+		}
+		if existing, seen := clauses[clause]; seen && existing != label {
+			return nil, fmt.Errorf("clause %s is filed twice, as %q and %q", clause, existing, label)
+		}
+		clauses[clause] = label
+	}
+	return clauses, nil
+}
+
+func readAdjudication(path string) (map[string]string, error) {
+	body, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	resolved, err := readClauseRows(string(body))
+	if err != nil {
+		return nil, err
+	}
+	if len(resolved) == 0 {
+		return nil, fmt.Errorf("no resolved clause rows")
+	}
+	return resolved, nil
+}
+
+func canonicalClauseVerdict(value string) (string, bool) {
+	switch strings.ToLower(strings.TrimSpace(value)) {
+	case clauseMeets, "meet", "met":
+		return clauseMeets, true
+	case clauseViolates, "violate", "violated":
+		return clauseViolates, true
+	case clauseCannotTell, "cannot-tell", "cannot_tell", "can't tell", "cant tell":
+		return clauseCannotTell, true
+	default:
+		return "", false
+	}
+}
+
+func canonicalOverall(value string) (string, bool) {
+	cleaned := strings.ToLower(strings.TrimSpace(value))
+	cleaned = strings.TrimSuffix(cleaned, ".")
+	switch cleaned {
+	case overallDefective:
+		return overallDefective, true
+	case overallNotDefective, "not-defective", "no defect", "clean":
+		return overallNotDefective, true
+	default:
+		return "", false
+	}
+}

cmd/internal-tools/confusion-matrix/table.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"regexp"
+	"strings"
+)
+
+// tableRow is one pipe-delimited markdown row with its 1-based line number, so
+// a malformed cell can name the line the author has to go and fix.
+type tableRow struct {
+	Line  int
+	Cells []string
+}
+
+var separatorCell = regexp.MustCompile(`^:?-{2,}:?$`)
+
+func parseTableRows(body string) []tableRow {
+	var rows []tableRow
+	for index, raw := range strings.Split(body, "\n") {
+		line := strings.TrimSpace(raw)
+		if !strings.HasPrefix(line, "|") {
+			continue
+		}
+		cells := splitCells(line)
+		if len(cells) == 0 || isSeparatorRow(cells) {
+			continue
+		}
+		rows = append(rows, tableRow{Line: index + 1, Cells: cells})
+	}
+	return rows
+}
+
+func splitCells(line string) []string {
+	trimmed := strings.Trim(line, "|")
+	parts := strings.Split(trimmed, "|")
+	cells := make([]string, 0, len(parts))
+	for _, part := range parts {
+		cells = append(cells, normalizeCell(part))
+	}
+	return cells
+}
+
+func isSeparatorRow(cells []string) bool {
+	for _, cell := range cells {
+		if !separatorCell.MatchString(cell) {
+			return false
+		}
+	}
+	return true
+}
+
+func normalizeCell(value string) string {
+	cleaned := strings.ReplaceAll(value, "**", "")
+	cleaned = strings.ReplaceAll(cleaned, "`", "")
+	return strings.Join(strings.Fields(cleaned), " ")
+}
+
+func (row tableRow) cell(index int) string {
+	if index >= len(row.Cells) {
+		return ""
+	}
+	return row.Cells[index]
+}
+
+var listSeparators = regexp.MustCompile(`[,\s]+`)
+
+func splitList(value string) []string {
+	trimmed := strings.TrimSpace(value)
+	if trimmed == "" {
+		return nil
+	}
+	var items []string
+	for _, item := range listSeparators.Split(trimmed, -1) {
+		if item != "" {
+			items = append(items, item)
+		}
+	}
+	return items
+}