mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
pj
56a002d87a
Cross-tabulates the properties that fired against the human verdict, one cell per implementation, over a sweep whose implementations all passed their own generated tests. An implementation that failed to build, has no usable run, or carries no filed verdict is listed as missing data rather than counted as a clean cell. Landing the package in one commit because the intermediate splits would not link.
56 lines
1.5 KiB
Go
56 lines
1.5 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"regexp"
|
|
"strings"
|
|
)
|
|
|
|
// The three models the sample is drawn from, in the capability order
|
|
// model-implementations.md fixed before any implementation was generated.
|
|
var capabilityOrder = []string{"Sonnet 5", "Opus 5", "Fable 5"}
|
|
|
|
var (
|
|
implementationName = regexp.MustCompile(`^impl-\d+$`)
|
|
nonAlphanumeric = regexp.MustCompile(`[^a-z0-9]`)
|
|
)
|
|
|
|
func loadAssignment(path string) (map[string]string, error) {
|
|
body, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read assignment: %w", err)
|
|
}
|
|
assignments := map[string]string{}
|
|
for _, row := range parseTableRows(string(body)) {
|
|
name := row.cell(0)
|
|
if !implementationName.MatchString(name) {
|
|
continue
|
|
}
|
|
model, ok := canonicalModel(row.cell(1))
|
|
if !ok {
|
|
return nil, fmt.Errorf("%s line %d: %s is assigned model %q, which is none of %s",
|
|
path, row.Line, name, row.cell(1), strings.Join(capabilityOrder, ", "))
|
|
}
|
|
if existing, seen := assignments[name]; seen && existing != model {
|
|
return nil, fmt.Errorf("%s line %d: %s is assigned to both %s and %s",
|
|
path, row.Line, name, existing, model)
|
|
}
|
|
assignments[name] = model
|
|
}
|
|
if len(assignments) == 0 {
|
|
return nil, fmt.Errorf("%s maps no implementation to a model", path)
|
|
}
|
|
return assignments, nil
|
|
}
|
|
|
|
func canonicalModel(value string) (string, bool) {
|
|
key := nonAlphanumeric.ReplaceAllString(strings.ToLower(value), "")
|
|
for _, model := range capabilityOrder {
|
|
if key == nonAlphanumeric.ReplaceAllString(strings.ToLower(model), "") {
|
|
return model, true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|