feat(trace): emit extractor_changes per step

Add ExtractorChanges to trace.Step and a runner helper that converts
the verifier's diff map into the trace shape. The inspect UI keys
its violation breadcrumbs off this field.
This commit is contained in:
pj committed 2026-05-30 21:58:08 +05:30
1 parent c66c4e0ad4
commit e44ad56749
2 files changed
+43 -18

No files matched your search

+23 -8
View File
@@ -187,14 +187,15 @@ func Run(ctx context.Context, options Options) (Summary, error) {
}
step := trace.Step{
Index: stepIndex,
Timestamp: stepStart,
Screen: screen,
Action: traceAction,
Violations: violations,
Hierarchy: tree,
Residuals: residuals,
Metrics: metrics,
Index: stepIndex,
Timestamp: stepStart,
Screen: screen,
Action: traceAction,
Violations: violations,
Hierarchy: tree,
Residuals: residuals,
Metrics: metrics,
ExtractorChanges: encodeExtractorChanges(options.Verifier.ChangedExtractors()),
}
if err := options.TraceWriter.WriteStep(step); err != nil {
return summary, fmt.Errorf("step %d trace: %w", stepIndex, err)
@@ -554,6 +555,20 @@ func captureScreenshot(ctx context.Context, options Options, logger *slog.Logger
}
}
func encodeExtractorChanges(changes map[string]verifier.ExtractorChange) map[string]trace.ExtractorChange {
if len(changes) == 0 {
return nil
}
out := make(map[string]trace.ExtractorChange, len(changes))
for name, change := range changes {
out[name] = trace.ExtractorChange{
Prev: json.RawMessage(change.Prev),
Curr: json.RawMessage(change.Curr),
}
}
return out
}
func encodeResiduals(residuals map[string]ltl.Formula) (map[string]json.RawMessage, error) {
if len(residuals) == 0 {
return nil, nil
+20 -10
View File
@@ -13,16 +13,26 @@ import (
)
type Step struct {
Index int `json:"step"`
Timestamp time.Time `json:"timestamp"`
Screen string `json:"screen,omitempty"`
Snapshots map[string]json.RawMessage `json:"snapshots,omitempty"`
Action *Action `json:"action,omitempty"`
Exceptions []Exception `json:"exceptions,omitempty"`
Violations []string `json:"violations,omitempty"`
Hierarchy *hierarchy.Tree `json:"hierarchy,omitempty"`
Residuals map[string]json.RawMessage `json:"residuals,omitempty"`
Metrics *Metrics `json:"metrics,omitempty"`
Index int `json:"step"`
Timestamp time.Time `json:"timestamp"`
Screen string `json:"screen,omitempty"`
Snapshots map[string]json.RawMessage `json:"snapshots,omitempty"`
Action *Action `json:"action,omitempty"`
Exceptions []Exception `json:"exceptions,omitempty"`
Violations []string `json:"violations,omitempty"`
Hierarchy *hierarchy.Tree `json:"hierarchy,omitempty"`
Residuals map[string]json.RawMessage `json:"residuals,omitempty"`
Metrics *Metrics `json:"metrics,omitempty"`
ExtractorChanges map[string]ExtractorChange `json:"extractor_changes,omitempty"`
}
// ExtractorChange records the prev/curr JSON values of an extractor whose
// observation differed between two consecutive steps. Surfaced under
// violation rows in the inspect UI as a "what changed at this step"
// breadcrumb.
type ExtractorChange struct {
Prev json.RawMessage `json:"prev"`
Curr json.RawMessage `json:"curr"`
}
type Metrics struct {