feat(trace): record one typed outcome per model-driven step

llm-calls.jsonl carries the prompts as sent, the candidate list as the model
saw it, the screenshot reference, the raw response, tokens, latency and how the
step ended. It sits beside trace.jsonl rather than inside it because every trace
line already carries a full hierarchy and both the replay server and the
campaign summarizer scan all of them; folding prompts in would grow the lines
those readers parse for data neither reads.

Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
pj committed 2026-08-12 22:56:58 +05:30
1 parent b9cdf7a571
commit ec5872ac6e
3 files changed
+293 -4

No files matched your search

+33 -4
View File
@@ -31,6 +31,10 @@ type Step struct {
// retry budget. The verifier is skipped for these steps so transient
// state does not poison the previous/current extractor advance.
Transitional bool `json:"transitional,omitempty"`
// ActionSkipped names why NextAction was chosen but never dispatched, so a
// count of executed actions cannot be inflated by steps whose action the
// runner threw away. Empty when the action ran (or when none was chosen).
ActionSkipped string `json:"action_skipped,omitempty"`
// SkippedVerification is set true exactly when the verifier was skipped
// for this step, so downstream tooling can tell a deliberately-skipped
// step from one that was verified and came back clean.
@@ -151,6 +155,10 @@ type Writer struct {
mutex sync.Mutex
file io.WriteCloser
encoder *json.Encoder
// llmCallFile is opened on the first WriteLLMCall, so a run whose picker
// never called a model leaves no llm-calls.jsonl behind at all.
llmCallFile io.WriteCloser
llmCallEncoder *json.Encoder
}
// NewWriter ensures `directory` exists and opens trace.jsonl for append.
@@ -194,14 +202,27 @@ func (w *Writer) WriteStep(step Step) error {
// file via os.WriteFile and touches no field of Writer, so concurrent calls
// never contend.
func (w *Writer) WriteScreenshot(stepIndex int, png []byte) error {
return w.writePNG(fmt.Sprintf("step-%05d.png", stepIndex), png)
return w.writePNG(screenshotName(stepIndex), png)
}
func screenshotName(stepIndex int) string {
return fmt.Sprintf("step-%05d.png", stepIndex)
}
// ScreenshotReference is the run-relative path WriteScreenshot puts a step's
// screenshot at. Records that describe an image point at it instead of copying
// the bytes.
func ScreenshotReference(stepIndex int) string {
return screenshotDirectory + "/" + screenshotName(stepIndex)
}
const screenshotDirectory = "screenshots"
func (w *Writer) writePNG(name string, png []byte) error {
if len(png) == 0 {
return nil
}
directory := filepath.Join(w.directory, "screenshots")
directory := filepath.Join(w.directory, screenshotDirectory)
if err := os.MkdirAll(directory, 0o755); err != nil {
return fmt.Errorf("mkdir screenshots: %w", err)
}
@@ -211,10 +232,18 @@ func (w *Writer) writePNG(name string, png []byte) error {
func (w *Writer) Close() error {
w.mutex.Lock()
defer w.mutex.Unlock()
var err error
if w.llmCallFile != nil {
err = w.llmCallFile.Close()
w.llmCallFile = nil
w.llmCallEncoder = nil
}
if w.file == nil {
return nil
return err
}
if closeErr := w.file.Close(); closeErr != nil {
err = closeErr
}
err := w.file.Close()
w.file = nil
return err
}