mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-04 12:07:09 +00:00
feat(trace): record one typed outcome per model-driven step
llm-calls.jsonl carries the prompts as sent, the candidate list as the model saw it, the screenshot reference, the raw response, tokens, latency and how the step ended. It sits beside trace.jsonl rather than inside it because every trace line already carries a full hierarchy and both the replay server and the campaign summarizer scan all of them; folding prompts in would grow the lines those readers parse for data neither reads. Claude-Session: https://claude.ai/code/session_01A5KmftdEJ49A9z5mF5ESrX
This commit is contained in:
1 parent
b9cdf7a571
commit
ec5872ac6e
3 files changed
+293
-4
No files matched your search
@@ -31,6 +31,10 @@ type Step struct {
|
||||
// retry budget. The verifier is skipped for these steps so transient
|
||||
// state does not poison the previous/current extractor advance.
|
||||
Transitional bool `json:"transitional,omitempty"`
|
||||
// ActionSkipped names why NextAction was chosen but never dispatched, so a
|
||||
// count of executed actions cannot be inflated by steps whose action the
|
||||
// runner threw away. Empty when the action ran (or when none was chosen).
|
||||
ActionSkipped string `json:"action_skipped,omitempty"`
|
||||
// SkippedVerification is set true exactly when the verifier was skipped
|
||||
// for this step, so downstream tooling can tell a deliberately-skipped
|
||||
// step from one that was verified and came back clean.
|
||||
@@ -151,6 +155,10 @@ type Writer struct {
|
||||
mutex sync.Mutex
|
||||
file io.WriteCloser
|
||||
encoder *json.Encoder
|
||||
// llmCallFile is opened on the first WriteLLMCall, so a run whose picker
|
||||
// never called a model leaves no llm-calls.jsonl behind at all.
|
||||
llmCallFile io.WriteCloser
|
||||
llmCallEncoder *json.Encoder
|
||||
}
|
||||
|
||||
// NewWriter ensures `directory` exists and opens trace.jsonl for append.
|
||||
@@ -194,14 +202,27 @@ func (w *Writer) WriteStep(step Step) error {
|
||||
// file via os.WriteFile and touches no field of Writer, so concurrent calls
|
||||
// never contend.
|
||||
func (w *Writer) WriteScreenshot(stepIndex int, png []byte) error {
|
||||
return w.writePNG(fmt.Sprintf("step-%05d.png", stepIndex), png)
|
||||
return w.writePNG(screenshotName(stepIndex), png)
|
||||
}
|
||||
|
||||
func screenshotName(stepIndex int) string {
|
||||
return fmt.Sprintf("step-%05d.png", stepIndex)
|
||||
}
|
||||
|
||||
// ScreenshotReference is the run-relative path WriteScreenshot puts a step's
|
||||
// screenshot at. Records that describe an image point at it instead of copying
|
||||
// the bytes.
|
||||
func ScreenshotReference(stepIndex int) string {
|
||||
return screenshotDirectory + "/" + screenshotName(stepIndex)
|
||||
}
|
||||
|
||||
const screenshotDirectory = "screenshots"
|
||||
|
||||
func (w *Writer) writePNG(name string, png []byte) error {
|
||||
if len(png) == 0 {
|
||||
return nil
|
||||
}
|
||||
directory := filepath.Join(w.directory, "screenshots")
|
||||
directory := filepath.Join(w.directory, screenshotDirectory)
|
||||
if err := os.MkdirAll(directory, 0o755); err != nil {
|
||||
return fmt.Errorf("mkdir screenshots: %w", err)
|
||||
}
|
||||
@@ -211,10 +232,18 @@ func (w *Writer) writePNG(name string, png []byte) error {
|
||||
func (w *Writer) Close() error {
|
||||
w.mutex.Lock()
|
||||
defer w.mutex.Unlock()
|
||||
var err error
|
||||
if w.llmCallFile != nil {
|
||||
err = w.llmCallFile.Close()
|
||||
w.llmCallFile = nil
|
||||
w.llmCallEncoder = nil
|
||||
}
|
||||
if w.file == nil {
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
if closeErr := w.file.Close(); closeErr != nil {
|
||||
err = closeErr
|
||||
}
|
||||
err := w.file.Close()
|
||||
w.file = nil
|
||||
return err
|
||||
}
|
||||
Reference in new issue
Block a user