sanderling/internal/runner/composition_reread_test.go

package runner

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"github.com/priyanshujain/sanderling/internal/driver"
	mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock"
)

// homeWithRows is one settled route whose list holds rows. A row arriving
// between two reads is what a Compose lazy list mounting over several frames
// looks like from the runner's side.
func homeWithRows(rows int) string {
	var children strings.Builder
	for row := range rows {
		fmt.Fprintf(&children,
			`,{"attributes":{"resource-id":"TxnRow%d","class":"android.view.View"},"children":[]}`, row)
	}
	return fmt.Sprintf(
		`{"attributes":{"resource-id":"HomeScreen","class":"android.view.View"},"children":[
		  {"attributes":{"resource-id":"TxnList","class":"android.view.View"},"children":[]}%s
		]}`, children.String())
}

// composesLateDriver answers the paired Snapshot with the frame the step
// records and the hierarchy read that follows with a tree that has grown a row,
// for the first composingReads reads of the run. After that both reads describe
// the same screen.
type composesLateDriver struct {
	*mockdriver.Driver
	composingReads int64
	reads          atomic.Int64
}

func (d *composesLateDriver) Snapshot(context.Context) (string, driver.Image, error) {
	return homeWithRows(1), driver.Image{PNG: []byte("png"), Width: 1, Height: 1}, nil
}

func (d *composesLateDriver) Hierarchy(context.Context) (string, error) {
	if d.reads.Add(1) <= d.composingReads {
		return homeWithRows(2), nil
	}
	return homeWithRows(1), nil
}

// A route can settle before its content composes, so a tree read the moment the
// route arrives can describe a screen that is still filling in. Verifying that
// step compares a half-composed frame against a settled one and convicts an app
// that did nothing wrong. Two reads a read apart see it happening, and the step
// they disagree on is one the verifier must never be handed.
//
// The always-false property is the witness: it fires on the first step the
// verifier evaluates, so the step index of its violation says exactly which
// step reached the verifier.
func TestRunner_AStepWhoseTreeChangedBetweenReadsIsNotVerified(t *testing.T) {
	run := func(t *testing.T, composingReads int64) (Summary, string) {
		t.Helper()
		state := newHarnessWithSpec(t, violationSpec)
		device := &composesLateDriver{Driver: state.mock, composingReads: composingReads}

		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()
		summary, err := Run(ctx, Options{
			Duration:    time.Hour,
			IdleTimeout: 20 * time.Millisecond,
			MaxSteps:    3,
			Driver:      device,
			Verifier:    state.verifier,
			TraceWriter: state.writer,
		})
		if err != nil {
			t.Fatalf("Run: %v", err)
		}
		if summary.Steps != 3 {
			t.Fatalf("steps = %d, want 3", summary.Steps)
		}
		return summary, state.writer.Directory()
	}

	t.Run("the step it changed on is skipped, the next one is judged", func(t *testing.T) {
		summary, directory := run(t, 1)
		if len(summary.Violations) != 1 {
			t.Fatalf("violations = %v, want exactly one", summary.Violations)
		}
		violation := summary.Violations[0]
		if violation.Properties[0] != "balanceNonNegative" {
			t.Fatalf("violated %v, want balanceNonNegative", violation.Properties)
		}
		if violation.StepIndex != 2 {
			t.Errorf("the property first judged step %d, want 2; the verifier was handed "+
				"a screen that grew a row while the runner was reading it",
				violation.StepIndex)
		}
		if summary.SkippedVerification != 1 {
			t.Errorf("the run reports %d step(s) judged by nothing, want 1",
				summary.SkippedVerification)
		}
		// Skipped is not lost: the step is still recorded, screenshot and all,
		// so the run can be replayed over the frame nothing judged.
		steps := traceSteps(t, directory)
		if len(steps) != 3 {
			t.Fatalf("trace holds %d step(s), want 3", len(steps))
		}
		if len(steps[0].Violations) != 0 {
			t.Errorf("step 1 recorded violations %v; it was never verified", steps[0].Violations)
		}
		screenshot := filepath.Join(directory, "screenshots", "step-00001.png")
		if _, err := os.Stat(screenshot); err != nil {
			t.Errorf("expected the skipped step's screenshot at %s: %v", screenshot, err)
		}
	})

	// The control. Two reads that agree must verify as they always did,
	// otherwise the case above is just a runner that verifies nothing.
	t.Run("two reads that agree verify the step", func(t *testing.T) {
		summary, _ := run(t, 0)
		if len(summary.Violations) != 1 {
			t.Fatalf("violations = %v, want exactly one", summary.Violations)
		}
		if got := summary.Violations[0].StepIndex; got != 1 {
			t.Errorf("the property first judged step %d, want 1; a settled screen must be "+
				"verified on the step it was read", got)
		}
	})
}

type traceLine struct {
	Step       int      `json:"step"`
	Violations []string `json:"violations"`
}

func traceSteps(t *testing.T, directory string) []traceLine {
	t.Helper()
	body, err := os.ReadFile(filepath.Join(directory, "trace.jsonl"))
	if err != nil {
		t.Fatal(err)
	}
	var steps []traceLine
	for _, raw := range bytes.Split(bytes.TrimSpace(body), []byte("\n")) {
		var line traceLine
		if err := json.Unmarshal(raw, &line); err != nil {
			t.Fatalf("decode trace line: %v", err)
		}
		steps = append(steps, line)
	}
	return steps
}