sanderling/internal/runner/web_extractor_trace_test.go

package runner

import (
	"bufio"
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock"
	"github.com/priyanshujain/sanderling/internal/trace"
)

// engineDisagreementSpec makes the two runtimes disagree on purpose: the
// extractor body returns "goja" when it runs in-process, and the web driver
// below reports "v8" for the same extractor. The property is true of one value
// and false of the other, so the verdict names the engine the verdict used.
const engineDisagreementSpec = `
import { actions, always, extract } from "@sanderling/spec";
const engine = extract("engine", () => "goja");
globalThis.properties = {
  ranInGoja: always(() => engine.current === "goja"),
};
globalThis.actions = actions(() => []);
`

// webMockDriver presents the mock device driver as a web target so the runner
// takes the V8 path, where extractor values come from the page rather than
// from goja.
type webMockDriver struct {
	*mockdriver.Driver
	overrides map[int]json.RawMessage
}

func (d *webMockDriver) InstallBundle(context.Context, []byte) error { return nil }

func (d *webMockDriver) EvaluateExtractors(context.Context) (map[int]json.RawMessage, error) {
	return d.overrides, nil
}

func (d *webMockDriver) NextActionFromV8(context.Context) (json.RawMessage, error) {
	return nil, nil
}

func (d *webMockDriver) SetLastAction(context.Context, json.RawMessage) error { return nil }

func (d *webMockDriver) SetLogs(context.Context, json.RawMessage) error { return nil }

// TestRunner_TraceRecordsTheValueTheVerdictUsed fails if the trace and the
// verdict disagree about an extractor. A witness is only an explanation of a
// violation if it holds the state the violated property was evaluated against.
func TestRunner_TraceRecordsTheValueTheVerdictUsed(t *testing.T) {
	state := newHarnessWithSpec(t, engineDisagreementSpec)
	const pageValue = `"v8"`
	web := &webMockDriver{
		Driver:    state.mock,
		overrides: map[int]json.RawMessage{0: json.RawMessage(pageValue)},
	}

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	summary, err := Run(ctx, Options{
		Duration:    100 * time.Millisecond,
		IdleTimeout: 20 * time.Millisecond,
		Driver:      web,
		Verifier:    state.verifier,
		TraceWriter: state.writer,
	})
	if err != nil {
		t.Fatalf("Run: %v", err)
	}
	if !containsProperty(summary.Violations, "ranInGoja") {
		t.Fatalf("ranInGoja did not violate, so the page value never reached the verdict: %v",
			summary.Violations)
	}

	file, err := os.Open(filepath.Join(state.writer.Directory(), "trace.jsonl"))
	if err != nil {
		t.Fatal(err)
	}
	defer file.Close()

	type traceLine struct {
		Step             int                              `json:"step"`
		ExtractorChanges map[string]trace.ExtractorChange `json:"extractor_changes"`
		Witnesses        map[string]trace.Witness         `json:"witnesses"`
	}
	changes, witnesses := 0, 0
	scanner := bufio.NewScanner(file)
	scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
	for scanner.Scan() {
		var line traceLine
		if err := json.Unmarshal(scanner.Bytes(), &line); err != nil {
			t.Fatalf("trace line decode: %v", err)
		}
		if change, ok := line.ExtractorChanges["engine"]; ok {
			changes++
			if got := string(change.Curr); got != pageValue {
				t.Errorf("step %d: trace records engine=%s, verdict used %s",
					line.Step, got, pageValue)
			}
		}
		for name, witness := range line.Witnesses {
			witnesses++
			if got := string(witness.Extractors["engine"]); got != pageValue {
				t.Errorf("step %d: %s witness records engine=%s, verdict used %s",
					line.Step, name, got, pageValue)
			}
		}
	}
	if err := scanner.Err(); err != nil {
		t.Fatalf("scan trace: %v", err)
	}
	if changes == 0 {
		t.Error("no extractor change reached the trace; nothing was compared")
	}
	if witnesses == 0 {
		t.Error("no witness reached the trace; nothing was compared")
	}
}

// splitTableSpec registers two extractors whose goja bodies both answer "goja".
// The page below reports only the first, so index 1 keeps goja's dump-derived
// reading while index 0 holds the page's.
const splitTableSpec = `
import { actions, extract } from "@sanderling/spec";
extract("first", () => "goja");
extract("second", () => "goja");
globalThis.properties = {};
globalThis.actions = actions(() => []);
`

// TestRunner_PartialExtractorTableIsFatal pins the failure the runner used to
// let through. JSON.stringify drops an undefined-valued key, so a page whose
// extractors are mostly undefined off their own screen reported a table with
// holes in it, and the run completed with half the extractors reading from V8
// and half from goja. A delta property spanning that split convicts an app that
// did nothing wrong, which is worse than a crash: it is a green report of a bug
// that is not there, or a red one for a bug nobody can reproduce.
func TestRunner_PartialExtractorTableIsFatal(t *testing.T) {
	state := newHarnessWithSpec(t, splitTableSpec)
	web := &webMockDriver{
		Driver:    state.mock,
		overrides: map[int]json.RawMessage{0: json.RawMessage(`"v8"`)},
	}

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	_, err := Run(ctx, Options{
		Duration:    time.Hour,
		IdleTimeout: 20 * time.Millisecond,
		MaxSteps:    2,
		Driver:      web,
		Verifier:    state.verifier,
		TraceWriter: state.writer,
	})
	if err == nil {
		t.Fatal("the run completed on a page that reported 1 of 2 extractors; " +
			"the second extractor silently kept goja's value")
	}
	const want = "the page reported values for 1 of the spec's 2 extractors"
	if !strings.Contains(err.Error(), want) {
		t.Errorf("Run failed with %q, want it to name the split: %q", err, want)
	}
}