From 7bafed8af4fbc33ef92826f29b86c9f40902395c Mon Sep 17 00:00:00 2001 From: PJ Date: Mon, 17 Aug 2026 23:27:21 +0530 Subject: [PATCH] feat(testrun): refuse a run against a spec that registers no properties A spec with no properties drove the app and reported no violations, which is indistinguishable from a spec that judged something and found nothing. Execute now aborts after loading the spec unless the run asks for the opt-out by name. --- internal/testrun/testrun.go | 27 +++++++ test/browser/no_properties_test.go | 77 +++++++++++++++++++ .../browser/testdata/no-properties/index.html | 19 +++++ test/browser/testdata/no-properties/spec.ts | 10 +++ 4 files changed, 133 insertions(+) create mode 100644 test/browser/no_properties_test.go create mode 100644 test/browser/testdata/no-properties/index.html create mode 100644 test/browser/testdata/no-properties/spec.ts diff --git a/internal/testrun/testrun.go b/internal/testrun/testrun.go index c03d3d8..acced71 100644 --- a/internal/testrun/testrun.go +++ b/internal/testrun/testrun.go @@ -62,6 +62,11 @@ type Options struct { // recorded violations as a ViolationsError, so a caller (CI) can tell // "the run found the bug" from "the run finished clean". ExitOnViolation bool + // AllowNoProperties lets a run proceed against a spec that registers no + // properties. The extraction and portability sweeps pass it: they measure + // what a spec can read and where the generator reaches, and they report no + // detection count. Every other run without it is a false green. + AllowNoProperties bool // Generator selects the action picker: "llm" or the default seeded picker. Generator string // LabelSource selects how candidates are named to the model picker, and is @@ -190,6 +195,9 @@ func Execute(ctx context.Context, options Options, stdout io.Writer) error { if err := verifierInstance.Load(string(bundle.JavaScript)); err != nil { return fmt.Errorf("load spec: %w", err) } + if !options.AllowNoProperties && len(verifierInstance.PropertyNames()) == 0 { + return NoPropertiesError{Spec: options.Spec} + } fmt.Fprintln(stdout, "spec loaded into verifier") runDirectory := filepath.Join(options.Output, time.Now().UTC().Format("20060102-150405")) @@ -307,6 +315,25 @@ func (e VacuousRunError) Error() string { e.Steps) } +// NoPropertiesError reports a spec that bundled and loaded cleanly and holds no +// properties. Nothing is broken: the run would drive the app, fill a trace and +// report no violations having judged nothing, and that green says as much about +// the app as an unplugged meter says about a wire. It stays untyped to the CLI's +// violation path like VacuousRunError, so it exits 1 as a run that cannot +// produce a verdict rather than 2. +type NoPropertiesError struct { + Spec string +} + +func (e NoPropertiesError) Error() string { + return fmt.Sprintf( + "%s bundled and loaded into the verifier cleanly and registers no properties: "+ + "nothing is wrong with the spec and nothing is wrong with the run, but this run "+ + "would check nothing and report no violations. Pass --allow-no-properties for a "+ + "run that measures extraction or exploration instead of judging the app", + e.Spec) +} + // bundleInputs holds the pre-driver assembly: alias map, seed, esbuild defines, // and the resolved spec-API/goja-runtime paths the bundler consumes. type bundleInputs struct { diff --git a/test/browser/no_properties_test.go b/test/browser/no_properties_test.go new file mode 100644 index 0000000..0f365f0 --- /dev/null +++ b/test/browser/no_properties_test.go @@ -0,0 +1,77 @@ +//go:build browser + +package browser_test + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/priyanshujain/sanderling/internal/testrun" +) + +// TestBrowserZeroPropertySpecFailsTheRun drives the whole pipeline against a +// spec that bundles and loads cleanly and registers nothing to judge with. Such +// a run reaches the verifier on every step, evaluates an empty property set and +// reports no violations: a confident green from an instrument measuring nothing, +// which is the one outcome a fuzzer must never hand back. +func TestBrowserZeroPropertySpecFailsTheRun(t *testing.T) { + err := executeFixture(t, "no-properties", false) + + var noProperties testrun.NoPropertiesError + if !errors.As(err, &noProperties) { + t.Fatalf("a spec registering no properties came back %v, want a NoPropertiesError", err) + } + for _, phrase := range []string{"loaded into the verifier cleanly", "registers no properties", "--allow-no-properties"} { + if !strings.Contains(noProperties.Error(), phrase) { + t.Errorf("the error never says %q, so it reads as a broken spec: %v", phrase, noProperties) + } + } +} + +// TestBrowserZeroPropertySpecRunsUnderTheOptOut covers the extraction and +// portability sweeps, which run a property-free spec on purpose to measure what +// it can read rather than to judge an app. They ask for it by name and the run +// proceeds. +func TestBrowserZeroPropertySpecRunsUnderTheOptOut(t *testing.T) { + if err := executeFixture(t, "no-properties", true); err != nil { + t.Fatalf("the opt-out did not carry a property-free run through: %v", err) + } +} + +// TestBrowserSpecWithPropertiesRunsUnaffected pins the guard to the empty case +// alone: a spec that registers one property runs to its budget as before. +func TestBrowserSpecWithPropertiesRunsUnaffected(t *testing.T) { + if err := executeFixture(t, "counter", false); err != nil { + t.Fatalf("a spec with a property no longer runs: %v", err) + } +} + +// executeFixture serves the named testdata case and drives it through +// testrun.Execute, the pipeline `sanderling test` itself runs. +func executeFixture(t *testing.T, name string, allowNoProperties bool) error { + t.Helper() + + server := httptest.NewServer(http.FileServer(http.Dir(testdataDir(t)))) + t.Cleanup(server.Close) + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + t.Cleanup(cancel) + + return testrun.Execute(ctx, testrun.Options{ + Spec: filepath.Join(testdataDir(t), name, "spec.ts"), + BundleID: server.URL + "/" + name + "/", + Platform: "web", + Seed: fixtureSeed, + MaxSteps: fixtureMaxSteps, + Duration: 90 * time.Second, + Output: t.TempDir(), + AllowNoProperties: allowNoProperties, + }, io.Discard) +} diff --git a/test/browser/testdata/no-properties/index.html b/test/browser/testdata/no-properties/index.html new file mode 100644 index 0000000..73d92b3 --- /dev/null +++ b/test/browser/testdata/no-properties/index.html @@ -0,0 +1,19 @@ + + + + + no-properties + + + +
0
+ + + diff --git a/test/browser/testdata/no-properties/spec.ts b/test/browser/testdata/no-properties/spec.ts new file mode 100644 index 0000000..b33d3c9 --- /dev/null +++ b/test/browser/testdata/no-properties/spec.ts @@ -0,0 +1,10 @@ +import { extract, taps } from "@sanderling/spec"; + +export const counter = extract((s) => { + const el = s.ax.find({ id: "counter" }); + return el ? parseInt(el.text, 10) || 0 : 0; +}).named("counter"); + +export const properties = {}; + +export const actionsRoot = taps;