test(runner): cover transient apply error resilience

TestRunner_TransientApplyErrorMarksTransitional drives the runner
through a wrapper that fails the first TapSelector with a gRPC
DeadlineExceeded then succeeds. Asserts the run does not exit, the
failed step is marked transitional with no violations, and the next
step runs cleanly. TestIsTransientApplyError_Classification covers the
helper's matching rules directly so future code changes don't quietly
drop a transient case.
This commit is contained in:
pj committed 2026-05-31 21:33:09 +05:30
1 parent 113282ffb2
commit 52865a1d7b
1 file changed
+112
+112
View File
@@ -15,6 +15,9 @@ import (
"testing" "testing"
"time" "time"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/priyanshujain/sanderling/internal/driver" "github.com/priyanshujain/sanderling/internal/driver"
mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock" mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock"
"github.com/priyanshujain/sanderling/internal/hierarchy" "github.com/priyanshujain/sanderling/internal/hierarchy"
@@ -846,6 +849,115 @@ func TestRunner_NilHierarchyMarksTransitional(t *testing.T) {
} }
} }
// tapSelectorFailFirst wraps a mock driver so the first TapSelector call
// returns a gRPC DeadlineExceeded error (mimicking a sidecar-side RPC hang),
// then delegates every subsequent call back to the mock.
type tapSelectorFailFirst struct {
*mockdriver.Driver
calls int
}
func (d *tapSelectorFailFirst) TapSelector(ctx context.Context, selector string) error {
d.calls++
if d.calls == 1 {
return status.Error(codes.DeadlineExceeded, "boom")
}
return d.Driver.TapSelector(ctx, selector)
}
// TestRunner_TransientApplyErrorMarksTransitional verifies that a transient
// gRPC error from applyAction (e.g. sidecar RPC deadline) does not kill the
// run: the step is marked transitional, the verifier is skipped for it, and
// the loop continues with the next step running cleanly.
func TestRunner_TransientApplyErrorMarksTransitional(t *testing.T) {
state := newHarness(t)
wrapped := &tapSelectorFailFirst{Driver: state.mock}
var logBuf bytes.Buffer
logger := slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
summary, err := Run(ctx, Options{
Duration: 300 * time.Millisecond,
IdleTimeout: 20 * time.Millisecond,
Driver: wrapped,
Verifier: state.verifier,
TraceWriter: state.writer,
Logger: logger,
})
if err != nil {
t.Fatalf("Run must not return on transient apply error, got %v", err)
}
if summary.Steps < 2 {
t.Fatalf("need at least 2 steps to prove the loop continued past the failed apply, got %d", summary.Steps)
}
if len(summary.Violations) != 0 {
t.Errorf("transient apply error must not surface as a violation, got %v", summary.Violations)
}
if !strings.Contains(logBuf.String(), "transient apply error") {
t.Errorf("expected transient-apply WARN log, got %q", logBuf.String())
}
type traceLine struct {
Step int `json:"step"`
Transitional bool `json:"transitional"`
Violations []string `json:"violations"`
}
body, err := os.ReadFile(filepath.Join(state.writer.Directory(), "trace.jsonl"))
if err != nil {
t.Fatal(err)
}
lines := bytes.Split(bytes.TrimSpace(body), []byte("\n"))
var first, second traceLine
if err := json.Unmarshal(lines[0], &first); err != nil {
t.Fatalf("decode first trace line: %v", err)
}
if err := json.Unmarshal(lines[1], &second); err != nil {
t.Fatalf("decode second trace line: %v", err)
}
if first.Step != 1 || !first.Transitional {
t.Errorf("step 1 must be transitional after transient apply error, got step=%d transitional=%v", first.Step, first.Transitional)
}
if len(first.Violations) != 0 {
t.Errorf("transient apply step must have no violations, got %v", first.Violations)
}
if second.Step != 2 || second.Transitional {
t.Errorf("step 2 must run cleanly after the transient step, got step=%d transitional=%v", second.Step, second.Transitional)
}
}
// TestIsTransientApplyError_Classification covers the helper's matching rules
// directly so future code changes don't quietly drop a transient case.
func TestIsTransientApplyError_Classification(t *testing.T) {
cleanCtx := context.Background()
cancelledCtx, cancel := context.WithCancel(context.Background())
cancel()
cases := []struct {
name string
ctx context.Context
err error
want bool
}{
{"nil error", cleanCtx, nil, false},
{"deadline exceeded", cleanCtx, status.Error(codes.DeadlineExceeded, "boom"), true},
{"unavailable", cleanCtx, status.Error(codes.Unavailable, "boom"), true},
{"internal wrapping deadline", cleanCtx, status.Error(codes.Internal, "io.grpc.StatusRuntimeException: DEADLINE_EXCEEDED: ..."), true},
{"internal wrapping unavailable", cleanCtx, status.Error(codes.Internal, "io.grpc.StatusRuntimeException: UNAVAILABLE: ..."), true},
{"internal generic", cleanCtx, status.Error(codes.Internal, "boom"), false},
{"raw context deadline", cleanCtx, context.DeadlineExceeded, true},
{"run context cancelled overrides", cancelledCtx, status.Error(codes.DeadlineExceeded, "boom"), false},
}
for _, testCase := range cases {
t.Run(testCase.name, func(t *testing.T) {
if got := isTransientApplyError(testCase.ctx, testCase.err); got != testCase.want {
t.Errorf("got %v, want %v", got, testCase.want)
}
})
}
}
// TestRunner_WaitActionSkipsIdle ensures the runner does not call WaitForIdle // TestRunner_WaitActionSkipsIdle ensures the runner does not call WaitForIdle
// after a Wait action - the action already provides settling time. // after a Wait action - the action already provides settling time.
func TestRunner_WaitActionSkipsIdle(t *testing.T) { func TestRunner_WaitActionSkipsIdle(t *testing.T) {