mirror of
https://github.com/priyanshujain/sanderling.git
synced 2026-10-02 19:17:10 +00:00
test(runner): cover transient apply error resilience
TestRunner_TransientApplyErrorMarksTransitional drives the runner through a wrapper that fails the first TapSelector with a gRPC DeadlineExceeded then succeeds. Asserts the run does not exit, the failed step is marked transitional with no violations, and the next step runs cleanly. TestIsTransientApplyError_Classification covers the helper's matching rules directly so future code changes don't quietly drop a transient case.
This commit is contained in:
1 parent
113282ffb2
commit
52865a1d7b
1 file changed
+112
@@ -15,6 +15,9 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"google.golang.org/grpc/codes"
|
||||||
|
"google.golang.org/grpc/status"
|
||||||
|
|
||||||
"github.com/priyanshujain/sanderling/internal/driver"
|
"github.com/priyanshujain/sanderling/internal/driver"
|
||||||
mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock"
|
mockdriver "github.com/priyanshujain/sanderling/internal/driver/mock"
|
||||||
"github.com/priyanshujain/sanderling/internal/hierarchy"
|
"github.com/priyanshujain/sanderling/internal/hierarchy"
|
||||||
@@ -846,6 +849,115 @@ func TestRunner_NilHierarchyMarksTransitional(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// tapSelectorFailFirst wraps a mock driver so the first TapSelector call
|
||||||
|
// returns a gRPC DeadlineExceeded error (mimicking a sidecar-side RPC hang),
|
||||||
|
// then delegates every subsequent call back to the mock.
|
||||||
|
type tapSelectorFailFirst struct {
|
||||||
|
*mockdriver.Driver
|
||||||
|
calls int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *tapSelectorFailFirst) TapSelector(ctx context.Context, selector string) error {
|
||||||
|
d.calls++
|
||||||
|
if d.calls == 1 {
|
||||||
|
return status.Error(codes.DeadlineExceeded, "boom")
|
||||||
|
}
|
||||||
|
return d.Driver.TapSelector(ctx, selector)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunner_TransientApplyErrorMarksTransitional verifies that a transient
|
||||||
|
// gRPC error from applyAction (e.g. sidecar RPC deadline) does not kill the
|
||||||
|
// run: the step is marked transitional, the verifier is skipped for it, and
|
||||||
|
// the loop continues with the next step running cleanly.
|
||||||
|
func TestRunner_TransientApplyErrorMarksTransitional(t *testing.T) {
|
||||||
|
state := newHarness(t)
|
||||||
|
wrapped := &tapSelectorFailFirst{Driver: state.mock}
|
||||||
|
|
||||||
|
var logBuf bytes.Buffer
|
||||||
|
logger := slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
summary, err := Run(ctx, Options{
|
||||||
|
Duration: 300 * time.Millisecond,
|
||||||
|
IdleTimeout: 20 * time.Millisecond,
|
||||||
|
Driver: wrapped,
|
||||||
|
Verifier: state.verifier,
|
||||||
|
TraceWriter: state.writer,
|
||||||
|
Logger: logger,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run must not return on transient apply error, got %v", err)
|
||||||
|
}
|
||||||
|
if summary.Steps < 2 {
|
||||||
|
t.Fatalf("need at least 2 steps to prove the loop continued past the failed apply, got %d", summary.Steps)
|
||||||
|
}
|
||||||
|
if len(summary.Violations) != 0 {
|
||||||
|
t.Errorf("transient apply error must not surface as a violation, got %v", summary.Violations)
|
||||||
|
}
|
||||||
|
if !strings.Contains(logBuf.String(), "transient apply error") {
|
||||||
|
t.Errorf("expected transient-apply WARN log, got %q", logBuf.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
type traceLine struct {
|
||||||
|
Step int `json:"step"`
|
||||||
|
Transitional bool `json:"transitional"`
|
||||||
|
Violations []string `json:"violations"`
|
||||||
|
}
|
||||||
|
body, err := os.ReadFile(filepath.Join(state.writer.Directory(), "trace.jsonl"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
lines := bytes.Split(bytes.TrimSpace(body), []byte("\n"))
|
||||||
|
var first, second traceLine
|
||||||
|
if err := json.Unmarshal(lines[0], &first); err != nil {
|
||||||
|
t.Fatalf("decode first trace line: %v", err)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(lines[1], &second); err != nil {
|
||||||
|
t.Fatalf("decode second trace line: %v", err)
|
||||||
|
}
|
||||||
|
if first.Step != 1 || !first.Transitional {
|
||||||
|
t.Errorf("step 1 must be transitional after transient apply error, got step=%d transitional=%v", first.Step, first.Transitional)
|
||||||
|
}
|
||||||
|
if len(first.Violations) != 0 {
|
||||||
|
t.Errorf("transient apply step must have no violations, got %v", first.Violations)
|
||||||
|
}
|
||||||
|
if second.Step != 2 || second.Transitional {
|
||||||
|
t.Errorf("step 2 must run cleanly after the transient step, got step=%d transitional=%v", second.Step, second.Transitional)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestIsTransientApplyError_Classification covers the helper's matching rules
|
||||||
|
// directly so future code changes don't quietly drop a transient case.
|
||||||
|
func TestIsTransientApplyError_Classification(t *testing.T) {
|
||||||
|
cleanCtx := context.Background()
|
||||||
|
cancelledCtx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
ctx context.Context
|
||||||
|
err error
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"nil error", cleanCtx, nil, false},
|
||||||
|
{"deadline exceeded", cleanCtx, status.Error(codes.DeadlineExceeded, "boom"), true},
|
||||||
|
{"unavailable", cleanCtx, status.Error(codes.Unavailable, "boom"), true},
|
||||||
|
{"internal wrapping deadline", cleanCtx, status.Error(codes.Internal, "io.grpc.StatusRuntimeException: DEADLINE_EXCEEDED: ..."), true},
|
||||||
|
{"internal wrapping unavailable", cleanCtx, status.Error(codes.Internal, "io.grpc.StatusRuntimeException: UNAVAILABLE: ..."), true},
|
||||||
|
{"internal generic", cleanCtx, status.Error(codes.Internal, "boom"), false},
|
||||||
|
{"raw context deadline", cleanCtx, context.DeadlineExceeded, true},
|
||||||
|
{"run context cancelled overrides", cancelledCtx, status.Error(codes.DeadlineExceeded, "boom"), false},
|
||||||
|
}
|
||||||
|
for _, testCase := range cases {
|
||||||
|
t.Run(testCase.name, func(t *testing.T) {
|
||||||
|
if got := isTransientApplyError(testCase.ctx, testCase.err); got != testCase.want {
|
||||||
|
t.Errorf("got %v, want %v", got, testCase.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestRunner_WaitActionSkipsIdle ensures the runner does not call WaitForIdle
|
// TestRunner_WaitActionSkipsIdle ensures the runner does not call WaitForIdle
|
||||||
// after a Wait action - the action already provides settling time.
|
// after a Wait action - the action already provides settling time.
|
||||||
func TestRunner_WaitActionSkipsIdle(t *testing.T) {
|
func TestRunner_WaitActionSkipsIdle(t *testing.T) {
|
||||||
|
|||||||
Reference in new issue
Block a user