package controller import ( "fmt" "sync" "testing" kcache "github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache" ) func TestRetryController_handleOneRetryableError(t *testing.T) { retried := false controller := &RetryController{ Handle: func(obj interface{}) error { return fmt.Errorf("retryable error") }, ShouldRetry: func(interface{}, error) bool { return true }, RetryManager: &testRetryManager{ RetryFunc: func(resource interface{}) { retried = true }, ForgetFunc: func(resource interface{}) { t.Fatalf("unexpected call to forget %v", resource) }, }, } controller.handleOne(struct{}{}) if !retried { t.Fatalf("expected a retry") } } func TestRetryController_handleOneFatalError(t *testing.T) { forgotten := false controller := &RetryController{ Handle: func(obj interface{}) error { return fmt.Errorf("fatal error") }, ShouldRetry: func(interface{}, error) bool { return false }, RetryManager: &testRetryManager{ RetryFunc: func(resource interface{}) { t.Fatalf("unexpected call to retry %v", resource) }, ForgetFunc: func(resource interface{}) { forgotten = true }, }, } controller.handleOne(struct{}{}) if !forgotten { t.Fatalf("expected to forget") } } func TestRetryController_handleOneNoError(t *testing.T) { forgotten := false controller := &RetryController{ Handle: func(obj interface{}) error { return nil }, ShouldRetry: func(interface{}, error) bool { t.Fatalf("unexpected retry check") return true }, RetryManager: &testRetryManager{ RetryFunc: func(resource interface{}) { t.Fatalf("unexpected call to retry %v", resource) }, ForgetFunc: func(resource interface{}) { forgotten = true }, }, } controller.handleOne(struct{}{}) if !forgotten { t.Fatalf("expected to forget") } } func TestQueueRetryManager_retries(t *testing.T) { retries := 5 requeued := map[string]int{} manager := &QueueRetryManager{ queue: &testFifo{ // Track re-queues AddIfNotPresentFunc: func(obj interface{}) error { id := obj.(testObj).id if _, exists := requeued[id]; !exists { requeued[id] = 0 } requeued[id] = requeued[id] + 1 return nil }, }, keyFunc: func(obj interface{}) (string, error) { return obj.(testObj).id, nil }, maxRetries: retries, retries: make(map[string]int), } objects := []testObj{ {"a", 1}, {"b", 2}, {"c", 3}, } // Retry one more than the max for _, obj := range objects { for i := 0; i < retries+1; i++ { manager.Retry(obj) } } // Should only have re-queued up to the max retry setting for _, obj := range objects { if e, a := retries, requeued[obj.id]; e != a { t.Fatalf("expected requeue count %d for obj %s, got %d", e, obj.id, a) } } // Should have no more state since all objects were retried beyond max if e, a := 0, len(manager.retries); e != a { t.Fatalf("expected retry len %d, got %d", e, a) } } // This test ensures that when an asynchronous state update is received // on the queue during failed event handling, that the updated state is // retried, NOT the event that failed (which is now stale). func TestRetryController_realFifoEventOrdering(t *testing.T) { keyFunc := func(obj interface{}) (string, error) { return obj.(testObj).id, nil } fifo := kcache.NewFIFO(keyFunc) wg := sync.WaitGroup{} wg.Add(1) controller := &RetryController{ Queue: fifo, RetryManager: NewQueueRetryManager(fifo, keyFunc, 1), ShouldRetry: func(interface{}, error) bool { return true }, Handle: func(obj interface{}) error { if e, a := 1, obj.(testObj).value; e != a { t.Fatalf("expected to handle test value %d, got %d") } go func() { fifo.Add(testObj{"a", 2}) wg.Done() }() wg.Wait() return fmt.Errorf("retryable error") }, } fifo.Add(testObj{"a", 1}) controller.handleOne(fifo.Pop()) if e, a := 1, len(fifo.List()); e != a { t.Fatalf("expected queue length %d, got %d", e, a) } obj := fifo.Pop() if e, a := 2, obj.(testObj).value; e != a { t.Fatalf("expected queued value %d, got %d", e, a) } } type testObj struct { id string value int } type testFifo struct { AddIfNotPresentFunc func(interface{}) error PopFunc func() interface{} } func (t *testFifo) AddIfNotPresent(obj interface{}) error { return t.AddIfNotPresentFunc(obj) } func (t *testFifo) Pop() interface{} { return t.PopFunc() } type testRetryManager struct { RetryFunc func(resource interface{}) ForgetFunc func(resource interface{}) } func (m *testRetryManager) Retry(resource interface{}) { m.RetryFunc(resource) } func (m *testRetryManager) Forget(resource interface{}) { m.ForgetFunc(resource) }