Browse code

Use IsServing to determine if c8d client is ready

Instead of sleeping an arbitrary amount of time, using the client to
tell us when it's ready so we can start processing events sooner.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

Brian Goff authored on 2020/07/18 05:34:57
Showing 1 changed files
... ...
@@ -711,6 +711,35 @@ func (c *client) processEvent(ctx context.Context, et libcontainerdtypes.EventTy
711 711
 	})
712 712
 }
713 713
 
714
+func (c *client) waitServe(ctx context.Context) bool {
715
+	t := 100 * time.Millisecond
716
+	delay := time.NewTimer(t)
717
+	if !delay.Stop() {
718
+		<-delay.C
719
+	}
720
+	defer delay.Stop()
721
+
722
+	// `IsServing` will actually block until the service is ready.
723
+	// However it can return early, so we'll loop with a delay to handle it.
724
+	for {
725
+		serving, _ := c.client.IsServing(ctx)
726
+		if ctx.Err() != nil {
727
+			return false
728
+		}
729
+
730
+		if serving {
731
+			return true
732
+		}
733
+
734
+		delay.Reset(t)
735
+		select {
736
+		case <-ctx.Done():
737
+			return false
738
+		case <-delay.C:
739
+		}
740
+	}
741
+}
742
+
714 743
 func (c *client) processEventStream(ctx context.Context, ns string) {
715 744
 	var (
716 745
 		err error
... ...
@@ -732,14 +761,11 @@ func (c *client) processEventStream(ctx context.Context, ns string) {
732 732
 			if err != nil {
733 733
 				errStatus, ok := status.FromError(err)
734 734
 				if !ok || errStatus.Code() != codes.Canceled {
735
-					c.logger.WithError(err).Error("failed to get event")
736
-
737
-					// rate limit
738
-					select {
739
-					case <-time.After(time.Second):
735
+					c.logger.WithError(err).Error("Failed to get event")
736
+					c.logger.Info("Waiting for containerd to be ready to restart event processing")
737
+					if c.waitServe(ctx) {
740 738
 						go c.processEventStream(ctx, ns)
741 739
 						return
742
-					case <-ctx.Done():
743 740
 					}
744 741
 				}
745 742
 				c.logger.WithError(ctx.Err()).Info("stopping event stream following graceful shutdown")