Browse code

Delay Endpoint config until the osSbox exists

When the SetKey hook is used (by a build container) it's called after
Endpoint.sbJoin, which will have called Sandbox.populateNetworkResources
to set up address, routes, sysctls and so on - but it's not able to do
any config until the osSbox exists. So, Sandbox.populateNetworkResources
is called again by SetKey to finish that config.

But, that means the rest of Endpoint.sbJoin has already happened before
the osSbox existed - it will have configured DNS, /etc/hosts, gateways
and so on before anything was set up for the OS.

So, if the osSbox configuration isn't applied as expected (for example,
a sysctl disables IPv6 on the endpoint), that sbJoin configuration is
incorrect.

To avoid unnecessary config+cleanup in thoses cases - delay the config
currently done by sbJoin until the osSbox exists.

Signed-off-by: Rob Murray <rob.murray@docker.com>

Rob Murray authored on 2025/01/15 01:36:49
Showing 4 changed files
... ...
@@ -560,23 +560,43 @@ func (ep *Endpoint) sbJoin(ctx context.Context, sb *Sandbox, options ...Endpoint
560 560
 		return err
561 561
 	}
562 562
 
563
-	// Current endpoint(s) providing external connectivity for the sandbox
563
+	// Current endpoint(s) providing external connectivity for the Sandbox.
564
+	// If ep is selected as a gateway endpoint once it's been added to the Sandbox,
565
+	// these are the endpoints that need to be un-gateway'd.
564 566
 	gwepBefore4, gwepBefore6 := sb.getGatewayEndpoint()
565 567
 
566 568
 	sb.addEndpoint(ep)
567 569
 
568
-	if err := sb.populateNetworkResources(ctx, ep); err != nil {
570
+	// For Linux, at this point, in most cases, the container task has been created
571
+	// and the container's network namespace (sb.osSbox) is ready to be configured
572
+	// with addresses, routes and so on. The exception is when the SetKey re-exec is
573
+	// used by a build container. In that case, the osSbox doesn't exist yet. So,
574
+	// stop here and SetKey will finish off the configuration when it's ready.
575
+	// For Windows, canPopulateNetworkResources() is always true.
576
+	if sb.canPopulateNetworkResources() {
577
+		if err := sb.populateNetworkResources(ctx, ep); err != nil {
578
+			return err
579
+		}
580
+		if err := ep.populateNetworkResources(ctx, sb); err != nil {
581
+			return err
582
+		}
583
+		if err := ep.updateExternalConnectivity(ctx, sb, gwepBefore4, gwepBefore6); err != nil {
584
+			return err
585
+		}
586
+	}
587
+
588
+	if err := n.getController().storeEndpoint(ctx, ep); err != nil {
569 589
 		return err
570 590
 	}
591
+	return nil
592
+}
571 593
 
594
+func (ep *Endpoint) populateNetworkResources(ctx context.Context, sb *Sandbox) (retErr error) {
595
+	n := ep.getNetwork()
572 596
 	if err := addEpToResolver(ctx, n.Name(), ep.Name(), &sb.config, ep.iface, n.Resolvers()); err != nil {
573 597
 		return errdefs.System(err)
574 598
 	}
575 599
 
576
-	if err := n.getController().storeEndpoint(ctx, ep); err != nil {
577
-		return err
578
-	}
579
-
580 600
 	if err := ep.addDriverInfoToCluster(); err != nil {
581 601
 		return err
582 602
 	}
... ...
@@ -603,7 +623,14 @@ func (ep *Endpoint) sbJoin(ctx context.Context, sb *Sandbox, options ...Endpoint
603 603
 	if sb.resolver != nil {
604 604
 		sb.resolver.SetForwardingPolicy(sb.hasExternalAccess())
605 605
 	}
606
+	return nil
607
+}
606 608
 
609
+// updateExternalConnectivity configures an Endpoint when it becomes the gateway
610
+// endpoint for a network, revoking external connectivity from the previous gateway
611
+// endpoints, if necessary. (It does not update the Sandbox's default gateway, the
612
+// Sandbox takes care of that. This is just about network driver config.)
613
+func (ep *Endpoint) updateExternalConnectivity(ctx context.Context, sb *Sandbox, gwepBefore4, gwepBefore6 *Endpoint) (retErr error) {
607 614
 	gwepAfter4, gwepAfter6 := sb.getGatewayEndpoint()
608 615
 
609 616
 	log.G(ctx).Infof("sbJoin: gwep4 '%s'->'%s', gwep6 '%s'->'%s'",
... ...
@@ -17,9 +17,7 @@ func (sb *Sandbox) restoreHostsPath() {}
17 17
 
18 18
 func (sb *Sandbox) restoreResolvConfPath() {}
19 19
 
20
-func (sb *Sandbox) addHostsEntries(_ context.Context, ifaceIP []netip.Addr) error {
21
-	return nil
22
-}
20
+func (sb *Sandbox) addHostsEntries(_ context.Context, ifaceIP []netip.Addr) {}
23 21
 
24 22
 func (sb *Sandbox) deleteHostsEntries(ifaceAddrs []netip.Addr) {}
25 23
 
... ...
@@ -161,6 +161,16 @@ func (sb *Sandbox) SetKey(ctx context.Context, basePath string) error {
161 161
 	oldosSbox := sb.osSbox
162 162
 	sb.mu.Unlock()
163 163
 
164
+	osSbox, err := osl.GetSandboxForExternalKey(basePath, sb.Key())
165
+	if err != nil {
166
+		return err
167
+	}
168
+
169
+	// Make sure the list of endpoints is stable while configuring them and selecting a
170
+	// gateway endpoint. Endpoints added after sbJoin will handle their setup.
171
+	sb.joinLeaveMu.Lock()
172
+	defer sb.joinLeaveMu.Unlock()
173
+
164 174
 	if oldosSbox != nil {
165 175
 		// If we already have an OS sandbox, release the network resources from that
166 176
 		// and destroy the OS snab. We are moving into a new home further down. Note that none
... ...
@@ -170,11 +180,6 @@ func (sb *Sandbox) SetKey(ctx context.Context, basePath string) error {
170 170
 		}
171 171
 	}
172 172
 
173
-	osSbox, err := osl.GetSandboxForExternalKey(basePath, sb.Key())
174
-	if err != nil {
175
-		return err
176
-	}
177
-
178 173
 	sb.mu.Lock()
179 174
 	sb.osSbox = osSbox
180 175
 	sb.mu.Unlock()
... ...
@@ -198,11 +203,11 @@ func (sb *Sandbox) SetKey(ctx context.Context, basePath string) error {
198 198
 		return err
199 199
 	}
200 200
 
201
-	for _, ep := range sb.Endpoints() {
202
-		if err = sb.populateNetworkResources(ctx, ep); err != nil {
203
-			return err
204
-		}
205
-	}
201
+	// If the Sandbox already has endpoints it's because sbJoin has been called for
202
+	// them - but configuration of addresses/routes (and so on) didn't complete because
203
+	// there was nowhere for it to go before the osSbox was set up. So, finish that
204
+	// configuration now.
205
+	sb.finishEndpointConfig(ctx)
206 206
 
207 207
 	return nil
208 208
 }
... ...
@@ -314,6 +319,42 @@ func (sb *Sandbox) restoreOslSandbox() error {
314 314
 	return nil
315 315
 }
316 316
 
317
+// finishEndpointConfig is to finish configuration of any Endpoint that was added to the
318
+// Sandbox (via sbJoin) before sb.osSbox had been set up.
319
+func (sb *Sandbox) finishEndpointConfig(ctx context.Context) error {
320
+	eps := sb.Endpoints()
321
+	if len(eps) == 0 {
322
+		return nil
323
+	}
324
+	for _, ep := range eps {
325
+		if err := sb.populateNetworkResources(ctx, ep); err != nil {
326
+			return err
327
+		}
328
+		if err := ep.populateNetworkResources(ctx, sb); err != nil {
329
+			return err
330
+		}
331
+	}
332
+
333
+	gwep4, gwep6 := sb.getGatewayEndpoint()
334
+	if gwep4 != nil {
335
+		if err := gwep4.updateExternalConnectivity(ctx, sb, nil, nil); err != nil {
336
+			return err
337
+		}
338
+	}
339
+	if gwep6 != nil && gwep6 != gwep4 {
340
+		if err := gwep6.updateExternalConnectivity(ctx, sb, nil, nil); err != nil {
341
+			return err
342
+		}
343
+	}
344
+	return nil
345
+}
346
+
347
+func (sb *Sandbox) canPopulateNetworkResources() bool {
348
+	sb.mu.Lock()
349
+	defer sb.mu.Unlock()
350
+	return sb.osSbox != nil
351
+}
352
+
317 353
 func (sb *Sandbox) populateNetworkResources(ctx context.Context, ep *Endpoint) error {
318 354
 	ctx, span := otel.Tracer("").Start(ctx, "libnetwork.Sandbox.populateNetworkResources", trace.WithAttributes(
319 355
 		attribute.String("endpoint.Name", ep.Name())))
... ...
@@ -322,7 +363,7 @@ func (sb *Sandbox) populateNetworkResources(ctx context.Context, ep *Endpoint) e
322 322
 	sb.mu.Lock()
323 323
 	if sb.osSbox == nil {
324 324
 		sb.mu.Unlock()
325
-		return nil
325
+		return fmt.Errorf("cannot populate network resources for container %s, no osSbox", sb.ContainerID())
326 326
 	}
327 327
 	inDelete := sb.inDelete
328 328
 	sb.mu.Unlock()
... ...
@@ -33,6 +33,10 @@ func (sb *Sandbox) NetnsPath() (path string, ok bool) {
33 33
 	return "", false
34 34
 }
35 35
 
36
+func (sb *Sandbox) canPopulateNetworkResources() bool {
37
+	return true
38
+}
39
+
36 40
 func (sb *Sandbox) populateNetworkResources(context.Context, *Endpoint) error {
37 41
 	// not implemented on Windows (Sandbox.osSbox is always nil)
38 42
 	return nil