Browse code

overlay: Reload Ingress iptables rules in swarm mode

- Implement firewalld reload handling for Ingress rules restoration
- Add TestRestoreIngressRulesOnFirewalldReload() integration test

Signed-off-by: Andrey Epifanov <aepifanov@mirantis.com>

Andrey Epifanov authored on 2025/02/28 08:27:03
Showing 5 changed files
... ...
@@ -223,7 +223,7 @@ func New(ctx context.Context, cfgOptions ...config.Option) (_ *Controller, retEr
223 223
 		return nil, err
224 224
 	}
225 225
 
226
-	c.setupUserChains()
226
+	c.setupPlatformFirewall()
227 227
 	return c, nil
228 228
 }
229 229
 
... ...
@@ -4,6 +4,7 @@ import (
4 4
 	"context"
5 5
 	"errors"
6 6
 	"fmt"
7
+	"net"
7 8
 
8 9
 	"github.com/containerd/log"
9 10
 	"github.com/docker/docker/daemon/libnetwork/internal/nftables"
... ...
@@ -27,6 +28,12 @@ func (c *Controller) selectFirewallBackend() error {
27 27
 	return nil
28 28
 }
29 29
 
30
+func (c *Controller) setupPlatformFirewall() {
31
+	c.setupUserChains()
32
+	// Add handler for iptables rules restoration in case of a firewalld reload
33
+	c.handleFirewalldReload()
34
+}
35
+
30 36
 // Sets up the DOCKER-USER chain for each iptables version (IPv4, IPv6) that's
31 37
 // enabled in the controller's configuration.
32 38
 func (c *Controller) setupUserChains() {
... ...
@@ -71,3 +78,56 @@ func setupUserChain(ipVersion iptables.IPVersion) error {
71 71
 	}
72 72
 	return nil
73 73
 }
74
+
75
+func (c *Controller) handleFirewalldReload() {
76
+	handler := func() {
77
+		services := make(map[serviceKey]*service)
78
+
79
+		c.mu.Lock()
80
+		for k, s := range c.serviceBindings {
81
+			if k.ports != "" && len(s.ingressPorts) != 0 {
82
+				services[k] = s
83
+			}
84
+		}
85
+		c.mu.Unlock()
86
+
87
+		for _, s := range services {
88
+			c.handleFirewallReloadService(s)
89
+		}
90
+	}
91
+	// Add handler for iptables rules restoration in case of a firewalld reload
92
+	iptables.OnReloaded(handler)
93
+}
94
+
95
+func (c *Controller) handleFirewallReloadService(s *service) {
96
+	s.Lock()
97
+	defer s.Unlock()
98
+	if s.deleted {
99
+		log.G(context.TODO()).Debugf("handleFirewallReloadService called for deleted service %s/%s", s.id, s.name)
100
+		return
101
+	}
102
+	for nid := range s.loadBalancers {
103
+		n, err := c.NetworkByID(nid)
104
+		if err != nil {
105
+			continue
106
+		}
107
+		ep, sb, err := n.findLBEndpointSandbox()
108
+		if err != nil {
109
+			log.G(context.TODO()).Warnf("handleFirewallReloadService failed to find LB Endpoint Sandbox for %s/%s: %v -- ", n.ID(), n.Name(), err)
110
+			continue
111
+		}
112
+		if sb.osSbox == nil {
113
+			return
114
+		}
115
+		if ep != nil {
116
+			var gwIP net.IP
117
+			if gwEP, _ := sb.getGatewayEndpoint(); gwEP != nil {
118
+				gwIP = gwEP.Iface().Address().IP
119
+			}
120
+			if err := restoreIngressPorts(gwIP, s.ingressPorts); err != nil {
121
+				log.G(context.TODO()).Errorf("Failed to add ingress: %v", err)
122
+				return
123
+			}
124
+		}
125
+	}
126
+}
... ...
@@ -6,4 +6,4 @@ func (c *Controller) selectFirewallBackend() error {
6 6
 	return nil
7 7
 }
8 8
 
9
-func (c *Controller) setupUserChains() {}
9
+func (c *Controller) setupPlatformFirewall() {}
... ...
@@ -403,6 +403,24 @@ func addIngressPorts(gwIP net.IP, ingressPorts []*PortConfig) error {
403 403
 	return nil
404 404
 }
405 405
 
406
+func restoreIngressPorts(gwIP net.IP, ingressPorts []*PortConfig) error {
407
+	// TODO IPv6 support
408
+	iptable := iptables.GetIptable(iptables.IPv4)
409
+
410
+	ingressMu.Lock()
411
+	defer ingressMu.Unlock()
412
+
413
+	if err := initIngressConfiguration(gwIP, iptable); err != nil {
414
+		return err
415
+	}
416
+
417
+	if err := programIngressPortsRules(gwIP, ingressPorts); err != nil {
418
+		return fmt.Errorf("failed to program ingress ports: %v", err)
419
+	}
420
+
421
+	return nil
422
+}
423
+
406 424
 func generateIngressRules(port *PortConfig, destIP net.IP) []iptables.Rule {
407 425
 	var (
408 426
 		protocol      = strings.ToLower(port.Protocol.String())
... ...
@@ -267,3 +267,58 @@ func TestDockerIngressChainPosition(t *testing.T) {
267 267
 		checkChain()
268 268
 	})
269 269
 }
270
+
271
+func TestRestoreIngressRulesOnFirewalldReload(t *testing.T) {
272
+	skip.If(t, testEnv.IsRemoteDaemon)
273
+	skip.If(t, testEnv.IsRootless, "rootless mode doesn't support Swarm-mode")
274
+	skip.If(t, testEnv.FirewallBackendDriver() != "iptables+firewalld", "nftables backend doesn't support Swarm-mode")
275
+	skip.If(t, !networking.FirewalldRunning(), "Need firewalld to test restoration ingress rules")
276
+	ctx := setupTest(t)
277
+
278
+	// Check the published port is accessible.
279
+	checkHTTP := func(_ poll.LogT) poll.Result {
280
+		res := icmd.RunCommand("curl", "-v", "-o", "/dev/null", "-w", "%{http_code}\n",
281
+			"http://"+stdnet.JoinHostPort("localhost", "8080"))
282
+		// A "404 Not Found" means the server responded, but it's got nothing to serve.
283
+		if !strings.Contains(res.Stdout(), "404") {
284
+			return poll.Continue("404 - not found in: %s, %+v", res.Stdout(), res)
285
+		}
286
+		return poll.Success()
287
+	}
288
+
289
+	d := swarm.NewSwarm(ctx, t, testEnv, daemon.WithSwarmIptables(true))
290
+	defer d.Stop(t)
291
+	c := d.NewClientT(t)
292
+	defer c.Close()
293
+
294
+	serviceID := swarm.CreateService(ctx, t, d,
295
+		swarm.ServiceWithName("test-ingress-on-firewalld-reload"),
296
+		swarm.ServiceWithCommand([]string{"httpd", "-f"}),
297
+		swarm.ServiceWithEndpoint(&swarmtypes.EndpointSpec{
298
+			Ports: []swarmtypes.PortConfig{
299
+				{
300
+					Protocol:      "tcp",
301
+					TargetPort:    80,
302
+					PublishedPort: 8080,
303
+					PublishMode:   swarmtypes.PortConfigPublishModeIngress,
304
+				},
305
+			},
306
+		}),
307
+	)
308
+	defer func() {
309
+		err := c.ServiceRemove(ctx, serviceID)
310
+		assert.NilError(t, err)
311
+	}()
312
+
313
+	t.Log("Waiting for the service to start")
314
+	poll.WaitOn(t, swarm.RunningTasksCount(ctx, c, serviceID, 1), swarm.ServicePoll)
315
+	t.Log("Checking http access to the service")
316
+	poll.WaitOn(t, checkHTTP, poll.WithTimeout(30*time.Second))
317
+
318
+	t.Log("Firewalld reload")
319
+	networking.FirewalldReload(t, d)
320
+
321
+	t.Log("Checking http access to the service")
322
+	// It takes a while before this works ...
323
+	poll.WaitOn(t, checkHTTP, poll.WithTimeout(30*time.Second))
324
+}