- Implement firewalld reload handling for Ingress rules restoration
- Add TestRestoreIngressRulesOnFirewalldReload() integration test
Signed-off-by: Andrey Epifanov <aepifanov@mirantis.com>
| ... | ... |
@@ -4,6 +4,7 @@ import ( |
| 4 | 4 |
"context" |
| 5 | 5 |
"errors" |
| 6 | 6 |
"fmt" |
| 7 |
+ "net" |
|
| 7 | 8 |
|
| 8 | 9 |
"github.com/containerd/log" |
| 9 | 10 |
"github.com/docker/docker/daemon/libnetwork/internal/nftables" |
| ... | ... |
@@ -27,6 +28,12 @@ func (c *Controller) selectFirewallBackend() error {
|
| 27 | 27 |
return nil |
| 28 | 28 |
} |
| 29 | 29 |
|
| 30 |
+func (c *Controller) setupPlatformFirewall() {
|
|
| 31 |
+ c.setupUserChains() |
|
| 32 |
+ // Add handler for iptables rules restoration in case of a firewalld reload |
|
| 33 |
+ c.handleFirewalldReload() |
|
| 34 |
+} |
|
| 35 |
+ |
|
| 30 | 36 |
// Sets up the DOCKER-USER chain for each iptables version (IPv4, IPv6) that's |
| 31 | 37 |
// enabled in the controller's configuration. |
| 32 | 38 |
func (c *Controller) setupUserChains() {
|
| ... | ... |
@@ -71,3 +78,56 @@ func setupUserChain(ipVersion iptables.IPVersion) error {
|
| 71 | 71 |
} |
| 72 | 72 |
return nil |
| 73 | 73 |
} |
| 74 |
+ |
|
| 75 |
+func (c *Controller) handleFirewalldReload() {
|
|
| 76 |
+ handler := func() {
|
|
| 77 |
+ services := make(map[serviceKey]*service) |
|
| 78 |
+ |
|
| 79 |
+ c.mu.Lock() |
|
| 80 |
+ for k, s := range c.serviceBindings {
|
|
| 81 |
+ if k.ports != "" && len(s.ingressPorts) != 0 {
|
|
| 82 |
+ services[k] = s |
|
| 83 |
+ } |
|
| 84 |
+ } |
|
| 85 |
+ c.mu.Unlock() |
|
| 86 |
+ |
|
| 87 |
+ for _, s := range services {
|
|
| 88 |
+ c.handleFirewallReloadService(s) |
|
| 89 |
+ } |
|
| 90 |
+ } |
|
| 91 |
+ // Add handler for iptables rules restoration in case of a firewalld reload |
|
| 92 |
+ iptables.OnReloaded(handler) |
|
| 93 |
+} |
|
| 94 |
+ |
|
| 95 |
+func (c *Controller) handleFirewallReloadService(s *service) {
|
|
| 96 |
+ s.Lock() |
|
| 97 |
+ defer s.Unlock() |
|
| 98 |
+ if s.deleted {
|
|
| 99 |
+ log.G(context.TODO()).Debugf("handleFirewallReloadService called for deleted service %s/%s", s.id, s.name)
|
|
| 100 |
+ return |
|
| 101 |
+ } |
|
| 102 |
+ for nid := range s.loadBalancers {
|
|
| 103 |
+ n, err := c.NetworkByID(nid) |
|
| 104 |
+ if err != nil {
|
|
| 105 |
+ continue |
|
| 106 |
+ } |
|
| 107 |
+ ep, sb, err := n.findLBEndpointSandbox() |
|
| 108 |
+ if err != nil {
|
|
| 109 |
+ log.G(context.TODO()).Warnf("handleFirewallReloadService failed to find LB Endpoint Sandbox for %s/%s: %v -- ", n.ID(), n.Name(), err)
|
|
| 110 |
+ continue |
|
| 111 |
+ } |
|
| 112 |
+ if sb.osSbox == nil {
|
|
| 113 |
+ return |
|
| 114 |
+ } |
|
| 115 |
+ if ep != nil {
|
|
| 116 |
+ var gwIP net.IP |
|
| 117 |
+ if gwEP, _ := sb.getGatewayEndpoint(); gwEP != nil {
|
|
| 118 |
+ gwIP = gwEP.Iface().Address().IP |
|
| 119 |
+ } |
|
| 120 |
+ if err := restoreIngressPorts(gwIP, s.ingressPorts); err != nil {
|
|
| 121 |
+ log.G(context.TODO()).Errorf("Failed to add ingress: %v", err)
|
|
| 122 |
+ return |
|
| 123 |
+ } |
|
| 124 |
+ } |
|
| 125 |
+ } |
|
| 126 |
+} |
| ... | ... |
@@ -403,6 +403,24 @@ func addIngressPorts(gwIP net.IP, ingressPorts []*PortConfig) error {
|
| 403 | 403 |
return nil |
| 404 | 404 |
} |
| 405 | 405 |
|
| 406 |
+func restoreIngressPorts(gwIP net.IP, ingressPorts []*PortConfig) error {
|
|
| 407 |
+ // TODO IPv6 support |
|
| 408 |
+ iptable := iptables.GetIptable(iptables.IPv4) |
|
| 409 |
+ |
|
| 410 |
+ ingressMu.Lock() |
|
| 411 |
+ defer ingressMu.Unlock() |
|
| 412 |
+ |
|
| 413 |
+ if err := initIngressConfiguration(gwIP, iptable); err != nil {
|
|
| 414 |
+ return err |
|
| 415 |
+ } |
|
| 416 |
+ |
|
| 417 |
+ if err := programIngressPortsRules(gwIP, ingressPorts); err != nil {
|
|
| 418 |
+ return fmt.Errorf("failed to program ingress ports: %v", err)
|
|
| 419 |
+ } |
|
| 420 |
+ |
|
| 421 |
+ return nil |
|
| 422 |
+} |
|
| 423 |
+ |
|
| 406 | 424 |
func generateIngressRules(port *PortConfig, destIP net.IP) []iptables.Rule {
|
| 407 | 425 |
var ( |
| 408 | 426 |
protocol = strings.ToLower(port.Protocol.String()) |
| ... | ... |
@@ -267,3 +267,58 @@ func TestDockerIngressChainPosition(t *testing.T) {
|
| 267 | 267 |
checkChain() |
| 268 | 268 |
}) |
| 269 | 269 |
} |
| 270 |
+ |
|
| 271 |
+func TestRestoreIngressRulesOnFirewalldReload(t *testing.T) {
|
|
| 272 |
+ skip.If(t, testEnv.IsRemoteDaemon) |
|
| 273 |
+ skip.If(t, testEnv.IsRootless, "rootless mode doesn't support Swarm-mode") |
|
| 274 |
+ skip.If(t, testEnv.FirewallBackendDriver() != "iptables+firewalld", "nftables backend doesn't support Swarm-mode") |
|
| 275 |
+ skip.If(t, !networking.FirewalldRunning(), "Need firewalld to test restoration ingress rules") |
|
| 276 |
+ ctx := setupTest(t) |
|
| 277 |
+ |
|
| 278 |
+ // Check the published port is accessible. |
|
| 279 |
+ checkHTTP := func(_ poll.LogT) poll.Result {
|
|
| 280 |
+ res := icmd.RunCommand("curl", "-v", "-o", "/dev/null", "-w", "%{http_code}\n",
|
|
| 281 |
+ "http://"+stdnet.JoinHostPort("localhost", "8080"))
|
|
| 282 |
+ // A "404 Not Found" means the server responded, but it's got nothing to serve. |
|
| 283 |
+ if !strings.Contains(res.Stdout(), "404") {
|
|
| 284 |
+ return poll.Continue("404 - not found in: %s, %+v", res.Stdout(), res)
|
|
| 285 |
+ } |
|
| 286 |
+ return poll.Success() |
|
| 287 |
+ } |
|
| 288 |
+ |
|
| 289 |
+ d := swarm.NewSwarm(ctx, t, testEnv, daemon.WithSwarmIptables(true)) |
|
| 290 |
+ defer d.Stop(t) |
|
| 291 |
+ c := d.NewClientT(t) |
|
| 292 |
+ defer c.Close() |
|
| 293 |
+ |
|
| 294 |
+ serviceID := swarm.CreateService(ctx, t, d, |
|
| 295 |
+ swarm.ServiceWithName("test-ingress-on-firewalld-reload"),
|
|
| 296 |
+ swarm.ServiceWithCommand([]string{"httpd", "-f"}),
|
|
| 297 |
+ swarm.ServiceWithEndpoint(&swarmtypes.EndpointSpec{
|
|
| 298 |
+ Ports: []swarmtypes.PortConfig{
|
|
| 299 |
+ {
|
|
| 300 |
+ Protocol: "tcp", |
|
| 301 |
+ TargetPort: 80, |
|
| 302 |
+ PublishedPort: 8080, |
|
| 303 |
+ PublishMode: swarmtypes.PortConfigPublishModeIngress, |
|
| 304 |
+ }, |
|
| 305 |
+ }, |
|
| 306 |
+ }), |
|
| 307 |
+ ) |
|
| 308 |
+ defer func() {
|
|
| 309 |
+ err := c.ServiceRemove(ctx, serviceID) |
|
| 310 |
+ assert.NilError(t, err) |
|
| 311 |
+ }() |
|
| 312 |
+ |
|
| 313 |
+ t.Log("Waiting for the service to start")
|
|
| 314 |
+ poll.WaitOn(t, swarm.RunningTasksCount(ctx, c, serviceID, 1), swarm.ServicePoll) |
|
| 315 |
+ t.Log("Checking http access to the service")
|
|
| 316 |
+ poll.WaitOn(t, checkHTTP, poll.WithTimeout(30*time.Second)) |
|
| 317 |
+ |
|
| 318 |
+ t.Log("Firewalld reload")
|
|
| 319 |
+ networking.FirewalldReload(t, d) |
|
| 320 |
+ |
|
| 321 |
+ t.Log("Checking http access to the service")
|
|
| 322 |
+ // It takes a while before this works ... |
|
| 323 |
+ poll.WaitOn(t, checkHTTP, poll.WithTimeout(30*time.Second)) |
|
| 324 |
+} |