Browse code

Allow user to modify ingress network

Signed-off-by: Alessandro Boch <aboch@docker.com>

Alessandro Boch authored on 2017/03/10 04:52:25
Showing 18 changed files
... ...
@@ -294,6 +294,7 @@ func (n *networkRouter) buildNetworkResource(nw libnetwork.Network) *types.Netwo
294 294
 	r.EnableIPv6 = info.IPv6Enabled()
295 295
 	r.Internal = info.Internal()
296 296
 	r.Attachable = info.Attachable()
297
+	r.Ingress = info.Ingress()
297 298
 	r.Options = info.DriverOptions()
298 299
 	r.Containers = make(map[string]types.EndpointResource)
299 300
 	buildIpamResources(r, info)
... ...
@@ -1117,6 +1117,8 @@ definitions:
1117 1117
         type: "boolean"
1118 1118
       Attachable:
1119 1119
         type: "boolean"
1120
+      Ingress:
1121
+        type: "boolean"
1120 1122
       Containers:
1121 1123
         type: "object"
1122 1124
         additionalProperties:
... ...
@@ -1145,6 +1147,7 @@ definitions:
1145 1145
           foo: "bar"
1146 1146
       Internal: false
1147 1147
       Attachable: false
1148
+      Ingress: false
1148 1149
       Containers:
1149 1150
         19a4d5d687db25203351ed79d478946f861258f018fe384f229f2efa4b23513c:
1150 1151
           Name: "test"
... ...
@@ -6211,6 +6214,7 @@ paths:
6211 6211
                 EnableIPv6: false
6212 6212
                 Internal: false
6213 6213
                 Attachable: false
6214
+                Ingress: false
6214 6215
                 IPAM:
6215 6216
                   Driver: "default"
6216 6217
                   Config:
... ...
@@ -6237,6 +6241,7 @@ paths:
6237 6237
                 EnableIPv6: false
6238 6238
                 Internal: false
6239 6239
                 Attachable: false
6240
+                Ingress: false
6240 6241
                 IPAM:
6241 6242
                   Driver: "default"
6242 6243
                   Config: []
... ...
@@ -6250,6 +6255,7 @@ paths:
6250 6250
                 EnableIPv6: false
6251 6251
                 Internal: false
6252 6252
                 Attachable: false
6253
+                Ingress: false
6253 6254
                 IPAM:
6254 6255
                   Driver: "default"
6255 6256
                   Config: []
... ...
@@ -6383,6 +6389,9 @@ paths:
6383 6383
               Attachable:
6384 6384
                 description: "Globally scoped network is manually attachable by regular containers from workers in swarm mode."
6385 6385
                 type: "boolean"
6386
+              Ingress:
6387
+                description: "Ingress network is the network which provides the routing-mesh in swarm mode."
6388
+                type: "boolean"
6386 6389
               IPAM:
6387 6390
                 description: "Optional custom IP scheme for the network."
6388 6391
                 $ref: "#/definitions/IPAM"
... ...
@@ -6416,6 +6425,7 @@ paths:
6416 6416
                   foo: "bar"
6417 6417
               Internal: true
6418 6418
               Attachable: false
6419
+              Ingress: false
6419 6420
               Options:
6420 6421
                 com.docker.network.bridge.default_bridge: "true"
6421 6422
                 com.docker.network.bridge.enable_icc: "true"
... ...
@@ -82,6 +82,7 @@ type NetworkSpec struct {
82 82
 	IPv6Enabled         bool         `json:",omitempty"`
83 83
 	Internal            bool         `json:",omitempty"`
84 84
 	Attachable          bool         `json:",omitempty"`
85
+	Ingress             bool         `json:",omitempty"`
85 86
 	IPAMOptions         *IPAMOptions `json:",omitempty"`
86 87
 }
87 88
 
... ...
@@ -400,6 +400,7 @@ type NetworkResource struct {
400 400
 	IPAM       network.IPAM                   // IPAM is the network's IP Address Management
401 401
 	Internal   bool                           // Internal represents if the network is used internal only
402 402
 	Attachable bool                           // Attachable represents if the global scope is manually attachable by regular containers from workers in swarm mode.
403
+	Ingress    bool                           // Ingress indicates the network is providing the routing-mesh for the swarm cluster.
403 404
 	Containers map[string]EndpointResource    // Containers contains endpoints belonging to the network
404 405
 	Options    map[string]string              // Options holds the network specific options to use for when creating the network
405 406
 	Labels     map[string]string              // Labels holds metadata specific to the network being created
... ...
@@ -431,6 +432,7 @@ type NetworkCreate struct {
431 431
 	IPAM           *network.IPAM
432 432
 	Internal       bool
433 433
 	Attachable     bool
434
+	Ingress        bool
434 435
 	Options        map[string]string
435 436
 	Labels         map[string]string
436 437
 }
... ...
@@ -24,6 +24,7 @@ type createOptions struct {
24 24
 	internal   bool
25 25
 	ipv6       bool
26 26
 	attachable bool
27
+	ingress    bool
27 28
 
28 29
 	ipamDriver  string
29 30
 	ipamSubnet  []string
... ...
@@ -59,6 +60,8 @@ func newCreateCommand(dockerCli *command.DockerCli) *cobra.Command {
59 59
 	flags.BoolVar(&opts.ipv6, "ipv6", false, "Enable IPv6 networking")
60 60
 	flags.BoolVar(&opts.attachable, "attachable", false, "Enable manual container attachment")
61 61
 	flags.SetAnnotation("attachable", "version", []string{"1.25"})
62
+	flags.BoolVar(&opts.ingress, "ingress", false, "Create swarm routing-mesh network")
63
+	flags.SetAnnotation("ingress", "version", []string{"1.29"})
62 64
 
63 65
 	flags.StringVar(&opts.ipamDriver, "ipam-driver", "default", "IP Address Management Driver")
64 66
 	flags.StringSliceVar(&opts.ipamSubnet, "subnet", []string{}, "Subnet in CIDR format that represents a network segment")
... ...
@@ -92,6 +95,7 @@ func runCreate(dockerCli *command.DockerCli, opts createOptions) error {
92 92
 		Internal:       opts.internal,
93 93
 		EnableIPv6:     opts.ipv6,
94 94
 		Attachable:     opts.attachable,
95
+		Ingress:        opts.ingress,
95 96
 		Labels:         runconfigopts.ConvertKVStringsToMap(opts.labels.GetAll()),
96 97
 	}
97 98
 
... ...
@@ -22,12 +22,22 @@ func newRemoveCommand(dockerCli *command.DockerCli) *cobra.Command {
22 22
 	}
23 23
 }
24 24
 
25
+const ingressWarning = "WARNING! Before removing the routing-mesh network, " +
26
+	"make sure all the nodes in your swarm run the same docker engine version. " +
27
+	"Otherwise, removal may not be effective and functionality of newly create " +
28
+	"ingress networks will be impaired.\nAre you sure you want to continue?"
29
+
25 30
 func runRemove(dockerCli *command.DockerCli, networks []string) error {
26 31
 	client := dockerCli.Client()
27 32
 	ctx := context.Background()
28 33
 	status := 0
29 34
 
30 35
 	for _, name := range networks {
36
+		if nw, _, err := client.NetworkInspectWithRaw(ctx, name, false); err == nil &&
37
+			nw.Ingress &&
38
+			!command.PromptForConfirmation(dockerCli.In(), dockerCli.Out(), ingressWarning) {
39
+			continue
40
+		}
31 41
 		if err := client.NetworkRemove(ctx, name); err != nil {
32 42
 			fmt.Fprintf(dockerCli.Err(), "%s\n", err)
33 43
 			status = 1
... ...
@@ -28,6 +28,7 @@ func networkFromGRPC(n *swarmapi.Network) types.Network {
28 28
 				IPv6Enabled: n.Spec.Ipv6Enabled,
29 29
 				Internal:    n.Spec.Internal,
30 30
 				Attachable:  n.Spec.Attachable,
31
+				Ingress:     n.Spec.Ingress,
31 32
 				IPAMOptions: ipamFromGRPC(n.Spec.IPAM),
32 33
 			},
33 34
 			IPAMOptions: ipamFromGRPC(n.IPAM),
... ...
@@ -156,6 +157,7 @@ func BasicNetworkFromGRPC(n swarmapi.Network) basictypes.NetworkResource {
156 156
 		IPAM:       ipam,
157 157
 		Internal:   spec.Internal,
158 158
 		Attachable: spec.Attachable,
159
+		Ingress:    spec.Ingress,
159 160
 		Labels:     n.Spec.Annotations.Labels,
160 161
 	}
161 162
 
... ...
@@ -181,6 +183,7 @@ func BasicNetworkCreateToGRPC(create basictypes.NetworkCreateRequest) swarmapi.N
181 181
 		Ipv6Enabled: create.EnableIPv6,
182 182
 		Internal:    create.Internal,
183 183
 		Attachable:  create.Attachable,
184
+		Ingress:     create.Ingress,
184 185
 	}
185 186
 	if create.IPAM != nil {
186 187
 		driver := create.IPAM.Driver
... ...
@@ -28,6 +28,7 @@ type Backend interface {
28 28
 	DeleteManagedNetwork(name string) error
29 29
 	FindNetwork(idName string) (libnetwork.Network, error)
30 30
 	SetupIngress(req clustertypes.NetworkCreateRequest, nodeIP string) error
31
+	ReleaseIngress() error
31 32
 	PullImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
32 33
 	CreateManagedContainer(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error)
33 34
 	ContainerStart(name string, hostConfig *container.HostConfig, checkpoint string, checkpointDir string) error
... ...
@@ -575,6 +575,7 @@ func (c *containerConfig) networkCreateRequest(name string) (clustertypes.Networ
575 575
 		Labels:         na.Network.Spec.Annotations.Labels,
576 576
 		Internal:       na.Network.Spec.Internal,
577 577
 		Attachable:     na.Network.Spec.Attachable,
578
+		Ingress:        na.Network.Spec.Ingress,
578 579
 		EnableIPv6:     na.Network.Spec.Ipv6Enabled,
579 580
 		CheckDuplicate: true,
580 581
 	}
... ...
@@ -116,6 +116,7 @@ func (e *executor) Describe(ctx context.Context) (*api.NodeDescription, error) {
116 116
 func (e *executor) Configure(ctx context.Context, node *api.Node) error {
117 117
 	na := node.Attachment
118 118
 	if na == nil {
119
+		e.backend.ReleaseIngress()
119 120
 		return nil
120 121
 	}
121 122
 
... ...
@@ -125,6 +126,7 @@ func (e *executor) Configure(ctx context.Context, node *api.Node) error {
125 125
 			Driver: na.Network.IPAM.Driver.Name,
126 126
 		},
127 127
 		Options:        na.Network.DriverState.Options,
128
+		Ingress:        true,
128 129
 		CheckDuplicate: true,
129 130
 	}
130 131
 
... ...
@@ -6,6 +6,7 @@ import (
6 6
 	"runtime"
7 7
 	"sort"
8 8
 	"strings"
9
+	"sync"
9 10
 
10 11
 	"github.com/Sirupsen/logrus"
11 12
 	apierrors "github.com/docker/docker/api/errors"
... ...
@@ -99,15 +100,40 @@ func (daemon *Daemon) getAllNetworks() []libnetwork.Network {
99 99
 	return daemon.netController.Networks()
100 100
 }
101 101
 
102
-func isIngressNetwork(name string) bool {
103
-	return name == "ingress"
102
+type ingressJob struct {
103
+	create *clustertypes.NetworkCreateRequest
104
+	ip     net.IP
104 105
 }
105 106
 
106
-var ingressChan = make(chan struct{}, 1)
107
+var (
108
+	ingressWorkerOnce  sync.Once
109
+	ingressJobsChannel chan *ingressJob
110
+	ingressID          string
111
+)
112
+
113
+func (daemon *Daemon) startIngressWorker() {
114
+	ingressJobsChannel = make(chan *ingressJob, 100)
115
+	go func() {
116
+		for {
117
+			select {
118
+			case r := <-ingressJobsChannel:
119
+				if r.create != nil {
120
+					daemon.setupIngress(r.create, r.ip, ingressID)
121
+					ingressID = r.create.ID
122
+				} else {
123
+					daemon.releaseIngress(ingressID)
124
+					ingressID = ""
125
+				}
126
+			}
127
+		}
128
+	}()
129
+}
107 130
 
108
-func ingressWait() func() {
109
-	ingressChan <- struct{}{}
110
-	return func() { <-ingressChan }
131
+// enqueueIngressJob adds a ingress add/rm request to the worker queue.
132
+// It guarantees the worker is started.
133
+func (daemon *Daemon) enqueueIngressJob(job *ingressJob) {
134
+	ingressWorkerOnce.Do(daemon.startIngressWorker)
135
+	ingressJobsChannel <- job
111 136
 }
112 137
 
113 138
 // SetupIngress setups ingress networking.
... ...
@@ -116,72 +142,93 @@ func (daemon *Daemon) SetupIngress(create clustertypes.NetworkCreateRequest, nod
116 116
 	if err != nil {
117 117
 		return err
118 118
 	}
119
+	daemon.enqueueIngressJob(&ingressJob{&create, ip})
120
+	return nil
121
+}
119 122
 
120
-	go func() {
121
-		controller := daemon.netController
122
-		controller.AgentInitWait()
123
+// ReleaseIngress releases the ingress networking.
124
+func (daemon *Daemon) ReleaseIngress() error {
125
+	daemon.enqueueIngressJob(&ingressJob{nil, nil})
126
+	return nil
127
+}
123 128
 
124
-		if n, err := daemon.GetNetworkByName(create.Name); err == nil && n != nil && n.ID() != create.ID {
125
-			if err := controller.SandboxDestroy("ingress-sbox"); err != nil {
126
-				logrus.Errorf("Failed to delete stale ingress sandbox: %v", err)
127
-				return
128
-			}
129
+func (daemon *Daemon) setupIngress(create *clustertypes.NetworkCreateRequest, ip net.IP, staleID string) {
130
+	controller := daemon.netController
131
+	controller.AgentInitWait()
129 132
 
130
-			// Cleanup any stale endpoints that might be left over during previous iterations
131
-			epList := n.Endpoints()
132
-			for _, ep := range epList {
133
-				if err := ep.Delete(true); err != nil {
134
-					logrus.Errorf("Failed to delete endpoint %s (%s): %v", ep.Name(), ep.ID(), err)
135
-				}
136
-			}
133
+	if staleID != "" && staleID != create.ID {
134
+		daemon.releaseIngress(staleID)
135
+	}
137 136
 
138
-			if err := n.Delete(); err != nil {
139
-				logrus.Errorf("Failed to delete stale ingress network %s: %v", n.ID(), err)
140
-				return
141
-			}
137
+	if _, err := daemon.createNetwork(create.NetworkCreateRequest, create.ID, true); err != nil {
138
+		// If it is any other error other than already
139
+		// exists error log error and return.
140
+		if _, ok := err.(libnetwork.NetworkNameError); !ok {
141
+			logrus.Errorf("Failed creating ingress network: %v", err)
142
+			return
142 143
 		}
144
+		// Otherwise continue down the call to create or recreate sandbox.
145
+	}
143 146
 
144
-		if _, err := daemon.createNetwork(create.NetworkCreateRequest, create.ID, true); err != nil {
145
-			// If it is any other error other than already
146
-			// exists error log error and return.
147
-			if _, ok := err.(libnetwork.NetworkNameError); !ok {
148
-				logrus.Errorf("Failed creating ingress network: %v", err)
149
-				return
150
-			}
147
+	n, err := daemon.GetNetworkByID(create.ID)
148
+	if err != nil {
149
+		logrus.Errorf("Failed getting ingress network by id after creating: %v", err)
150
+	}
151 151
 
152
-			// Otherwise continue down the call to create or recreate sandbox.
152
+	sb, err := controller.NewSandbox("ingress-sbox", libnetwork.OptionIngress())
153
+	if err != nil {
154
+		if _, ok := err.(networktypes.ForbiddenError); !ok {
155
+			logrus.Errorf("Failed creating ingress sandbox: %v", err)
153 156
 		}
157
+		return
158
+	}
154 159
 
155
-		n, err := daemon.GetNetworkByID(create.ID)
156
-		if err != nil {
157
-			logrus.Errorf("Failed getting ingress network by id after creating: %v", err)
158
-			return
159
-		}
160
+	ep, err := n.CreateEndpoint("ingress-endpoint", libnetwork.CreateOptionIpam(ip, nil, nil, nil))
161
+	if err != nil {
162
+		logrus.Errorf("Failed creating ingress endpoint: %v", err)
163
+		return
164
+	}
160 165
 
161
-		sb, err := controller.NewSandbox("ingress-sbox", libnetwork.OptionIngress())
162
-		if err != nil {
163
-			if _, ok := err.(networktypes.ForbiddenError); !ok {
164
-				logrus.Errorf("Failed creating ingress sandbox: %v", err)
165
-			}
166
-			return
167
-		}
166
+	if err := ep.Join(sb, nil); err != nil {
167
+		logrus.Errorf("Failed joining ingress sandbox to ingress endpoint: %v", err)
168
+		return
169
+	}
168 170
 
169
-		ep, err := n.CreateEndpoint("ingress-endpoint", libnetwork.CreateOptionIpam(ip, nil, nil, nil))
170
-		if err != nil {
171
-			logrus.Errorf("Failed creating ingress endpoint: %v", err)
172
-			return
173
-		}
171
+	if err := sb.EnableService(); err != nil {
172
+		logrus.Errorf("Failed enabling service for ingress sandbox")
173
+	}
174
+}
174 175
 
175
-		if err := ep.Join(sb, nil); err != nil {
176
-			logrus.Errorf("Failed joining ingress sandbox to ingress endpoint: %v", err)
177
-		}
176
+func (daemon *Daemon) releaseIngress(id string) {
177
+	controller := daemon.netController
178 178
 
179
-		if err := sb.EnableService(); err != nil {
180
-			logrus.WithError(err).Error("Failed enabling service for ingress sandbox")
179
+	if err := controller.SandboxDestroy("ingress-sbox"); err != nil {
180
+		logrus.Errorf("Failed to delete ingress sandbox: %v", err)
181
+	}
182
+
183
+	if id == "" {
184
+		return
185
+	}
186
+
187
+	n, err := controller.NetworkByID(id)
188
+	if err != nil {
189
+		logrus.Errorf("failed to retrieve ingress network %s: %v", id, err)
190
+		return
191
+	}
192
+
193
+	for _, ep := range n.Endpoints() {
194
+		if err := ep.Delete(true); err != nil {
195
+			logrus.Errorf("Failed to delete endpoint %s (%s): %v", ep.Name(), ep.ID(), err)
196
+			return
181 197
 		}
182
-	}()
198
+	}
183 199
 
184
-	return nil
200
+	if err := n.Delete(); err != nil {
201
+		logrus.Errorf("Failed to delete ingress network %s: %v", n.ID(), err)
202
+		return
203
+	}
204
+
205
+	return
185 206
 }
186 207
 
187 208
 // SetNetworkBootstrapKeys sets the bootstrap keys.
... ...
@@ -228,13 +275,6 @@ func (daemon *Daemon) CreateNetwork(create types.NetworkCreateRequest) (*types.N
228 228
 }
229 229
 
230 230
 func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string, agent bool) (*types.NetworkCreateResponse, error) {
231
-	// If there is a pending ingress network creation wait here
232
-	// since ingress network creation can happen via node download
233
-	// from manager or task download.
234
-	if isIngressNetwork(create.Name) {
235
-		defer ingressWait()()
236
-	}
237
-
238 231
 	if runconfig.IsPreDefinedNetwork(create.Name) && !agent {
239 232
 		err := fmt.Errorf("%s is a pre-defined network and cannot be created", create.Name)
240 233
 		return nil, apierrors.NewRequestForbiddenError(err)
... ...
@@ -267,6 +307,7 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string
267 267
 		libnetwork.NetworkOptionDriverOpts(create.Options),
268 268
 		libnetwork.NetworkOptionLabels(create.Labels),
269 269
 		libnetwork.NetworkOptionAttachable(create.Attachable),
270
+		libnetwork.NetworkOptionIngress(create.Ingress),
270 271
 	}
271 272
 
272 273
 	if create.IPAM != nil {
... ...
@@ -286,10 +327,6 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string
286 286
 		nwOptions = append(nwOptions, libnetwork.NetworkOptionPersist(false))
287 287
 	}
288 288
 
289
-	if isIngressNetwork(create.Name) {
290
-		nwOptions = append(nwOptions, libnetwork.NetworkOptionIngress())
291
-	}
292
-
293 289
 	n, err := c.NewNetwork(driver, create.Name, id, nwOptions...)
294 290
 	if err != nil {
295 291
 		return nil, err
... ...
@@ -231,7 +231,8 @@ func (daemon *Daemon) clusterNetworksPrune(pruneFilters filters.Args) (*types.Ne
231 231
 	}
232 232
 	networkIsInUse := regexp.MustCompile(`network ([[:alnum:]]+) is in use`)
233 233
 	for _, nw := range networks {
234
-		if nw.Name == "ingress" {
234
+		if nw.Ingress {
235
+			// Routing-mesh network removal has to be explicitly invoked by user
235 236
 			continue
236 237
 		}
237 238
 		if !until.IsZero() && nw.Created.After(until) {
... ...
@@ -17,6 +17,10 @@ keywords: "API, Docker, rcli, REST, documentation"
17 17
 
18 18
 [Docker Engine API v1.29](https://docs.docker.com/engine/api/v1.29/) documentation
19 19
 
20
+
21
+* `DELETE /networks/(name)` now allows to remove the ingress network, the one used to provide the routing-mesh.
22
+* `POST /networks/create` now supports creating the ingress network, by specifying an `Ingress` boolean field. As of now this is supported only when using the overlay network driver.
23
+* `GET /networks/(name)` now returns an `Ingress` field showing whether the network is the ingress one.
20 24
 * `GET /networks/` now supports a `scope` filter to filter networks based on the network mode (`swarm`, `global`, or `local`).
21 25
 
22 26
 ## v1.28 API changes
... ...
@@ -22,6 +22,7 @@ Create a network
22 22
 
23 23
 Options:
24 24
       --attachable           Enable manual container attachment
25
+      --ingress              Specify the network provides the routing-mesh
25 26
       --aux-address value    Auxiliary IPv4 or IPv6 addresses used by Network
26 27
                              driver (default map[])
27 28
   -d, --driver string        Driver to manage the Network (default "bridge")
... ...
@@ -195,6 +196,23 @@ connects a bridge network to it to provide external connectivity. If you want
195 195
 to create an externally isolated `overlay` network, you can specify the
196 196
 `--internal` option.
197 197
 
198
+### Network ingress mode
199
+
200
+You can create the network which will be used to provide the routing-mesh in the
201
+swarm cluster. You do so by specifying `--ingress` when creating the network. Only
202
+one ingress network can be created at the time. The network can be removed only
203
+if no services depend on it. Any option available when creating a overlay network
204
+is also available when creating the ingress network, besides the `--attachable` option.
205
+
206
+```bash
207
+$ docker network create -d overlay \
208
+  --subnet=10.11.0.0/16 \
209
+  --ingress \
210
+  --opt com.docker.network.mtu=9216 \
211
+  --opt encrypted=true \
212
+  my-ingress-network
213
+```
214
+
198 215
 ## Related commands
199 216
 
200 217
 * [network inspect](network_inspect.md)
... ...
@@ -10,6 +10,7 @@ import (
10 10
 	"net/http"
11 11
 	"net/http/httptest"
12 12
 	"os"
13
+	"os/exec"
13 14
 	"path/filepath"
14 15
 	"strings"
15 16
 	"time"
... ...
@@ -19,6 +20,7 @@ import (
19 19
 	"github.com/docker/docker/integration-cli/checker"
20 20
 	"github.com/docker/docker/integration-cli/cli"
21 21
 	"github.com/docker/docker/integration-cli/daemon"
22
+	"github.com/docker/docker/pkg/testutil"
22 23
 	icmd "github.com/docker/docker/pkg/testutil/cmd"
23 24
 	"github.com/docker/libnetwork/driverapi"
24 25
 	"github.com/docker/libnetwork/ipamapi"
... ...
@@ -413,14 +415,57 @@ func (s *DockerSwarmSuite) TestOverlayAttachableReleaseResourcesOnFailure(c *che
413 413
 	c.Assert(err, checker.IsNil, check.Commentf(out))
414 414
 }
415 415
 
416
-func (s *DockerSwarmSuite) TestSwarmRemoveInternalNetwork(c *check.C) {
416
+func (s *DockerSwarmSuite) TestSwarmIngressNetwork(c *check.C) {
417 417
 	d := s.AddDaemon(c, true, true)
418 418
 
419
-	name := "ingress"
420
-	out, err := d.Cmd("network", "rm", name)
419
+	// Ingress network can be removed
420
+	out, _, err := testutil.RunCommandPipelineWithOutput(
421
+		exec.Command("echo", "Y"),
422
+		exec.Command("docker", "-H", d.Sock(), "network", "rm", "ingress"),
423
+	)
424
+	c.Assert(err, checker.IsNil, check.Commentf(out))
425
+
426
+	// And recreated
427
+	out, err = d.Cmd("network", "create", "-d", "overlay", "--ingress", "new-ingress")
428
+	c.Assert(err, checker.IsNil, check.Commentf(out))
429
+
430
+	// But only one is allowed
431
+	out, err = d.Cmd("network", "create", "-d", "overlay", "--ingress", "another-ingress")
421 432
 	c.Assert(err, checker.NotNil)
422
-	c.Assert(strings.TrimSpace(out), checker.Contains, name)
423
-	c.Assert(strings.TrimSpace(out), checker.Contains, "is a pre-defined network and cannot be removed")
433
+	c.Assert(strings.TrimSpace(out), checker.Contains, "is already present")
434
+
435
+	// It cannot be removed if it is being used
436
+	out, err = d.Cmd("service", "create", "--name", "srv1", "-p", "9000:8000", "busybox", "top")
437
+	c.Assert(err, checker.IsNil, check.Commentf(out))
438
+	out, _, err = testutil.RunCommandPipelineWithOutput(
439
+		exec.Command("echo", "Y"),
440
+		exec.Command("docker", "-H", d.Sock(), "network", "rm", "new-ingress"),
441
+	)
442
+	c.Assert(err, checker.NotNil)
443
+	c.Assert(strings.TrimSpace(out), checker.Contains, "ingress network cannot be removed because service")
444
+
445
+	// But it can be removed once no more services depend on it
446
+	out, err = d.Cmd("service", "update", "--publish-rm", "9000:8000", "srv1")
447
+	c.Assert(err, checker.IsNil, check.Commentf(out))
448
+	out, _, err = testutil.RunCommandPipelineWithOutput(
449
+		exec.Command("echo", "Y"),
450
+		exec.Command("docker", "-H", d.Sock(), "network", "rm", "new-ingress"),
451
+	)
452
+	c.Assert(err, checker.IsNil, check.Commentf(out))
453
+
454
+	// A service which needs the ingress network cannot be created if no ingress is present
455
+	out, err = d.Cmd("service", "create", "--name", "srv2", "-p", "500:500", "busybox", "top")
456
+	c.Assert(err, checker.NotNil)
457
+	c.Assert(strings.TrimSpace(out), checker.Contains, "no ingress network is present")
458
+
459
+	// An existing service cannot be updated to use the ingress nw if the nw is not present
460
+	out, err = d.Cmd("service", "update", "--publish-add", "9000:8000", "srv1")
461
+	c.Assert(err, checker.NotNil)
462
+	c.Assert(strings.TrimSpace(out), checker.Contains, "no ingress network is present")
463
+
464
+	// But services which do not need routing mesh can be created regardless
465
+	out, err = d.Cmd("service", "create", "--name", "srv3", "--endpoint-mode", "dnsrr", "busybox", "top")
466
+	c.Assert(err, checker.IsNil, check.Commentf(out))
424 467
 }
425 468
 
426 469
 // Test case for #24108, also the case from:
... ...
@@ -117,3 +117,20 @@ By default, when you connect a container to an `overlay` network, Docker also
117 117
 connects a bridge network to it to provide external connectivity. If you want
118 118
 to create an externally isolated `overlay` network, you can specify the
119 119
 `--internal` option.
120
+
121
+### Network ingress mode
122
+
123
+You can create the network which will be used to provide the routing-mesh in the
124
+swarm cluster. You do so by specifying `--ingress` when creating the network. Only
125
+one ingress network can be created at the time. The network can be removed only
126
+if no services depend on it. Any option available when creating a overlay network
127
+is also available when creating the ingress network, besides the `--attachable` option.
128
+
129
+```bash
130
+$ docker network create -d overlay \
131
+  --subnet=10.11.0.0/16 \
132
+  --ingress \
133
+  --opt com.docker.network.mtu=9216 \
134
+  --opt encrypted=true \
135
+  my-ingress-network
136
+```
... ...
@@ -32,6 +32,7 @@ $ sudo docker network inspect bridge
32 32
             ]
33 33
         },
34 34
         "Internal": false,
35
+        "Ingress": false,
35 36
         "Containers": {
36 37
             "bda12f8922785d1f160be70736f26c1e331ab8aaf8ed8d56728508f2e2fd4727": {
37 38
                 "Name": "container2",
... ...
@@ -116,6 +117,7 @@ $ docker network inspect --verbose ov1
116 116
         },
117 117
         "Internal": false,
118 118
         "Attachable": false,
119
+        "Ingress": false,
119 120
         "Containers": {
120 121
             "020403bd88a15f60747fd25d1ad5fa1272eb740e8a97fc547d8ad07b2f721c5e": {
121 122
                 "Name": "s1.1.pjn2ik0sfgkfzed3h0s00gs9o",
... ...
@@ -19,7 +19,7 @@ func DefaultDaemonNetworkMode() container.NetworkMode {
19 19
 // IsPreDefinedNetwork indicates if a network is predefined by the daemon
20 20
 func IsPreDefinedNetwork(network string) bool {
21 21
 	n := container.NetworkMode(network)
22
-	return n.IsBridge() || n.IsHost() || n.IsNone() || n.IsDefault() || network == "ingress"
22
+	return n.IsBridge() || n.IsHost() || n.IsNone() || n.IsDefault()
23 23
 }
24 24
 
25 25
 // validateNetMode ensures that the various combinations of requested