Browse code

integration-cli/Test*Swarm*: use same args on restart

When starting docker daemons for swarm testing, we disable iptables
and use lo for communication (in order to avoid network conflicts).

The problem is, these options are lost on restart, that can lead
to any sorts of network conflicts and thus connectivity issues
between swarm nodes.

Fix this. This does not fix issues with swarm test failures, but
it seems they appear are less often after this one.

Signed-off-by: Kir Kolyshkin <kolyshkin@gmail.com>

Kir Kolyshkin authored on 2018/10/26 03:47:56
Showing 5 changed files
... ...
@@ -333,7 +333,7 @@ func (s *DockerSwarmSuite) AddDaemon(c *check.C, joinSwarm, manager bool) *daemo
333 333
 			d.StartAndSwarmInit(c)
334 334
 		}
335 335
 	} else {
336
-		d.StartWithBusybox(c, "--iptables=false", "--swarm-default-advertise-addr=lo")
336
+		d.StartNode(c)
337 337
 	}
338 338
 
339 339
 	s.portIndex++
... ...
@@ -62,7 +62,7 @@ func (s *DockerSwarmSuite) TestAPISwarmNodeRemove(c *check.C) {
62 62
 	c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes))
63 63
 
64 64
 	// Restart the node that was removed
65
-	d2.Restart(c)
65
+	d2.RestartNode(c)
66 66
 
67 67
 	// Give some time for the node to rejoin
68 68
 	time.Sleep(1 * time.Second)
... ...
@@ -66,8 +66,8 @@ func (s *DockerSwarmSuite) TestAPISwarmInit(c *check.C) {
66 66
 	d1.Stop(c)
67 67
 	d2.Stop(c)
68 68
 
69
-	d1.Start(c)
70
-	d2.Start(c)
69
+	d1.StartNode(c)
70
+	d2.StartNode(c)
71 71
 
72 72
 	info = d1.SwarmInfo(c)
73 73
 	c.Assert(info.ControlAvailable, checker.True)
... ...
@@ -342,7 +342,7 @@ func (s *DockerSwarmSuite) TestAPISwarmLeaderElection(c *check.C) {
342 342
 	stableleader := leader
343 343
 
344 344
 	// add the d1, the initial leader, back
345
-	d1.Start(c)
345
+	d1.StartNode(c)
346 346
 
347 347
 	// wait for possible election
348 348
 	c.Logf("Waiting for possible election...")
... ...
@@ -387,7 +387,7 @@ func (s *DockerSwarmSuite) TestAPISwarmRaftQuorum(c *check.C) {
387 387
 		return err.Error(), nil
388 388
 	}, checker.Contains, "Make sure more than half of the managers are online.")
389 389
 
390
-	d2.Start(c)
390
+	d2.StartNode(c)
391 391
 
392 392
 	// make sure there is a leader
393 393
 	waitAndAssert(c, defaultReconciliationTimeout, d1.CheckLeader, checker.IsNil)
... ...
@@ -463,8 +463,7 @@ func (s *DockerSwarmSuite) TestAPISwarmRestoreOnPendingJoin(c *check.C) {
463 463
 
464 464
 	waitAndAssert(c, defaultReconciliationTimeout, d.CheckLocalNodeState, checker.Equals, swarm.LocalNodeStatePending)
465 465
 
466
-	d.Stop(c)
467
-	d.Start(c)
466
+	d.RestartNode(c)
468 467
 
469 468
 	info := d.SwarmInfo(c)
470 469
 	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive)
... ...
@@ -477,25 +476,22 @@ func (s *DockerSwarmSuite) TestAPISwarmManagerRestore(c *check.C) {
477 477
 	id := d1.CreateService(c, simpleTestService, setInstances(instances))
478 478
 
479 479
 	d1.GetService(c, id)
480
-	d1.Stop(c)
481
-	d1.Start(c)
480
+	d1.RestartNode(c)
482 481
 	d1.GetService(c, id)
483 482
 
484 483
 	d2 := s.AddDaemon(c, true, true)
485 484
 	d2.GetService(c, id)
486
-	d2.Stop(c)
487
-	d2.Start(c)
485
+	d2.RestartNode(c)
488 486
 	d2.GetService(c, id)
489 487
 
490 488
 	d3 := s.AddDaemon(c, true, true)
491 489
 	d3.GetService(c, id)
492
-	d3.Stop(c)
493
-	d3.Start(c)
490
+	d3.RestartNode(c)
494 491
 	d3.GetService(c, id)
495 492
 
496 493
 	d3.Kill()
497 494
 	time.Sleep(1 * time.Second) // time to handle signal
498
-	d3.Start(c)
495
+	d3.StartNode(c)
499 496
 	d3.GetService(c, id)
500 497
 }
501 498
 
... ...
@@ -163,7 +163,7 @@ func (s *DockerSwarmSuite) TestSwarmIncompatibleDaemon(c *check.C) {
163 163
 	c.Assert(err, checker.IsNil)
164 164
 	c.Assert(string(content), checker.Contains, "--live-restore daemon configuration is incompatible with swarm mode")
165 165
 	// restart for teardown
166
-	d.Start(c)
166
+	d.StartNode(c)
167 167
 }
168 168
 
169 169
 func (s *DockerSwarmSuite) TestSwarmServiceTemplatingHostname(c *check.C) {
... ...
@@ -330,7 +330,7 @@ func (s *DockerSwarmSuite) TestSwarmContainerAutoStart(c *check.C) {
330 330
 	c.Assert(err, checker.IsNil, check.Commentf("%s", out))
331 331
 	c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "")
332 332
 
333
-	d.Restart(c)
333
+	d.RestartNode(c)
334 334
 
335 335
 	out, err = d.Cmd("ps", "-q")
336 336
 	c.Assert(err, checker.IsNil, check.Commentf("%s", out))
... ...
@@ -1013,7 +1013,7 @@ func checkSwarmLockedToUnlocked(c *check.C, d *daemon.Daemon) {
1013 1013
 	// Wait for the PEM file to become unencrypted
1014 1014
 	waitAndAssert(c, defaultReconciliationTimeout, checkKeyIsEncrypted(d), checker.Equals, false)
1015 1015
 
1016
-	d.Restart(c)
1016
+	d.RestartNode(c)
1017 1017
 	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
1018 1018
 }
1019 1019
 
... ...
@@ -1021,7 +1021,7 @@ func checkSwarmUnlockedToLocked(c *check.C, d *daemon.Daemon) {
1021 1021
 	// Wait for the PEM file to become encrypted
1022 1022
 	waitAndAssert(c, defaultReconciliationTimeout, checkKeyIsEncrypted(d), checker.Equals, true)
1023 1023
 
1024
-	d.Restart(c)
1024
+	d.RestartNode(c)
1025 1025
 	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
1026 1026
 }
1027 1027
 
... ...
@@ -1060,7 +1060,7 @@ func (s *DockerSwarmSuite) TestSwarmInitLocked(c *check.C) {
1060 1060
 	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive)
1061 1061
 
1062 1062
 	// It starts off locked
1063
-	d.Restart(c)
1063
+	d.RestartNode(c)
1064 1064
 	c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
1065 1065
 
1066 1066
 	cmd := d.Command("swarm", "unlock")
... ...
@@ -1099,7 +1099,7 @@ func (s *DockerSwarmSuite) TestSwarmLeaveLocked(c *check.C) {
1099 1099
 	c.Assert(err, checker.IsNil, check.Commentf("%s", outs))
1100 1100
 
1101 1101
 	// It starts off locked
1102
-	d.Restart(c, "--swarm-default-advertise-addr=lo")
1102
+	d.RestartNode(c)
1103 1103
 
1104 1104
 	info := d.SwarmInfo(c)
1105 1105
 	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateLocked)
... ...
@@ -1131,7 +1131,7 @@ func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) {
1131 1131
 	d3 := s.AddDaemon(c, true, true)
1132 1132
 
1133 1133
 	// they start off unlocked
1134
-	d2.Restart(c)
1134
+	d2.RestartNode(c)
1135 1135
 	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive)
1136 1136
 
1137 1137
 	// stop this one so it does not get autolock info
... ...
@@ -1153,7 +1153,7 @@ func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) {
1153 1153
 	}
1154 1154
 
1155 1155
 	// d2 never got the cluster update, so it is still set to unlocked
1156
-	d2.Start(c)
1156
+	d2.StartNode(c)
1157 1157
 	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive)
1158 1158
 
1159 1159
 	// d2 is now set to lock
... ...
@@ -1182,7 +1182,7 @@ func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) {
1182 1182
 
1183 1183
 	// managers who join now are never set to locked in the first place
1184 1184
 	d4 := s.AddDaemon(c, true, true)
1185
-	d4.Restart(c)
1185
+	d4.RestartNode(c)
1186 1186
 	c.Assert(getNodeStatus(c, d4), checker.Equals, swarm.LocalNodeStateActive)
1187 1187
 }
1188 1188
 
... ...
@@ -1196,7 +1196,7 @@ func (s *DockerSwarmSuite) TestSwarmJoinPromoteLocked(c *check.C) {
1196 1196
 
1197 1197
 	// joined workers start off unlocked
1198 1198
 	d2 := s.AddDaemon(c, true, false)
1199
-	d2.Restart(c)
1199
+	d2.RestartNode(c)
1200 1200
 	c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive)
1201 1201
 
1202 1202
 	// promote worker
... ...
@@ -1241,7 +1241,7 @@ func (s *DockerSwarmSuite) TestSwarmJoinPromoteLocked(c *check.C) {
1241 1241
 	}, checker.Equals, "swarm-worker")
1242 1242
 
1243 1243
 	// by now, it should *never* be locked on restart
1244
-	d3.Restart(c)
1244
+	d3.RestartNode(c)
1245 1245
 	c.Assert(getNodeStatus(c, d3), checker.Equals, swarm.LocalNodeStateActive)
1246 1246
 }
1247 1247
 
... ...
@@ -1261,7 +1261,7 @@ func (s *DockerSwarmSuite) TestSwarmRotateUnlockKey(c *check.C) {
1261 1261
 		c.Assert(newUnlockKey, checker.Not(checker.Equals), "")
1262 1262
 		c.Assert(newUnlockKey, checker.Not(checker.Equals), unlockKey)
1263 1263
 
1264
-		d.Restart(c)
1264
+		d.RestartNode(c)
1265 1265
 		c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
1266 1266
 
1267 1267
 		outs, _ = d.Cmd("node", "ls")
... ...
@@ -1282,7 +1282,7 @@ func (s *DockerSwarmSuite) TestSwarmRotateUnlockKey(c *check.C) {
1282 1282
 
1283 1283
 			time.Sleep(3 * time.Second)
1284 1284
 
1285
-			d.Restart(c)
1285
+			d.RestartNode(c)
1286 1286
 
1287 1287
 			cmd = d.Command("swarm", "unlock")
1288 1288
 			cmd.Stdin = bytes.NewBufferString(unlockKey)
... ...
@@ -1331,8 +1331,8 @@ func (s *DockerSwarmSuite) TestSwarmClusterRotateUnlockKey(c *check.C) {
1331 1331
 		c.Assert(newUnlockKey, checker.Not(checker.Equals), "")
1332 1332
 		c.Assert(newUnlockKey, checker.Not(checker.Equals), unlockKey)
1333 1333
 
1334
-		d2.Restart(c)
1335
-		d3.Restart(c)
1334
+		d2.RestartNode(c)
1335
+		d3.RestartNode(c)
1336 1336
 
1337 1337
 		for _, d := range []*daemon.Daemon{d2, d3} {
1338 1338
 			c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked)
... ...
@@ -1355,7 +1355,7 @@ func (s *DockerSwarmSuite) TestSwarmClusterRotateUnlockKey(c *check.C) {
1355 1355
 
1356 1356
 				time.Sleep(3 * time.Second)
1357 1357
 
1358
-				d.Restart(c)
1358
+				d.RestartNode(c)
1359 1359
 
1360 1360
 				cmd = d.Command("swarm", "unlock")
1361 1361
 				cmd.Stdin = bytes.NewBufferString(unlockKey)
... ...
@@ -16,26 +16,38 @@ const (
16 16
 	defaultSwarmListenAddr = "0.0.0.0"
17 17
 )
18 18
 
19
-// StartAndSwarmInit starts the daemon (with busybox) and init the swarm
20
-func (d *Daemon) StartAndSwarmInit(t testingT) {
19
+var (
20
+	startArgs = []string{"--iptables=false", "--swarm-default-advertise-addr=lo"}
21
+)
22
+
23
+// StartNode starts daemon to be used as a swarm node
24
+func (d *Daemon) StartNode(t testingT) {
21 25
 	if ht, ok := t.(test.HelperT); ok {
22 26
 		ht.Helper()
23 27
 	}
24 28
 	// avoid networking conflicts
25
-	args := []string{"--iptables=false", "--swarm-default-advertise-addr=lo"}
26
-	d.StartWithBusybox(t, args...)
27
-
28
-	d.SwarmInit(t, swarm.InitRequest{})
29
+	d.StartWithBusybox(t, startArgs...)
29 30
 }
30 31
 
31
-// StartAndSwarmJoin starts the daemon (with busybox) and join the specified swarm as worker or manager
32
-func (d *Daemon) StartAndSwarmJoin(t testingT, leader *Daemon, manager bool) {
32
+// RestartNode restarts a daemon to be used as a swarm node
33
+func (d *Daemon) RestartNode(t testingT) {
33 34
 	if ht, ok := t.(test.HelperT); ok {
34 35
 		ht.Helper()
35 36
 	}
36 37
 	// avoid networking conflicts
37
-	args := []string{"--iptables=false", "--swarm-default-advertise-addr=lo"}
38
-	d.StartWithBusybox(t, args...)
38
+	d.Stop(t)
39
+	d.StartWithBusybox(t, startArgs...)
40
+}
41
+
42
+// StartAndSwarmInit starts the daemon (with busybox) and init the swarm
43
+func (d *Daemon) StartAndSwarmInit(t testingT) {
44
+	d.StartNode(t)
45
+	d.SwarmInit(t, swarm.InitRequest{})
46
+}
47
+
48
+// StartAndSwarmJoin starts the daemon (with busybox) and join the specified swarm as worker or manager
49
+func (d *Daemon) StartAndSwarmJoin(t testingT, leader *Daemon, manager bool) {
50
+	d.StartNode(t)
39 51
 
40 52
 	tokens := leader.JoinTokens(t)
41 53
 	token := tokens.Worker