Browse code
Retry service updates on out of sequence errors
Code retrying service update operations when receiving "update out of
sequence" errors was removed because of a misunderstanding, which has
made tests flaky. This re-adds the "CmdRetryOutOfSequence" method, and
uses it in TestSwarmPublishAdd to avoid flaky behavior.
Signed-off-by: Drew Erny <drew.erny@docker.com>
(cherry picked from commit 1de914695b7d0c9affc97a6da4198548da2f5f78)
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
Drew Erny authored on 2019/07/19 02:58:21Showing 2 changed files
| ... | ... |
@@ -189,3 +189,25 @@ func (d *Daemon) CheckLeader(c *check.C) (interface{}, check.CommentInterface) {
|
| 189 | 189 |
} |
| 190 | 190 |
return fmt.Errorf("no leader"), check.Commentf("could not find leader")
|
| 191 | 191 |
} |
| 192 |
+ |
|
| 193 |
+// CmdRetryOutOfSequence tries the specified command against the current daemon |
|
| 194 |
+// up to 10 times, retrying if it encounters an "update out of sequence" error. |
|
| 195 |
+func (d *Daemon) CmdRetryOutOfSequence(args ...string) (string, error) {
|
|
| 196 |
+ var ( |
|
| 197 |
+ output string |
|
| 198 |
+ err error |
|
| 199 |
+ ) |
|
| 200 |
+ |
|
| 201 |
+ for i := 0; i < 10; i++ {
|
|
| 202 |
+ output, err = d.Cmd(args...) |
|
| 203 |
+ // error, no error, whatever. if we don't have "update out of |
|
| 204 |
+ // sequence", we don't retry, we just return. |
|
| 205 |
+ if !strings.Contains(output, "update out of sequence") {
|
|
| 206 |
+ return output, err |
|
| 207 |
+ } |
|
| 208 |
+ } |
|
| 209 |
+ |
|
| 210 |
+ // otherwise, once all of our attempts have been exhausted, just return |
|
| 211 |
+ // whatever the last values were. |
|
| 212 |
+ return output, err |
|
| 213 |
+} |
| ... | ... |
@@ -277,19 +277,23 @@ func (s *DockerSwarmSuite) TestSwarmPublishAdd(c *check.C) {
|
| 277 | 277 |
d := s.AddDaemon(c, true, true) |
| 278 | 278 |
|
| 279 | 279 |
name := "top" |
| 280 |
+ // this first command does not have to be retried because service creates |
|
| 281 |
+ // don't return out of sequence errors. |
|
| 280 | 282 |
out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--label", "x=y", "busybox", "top")
|
| 281 | 283 |
assert.NilError(c, err, out) |
| 282 | 284 |
assert.Assert(c, strings.TrimSpace(out) != "") |
| 283 | 285 |
|
| 284 |
- out, err = d.Cmd("service", "update", "--detach", "--publish-add", "80:80", name)
|
|
| 286 |
+ out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", name)
|
|
| 285 | 287 |
assert.NilError(c, err, out) |
| 286 | 288 |
|
| 287 |
- out, err = d.Cmd("service", "update", "--detach", "--publish-add", "80:80", name)
|
|
| 289 |
+ out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", name)
|
|
| 288 | 290 |
assert.NilError(c, err, out) |
| 289 | 291 |
|
| 290 |
- _, err = d.Cmd("service", "update", "--detach", "--publish-add", "80:80", "--publish-add", "80:20", name)
|
|
| 292 |
+ _, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", "--publish-add", "80:20", name)
|
|
| 291 | 293 |
assert.ErrorContains(c, err, "") |
| 292 | 294 |
|
| 295 |
+ // this last command does not have to be retried because service inspect |
|
| 296 |
+ // does not return out of sequence errors. |
|
| 293 | 297 |
out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.EndpointSpec.Ports }}", name)
|
| 294 | 298 |
assert.NilError(c, err, out) |
| 295 | 299 |
assert.Equal(c, strings.TrimSpace(out), "[{ tcp 80 80 ingress}]")
|