Browse code
Avoid redundant HEAD requests for identical layers on push
pushV2Tag already deduplicates layers, but the scope of this
deduplication is only for a particular tag. If we are pushing all tags
in a repository, we may check layers several times. Fix this by moving
the layersSeen map from the pushV2Tag function to the v2Pusher struct.
In addition to avoiding some useless round-trips, this makes the "docker
push" output less confusing. It formerly could contain many repeated
lines like:
124e2127157f: Image already exists
124e2127157f: Image already exists
...
Add test coverage based on the "docker push" output: a hash should not
appear multiple times when pushing multiple tags.
Fixes #14873
Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>
Aaron Lehmann authored on 2015/07/23 09:29:36Showing 3 changed files
| ... | ... |
@@ -29,12 +29,13 @@ func (s *TagStore) NewPusher(endpoint registry.APIEndpoint, localRepo Repository |
| 29 | 29 |
switch endpoint.Version {
|
| 30 | 30 |
case registry.APIVersion2: |
| 31 | 31 |
return &v2Pusher{
|
| 32 |
- TagStore: s, |
|
| 33 |
- endpoint: endpoint, |
|
| 34 |
- localRepo: localRepo, |
|
| 35 |
- repoInfo: repoInfo, |
|
| 36 |
- config: imagePushConfig, |
|
| 37 |
- sf: sf, |
|
| 32 |
+ TagStore: s, |
|
| 33 |
+ endpoint: endpoint, |
|
| 34 |
+ localRepo: localRepo, |
|
| 35 |
+ repoInfo: repoInfo, |
|
| 36 |
+ config: imagePushConfig, |
|
| 37 |
+ sf: sf, |
|
| 38 |
+ layersSeen: make(map[string]bool), |
|
| 38 | 39 |
}, nil |
| 39 | 40 |
case registry.APIVersion1: |
| 40 | 41 |
return &v1Pusher{
|
| ... | ... |
@@ -26,6 +26,11 @@ type v2Pusher struct {
|
| 26 | 26 |
config *ImagePushConfig |
| 27 | 27 |
sf *streamformatter.StreamFormatter |
| 28 | 28 |
repo distribution.Repository |
| 29 |
+ |
|
| 30 |
+ // layersSeen is the set of layers known to exist on the remote side. |
|
| 31 |
+ // This avoids redundant queries when pushing multiple tags that |
|
| 32 |
+ // involve the same layers. |
|
| 33 |
+ layersSeen map[string]bool |
|
| 29 | 34 |
} |
| 30 | 35 |
|
| 31 | 36 |
func (p *v2Pusher) Push() (fallback bool, err error) {
|
| ... | ... |
@@ -86,8 +91,6 @@ func (p *v2Pusher) pushV2Tag(tag string) error {
|
| 86 | 86 |
return fmt.Errorf("tag does not exist: %s", tag)
|
| 87 | 87 |
} |
| 88 | 88 |
|
| 89 |
- layersSeen := make(map[string]bool) |
|
| 90 |
- |
|
| 91 | 89 |
layer, err := p.graph.Get(layerId) |
| 92 | 90 |
if err != nil {
|
| 93 | 91 |
return err |
| ... | ... |
@@ -116,7 +119,7 @@ func (p *v2Pusher) pushV2Tag(tag string) error {
|
| 116 | 116 |
return err |
| 117 | 117 |
} |
| 118 | 118 |
|
| 119 |
- if layersSeen[layer.ID] {
|
|
| 119 |
+ if p.layersSeen[layer.ID] {
|
|
| 120 | 120 |
break |
| 121 | 121 |
} |
| 122 | 122 |
|
| ... | ... |
@@ -171,7 +174,7 @@ func (p *v2Pusher) pushV2Tag(tag string) error {
|
| 171 | 171 |
m.FSLayers = append(m.FSLayers, manifest.FSLayer{BlobSum: dgst})
|
| 172 | 172 |
m.History = append(m.History, manifest.History{V1Compatibility: string(jsonData)})
|
| 173 | 173 |
|
| 174 |
- layersSeen[layer.ID] = true |
|
| 174 |
+ p.layersSeen[layer.ID] = true |
|
| 175 | 175 |
} |
| 176 | 176 |
|
| 177 | 177 |
logrus.Infof("Signed manifest for %s:%s using daemon's key: %s", p.repo.Name(), tag, p.trustKey.KeyID())
|
| ... | ... |
@@ -60,7 +60,32 @@ func (s *DockerRegistrySuite) TestPushMultipleTags(c *check.C) {
|
| 60 | 60 |
|
| 61 | 61 |
dockerCmd(c, "tag", "busybox", repoTag2) |
| 62 | 62 |
|
| 63 |
- dockerCmd(c, "push", repoName) |
|
| 63 |
+ out, _ := dockerCmd(c, "push", repoName) |
|
| 64 |
+ |
|
| 65 |
+ // There should be no duplicate hashes in the output |
|
| 66 |
+ imageSuccessfullyPushed := ": Image successfully pushed" |
|
| 67 |
+ imageAlreadyExists := ": Image already exists" |
|
| 68 |
+ imagePushHashes := make(map[string]struct{})
|
|
| 69 |
+ outputLines := strings.Split(out, "\n") |
|
| 70 |
+ for _, outputLine := range outputLines {
|
|
| 71 |
+ if strings.Contains(outputLine, imageSuccessfullyPushed) {
|
|
| 72 |
+ hash := strings.TrimSuffix(outputLine, imageSuccessfullyPushed) |
|
| 73 |
+ if _, present := imagePushHashes[hash]; present {
|
|
| 74 |
+ c.Fatalf("Duplicate image push: %s", outputLine)
|
|
| 75 |
+ } |
|
| 76 |
+ imagePushHashes[hash] = struct{}{}
|
|
| 77 |
+ } else if strings.Contains(outputLine, imageAlreadyExists) {
|
|
| 78 |
+ hash := strings.TrimSuffix(outputLine, imageAlreadyExists) |
|
| 79 |
+ if _, present := imagePushHashes[hash]; present {
|
|
| 80 |
+ c.Fatalf("Duplicate image push: %s", outputLine)
|
|
| 81 |
+ } |
|
| 82 |
+ imagePushHashes[hash] = struct{}{}
|
|
| 83 |
+ } |
|
| 84 |
+ } |
|
| 85 |
+ |
|
| 86 |
+ if len(imagePushHashes) == 0 {
|
|
| 87 |
+ c.Fatal(`Expected at least one line containing "Image successfully pushed"`) |
|
| 88 |
+ } |
|
| 64 | 89 |
} |
| 65 | 90 |
|
| 66 | 91 |
func (s *DockerRegistrySuite) TestPushInterrupt(c *check.C) {
|
| ... | ... |
@@ -79,8 +104,7 @@ func (s *DockerRegistrySuite) TestPushInterrupt(c *check.C) {
|
| 79 | 79 |
c.Fatalf("Failed to kill push process: %v", err)
|
| 80 | 80 |
} |
| 81 | 81 |
if out, _, err := dockerCmdWithError(c, "push", repoName); err == nil {
|
| 82 |
- str := string(out) |
|
| 83 |
- if !strings.Contains(str, "already in progress") {
|
|
| 82 |
+ if !strings.Contains(out, "already in progress") {
|
|
| 84 | 83 |
c.Fatalf("Push should be continued on daemon side, but seems ok: %v, %s", err, out)
|
| 85 | 84 |
} |
| 86 | 85 |
} |