This patch switches image pruning to use image.DockerImageLayers instead
of parsing image.DockerImageManifest. Additionally all log levels should
be consistent in that all errors finding objects in graph are logged at
level 2, the entire debug messages at level 4.
... | ... |
@@ -14,7 +14,6 @@ import ( |
14 | 14 |
"k8s.io/kubernetes/pkg/api/resource" |
15 | 15 |
"k8s.io/kubernetes/pkg/api/unversioned" |
16 | 16 |
kerrors "k8s.io/kubernetes/pkg/util/errors" |
17 |
- utilruntime "k8s.io/kubernetes/pkg/util/runtime" |
|
18 | 17 |
"k8s.io/kubernetes/pkg/util/sets" |
19 | 18 |
|
20 | 19 |
"github.com/openshift/origin/pkg/api/graph" |
... | ... |
@@ -307,23 +306,9 @@ func addImagesToGraph(g graph.Graph, images *imageapi.ImageList, algorithm prune |
307 | 307 |
glog.V(4).Infof("Adding image %q to graph", image.Name) |
308 | 308 |
imageNode := imagegraph.EnsureImageNode(g, image) |
309 | 309 |
|
310 |
- manifest := imageapi.DockerImageManifest{} |
|
311 |
- if err := json.Unmarshal([]byte(image.DockerImageManifest), &manifest); err != nil { |
|
312 |
- utilruntime.HandleError(fmt.Errorf("unable to extract manifest from image: %v. This image's layers won't be pruned if the image is pruned now.", err)) |
|
313 |
- continue |
|
314 |
- } |
|
315 |
- |
|
316 |
- // schema1 layers |
|
317 |
- for _, layer := range manifest.FSLayers { |
|
318 |
- glog.V(4).Infof("Adding image layer v1 %q to graph", layer.DockerBlobSum) |
|
319 |
- layerNode := imagegraph.EnsureImageLayerNode(g, layer.DockerBlobSum) |
|
320 |
- g.AddEdge(imageNode, layerNode, ReferencedImageLayerEdgeKind) |
|
321 |
- } |
|
322 |
- |
|
323 |
- // schema2 layers |
|
324 |
- for _, layer := range manifest.Layers { |
|
325 |
- glog.V(4).Infof("Adding image layer v2 %q to graph", layer.Digest) |
|
326 |
- layerNode := imagegraph.EnsureImageLayerNode(g, layer.Digest) |
|
310 |
+ for _, layer := range image.DockerImageLayers { |
|
311 |
+ glog.V(4).Infof("Adding image layer %q to graph", layer.Name) |
|
312 |
+ layerNode := imagegraph.EnsureImageLayerNode(g, layer.Name) |
|
327 | 313 |
g.AddEdge(imageNode, layerNode, ReferencedImageLayerEdgeKind) |
328 | 314 |
} |
329 | 315 |
} |
... | ... |
@@ -365,7 +350,8 @@ func addImageStreamsToGraph(g graph.Graph, streams *imageapi.ImageStreamList, li |
365 | 365 |
for i := range history.Items { |
366 | 366 |
n := imagegraph.FindImage(g, history.Items[i].Image) |
367 | 367 |
if n == nil { |
368 |
- glog.V(2).Infof("Unable to find image %q in graph (from tag=%q, revision=%d, dockerImageReference=%s)", history.Items[i].Image, tag, i, history.Items[i].DockerImageReference) |
|
368 |
+ glog.V(2).Infof("Unable to find image %q in graph (from tag=%q, revision=%d, dockerImageReference=%s) - skipping", |
|
369 |
+ history.Items[i].Image, tag, i, history.Items[i].DockerImageReference) |
|
369 | 370 |
continue |
370 | 371 |
} |
371 | 372 |
imageNode := n.(*imagegraph.ImageNode) |
... | ... |
@@ -482,7 +468,7 @@ func addPodSpecToGraph(g graph.Graph, spec *kapi.PodSpec, predecessor gonum.Node |
482 | 482 |
|
483 | 483 |
ref, err := imageapi.ParseDockerImageReference(container.Image) |
484 | 484 |
if err != nil { |
485 |
- utilruntime.HandleError(fmt.Errorf("unable to parse DockerImageReference %q: %v", container.Image, err)) |
|
485 |
+ glog.V(2).Infof("Unable to parse DockerImageReference %q: %v - skipping", container.Image, err) |
|
486 | 486 |
continue |
487 | 487 |
} |
488 | 488 |
|
... | ... |
@@ -493,7 +479,7 @@ func addPodSpecToGraph(g graph.Graph, spec *kapi.PodSpec, predecessor gonum.Node |
493 | 493 |
|
494 | 494 |
imageNode := imagegraph.FindImage(g, ref.ID) |
495 | 495 |
if imageNode == nil { |
496 |
- glog.Infof("Unable to find image %q in the graph", ref.ID) |
|
496 |
+ glog.V(2).Infof("Unable to find image %q in the graph - skipping", ref.ID) |
|
497 | 497 |
continue |
498 | 498 |
} |
499 | 499 |
|
... | ... |
@@ -850,12 +836,10 @@ func layerIsPrunable(g graph.Graph, layerNode *imagegraph.ImageLayerNode) bool { |
850 | 850 |
// given ImageLayerNode. |
851 | 851 |
func streamLayerReferences(g graph.Graph, layerNode *imagegraph.ImageLayerNode) []*imagegraph.ImageStreamNode { |
852 | 852 |
ret := []*imagegraph.ImageStreamNode{} |
853 |
- |
|
854 | 853 |
for _, predecessor := range g.To(layerNode) { |
855 | 854 |
if g.Kind(predecessor) != imagegraph.ImageStreamNodeKind { |
856 | 855 |
continue |
857 | 856 |
} |
858 |
- |
|
859 | 857 |
ret = append(ret, predecessor.(*imagegraph.ImageStreamNode)) |
860 | 858 |
} |
861 | 859 |
|
... | ... |
@@ -2,7 +2,6 @@ package prune |
2 | 2 |
|
3 | 3 |
import ( |
4 | 4 |
"bytes" |
5 |
- "encoding/json" |
|
6 | 5 |
"errors" |
7 | 6 |
"flag" |
8 | 7 |
"fmt" |
... | ... |
@@ -51,7 +50,7 @@ const ( |
51 | 51 |
) |
52 | 52 |
|
53 | 53 |
func agedImage(id, ref string, ageInMinutes int64) imageapi.Image { |
54 |
- image := imageWithLayers(id, ref, false, layer1, layer2, layer3, layer4, layer5) |
|
54 |
+ image := imageWithLayers(id, ref, layer1, layer2, layer3, layer4, layer5) |
|
55 | 55 |
|
56 | 56 |
if ageInMinutes >= 0 { |
57 | 57 |
image.CreationTimestamp = unversioned.NewTime(unversioned.Now().Add(time.Duration(-1*ageInMinutes) * time.Minute)) |
... | ... |
@@ -61,7 +60,7 @@ func agedImage(id, ref string, ageInMinutes int64) imageapi.Image { |
61 | 61 |
} |
62 | 62 |
|
63 | 63 |
func sizedImage(id, ref string, size int64) imageapi.Image { |
64 |
- image := imageWithLayers(id, ref, false, layer1, layer2, layer3, layer4, layer5) |
|
64 |
+ image := imageWithLayers(id, ref, layer1, layer2, layer3, layer4, layer5) |
|
65 | 65 |
image.CreationTimestamp = unversioned.NewTime(unversioned.Now().Add(time.Duration(-1) * time.Minute)) |
66 | 66 |
image.DockerImageMetadata.Size = size |
67 | 67 |
|
... | ... |
@@ -72,7 +71,7 @@ func image(id, ref string) imageapi.Image { |
72 | 72 |
return agedImage(id, ref, -1) |
73 | 73 |
} |
74 | 74 |
|
75 |
-func imageWithLayers(id, ref string, v2 bool, layers ...string) imageapi.Image { |
|
75 |
+func imageWithLayers(id, ref string, layers ...string) imageapi.Image { |
|
76 | 76 |
image := imageapi.Image{ |
77 | 77 |
ObjectMeta: kapi.ObjectMeta{ |
78 | 78 |
Name: id, |
... | ... |
@@ -83,59 +82,16 @@ func imageWithLayers(id, ref string, v2 bool, layers ...string) imageapi.Image { |
83 | 83 |
DockerImageReference: ref, |
84 | 84 |
} |
85 | 85 |
|
86 |
- manifest := imageapi.DockerImageManifest{} |
|
87 |
- if v2 { |
|
88 |
- manifest.Layers = []imageapi.Descriptor{} |
|
89 |
- for _, layer := range layers { |
|
90 |
- manifest.Layers = append(manifest.Layers, imageapi.Descriptor{Digest: layer}) |
|
91 |
- } |
|
92 |
- } else { |
|
93 |
- manifest.FSLayers = []imageapi.DockerFSLayer{} |
|
94 |
- for _, layer := range layers { |
|
95 |
- manifest.FSLayers = append(manifest.FSLayers, imageapi.DockerFSLayer{DockerBlobSum: layer}) |
|
96 |
- } |
|
97 |
- } |
|
98 |
- |
|
99 |
- manifestBytes, err := json.Marshal(&manifest) |
|
100 |
- if err != nil { |
|
101 |
- panic(err) |
|
102 |
- } |
|
103 |
- |
|
104 |
- image.DockerImageManifest = string(manifestBytes) |
|
105 |
- |
|
106 |
- return image |
|
107 |
-} |
|
108 |
- |
|
109 |
-func imageWithLayersV1(id, ref string, layers ...string) imageapi.Image { |
|
110 |
- image := imageapi.Image{ |
|
111 |
- ObjectMeta: kapi.ObjectMeta{ |
|
112 |
- Name: id, |
|
113 |
- Annotations: map[string]string{ |
|
114 |
- imageapi.ManagedByOpenShiftAnnotation: "true", |
|
115 |
- }, |
|
116 |
- }, |
|
117 |
- DockerImageReference: ref, |
|
118 |
- } |
|
119 |
- |
|
120 |
- manifest := imageapi.DockerImageManifest{ |
|
121 |
- FSLayers: []imageapi.DockerFSLayer{}, |
|
122 |
- } |
|
86 |
+ image.DockerImageLayers = []imageapi.ImageLayer{} |
|
123 | 87 |
for _, layer := range layers { |
124 |
- manifest.FSLayers = append(manifest.FSLayers, imageapi.DockerFSLayer{DockerBlobSum: layer}) |
|
88 |
+ image.DockerImageLayers = append(image.DockerImageLayers, imageapi.ImageLayer{Name: layer}) |
|
125 | 89 |
} |
126 | 90 |
|
127 |
- manifestBytes, err := json.Marshal(&manifest) |
|
128 |
- if err != nil { |
|
129 |
- panic(err) |
|
130 |
- } |
|
131 |
- |
|
132 |
- image.DockerImageManifest = string(manifestBytes) |
|
133 |
- |
|
134 | 91 |
return image |
135 | 92 |
} |
136 | 93 |
|
137 | 94 |
func unmanagedImage(id, ref string, hasAnnotations bool, annotation, value string) imageapi.Image { |
138 |
- image := imageWithLayers(id, ref, false) |
|
95 |
+ image := imageWithLayers(id, ref) |
|
139 | 96 |
if !hasAnnotations { |
140 | 97 |
image.Annotations = nil |
141 | 98 |
} else { |
... | ... |
@@ -145,12 +101,6 @@ func unmanagedImage(id, ref string, hasAnnotations bool, annotation, value strin |
145 | 145 |
return image |
146 | 146 |
} |
147 | 147 |
|
148 |
-func imageWithBadManifest(id, ref string) imageapi.Image { |
|
149 |
- image := image(id, ref) |
|
150 |
- image.DockerImageManifest = "asdf" |
|
151 |
- return image |
|
152 |
-} |
|
153 |
- |
|
154 | 148 |
func podList(pods ...kapi.Pod) kapi.PodList { |
155 | 149 |
return kapi.PodList{ |
156 | 150 |
Items: pods, |
... | ... |
@@ -766,51 +716,12 @@ func TestImagePruning(t *testing.T) { |
766 | 766 |
expectedImageDeletions: []string{}, |
767 | 767 |
expectedStreamUpdates: []string{}, |
768 | 768 |
}, |
769 |
- "image with bad manifest is pruned ok": { |
|
770 |
- images: imageList( |
|
771 |
- imageWithBadManifest("id", "someregistry/foo/bar@id"), |
|
772 |
- ), |
|
773 |
- expectedImageDeletions: []string{"id"}, |
|
774 |
- expectedStreamUpdates: []string{}, |
|
775 |
- }, |
|
776 |
- "image with v1 layers": { |
|
777 |
- images: imageList( |
|
778 |
- imageWithLayers("id1", registryURL+"/foo/bar@id1", false, "layer1", "layer2", "layer3", "layer4"), |
|
779 |
- imageWithLayers("id2", registryURL+"/foo/bar@id2", false, "layer1", "layer2", "layer3", "layer4"), |
|
780 |
- imageWithLayers("id3", registryURL+"/foo/bar@id3", false, "layer1", "layer2", "layer3", "layer4"), |
|
781 |
- imageWithLayers("id4", registryURL+"/foo/bar@id4", false, "layer5", "layer6", "layer7", "layer8"), |
|
782 |
- ), |
|
783 |
- streams: streamList( |
|
784 |
- stream(registryURL, "foo", "bar", tags( |
|
785 |
- tag("latest", |
|
786 |
- tagEvent("id1", registryURL+"/foo/bar@id1"), |
|
787 |
- tagEvent("id2", registryURL+"/foo/bar@id2"), |
|
788 |
- tagEvent("id3", registryURL+"/foo/bar@id3"), |
|
789 |
- tagEvent("id4", registryURL+"/foo/bar@id4"), |
|
790 |
- ), |
|
791 |
- )), |
|
792 |
- ), |
|
793 |
- expectedImageDeletions: []string{"id4"}, |
|
794 |
- expectedStreamUpdates: []string{"foo/bar|id4"}, |
|
795 |
- expectedLayerDeletions: []string{ |
|
796 |
- registryURL + "|foo/bar|layer5", |
|
797 |
- registryURL + "|foo/bar|layer6", |
|
798 |
- registryURL + "|foo/bar|layer7", |
|
799 |
- registryURL + "|foo/bar|layer8", |
|
800 |
- }, |
|
801 |
- expectedBlobDeletions: []string{ |
|
802 |
- registryURL + "|layer5", |
|
803 |
- registryURL + "|layer6", |
|
804 |
- registryURL + "|layer7", |
|
805 |
- registryURL + "|layer8", |
|
806 |
- }, |
|
807 |
- }, |
|
808 |
- "image with v2 layers": { |
|
769 |
+ "image with layers": { |
|
809 | 770 |
images: imageList( |
810 |
- imageWithLayers("id1", registryURL+"/foo/bar@id1", true, "layer1", "layer2", "layer3", "layer4"), |
|
811 |
- imageWithLayers("id2", registryURL+"/foo/bar@id2", true, "layer1", "layer2", "layer3", "layer4"), |
|
812 |
- imageWithLayers("id3", registryURL+"/foo/bar@id3", true, "layer1", "layer2", "layer3", "layer4"), |
|
813 |
- imageWithLayers("id4", registryURL+"/foo/bar@id4", true, "layer5", "layer6", "layer7", "layer8"), |
|
771 |
+ imageWithLayers("id1", registryURL+"/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"), |
|
772 |
+ imageWithLayers("id2", registryURL+"/foo/bar@id2", "layer1", "layer2", "layer3", "layer4"), |
|
773 |
+ imageWithLayers("id3", registryURL+"/foo/bar@id3", "layer1", "layer2", "layer3", "layer4"), |
|
774 |
+ imageWithLayers("id4", registryURL+"/foo/bar@id4", "layer5", "layer6", "layer7", "layer8"), |
|
814 | 775 |
), |
815 | 776 |
streams: streamList( |
816 | 777 |
stream(registryURL, "foo", "bar", tags( |
... | ... |
@@ -1052,8 +963,8 @@ func TestRegistryPruning(t *testing.T) { |
1052 | 1052 |
}{ |
1053 | 1053 |
"layers unique to id1 pruned": { |
1054 | 1054 |
images: imageList( |
1055 |
- imageWithLayers("id1", "registry1/foo/bar@id1", false, "layer1", "layer2", "layer3", "layer4"), |
|
1056 |
- imageWithLayers("id2", "registry1/foo/bar@id2", false, "layer3", "layer4", "layer5", "layer6"), |
|
1055 |
+ imageWithLayers("id1", "registry1/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"), |
|
1056 |
+ imageWithLayers("id2", "registry1/foo/bar@id2", "layer3", "layer4", "layer5", "layer6"), |
|
1057 | 1057 |
), |
1058 | 1058 |
streams: streamList( |
1059 | 1059 |
stream("registry1", "foo", "bar", tags( |
... | ... |
@@ -1082,7 +993,7 @@ func TestRegistryPruning(t *testing.T) { |
1082 | 1082 |
}, |
1083 | 1083 |
"no pruning when no images are pruned": { |
1084 | 1084 |
images: imageList( |
1085 |
- imageWithLayers("id1", "registry1/foo/bar@id1", false, "layer1", "layer2", "layer3", "layer4"), |
|
1085 |
+ imageWithLayers("id1", "registry1/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"), |
|
1086 | 1086 |
), |
1087 | 1087 |
streams: streamList( |
1088 | 1088 |
stream("registry1", "foo", "bar", tags( |
... | ... |
@@ -1097,8 +1008,8 @@ func TestRegistryPruning(t *testing.T) { |
1097 | 1097 |
}, |
1098 | 1098 |
"blobs pruned when streams have already been deleted": { |
1099 | 1099 |
images: imageList( |
1100 |
- imageWithLayers("id1", "registry1/foo/bar@id1", false, "layer1", "layer2", "layer3", "layer4"), |
|
1101 |
- imageWithLayers("id2", "registry1/foo/bar@id2", false, "layer3", "layer4", "layer5", "layer6"), |
|
1100 |
+ imageWithLayers("id1", "registry1/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"), |
|
1101 |
+ imageWithLayers("id2", "registry1/foo/bar@id2", "layer3", "layer4", "layer5", "layer6"), |
|
1102 | 1102 |
), |
1103 | 1103 |
expectedLayerDeletions: sets.NewString(), |
1104 | 1104 |
expectedBlobDeletions: sets.NewString( |
... | ... |
@@ -1113,8 +1024,8 @@ func TestRegistryPruning(t *testing.T) { |
1113 | 1113 |
}, |
1114 | 1114 |
"ping error": { |
1115 | 1115 |
images: imageList( |
1116 |
- imageWithLayers("id1", "registry1/foo/bar@id1", false, "layer1", "layer2", "layer3", "layer4"), |
|
1117 |
- imageWithLayers("id2", "registry1/foo/bar@id2", false, "layer3", "layer4", "layer5", "layer6"), |
|
1116 |
+ imageWithLayers("id1", "registry1/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"), |
|
1117 |
+ imageWithLayers("id2", "registry1/foo/bar@id2", "layer3", "layer4", "layer5", "layer6"), |
|
1118 | 1118 |
), |
1119 | 1119 |
streams: streamList( |
1120 | 1120 |
stream("registry1", "foo", "bar", tags( |