Browse code

Use image.DockerImageLayers for pruning

This patch switches image pruning to use image.DockerImageLayers instead
of parsing image.DockerImageManifest. Additionally all log levels should
be consistent in that all errors finding objects in graph are logged at
level 2, the entire debug messages at level 4.

Maciej Szulik authored on 2016/07/25 23:04:38
Showing 2 changed files
... ...
@@ -14,7 +14,6 @@ import (
14 14
 	"k8s.io/kubernetes/pkg/api/resource"
15 15
 	"k8s.io/kubernetes/pkg/api/unversioned"
16 16
 	kerrors "k8s.io/kubernetes/pkg/util/errors"
17
-	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
18 17
 	"k8s.io/kubernetes/pkg/util/sets"
19 18
 
20 19
 	"github.com/openshift/origin/pkg/api/graph"
... ...
@@ -307,23 +306,9 @@ func addImagesToGraph(g graph.Graph, images *imageapi.ImageList, algorithm prune
307 307
 		glog.V(4).Infof("Adding image %q to graph", image.Name)
308 308
 		imageNode := imagegraph.EnsureImageNode(g, image)
309 309
 
310
-		manifest := imageapi.DockerImageManifest{}
311
-		if err := json.Unmarshal([]byte(image.DockerImageManifest), &manifest); err != nil {
312
-			utilruntime.HandleError(fmt.Errorf("unable to extract manifest from image: %v. This image's layers won't be pruned if the image is pruned now.", err))
313
-			continue
314
-		}
315
-
316
-		// schema1 layers
317
-		for _, layer := range manifest.FSLayers {
318
-			glog.V(4).Infof("Adding image layer v1 %q to graph", layer.DockerBlobSum)
319
-			layerNode := imagegraph.EnsureImageLayerNode(g, layer.DockerBlobSum)
320
-			g.AddEdge(imageNode, layerNode, ReferencedImageLayerEdgeKind)
321
-		}
322
-
323
-		// schema2 layers
324
-		for _, layer := range manifest.Layers {
325
-			glog.V(4).Infof("Adding image layer v2 %q to graph", layer.Digest)
326
-			layerNode := imagegraph.EnsureImageLayerNode(g, layer.Digest)
310
+		for _, layer := range image.DockerImageLayers {
311
+			glog.V(4).Infof("Adding image layer %q to graph", layer.Name)
312
+			layerNode := imagegraph.EnsureImageLayerNode(g, layer.Name)
327 313
 			g.AddEdge(imageNode, layerNode, ReferencedImageLayerEdgeKind)
328 314
 		}
329 315
 	}
... ...
@@ -365,7 +350,8 @@ func addImageStreamsToGraph(g graph.Graph, streams *imageapi.ImageStreamList, li
365 365
 			for i := range history.Items {
366 366
 				n := imagegraph.FindImage(g, history.Items[i].Image)
367 367
 				if n == nil {
368
-					glog.V(2).Infof("Unable to find image %q in graph (from tag=%q, revision=%d, dockerImageReference=%s)", history.Items[i].Image, tag, i, history.Items[i].DockerImageReference)
368
+					glog.V(2).Infof("Unable to find image %q in graph (from tag=%q, revision=%d, dockerImageReference=%s) - skipping",
369
+						history.Items[i].Image, tag, i, history.Items[i].DockerImageReference)
369 370
 					continue
370 371
 				}
371 372
 				imageNode := n.(*imagegraph.ImageNode)
... ...
@@ -482,7 +468,7 @@ func addPodSpecToGraph(g graph.Graph, spec *kapi.PodSpec, predecessor gonum.Node
482 482
 
483 483
 		ref, err := imageapi.ParseDockerImageReference(container.Image)
484 484
 		if err != nil {
485
-			utilruntime.HandleError(fmt.Errorf("unable to parse DockerImageReference %q: %v", container.Image, err))
485
+			glog.V(2).Infof("Unable to parse DockerImageReference %q: %v - skipping", container.Image, err)
486 486
 			continue
487 487
 		}
488 488
 
... ...
@@ -493,7 +479,7 @@ func addPodSpecToGraph(g graph.Graph, spec *kapi.PodSpec, predecessor gonum.Node
493 493
 
494 494
 		imageNode := imagegraph.FindImage(g, ref.ID)
495 495
 		if imageNode == nil {
496
-			glog.Infof("Unable to find image %q in the graph", ref.ID)
496
+			glog.V(2).Infof("Unable to find image %q in the graph - skipping", ref.ID)
497 497
 			continue
498 498
 		}
499 499
 
... ...
@@ -850,12 +836,10 @@ func layerIsPrunable(g graph.Graph, layerNode *imagegraph.ImageLayerNode) bool {
850 850
 // given ImageLayerNode.
851 851
 func streamLayerReferences(g graph.Graph, layerNode *imagegraph.ImageLayerNode) []*imagegraph.ImageStreamNode {
852 852
 	ret := []*imagegraph.ImageStreamNode{}
853
-
854 853
 	for _, predecessor := range g.To(layerNode) {
855 854
 		if g.Kind(predecessor) != imagegraph.ImageStreamNodeKind {
856 855
 			continue
857 856
 		}
858
-
859 857
 		ret = append(ret, predecessor.(*imagegraph.ImageStreamNode))
860 858
 	}
861 859
 
... ...
@@ -2,7 +2,6 @@ package prune
2 2
 
3 3
 import (
4 4
 	"bytes"
5
-	"encoding/json"
6 5
 	"errors"
7 6
 	"flag"
8 7
 	"fmt"
... ...
@@ -51,7 +50,7 @@ const (
51 51
 )
52 52
 
53 53
 func agedImage(id, ref string, ageInMinutes int64) imageapi.Image {
54
-	image := imageWithLayers(id, ref, false, layer1, layer2, layer3, layer4, layer5)
54
+	image := imageWithLayers(id, ref, layer1, layer2, layer3, layer4, layer5)
55 55
 
56 56
 	if ageInMinutes >= 0 {
57 57
 		image.CreationTimestamp = unversioned.NewTime(unversioned.Now().Add(time.Duration(-1*ageInMinutes) * time.Minute))
... ...
@@ -61,7 +60,7 @@ func agedImage(id, ref string, ageInMinutes int64) imageapi.Image {
61 61
 }
62 62
 
63 63
 func sizedImage(id, ref string, size int64) imageapi.Image {
64
-	image := imageWithLayers(id, ref, false, layer1, layer2, layer3, layer4, layer5)
64
+	image := imageWithLayers(id, ref, layer1, layer2, layer3, layer4, layer5)
65 65
 	image.CreationTimestamp = unversioned.NewTime(unversioned.Now().Add(time.Duration(-1) * time.Minute))
66 66
 	image.DockerImageMetadata.Size = size
67 67
 
... ...
@@ -72,7 +71,7 @@ func image(id, ref string) imageapi.Image {
72 72
 	return agedImage(id, ref, -1)
73 73
 }
74 74
 
75
-func imageWithLayers(id, ref string, v2 bool, layers ...string) imageapi.Image {
75
+func imageWithLayers(id, ref string, layers ...string) imageapi.Image {
76 76
 	image := imageapi.Image{
77 77
 		ObjectMeta: kapi.ObjectMeta{
78 78
 			Name: id,
... ...
@@ -83,59 +82,16 @@ func imageWithLayers(id, ref string, v2 bool, layers ...string) imageapi.Image {
83 83
 		DockerImageReference: ref,
84 84
 	}
85 85
 
86
-	manifest := imageapi.DockerImageManifest{}
87
-	if v2 {
88
-		manifest.Layers = []imageapi.Descriptor{}
89
-		for _, layer := range layers {
90
-			manifest.Layers = append(manifest.Layers, imageapi.Descriptor{Digest: layer})
91
-		}
92
-	} else {
93
-		manifest.FSLayers = []imageapi.DockerFSLayer{}
94
-		for _, layer := range layers {
95
-			manifest.FSLayers = append(manifest.FSLayers, imageapi.DockerFSLayer{DockerBlobSum: layer})
96
-		}
97
-	}
98
-
99
-	manifestBytes, err := json.Marshal(&manifest)
100
-	if err != nil {
101
-		panic(err)
102
-	}
103
-
104
-	image.DockerImageManifest = string(manifestBytes)
105
-
106
-	return image
107
-}
108
-
109
-func imageWithLayersV1(id, ref string, layers ...string) imageapi.Image {
110
-	image := imageapi.Image{
111
-		ObjectMeta: kapi.ObjectMeta{
112
-			Name: id,
113
-			Annotations: map[string]string{
114
-				imageapi.ManagedByOpenShiftAnnotation: "true",
115
-			},
116
-		},
117
-		DockerImageReference: ref,
118
-	}
119
-
120
-	manifest := imageapi.DockerImageManifest{
121
-		FSLayers: []imageapi.DockerFSLayer{},
122
-	}
86
+	image.DockerImageLayers = []imageapi.ImageLayer{}
123 87
 	for _, layer := range layers {
124
-		manifest.FSLayers = append(manifest.FSLayers, imageapi.DockerFSLayer{DockerBlobSum: layer})
88
+		image.DockerImageLayers = append(image.DockerImageLayers, imageapi.ImageLayer{Name: layer})
125 89
 	}
126 90
 
127
-	manifestBytes, err := json.Marshal(&manifest)
128
-	if err != nil {
129
-		panic(err)
130
-	}
131
-
132
-	image.DockerImageManifest = string(manifestBytes)
133
-
134 91
 	return image
135 92
 }
136 93
 
137 94
 func unmanagedImage(id, ref string, hasAnnotations bool, annotation, value string) imageapi.Image {
138
-	image := imageWithLayers(id, ref, false)
95
+	image := imageWithLayers(id, ref)
139 96
 	if !hasAnnotations {
140 97
 		image.Annotations = nil
141 98
 	} else {
... ...
@@ -145,12 +101,6 @@ func unmanagedImage(id, ref string, hasAnnotations bool, annotation, value strin
145 145
 	return image
146 146
 }
147 147
 
148
-func imageWithBadManifest(id, ref string) imageapi.Image {
149
-	image := image(id, ref)
150
-	image.DockerImageManifest = "asdf"
151
-	return image
152
-}
153
-
154 148
 func podList(pods ...kapi.Pod) kapi.PodList {
155 149
 	return kapi.PodList{
156 150
 		Items: pods,
... ...
@@ -766,51 +716,12 @@ func TestImagePruning(t *testing.T) {
766 766
 			expectedImageDeletions: []string{},
767 767
 			expectedStreamUpdates:  []string{},
768 768
 		},
769
-		"image with bad manifest is pruned ok": {
770
-			images: imageList(
771
-				imageWithBadManifest("id", "someregistry/foo/bar@id"),
772
-			),
773
-			expectedImageDeletions: []string{"id"},
774
-			expectedStreamUpdates:  []string{},
775
-		},
776
-		"image with v1 layers": {
777
-			images: imageList(
778
-				imageWithLayers("id1", registryURL+"/foo/bar@id1", false, "layer1", "layer2", "layer3", "layer4"),
779
-				imageWithLayers("id2", registryURL+"/foo/bar@id2", false, "layer1", "layer2", "layer3", "layer4"),
780
-				imageWithLayers("id3", registryURL+"/foo/bar@id3", false, "layer1", "layer2", "layer3", "layer4"),
781
-				imageWithLayers("id4", registryURL+"/foo/bar@id4", false, "layer5", "layer6", "layer7", "layer8"),
782
-			),
783
-			streams: streamList(
784
-				stream(registryURL, "foo", "bar", tags(
785
-					tag("latest",
786
-						tagEvent("id1", registryURL+"/foo/bar@id1"),
787
-						tagEvent("id2", registryURL+"/foo/bar@id2"),
788
-						tagEvent("id3", registryURL+"/foo/bar@id3"),
789
-						tagEvent("id4", registryURL+"/foo/bar@id4"),
790
-					),
791
-				)),
792
-			),
793
-			expectedImageDeletions: []string{"id4"},
794
-			expectedStreamUpdates:  []string{"foo/bar|id4"},
795
-			expectedLayerDeletions: []string{
796
-				registryURL + "|foo/bar|layer5",
797
-				registryURL + "|foo/bar|layer6",
798
-				registryURL + "|foo/bar|layer7",
799
-				registryURL + "|foo/bar|layer8",
800
-			},
801
-			expectedBlobDeletions: []string{
802
-				registryURL + "|layer5",
803
-				registryURL + "|layer6",
804
-				registryURL + "|layer7",
805
-				registryURL + "|layer8",
806
-			},
807
-		},
808
-		"image with v2 layers": {
769
+		"image with layers": {
809 770
 			images: imageList(
810
-				imageWithLayers("id1", registryURL+"/foo/bar@id1", true, "layer1", "layer2", "layer3", "layer4"),
811
-				imageWithLayers("id2", registryURL+"/foo/bar@id2", true, "layer1", "layer2", "layer3", "layer4"),
812
-				imageWithLayers("id3", registryURL+"/foo/bar@id3", true, "layer1", "layer2", "layer3", "layer4"),
813
-				imageWithLayers("id4", registryURL+"/foo/bar@id4", true, "layer5", "layer6", "layer7", "layer8"),
771
+				imageWithLayers("id1", registryURL+"/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"),
772
+				imageWithLayers("id2", registryURL+"/foo/bar@id2", "layer1", "layer2", "layer3", "layer4"),
773
+				imageWithLayers("id3", registryURL+"/foo/bar@id3", "layer1", "layer2", "layer3", "layer4"),
774
+				imageWithLayers("id4", registryURL+"/foo/bar@id4", "layer5", "layer6", "layer7", "layer8"),
814 775
 			),
815 776
 			streams: streamList(
816 777
 				stream(registryURL, "foo", "bar", tags(
... ...
@@ -1052,8 +963,8 @@ func TestRegistryPruning(t *testing.T) {
1052 1052
 	}{
1053 1053
 		"layers unique to id1 pruned": {
1054 1054
 			images: imageList(
1055
-				imageWithLayers("id1", "registry1/foo/bar@id1", false, "layer1", "layer2", "layer3", "layer4"),
1056
-				imageWithLayers("id2", "registry1/foo/bar@id2", false, "layer3", "layer4", "layer5", "layer6"),
1055
+				imageWithLayers("id1", "registry1/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"),
1056
+				imageWithLayers("id2", "registry1/foo/bar@id2", "layer3", "layer4", "layer5", "layer6"),
1057 1057
 			),
1058 1058
 			streams: streamList(
1059 1059
 				stream("registry1", "foo", "bar", tags(
... ...
@@ -1082,7 +993,7 @@ func TestRegistryPruning(t *testing.T) {
1082 1082
 		},
1083 1083
 		"no pruning when no images are pruned": {
1084 1084
 			images: imageList(
1085
-				imageWithLayers("id1", "registry1/foo/bar@id1", false, "layer1", "layer2", "layer3", "layer4"),
1085
+				imageWithLayers("id1", "registry1/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"),
1086 1086
 			),
1087 1087
 			streams: streamList(
1088 1088
 				stream("registry1", "foo", "bar", tags(
... ...
@@ -1097,8 +1008,8 @@ func TestRegistryPruning(t *testing.T) {
1097 1097
 		},
1098 1098
 		"blobs pruned when streams have already been deleted": {
1099 1099
 			images: imageList(
1100
-				imageWithLayers("id1", "registry1/foo/bar@id1", false, "layer1", "layer2", "layer3", "layer4"),
1101
-				imageWithLayers("id2", "registry1/foo/bar@id2", false, "layer3", "layer4", "layer5", "layer6"),
1100
+				imageWithLayers("id1", "registry1/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"),
1101
+				imageWithLayers("id2", "registry1/foo/bar@id2", "layer3", "layer4", "layer5", "layer6"),
1102 1102
 			),
1103 1103
 			expectedLayerDeletions: sets.NewString(),
1104 1104
 			expectedBlobDeletions: sets.NewString(
... ...
@@ -1113,8 +1024,8 @@ func TestRegistryPruning(t *testing.T) {
1113 1113
 		},
1114 1114
 		"ping error": {
1115 1115
 			images: imageList(
1116
-				imageWithLayers("id1", "registry1/foo/bar@id1", false, "layer1", "layer2", "layer3", "layer4"),
1117
-				imageWithLayers("id2", "registry1/foo/bar@id2", false, "layer3", "layer4", "layer5", "layer6"),
1116
+				imageWithLayers("id1", "registry1/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"),
1117
+				imageWithLayers("id2", "registry1/foo/bar@id2", "layer3", "layer4", "layer5", "layer6"),
1118 1118
 			),
1119 1119
 			streams: streamList(
1120 1120
 				stream("registry1", "foo", "bar", tags(