Browse code

Refactor pruning images to follow one common flow with other pruners

Maciej Szulik authored on 2016/06/22 20:09:53
Showing 5 changed files
... ...
@@ -7,6 +7,7 @@ import (
7 7
 	"io"
8 8
 	"io/ioutil"
9 9
 	"net/http"
10
+	"net/url"
10 11
 	"os"
11 12
 	"strings"
12 13
 	"text/tabwriter"
... ...
@@ -16,7 +17,7 @@ import (
16 16
 	kapi "k8s.io/kubernetes/pkg/api"
17 17
 	"k8s.io/kubernetes/pkg/client/restclient"
18 18
 	kclient "k8s.io/kubernetes/pkg/client/unversioned"
19
-	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
19
+	kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
20 20
 	knet "k8s.io/kubernetes/pkg/util/net"
21 21
 
22 22
 	"github.com/openshift/origin/pkg/client"
... ...
@@ -47,21 +48,20 @@ images.`
47 47
   %[1]s %[2]s --keep-tag-revisions=3 --keep-younger-than=60m --confirm`
48 48
 )
49 49
 
50
-// PruneImagesOptions holds all the required options for prune images
50
+// PruneImagesOptions holds all the required options for pruning images.
51 51
 type PruneImagesOptions struct {
52
-	Pruner prune.ImageRegistryPruner
53
-	Client client.Interface
54
-	Out    io.Writer
55
-
56
-	Confirm          bool
57
-	KeepYoungerThan  time.Duration
58
-	KeepTagRevisions int
59
-
52
+	Confirm             bool
53
+	KeepYoungerThan     time.Duration
54
+	KeepTagRevisions    int
60 55
 	CABundle            string
61 56
 	RegistryUrlOverride string
57
+
58
+	Pruner prune.Pruner
59
+	Client client.Interface
60
+	Out    io.Writer
62 61
 }
63 62
 
64
-// NewCmdPruneImages implements the OpenShift cli prune images command
63
+// NewCmdPruneImages implements the OpenShift cli prune images command.
65 64
 func NewCmdPruneImages(f *clientcmd.Factory, parentName, name string, out io.Writer) *cobra.Command {
66 65
 	opts := &PruneImagesOptions{
67 66
 		Confirm:          false,
... ...
@@ -77,17 +77,9 @@ func NewCmdPruneImages(f *clientcmd.Factory, parentName, name string, out io.Wri
77 77
 		Example: fmt.Sprintf(imagesExample, parentName, name),
78 78
 
79 79
 		Run: func(cmd *cobra.Command, args []string) {
80
-			if err := opts.Complete(f, args, out); err != nil {
81
-				cmdutil.CheckErr(err)
82
-			}
83
-
84
-			if err := opts.Validate(); err != nil {
85
-				cmdutil.CheckErr(cmdutil.UsageError(cmd, err.Error()))
86
-			}
87
-
88
-			if err := opts.RunPruneImages(); err != nil {
89
-				cmdutil.CheckErr(err)
90
-			}
80
+			kcmdutil.CheckErr(opts.Complete(f, cmd, args, out))
81
+			kcmdutil.CheckErr(opts.Validate())
82
+			kcmdutil.CheckErr(opts.Run())
91 83
 		},
92 84
 	}
93 85
 
... ...
@@ -100,10 +92,11 @@ func NewCmdPruneImages(f *clientcmd.Factory, parentName, name string, out io.Wri
100 100
 	return cmd
101 101
 }
102 102
 
103
-// Complete the options for prune images
104
-func (o *PruneImagesOptions) Complete(f *clientcmd.Factory, args []string, out io.Writer) error {
103
+// Complete turns a partially defined PruneImagesOptions into a solvent structure
104
+// which can be validated and used for pruning images.
105
+func (o *PruneImagesOptions) Complete(f *clientcmd.Factory, cmd *cobra.Command, args []string, out io.Writer) error {
105 106
 	if len(args) > 0 {
106
-		return errors.New("no arguments are allowed to this command")
107
+		return kcmdutil.UsageError(cmd, "no arguments are allowed to this command")
107 108
 	}
108 109
 
109 110
 	o.Out = out
... ...
@@ -153,7 +146,7 @@ func (o *PruneImagesOptions) Complete(f *clientcmd.Factory, args []string, out i
153 153
 		return err
154 154
 	}
155 155
 
156
-	options := prune.ImageRegistryPrunerOptions{
156
+	options := prune.PrunerOptions{
157 157
 		KeepYoungerThan:  o.KeepYoungerThan,
158 158
 		KeepTagRevisions: o.KeepTagRevisions,
159 159
 		Images:           allImages,
... ...
@@ -168,61 +161,60 @@ func (o *PruneImagesOptions) Complete(f *clientcmd.Factory, args []string, out i
168 168
 		RegistryURL:      o.RegistryUrlOverride,
169 169
 	}
170 170
 
171
-	o.Pruner = prune.NewImageRegistryPruner(options)
171
+	o.Pruner = prune.NewPruner(options)
172 172
 
173 173
 	return nil
174 174
 }
175 175
 
176
-// Validate the options for prune images
177
-func (o *PruneImagesOptions) Validate() error {
178
-	if o.Pruner == nil && o.Confirm {
179
-		return errors.New("an image pruner needs to be specified")
176
+// Validate ensures that a PruneImagesOptions is valid and can be used to execute pruning.
177
+func (o PruneImagesOptions) Validate() error {
178
+	if o.KeepYoungerThan < 0 {
179
+		return fmt.Errorf("--keep-younger-than must be greater than or equal to 0")
180 180
 	}
181
-	if o.Client == nil {
182
-		return errors.New("a client needs to be specified")
181
+	if o.KeepTagRevisions < 0 {
182
+		return fmt.Errorf("--keep-tag-revisions must be greater than or equal to 0")
183 183
 	}
184
-	if o.Out == nil {
185
-		return errors.New("a writer needs to be specified")
184
+	if _, err := url.Parse(o.RegistryUrlOverride); err != nil {
185
+		return fmt.Errorf("invalid --registry-url flag: %v", err)
186 186
 	}
187 187
 	return nil
188 188
 }
189 189
 
190
-// RunPruneImages runs the prune images cli command
191
-func (o *PruneImagesOptions) RunPruneImages() error {
192
-	// this tabwriter is used by the describing*Pruners below for their output
190
+// Run contains all the necessary functionality for the OpenShift cli prune images command.
191
+func (o PruneImagesOptions) Run() error {
193 192
 	w := tabwriter.NewWriter(o.Out, 10, 4, 3, ' ', 0)
194 193
 	defer w.Flush()
195 194
 
196
-	imagePruner := &describingImagePruner{w: w}
197
-	imageStreamPruner := &describingImageStreamPruner{w: w}
198
-	layerPruner := &describingLayerPruner{w: w}
199
-	blobPruner := &describingBlobPruner{w: w}
200
-	manifestPruner := &describingManifestPruner{w: w}
195
+	imageDeleter := &describingImageDeleter{w: w}
196
+	imageStreamDeleter := &describingImageStreamDeleter{w: w}
197
+	layerDeleter := &describingLayerDeleter{w: w}
198
+	blobDeleter := &describingBlobDeleter{w: w}
199
+	manifestDeleter := &describingManifestDeleter{w: w}
201 200
 
202 201
 	if o.Confirm {
203
-		imagePruner.delegate = prune.NewDeletingImagePruner(o.Client.Images())
204
-		imageStreamPruner.delegate = prune.NewDeletingImageStreamPruner(o.Client)
205
-		layerPruner.delegate = prune.NewDeletingLayerPruner()
206
-		blobPruner.delegate = prune.NewDeletingBlobPruner()
207
-		manifestPruner.delegate = prune.NewDeletingManifestPruner()
202
+		imageDeleter.delegate = prune.NewImageDeleter(o.Client.Images())
203
+		imageStreamDeleter.delegate = prune.NewImageStreamDeleter(o.Client)
204
+		layerDeleter.delegate = prune.NewLayerDeleter()
205
+		blobDeleter.delegate = prune.NewBlobDeleter()
206
+		manifestDeleter.delegate = prune.NewManifestDeleter()
208 207
 	} else {
209 208
 		fmt.Fprintln(os.Stderr, "Dry run enabled - no modifications will be made. Add --confirm to remove images")
210 209
 	}
211 210
 
212
-	return o.Pruner.Prune(imagePruner, imageStreamPruner, layerPruner, blobPruner, manifestPruner)
211
+	return o.Pruner.Prune(imageDeleter, imageStreamDeleter, layerDeleter, blobDeleter, manifestDeleter)
213 212
 }
214 213
 
215
-// describingImageStreamPruner prints information about each image stream update.
216
-// If a delegate exists, its PruneImageStream function is invoked prior to returning.
217
-type describingImageStreamPruner struct {
214
+// describingImageStreamDeleter prints information about each image stream update.
215
+// If a delegate exists, its DeleteImageStream function is invoked prior to returning.
216
+type describingImageStreamDeleter struct {
218 217
 	w             io.Writer
219
-	delegate      prune.ImageStreamPruner
218
+	delegate      prune.ImageStreamDeleter
220 219
 	headerPrinted bool
221 220
 }
222 221
 
223
-var _ prune.ImageStreamPruner = &describingImageStreamPruner{}
222
+var _ prune.ImageStreamDeleter = &describingImageStreamDeleter{}
224 223
 
225
-func (p *describingImageStreamPruner) PruneImageStream(stream *imageapi.ImageStream, image *imageapi.Image, updatedTags []string) (*imageapi.ImageStream, error) {
224
+func (p *describingImageStreamDeleter) DeleteImageStream(stream *imageapi.ImageStream, image *imageapi.Image, updatedTags []string) (*imageapi.ImageStream, error) {
226 225
 	if !p.headerPrinted {
227 226
 		p.headerPrinted = true
228 227
 		fmt.Fprintln(p.w, "Deleting references from image streams to images ...")
... ...
@@ -235,7 +227,7 @@ func (p *describingImageStreamPruner) PruneImageStream(stream *imageapi.ImageStr
235 235
 		return stream, nil
236 236
 	}
237 237
 
238
-	updatedStream, err := p.delegate.PruneImageStream(stream, image, updatedTags)
238
+	updatedStream, err := p.delegate.DeleteImageStream(stream, image, updatedTags)
239 239
 	if err != nil {
240 240
 		fmt.Fprintf(os.Stderr, "error updating image stream %s/%s to remove references to image %s: %v\n", stream.Namespace, stream.Name, image.Name, err)
241 241
 	}
... ...
@@ -243,17 +235,17 @@ func (p *describingImageStreamPruner) PruneImageStream(stream *imageapi.ImageStr
243 243
 	return updatedStream, err
244 244
 }
245 245
 
246
-// describingImagePruner prints information about each image being deleted.
247
-// If a delegate exists, its PruneImage function is invoked prior to returning.
248
-type describingImagePruner struct {
246
+// describingImageDeleter prints information about each image being deleted.
247
+// If a delegate exists, its DeleteImage function is invoked prior to returning.
248
+type describingImageDeleter struct {
249 249
 	w             io.Writer
250
-	delegate      prune.ImagePruner
250
+	delegate      prune.ImageDeleter
251 251
 	headerPrinted bool
252 252
 }
253 253
 
254
-var _ prune.ImagePruner = &describingImagePruner{}
254
+var _ prune.ImageDeleter = &describingImageDeleter{}
255 255
 
256
-func (p *describingImagePruner) PruneImage(image *imageapi.Image) error {
256
+func (p *describingImageDeleter) DeleteImage(image *imageapi.Image) error {
257 257
 	if !p.headerPrinted {
258 258
 		p.headerPrinted = true
259 259
 		fmt.Fprintln(p.w, "\nDeleting images from server ...")
... ...
@@ -266,7 +258,7 @@ func (p *describingImagePruner) PruneImage(image *imageapi.Image) error {
266 266
 		return nil
267 267
 	}
268 268
 
269
-	err := p.delegate.PruneImage(image)
269
+	err := p.delegate.DeleteImage(image)
270 270
 	if err != nil {
271 271
 		fmt.Fprintf(os.Stderr, "error deleting image %s from server: %v\n", image.Name, err)
272 272
 	}
... ...
@@ -274,18 +266,18 @@ func (p *describingImagePruner) PruneImage(image *imageapi.Image) error {
274 274
 	return err
275 275
 }
276 276
 
277
-// describingLayerPruner prints information about each repo layer link being
278
-// deleted. If a delegate exists, its PruneLayer function is invoked prior to
277
+// describingLayerDeleter prints information about each repo layer link being
278
+// deleted. If a delegate exists, its DeleteLayer function is invoked prior to
279 279
 // returning.
280
-type describingLayerPruner struct {
280
+type describingLayerDeleter struct {
281 281
 	w             io.Writer
282
-	delegate      prune.LayerPruner
282
+	delegate      prune.LayerDeleter
283 283
 	headerPrinted bool
284 284
 }
285 285
 
286
-var _ prune.LayerPruner = &describingLayerPruner{}
286
+var _ prune.LayerDeleter = &describingLayerDeleter{}
287 287
 
288
-func (p *describingLayerPruner) PruneLayer(registryClient *http.Client, registryURL, repo, layer string) error {
288
+func (p *describingLayerDeleter) DeleteLayer(registryClient *http.Client, registryURL, repo, layer string) error {
289 289
 	if !p.headerPrinted {
290 290
 		p.headerPrinted = true
291 291
 		fmt.Fprintln(p.w, "\nDeleting registry repository layer links ...")
... ...
@@ -298,7 +290,7 @@ func (p *describingLayerPruner) PruneLayer(registryClient *http.Client, registry
298 298
 		return nil
299 299
 	}
300 300
 
301
-	err := p.delegate.PruneLayer(registryClient, registryURL, repo, layer)
301
+	err := p.delegate.DeleteLayer(registryClient, registryURL, repo, layer)
302 302
 	if err != nil {
303 303
 		fmt.Fprintf(os.Stderr, "error deleting repository %s layer link %s from the registry: %v\n", repo, layer, err)
304 304
 	}
... ...
@@ -306,17 +298,17 @@ func (p *describingLayerPruner) PruneLayer(registryClient *http.Client, registry
306 306
 	return err
307 307
 }
308 308
 
309
-// describingBlobPruner prints information about each blob being deleted. If a
310
-// delegate exists, its PruneBlob function is invoked prior to returning.
311
-type describingBlobPruner struct {
309
+// describingBlobDeleter prints information about each blob being deleted. If a
310
+// delegate exists, its DeleteBlob function is invoked prior to returning.
311
+type describingBlobDeleter struct {
312 312
 	w             io.Writer
313
-	delegate      prune.BlobPruner
313
+	delegate      prune.BlobDeleter
314 314
 	headerPrinted bool
315 315
 }
316 316
 
317
-var _ prune.BlobPruner = &describingBlobPruner{}
317
+var _ prune.BlobDeleter = &describingBlobDeleter{}
318 318
 
319
-func (p *describingBlobPruner) PruneBlob(registryClient *http.Client, registryURL, layer string) error {
319
+func (p *describingBlobDeleter) DeleteBlob(registryClient *http.Client, registryURL, layer string) error {
320 320
 	if !p.headerPrinted {
321 321
 		p.headerPrinted = true
322 322
 		fmt.Fprintln(p.w, "\nDeleting registry layer blobs ...")
... ...
@@ -329,7 +321,7 @@ func (p *describingBlobPruner) PruneBlob(registryClient *http.Client, registryUR
329 329
 		return nil
330 330
 	}
331 331
 
332
-	err := p.delegate.PruneBlob(registryClient, registryURL, layer)
332
+	err := p.delegate.DeleteBlob(registryClient, registryURL, layer)
333 333
 	if err != nil {
334 334
 		fmt.Fprintf(os.Stderr, "error deleting blob %s from the registry: %v\n", layer, err)
335 335
 	}
... ...
@@ -337,18 +329,18 @@ func (p *describingBlobPruner) PruneBlob(registryClient *http.Client, registryUR
337 337
 	return err
338 338
 }
339 339
 
340
-// describingManifestPruner prints information about each repo manifest being
341
-// deleted. If a delegate exists, its PruneManifest function is invoked prior
340
+// describingManifestDeleter prints information about each repo manifest being
341
+// deleted. If a delegate exists, its DeleteManifest function is invoked prior
342 342
 // to returning.
343
-type describingManifestPruner struct {
343
+type describingManifestDeleter struct {
344 344
 	w             io.Writer
345
-	delegate      prune.ManifestPruner
345
+	delegate      prune.ManifestDeleter
346 346
 	headerPrinted bool
347 347
 }
348 348
 
349
-var _ prune.ManifestPruner = &describingManifestPruner{}
349
+var _ prune.ManifestDeleter = &describingManifestDeleter{}
350 350
 
351
-func (p *describingManifestPruner) PruneManifest(registryClient *http.Client, registryURL, repo, manifest string) error {
351
+func (p *describingManifestDeleter) DeleteManifest(registryClient *http.Client, registryURL, repo, manifest string) error {
352 352
 	if !p.headerPrinted {
353 353
 		p.headerPrinted = true
354 354
 		fmt.Fprintln(p.w, "\nDeleting registry repository manifest data ...")
... ...
@@ -361,7 +353,7 @@ func (p *describingManifestPruner) PruneManifest(registryClient *http.Client, re
361 361
 		return nil
362 362
 	}
363 363
 
364
-	err := p.delegate.PruneManifest(registryClient, registryURL, repo, manifest)
364
+	err := p.delegate.DeleteManifest(registryClient, registryURL, repo, manifest)
365 365
 	if err != nil {
366 366
 		fmt.Fprintf(os.Stderr, "error deleting data for repository %s image manifest %s from the registry: %v\n", repo, manifest, err)
367 367
 	}
... ...
@@ -391,7 +383,7 @@ func getClients(f *clientcmd.Factory, caBundle string) (*client.Client, *kclient
391 391
 		}
392 392
 		token = clientConfig.BearerToken
393 393
 	default:
394
-		err = errors.New("You must use a client config with a token")
394
+		err = errors.New("you must use a client config with a token")
395 395
 		return nil, nil, nil, err
396 396
 	}
397 397
 
398 398
deleted file mode 100644
... ...
@@ -1,1010 +0,0 @@
1
-package prune
2
-
3
-import (
4
-	"encoding/json"
5
-	"fmt"
6
-	"net/http"
7
-	"time"
8
-
9
-	"github.com/docker/distribution/registry/api/errcode"
10
-	"github.com/golang/glog"
11
-	gonum "github.com/gonum/graph"
12
-
13
-	kapi "k8s.io/kubernetes/pkg/api"
14
-	"k8s.io/kubernetes/pkg/api/unversioned"
15
-	kerrors "k8s.io/kubernetes/pkg/util/errors"
16
-	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
17
-	"k8s.io/kubernetes/pkg/util/sets"
18
-
19
-	"github.com/openshift/origin/pkg/api/graph"
20
-	kubegraph "github.com/openshift/origin/pkg/api/kubegraph/nodes"
21
-	buildapi "github.com/openshift/origin/pkg/build/api"
22
-	buildgraph "github.com/openshift/origin/pkg/build/graph/nodes"
23
-	buildutil "github.com/openshift/origin/pkg/build/util"
24
-	"github.com/openshift/origin/pkg/client"
25
-	deployapi "github.com/openshift/origin/pkg/deploy/api"
26
-	deploygraph "github.com/openshift/origin/pkg/deploy/graph/nodes"
27
-	imageapi "github.com/openshift/origin/pkg/image/api"
28
-	imagegraph "github.com/openshift/origin/pkg/image/graph/nodes"
29
-)
30
-
31
-// TODO these edges should probably have an `Add***Edges` method in images/graph and be moved there
32
-const (
33
-	// ReferencedImageEdgeKind defines a "strong" edge where the tail is an
34
-	// ImageNode, with strong indicating that the ImageNode tail is not a
35
-	// candidate for pruning.
36
-	ReferencedImageEdgeKind = "ReferencedImage"
37
-	// WeakReferencedImageEdgeKind defines a "weak" edge where the tail is
38
-	// an ImageNode, with weak indicating that this particular edge does
39
-	// not keep an ImageNode from being a candidate for pruning.
40
-	WeakReferencedImageEdgeKind = "WeakReferencedImage"
41
-
42
-	// ReferencedImageLayerEdgeKind defines an edge from an ImageStreamNode or an
43
-	// ImageNode to an ImageLayerNode.
44
-	ReferencedImageLayerEdgeKind = "ReferencedImageLayer"
45
-)
46
-
47
-// pruneAlgorithm contains the various settings to use when evaluating images
48
-// and layers for pruning.
49
-type pruneAlgorithm struct {
50
-	keepYoungerThan  time.Duration
51
-	keepTagRevisions int
52
-}
53
-
54
-// ImagePruner knows how to delete images from OpenShift.
55
-type ImagePruner interface {
56
-	// PruneImage deletes the image from OpenShift's storage.
57
-	PruneImage(image *imageapi.Image) error
58
-}
59
-
60
-// ImageStreamPruner knows how to remove an image reference from an image
61
-// stream.
62
-type ImageStreamPruner interface {
63
-	// PruneImageStream deletes all references to the image from the image
64
-	// stream's status.tags. The updated image stream is returned.
65
-	PruneImageStream(stream *imageapi.ImageStream, image *imageapi.Image, updatedTags []string) (*imageapi.ImageStream, error)
66
-}
67
-
68
-// BlobPruner knows how to delete a blob from the Docker registry.
69
-type BlobPruner interface {
70
-	// PruneBlob uses registryClient to ask the registry at registryURL to delete
71
-	// the blob.
72
-	PruneBlob(registryClient *http.Client, registryURL, blob string) error
73
-}
74
-
75
-// LayerPruner knows how to delete a repository layer link from the Docker
76
-// registry.
77
-type LayerPruner interface {
78
-	// PruneLayer uses registryClient to ask the registry at registryURL to
79
-	// delete the repository layer link.
80
-	PruneLayer(registryClient *http.Client, registryURL, repo, layer string) error
81
-}
82
-
83
-// ManifestPruner knows how to delete image manifest data for a repository from
84
-// the Docker registry.
85
-type ManifestPruner interface {
86
-	// PruneManifest uses registryClient to ask the registry at registryURL to
87
-	// delete the repository's image manifest data.
88
-	PruneManifest(registryClient *http.Client, registryURL, repo, manifest string) error
89
-}
90
-
91
-// ImageRegistryPrunerOptions contains the fields used to initialize a new
92
-// ImageRegistryPruner.
93
-type ImageRegistryPrunerOptions struct {
94
-	// KeepYoungerThan indicates the minimum age an Image must be to be a
95
-	// candidate for pruning.
96
-	KeepYoungerThan time.Duration
97
-	// KeepTagRevisions is the minimum number of tag revisions to preserve;
98
-	// revisions older than this value are candidates for pruning.
99
-	KeepTagRevisions int
100
-	// Images is the entire list of images in OpenShift. An image must be in this
101
-	// list to be a candidate for pruning.
102
-	Images *imageapi.ImageList
103
-	// Streams is the entire list of image streams across all namespaces in the
104
-	// cluster.
105
-	Streams *imageapi.ImageStreamList
106
-	// Pods is the entire list of pods across all namespaces in the cluster.
107
-	Pods *kapi.PodList
108
-	// RCs is the entire list of replication controllers across all namespaces in
109
-	// the cluster.
110
-	RCs *kapi.ReplicationControllerList
111
-	// BCs is the entire list of build configs across all namespaces in the
112
-	// cluster.
113
-	BCs *buildapi.BuildConfigList
114
-	// Builds is the entire list of builds across all namespaces in the cluster.
115
-	Builds *buildapi.BuildList
116
-	// DCs is the entire list of deployment configs across all namespaces in the
117
-	// cluster.
118
-	DCs *deployapi.DeploymentConfigList
119
-	// DryRun indicates that no changes will be made to the cluster and nothing
120
-	// will be removed.
121
-	DryRun bool
122
-	// RegistryClient is the http.Client to use when contacting the registry.
123
-	RegistryClient *http.Client
124
-	// RegistryURL is the URL for the registry.
125
-	RegistryURL string
126
-}
127
-
128
-// ImageRegistryPruner knows how to prune images and layers.
129
-type ImageRegistryPruner interface {
130
-	// Prune uses imagePruner, streamPruner, layerPruner, blobPruner, and
131
-	// manifestPruner to remove images that have been identified as candidates
132
-	// for pruning based on the ImageRegistryPruner's internal pruning algorithm.
133
-	// Please see NewImageRegistryPruner for details on the algorithm.
134
-	Prune(imagePruner ImagePruner, streamPruner ImageStreamPruner, layerPruner LayerPruner, blobPruner BlobPruner, manifestPruner ManifestPruner) error
135
-}
136
-
137
-// imageRegistryPruner implements ImageRegistryPruner.
138
-type imageRegistryPruner struct {
139
-	g              graph.Graph
140
-	algorithm      pruneAlgorithm
141
-	registryPinger registryPinger
142
-	registryClient *http.Client
143
-	registryURL    string
144
-}
145
-
146
-var _ ImageRegistryPruner = &imageRegistryPruner{}
147
-
148
-// registryPinger performs a health check against a registry.
149
-type registryPinger interface {
150
-	// ping performs a health check against registry.
151
-	ping(registry string) error
152
-}
153
-
154
-// defaultRegistryPinger implements registryPinger.
155
-type defaultRegistryPinger struct {
156
-	client *http.Client
157
-}
158
-
159
-func (drp *defaultRegistryPinger) ping(registry string) error {
160
-	healthCheck := func(proto, registry string) error {
161
-		// TODO: `/healthz` route is deprecated by `/`; remove it in future versions
162
-		healthResponse, err := drp.client.Get(fmt.Sprintf("%s://%s/healthz", proto, registry))
163
-		if err != nil {
164
-			return err
165
-		}
166
-		defer healthResponse.Body.Close()
167
-
168
-		if healthResponse.StatusCode != http.StatusOK {
169
-			return fmt.Errorf("unexpected status code %d", healthResponse.StatusCode)
170
-		}
171
-
172
-		return nil
173
-	}
174
-
175
-	var err error
176
-	for _, proto := range []string{"https", "http"} {
177
-		glog.V(4).Infof("Trying %s for %s", proto, registry)
178
-		err = healthCheck(proto, registry)
179
-		if err == nil {
180
-			break
181
-		}
182
-		glog.V(4).Infof("Error with %s for %s: %v", proto, registry, err)
183
-	}
184
-
185
-	return err
186
-}
187
-
188
-// dryRunRegistryPinger implements registryPinger.
189
-type dryRunRegistryPinger struct {
190
-}
191
-
192
-func (*dryRunRegistryPinger) ping(registry string) error {
193
-	return nil
194
-}
195
-
196
-/*
197
-NewImageRegistryPruner creates a new ImageRegistryPruner.
198
-
199
-Images younger than keepYoungerThan and images referenced by image streams
200
-and/or pods younger than keepYoungerThan are preserved. All other images are
201
-candidates for pruning. For example, if keepYoungerThan is 60m, and an
202
-ImageStream is only 59 minutes old, none of the images it references are
203
-eligible for pruning.
204
-
205
-keepTagRevisions is the number of revisions per tag in an image stream's
206
-status.tags that are preserved and ineligible for pruning. Any revision older
207
-than keepTagRevisions is eligible for pruning.
208
-
209
-images, streams, pods, rcs, bcs, builds, and dcs are the resources used to run
210
-the pruning algorithm. These should be the full list for each type from the
211
-cluster; otherwise, the pruning algorithm might result in incorrect
212
-calculations and premature pruning.
213
-
214
-The ImagePruner performs the following logic: remove any image containing the
215
-annotation openshift.io/image.managed=true that was created at least *n*
216
-minutes ago and is *not* currently referenced by:
217
-
218
-- any pod created less than *n* minutes ago
219
-- any image stream created less than *n* minutes ago
220
-- any running pods
221
-- any pending pods
222
-- any replication controllers
223
-- any deployment configs
224
-- any build configs
225
-- any builds
226
-- the n most recent tag revisions in an image stream's status.tags
227
-
228
-When removing an image, remove all references to the image from all
229
-ImageStreams having a reference to the image in `status.tags`.
230
-
231
-Also automatically remove any image layer that is no longer referenced by any
232
-images.
233
-*/
234
-func NewImageRegistryPruner(options ImageRegistryPrunerOptions) ImageRegistryPruner {
235
-	g := graph.New()
236
-
237
-	glog.V(1).Infof("Creating image pruner with keepYoungerThan=%v, keepTagRevisions=%d", options.KeepYoungerThan, options.KeepTagRevisions)
238
-
239
-	algorithm := pruneAlgorithm{
240
-		keepYoungerThan:  options.KeepYoungerThan,
241
-		keepTagRevisions: options.KeepTagRevisions,
242
-	}
243
-
244
-	addImagesToGraph(g, options.Images, algorithm)
245
-	addImageStreamsToGraph(g, options.Streams, algorithm)
246
-	addPodsToGraph(g, options.Pods, algorithm)
247
-	addReplicationControllersToGraph(g, options.RCs)
248
-	addBuildConfigsToGraph(g, options.BCs)
249
-	addBuildsToGraph(g, options.Builds)
250
-	addDeploymentConfigsToGraph(g, options.DCs)
251
-
252
-	var rp registryPinger
253
-	if options.DryRun {
254
-		rp = &dryRunRegistryPinger{}
255
-	} else {
256
-		rp = &defaultRegistryPinger{options.RegistryClient}
257
-	}
258
-
259
-	return &imageRegistryPruner{
260
-		g:              g,
261
-		algorithm:      algorithm,
262
-		registryPinger: rp,
263
-		registryClient: options.RegistryClient,
264
-		registryURL:    options.RegistryURL,
265
-	}
266
-}
267
-
268
-// addImagesToGraph adds all images to the graph that belong to one of the
269
-// registries in the algorithm and are at least as old as the minimum age
270
-// threshold as specified by the algorithm. It also adds all the images' layers
271
-// to the graph.
272
-func addImagesToGraph(g graph.Graph, images *imageapi.ImageList, algorithm pruneAlgorithm) {
273
-	for i := range images.Items {
274
-		image := &images.Items[i]
275
-
276
-		glog.V(4).Infof("Examining image %q", image.Name)
277
-
278
-		if image.Annotations == nil {
279
-			glog.V(4).Infof("Image %q with DockerImageReference %q belongs to an external registry - skipping", image.Name, image.DockerImageReference)
280
-			continue
281
-		}
282
-		if value, ok := image.Annotations[imageapi.ManagedByOpenShiftAnnotation]; !ok || value != "true" {
283
-			glog.V(4).Infof("Image %q with DockerImageReference %q belongs to an external registry - skipping", image.Name, image.DockerImageReference)
284
-			continue
285
-		}
286
-
287
-		age := unversioned.Now().Sub(image.CreationTimestamp.Time)
288
-		if age < algorithm.keepYoungerThan {
289
-			glog.V(4).Infof("Image %q is younger than minimum pruning age, skipping (age=%v)", image.Name, age)
290
-			continue
291
-		}
292
-
293
-		glog.V(4).Infof("Adding image %q to graph", image.Name)
294
-		imageNode := imagegraph.EnsureImageNode(g, image)
295
-
296
-		manifest := imageapi.DockerImageManifest{}
297
-		if err := json.Unmarshal([]byte(image.DockerImageManifest), &manifest); err != nil {
298
-			utilruntime.HandleError(fmt.Errorf("unable to extract manifest from image: %v. This image's layers won't be pruned if the image is pruned now.", err))
299
-			continue
300
-		}
301
-
302
-		for _, layer := range manifest.FSLayers {
303
-			glog.V(4).Infof("Adding image layer %q to graph", layer.DockerBlobSum)
304
-			layerNode := imagegraph.EnsureImageLayerNode(g, layer.DockerBlobSum)
305
-			g.AddEdge(imageNode, layerNode, ReferencedImageLayerEdgeKind)
306
-		}
307
-	}
308
-}
309
-
310
-// addImageStreamsToGraph adds all the streams to the graph. The most recent n
311
-// image revisions for a tag will be preserved, where n is specified by the
312
-// algorithm's keepTagRevisions. Image revisions older than n are candidates
313
-// for pruning.  if the image stream's age is at least as old as the minimum
314
-// threshold in algorithm.  Otherwise, if the image stream is younger than the
315
-// threshold, all image revisions for that stream are ineligible for pruning.
316
-//
317
-// addImageStreamsToGraph also adds references from each stream to all the
318
-// layers it references (via each image a stream references).
319
-func addImageStreamsToGraph(g graph.Graph, streams *imageapi.ImageStreamList, algorithm pruneAlgorithm) {
320
-	for i := range streams.Items {
321
-		stream := &streams.Items[i]
322
-
323
-		glog.V(4).Infof("Examining ImageStream %s/%s", stream.Namespace, stream.Name)
324
-
325
-		// use a weak reference for old image revisions by default
326
-		oldImageRevisionReferenceKind := WeakReferencedImageEdgeKind
327
-
328
-		age := unversioned.Now().Sub(stream.CreationTimestamp.Time)
329
-		if age < algorithm.keepYoungerThan {
330
-			// stream's age is below threshold - use a strong reference for old image revisions instead
331
-			glog.V(4).Infof("Stream %s/%s is below age threshold - none of its images are eligible for pruning", stream.Namespace, stream.Name)
332
-			oldImageRevisionReferenceKind = ReferencedImageEdgeKind
333
-		}
334
-
335
-		glog.V(4).Infof("Adding ImageStream %s/%s to graph", stream.Namespace, stream.Name)
336
-		isNode := imagegraph.EnsureImageStreamNode(g, stream)
337
-		imageStreamNode := isNode.(*imagegraph.ImageStreamNode)
338
-
339
-		for tag, history := range stream.Status.Tags {
340
-			for i := range history.Items {
341
-				n := imagegraph.FindImage(g, history.Items[i].Image)
342
-				if n == nil {
343
-					glog.V(2).Infof("Unable to find image %q in graph (from tag=%q, revision=%d, dockerImageReference=%s)", history.Items[i].Image, tag, i, history.Items[i].DockerImageReference)
344
-					continue
345
-				}
346
-				imageNode := n.(*imagegraph.ImageNode)
347
-
348
-				var kind string
349
-				switch {
350
-				case i < algorithm.keepTagRevisions:
351
-					kind = ReferencedImageEdgeKind
352
-				default:
353
-					kind = oldImageRevisionReferenceKind
354
-				}
355
-
356
-				glog.V(4).Infof("Checking for existing strong reference from stream %s/%s to image %s", stream.Namespace, stream.Name, imageNode.Image.Name)
357
-				if edge := g.Edge(imageStreamNode, imageNode); edge != nil && g.EdgeKinds(edge).Has(ReferencedImageEdgeKind) {
358
-					glog.V(4).Infof("Strong reference found")
359
-					continue
360
-				}
361
-
362
-				glog.V(4).Infof("Adding edge (kind=%s) from %q to %q", kind, imageStreamNode.UniqueName(), imageNode.UniqueName())
363
-				g.AddEdge(imageStreamNode, imageNode, kind)
364
-
365
-				glog.V(4).Infof("Adding stream->layer references")
366
-				// add stream -> layer references so we can prune them later
367
-				for _, s := range g.From(imageNode) {
368
-					if g.Kind(s) != imagegraph.ImageLayerNodeKind {
369
-						continue
370
-					}
371
-					glog.V(4).Infof("Adding reference from stream %q to layer %q", stream.Name, s.(*imagegraph.ImageLayerNode).Layer)
372
-					g.AddEdge(imageStreamNode, s, ReferencedImageLayerEdgeKind)
373
-				}
374
-			}
375
-		}
376
-	}
377
-}
378
-
379
-// addPodsToGraph adds pods to the graph.
380
-//
381
-// A pod is only *excluded* from being added to the graph if its phase is not
382
-// pending or running and it is at least as old as the minimum age threshold
383
-// defined by algorithm.
384
-//
385
-// Edges are added to the graph from each pod to the images specified by that
386
-// pod's list of containers, as long as the image is managed by OpenShift.
387
-func addPodsToGraph(g graph.Graph, pods *kapi.PodList, algorithm pruneAlgorithm) {
388
-	for i := range pods.Items {
389
-		pod := &pods.Items[i]
390
-
391
-		glog.V(4).Infof("Examining pod %s/%s", pod.Namespace, pod.Name)
392
-
393
-		if pod.Status.Phase != kapi.PodRunning && pod.Status.Phase != kapi.PodPending {
394
-			age := unversioned.Now().Sub(pod.CreationTimestamp.Time)
395
-			if age >= algorithm.keepYoungerThan {
396
-				glog.V(4).Infof("Pod %s/%s is not running or pending and age is at least minimum pruning age - skipping", pod.Namespace, pod.Name)
397
-				// not pending or running, age is at least minimum pruning age, skip
398
-				continue
399
-			}
400
-		}
401
-
402
-		glog.V(4).Infof("Adding pod %s/%s to graph", pod.Namespace, pod.Name)
403
-		podNode := kubegraph.EnsurePodNode(g, pod)
404
-
405
-		addPodSpecToGraph(g, &pod.Spec, podNode)
406
-	}
407
-}
408
-
409
-// Edges are added to the graph from each predecessor (pod or replication
410
-// controller) to the images specified by the pod spec's list of containers, as
411
-// long as the image is managed by OpenShift.
412
-func addPodSpecToGraph(g graph.Graph, spec *kapi.PodSpec, predecessor gonum.Node) {
413
-	for j := range spec.Containers {
414
-		container := spec.Containers[j]
415
-
416
-		glog.V(4).Infof("Examining container image %q", container.Image)
417
-
418
-		ref, err := imageapi.ParseDockerImageReference(container.Image)
419
-		if err != nil {
420
-			utilruntime.HandleError(fmt.Errorf("unable to parse DockerImageReference %q: %v", container.Image, err))
421
-			continue
422
-		}
423
-
424
-		if len(ref.ID) == 0 {
425
-			glog.V(4).Infof("%q has no image ID", container.Image)
426
-			continue
427
-		}
428
-
429
-		imageNode := imagegraph.FindImage(g, ref.ID)
430
-		if imageNode == nil {
431
-			glog.Infof("Unable to find image %q in the graph", ref.ID)
432
-			continue
433
-		}
434
-
435
-		glog.V(4).Infof("Adding edge from pod to image")
436
-		g.AddEdge(predecessor, imageNode, ReferencedImageEdgeKind)
437
-	}
438
-}
439
-
440
-// addReplicationControllersToGraph adds replication controllers to the graph.
441
-//
442
-// Edges are added to the graph from each replication controller to the images
443
-// specified by its pod spec's list of containers, as long as the image is
444
-// managed by OpenShift.
445
-func addReplicationControllersToGraph(g graph.Graph, rcs *kapi.ReplicationControllerList) {
446
-	for i := range rcs.Items {
447
-		rc := &rcs.Items[i]
448
-		glog.V(4).Infof("Examining replication controller %s/%s", rc.Namespace, rc.Name)
449
-		rcNode := kubegraph.EnsureReplicationControllerNode(g, rc)
450
-		addPodSpecToGraph(g, &rc.Spec.Template.Spec, rcNode)
451
-	}
452
-}
453
-
454
-// addDeploymentConfigsToGraph adds deployment configs to the graph.
455
-//
456
-// Edges are added to the graph from each deployment config to the images
457
-// specified by its pod spec's list of containers, as long as the image is
458
-// managed by OpenShift.
459
-func addDeploymentConfigsToGraph(g graph.Graph, dcs *deployapi.DeploymentConfigList) {
460
-	for i := range dcs.Items {
461
-		dc := &dcs.Items[i]
462
-		glog.V(4).Infof("Examining DeploymentConfig %s/%s", dc.Namespace, dc.Name)
463
-		dcNode := deploygraph.EnsureDeploymentConfigNode(g, dc)
464
-		addPodSpecToGraph(g, &dc.Spec.Template.Spec, dcNode)
465
-	}
466
-}
467
-
468
-// addBuildConfigsToGraph adds build configs to the graph.
469
-//
470
-// Edges are added to the graph from each build config to the image specified by its strategy.from.
471
-func addBuildConfigsToGraph(g graph.Graph, bcs *buildapi.BuildConfigList) {
472
-	for i := range bcs.Items {
473
-		bc := &bcs.Items[i]
474
-		glog.V(4).Infof("Examining BuildConfig %s/%s", bc.Namespace, bc.Name)
475
-		bcNode := buildgraph.EnsureBuildConfigNode(g, bc)
476
-		addBuildStrategyImageReferencesToGraph(g, bc.Spec.Strategy, bcNode)
477
-	}
478
-}
479
-
480
-// addBuildsToGraph adds builds to the graph.
481
-//
482
-// Edges are added to the graph from each build to the image specified by its strategy.from.
483
-func addBuildsToGraph(g graph.Graph, builds *buildapi.BuildList) {
484
-	for i := range builds.Items {
485
-		build := &builds.Items[i]
486
-		glog.V(4).Infof("Examining build %s/%s", build.Namespace, build.Name)
487
-		buildNode := buildgraph.EnsureBuildNode(g, build)
488
-		addBuildStrategyImageReferencesToGraph(g, build.Spec.Strategy, buildNode)
489
-	}
490
-}
491
-
492
-// addBuildStrategyImageReferencesToGraph ads references from the build strategy's parent node to the image
493
-// the build strategy references.
494
-//
495
-// Edges are added to the graph from each predecessor (build or build config)
496
-// to the image specified by strategy.from, as long as the image is managed by
497
-// OpenShift.
498
-func addBuildStrategyImageReferencesToGraph(g graph.Graph, strategy buildapi.BuildStrategy, predecessor gonum.Node) {
499
-	from := buildutil.GetInputReference(strategy)
500
-	if from == nil {
501
-		glog.V(4).Infof("Unable to determine 'from' reference - skipping")
502
-		return
503
-	}
504
-
505
-	glog.V(4).Infof("Examining build strategy with from: %#v", from)
506
-
507
-	var imageID string
508
-
509
-	switch from.Kind {
510
-	case "ImageStreamImage":
511
-		_, id, err := imageapi.ParseImageStreamImageName(from.Name)
512
-		if err != nil {
513
-			glog.V(2).Infof("Error parsing ImageStreamImage name %q: %v - skipping", from.Name, err)
514
-			return
515
-		}
516
-		imageID = id
517
-	case "DockerImage":
518
-		ref, err := imageapi.ParseDockerImageReference(from.Name)
519
-		if err != nil {
520
-			glog.V(2).Infof("Error parsing DockerImage name %q: %v - skipping", from.Name, err)
521
-			return
522
-		}
523
-		imageID = ref.ID
524
-	default:
525
-		return
526
-	}
527
-
528
-	glog.V(4).Infof("Looking for image %q in graph", imageID)
529
-	imageNode := imagegraph.FindImage(g, imageID)
530
-	if imageNode == nil {
531
-		glog.V(4).Infof("Unable to find image %q in graph - skipping", imageID)
532
-		return
533
-	}
534
-
535
-	glog.V(4).Infof("Adding edge from %v to %v", predecessor, imageNode)
536
-	g.AddEdge(predecessor, imageNode, ReferencedImageEdgeKind)
537
-}
538
-
539
-// getImageNodes returns only nodes of type ImageNode.
540
-func getImageNodes(nodes []gonum.Node) []*imagegraph.ImageNode {
541
-	ret := []*imagegraph.ImageNode{}
542
-	for i := range nodes {
543
-		if node, ok := nodes[i].(*imagegraph.ImageNode); ok {
544
-			ret = append(ret, node)
545
-		}
546
-	}
547
-	return ret
548
-}
549
-
550
-// edgeKind returns true if the edge from "from" to "to" is of the desired kind.
551
-func edgeKind(g graph.Graph, from, to gonum.Node, desiredKind string) bool {
552
-	edge := g.Edge(from, to)
553
-	kinds := g.EdgeKinds(edge)
554
-	return kinds.Has(desiredKind)
555
-}
556
-
557
-// imageIsPrunable returns true iff the image node only has weak references
558
-// from its predecessors to it. A weak reference to an image is a reference
559
-// from an image stream to an image where the image is not the current image
560
-// for a tag and the image stream is at least as old as the minimum pruning
561
-// age.
562
-func imageIsPrunable(g graph.Graph, imageNode *imagegraph.ImageNode) bool {
563
-	onlyWeakReferences := true
564
-
565
-	for _, n := range g.To(imageNode) {
566
-		glog.V(4).Infof("Examining predecessor %#v", n)
567
-		if !edgeKind(g, n, imageNode, WeakReferencedImageEdgeKind) {
568
-			glog.V(4).Infof("Strong reference detected")
569
-			onlyWeakReferences = false
570
-			break
571
-		}
572
-	}
573
-
574
-	return onlyWeakReferences
575
-
576
-}
577
-
578
-// calculatePrunableImages returns the list of prunable images and a
579
-// graph.NodeSet containing the image node IDs.
580
-func calculatePrunableImages(g graph.Graph, imageNodes []*imagegraph.ImageNode) ([]*imagegraph.ImageNode, graph.NodeSet) {
581
-	prunable := []*imagegraph.ImageNode{}
582
-	ids := make(graph.NodeSet)
583
-
584
-	for _, imageNode := range imageNodes {
585
-		glog.V(4).Infof("Examining image %q", imageNode.Image.Name)
586
-
587
-		if imageIsPrunable(g, imageNode) {
588
-			glog.V(4).Infof("Image %q is prunable", imageNode.Image.Name)
589
-			prunable = append(prunable, imageNode)
590
-			ids.Add(imageNode.ID())
591
-		}
592
-	}
593
-
594
-	return prunable, ids
595
-}
596
-
597
-// subgraphWithoutPrunableImages creates a subgraph from g with prunable image
598
-// nodes excluded.
599
-func subgraphWithoutPrunableImages(g graph.Graph, prunableImageIDs graph.NodeSet) graph.Graph {
600
-	return g.Subgraph(
601
-		func(g graph.Interface, node gonum.Node) bool {
602
-			return !prunableImageIDs.Has(node.ID())
603
-		},
604
-		func(g graph.Interface, from, to gonum.Node, edgeKinds sets.String) bool {
605
-			if prunableImageIDs.Has(from.ID()) {
606
-				return false
607
-			}
608
-			if prunableImageIDs.Has(to.ID()) {
609
-				return false
610
-			}
611
-			return true
612
-		},
613
-	)
614
-}
615
-
616
-// calculatePrunableLayers returns the list of prunable layers.
617
-func calculatePrunableLayers(g graph.Graph) []*imagegraph.ImageLayerNode {
618
-	prunable := []*imagegraph.ImageLayerNode{}
619
-
620
-	nodes := g.Nodes()
621
-	for i := range nodes {
622
-		layerNode, ok := nodes[i].(*imagegraph.ImageLayerNode)
623
-		if !ok {
624
-			continue
625
-		}
626
-
627
-		glog.V(4).Infof("Examining layer %q", layerNode.Layer)
628
-
629
-		if layerIsPrunable(g, layerNode) {
630
-			glog.V(4).Infof("Layer %q is prunable", layerNode.Layer)
631
-			prunable = append(prunable, layerNode)
632
-		}
633
-	}
634
-
635
-	return prunable
636
-}
637
-
638
-// pruneStreams removes references from all image streams' status.tags entries
639
-// to prunable images, invoking streamPruner.PruneImageStream for each updated
640
-// stream.
641
-func pruneStreams(g graph.Graph, imageNodes []*imagegraph.ImageNode, streamPruner ImageStreamPruner) []error {
642
-	errs := []error{}
643
-
644
-	glog.V(4).Infof("Removing pruned image references from streams")
645
-	for _, imageNode := range imageNodes {
646
-		for _, n := range g.To(imageNode) {
647
-			streamNode, ok := n.(*imagegraph.ImageStreamNode)
648
-			if !ok {
649
-				continue
650
-			}
651
-
652
-			stream := streamNode.ImageStream
653
-			updatedTags := sets.NewString()
654
-
655
-			glog.V(4).Infof("Checking if ImageStream %s/%s has references to image %s in status.tags", stream.Namespace, stream.Name, imageNode.Image.Name)
656
-
657
-			for tag, history := range stream.Status.Tags {
658
-				glog.V(4).Infof("Checking tag %q", tag)
659
-
660
-				newHistory := imageapi.TagEventList{}
661
-
662
-				for i, tagEvent := range history.Items {
663
-					glog.V(4).Infof("Checking tag event %d with image %q", i, tagEvent.Image)
664
-
665
-					if tagEvent.Image != imageNode.Image.Name {
666
-						glog.V(4).Infof("Tag event doesn't match deleted image - keeping")
667
-						newHistory.Items = append(newHistory.Items, tagEvent)
668
-					} else {
669
-						glog.V(4).Infof("Tag event matches deleted image - removing reference")
670
-						updatedTags.Insert(tag)
671
-					}
672
-				}
673
-				if len(newHistory.Items) == 0 {
674
-					glog.V(4).Infof("Removing tag %q from status.tags of ImageStream %s/%s", tag, stream.Namespace, stream.Name)
675
-					delete(stream.Status.Tags, tag)
676
-				} else {
677
-					stream.Status.Tags[tag] = newHistory
678
-				}
679
-			}
680
-
681
-			updatedStream, err := streamPruner.PruneImageStream(stream, imageNode.Image, updatedTags.List())
682
-			if err != nil {
683
-				errs = append(errs, fmt.Errorf("error pruning image from stream: %v", err))
684
-				continue
685
-			}
686
-
687
-			streamNode.ImageStream = updatedStream
688
-		}
689
-	}
690
-	glog.V(4).Infof("Done removing pruned image references from streams")
691
-	return errs
692
-}
693
-
694
-// pruneImages invokes imagePruner.PruneImage with each image that is prunable.
695
-func pruneImages(g graph.Graph, imageNodes []*imagegraph.ImageNode, imagePruner ImagePruner) []error {
696
-	errs := []error{}
697
-
698
-	for _, imageNode := range imageNodes {
699
-		if err := imagePruner.PruneImage(imageNode.Image); err != nil {
700
-			errs = append(errs, fmt.Errorf("error pruning image %q: %v", imageNode.Image.Name, err))
701
-		}
702
-	}
703
-
704
-	return errs
705
-}
706
-
707
-func (p *imageRegistryPruner) determineRegistry(imageNodes []*imagegraph.ImageNode) (string, error) {
708
-	if len(p.registryURL) > 0 {
709
-		return p.registryURL, nil
710
-	}
711
-
712
-	// we only support a single internal registry, and all images have the same registry
713
-	// so we just take the 1st one and use it
714
-	pullSpec := imageNodes[0].Image.DockerImageReference
715
-
716
-	ref, err := imageapi.ParseDockerImageReference(pullSpec)
717
-	if err != nil {
718
-		return "", fmt.Errorf("unable to parse %q: %v", pullSpec, err)
719
-	}
720
-
721
-	if len(ref.Registry) == 0 {
722
-		return "", fmt.Errorf("%s does not include a registry", pullSpec)
723
-	}
724
-
725
-	return ref.Registry, nil
726
-}
727
-
728
-// Run identifies images eligible for pruning, invoking imagePruneFunc for each
729
-// image, and then it identifies layers eligible for pruning, invoking
730
-// layerPruneFunc for each registry URL that has layers that can be pruned.
731
-func (p *imageRegistryPruner) Prune(imagePruner ImagePruner, streamPruner ImageStreamPruner, layerPruner LayerPruner, blobPruner BlobPruner, manifestPruner ManifestPruner) error {
732
-	allNodes := p.g.Nodes()
733
-
734
-	imageNodes := getImageNodes(allNodes)
735
-	if len(imageNodes) == 0 {
736
-		return nil
737
-	}
738
-
739
-	registryURL, err := p.determineRegistry(imageNodes)
740
-	if err != nil {
741
-		return fmt.Errorf("unable to determine registry: %v", err)
742
-	}
743
-	glog.V(1).Infof("Using registry: %s", registryURL)
744
-
745
-	if err := p.registryPinger.ping(registryURL); err != nil {
746
-		return fmt.Errorf("error communicating with registry: %v", err)
747
-	}
748
-
749
-	prunableImageNodes, prunableImageIDs := calculatePrunableImages(p.g, imageNodes)
750
-	graphWithoutPrunableImages := subgraphWithoutPrunableImages(p.g, prunableImageIDs)
751
-	prunableLayers := calculatePrunableLayers(graphWithoutPrunableImages)
752
-
753
-	errs := []error{}
754
-
755
-	errs = append(errs, pruneStreams(p.g, prunableImageNodes, streamPruner)...)
756
-	errs = append(errs, pruneLayers(p.g, p.registryClient, registryURL, prunableLayers, layerPruner)...)
757
-	errs = append(errs, pruneBlobs(p.g, p.registryClient, registryURL, prunableLayers, blobPruner)...)
758
-	errs = append(errs, pruneManifests(p.g, p.registryClient, registryURL, prunableImageNodes, manifestPruner)...)
759
-
760
-	if len(errs) > 0 {
761
-		// If we had any errors removing image references from image streams or deleting
762
-		// layers, blobs, or manifest data from the registry, stop here and don't
763
-		// delete any images. This way, you can rerun prune and retry things that failed.
764
-		return kerrors.NewAggregate(errs)
765
-	}
766
-
767
-	errs = append(errs, pruneImages(p.g, prunableImageNodes, imagePruner)...)
768
-	return kerrors.NewAggregate(errs)
769
-}
770
-
771
-// layerIsPrunable returns true if the layer is not referenced by any images.
772
-func layerIsPrunable(g graph.Graph, layerNode *imagegraph.ImageLayerNode) bool {
773
-	for _, predecessor := range g.To(layerNode) {
774
-		glog.V(4).Infof("Examining layer predecessor %#v", predecessor)
775
-		if g.Kind(predecessor) == imagegraph.ImageNodeKind {
776
-			glog.V(4).Infof("Layer has an image predecessor")
777
-			return false
778
-		}
779
-	}
780
-
781
-	return true
782
-}
783
-
784
-// streamLayerReferences returns a list of ImageStreamNodes that reference a
785
-// given ImageLayerNode.
786
-func streamLayerReferences(g graph.Graph, layerNode *imagegraph.ImageLayerNode) []*imagegraph.ImageStreamNode {
787
-	ret := []*imagegraph.ImageStreamNode{}
788
-
789
-	for _, predecessor := range g.To(layerNode) {
790
-		if g.Kind(predecessor) != imagegraph.ImageStreamNodeKind {
791
-			continue
792
-		}
793
-
794
-		ret = append(ret, predecessor.(*imagegraph.ImageStreamNode))
795
-	}
796
-
797
-	return ret
798
-}
799
-
800
-// pruneLayers invokes layerPruner.PruneLayer for each repository layer link to
801
-// be deleted from the registry.
802
-func pruneLayers(g graph.Graph, registryClient *http.Client, registryURL string, layerNodes []*imagegraph.ImageLayerNode, layerPruner LayerPruner) []error {
803
-	errs := []error{}
804
-
805
-	for _, layerNode := range layerNodes {
806
-		// get streams that reference layer
807
-		streamNodes := streamLayerReferences(g, layerNode)
808
-
809
-		for _, streamNode := range streamNodes {
810
-			stream := streamNode.ImageStream
811
-			streamName := fmt.Sprintf("%s/%s", stream.Namespace, stream.Name)
812
-
813
-			glog.V(4).Infof("Pruning registry=%q, repo=%q, layer=%q", registryURL, streamName, layerNode.Layer)
814
-			if err := layerPruner.PruneLayer(registryClient, registryURL, streamName, layerNode.Layer); err != nil {
815
-				errs = append(errs, fmt.Errorf("error pruning repo %q layer link %q: %v", streamName, layerNode.Layer, err))
816
-			}
817
-		}
818
-	}
819
-
820
-	return errs
821
-}
822
-
823
-// pruneBlobs invokes blobPruner.PruneBlob for each blob to be deleted from the
824
-// registry.
825
-func pruneBlobs(g graph.Graph, registryClient *http.Client, registryURL string, layerNodes []*imagegraph.ImageLayerNode, blobPruner BlobPruner) []error {
826
-	errs := []error{}
827
-
828
-	for _, layerNode := range layerNodes {
829
-		glog.V(4).Infof("Pruning registry=%q, blob=%q", registryURL, layerNode.Layer)
830
-		if err := blobPruner.PruneBlob(registryClient, registryURL, layerNode.Layer); err != nil {
831
-			errs = append(errs, fmt.Errorf("error pruning blob %q: %v", layerNode.Layer, err))
832
-		}
833
-	}
834
-
835
-	return errs
836
-}
837
-
838
-// pruneManifests invokes manifestPruner.PruneManifest for each repository
839
-// manifest to be deleted from the registry.
840
-func pruneManifests(g graph.Graph, registryClient *http.Client, registryURL string, imageNodes []*imagegraph.ImageNode, manifestPruner ManifestPruner) []error {
841
-	errs := []error{}
842
-
843
-	for _, imageNode := range imageNodes {
844
-		for _, n := range g.To(imageNode) {
845
-			streamNode, ok := n.(*imagegraph.ImageStreamNode)
846
-			if !ok {
847
-				continue
848
-			}
849
-
850
-			stream := streamNode.ImageStream
851
-			repoName := fmt.Sprintf("%s/%s", stream.Namespace, stream.Name)
852
-
853
-			glog.V(4).Infof("Pruning manifest for registry %q, repo %q, image %q", registryURL, repoName, imageNode.Image.Name)
854
-			if err := manifestPruner.PruneManifest(registryClient, registryURL, repoName, imageNode.Image.Name); err != nil {
855
-				errs = append(errs, fmt.Errorf("error pruning manifest for registry %q, repo %q, image %q: %v", registryURL, repoName, imageNode.Image.Name, err))
856
-			}
857
-		}
858
-	}
859
-
860
-	return errs
861
-}
862
-
863
-// deletingImagePruner deletes an image from OpenShift.
864
-type deletingImagePruner struct {
865
-	images client.ImageInterface
866
-}
867
-
868
-var _ ImagePruner = &deletingImagePruner{}
869
-
870
-// NewDeletingImagePruner creates a new deletingImagePruner.
871
-func NewDeletingImagePruner(images client.ImageInterface) ImagePruner {
872
-	return &deletingImagePruner{
873
-		images: images,
874
-	}
875
-}
876
-
877
-func (p *deletingImagePruner) PruneImage(image *imageapi.Image) error {
878
-	glog.V(4).Infof("Deleting image %q", image.Name)
879
-	return p.images.Delete(image.Name)
880
-}
881
-
882
-// deletingImageStreamPruner updates an image stream in OpenShift.
883
-type deletingImageStreamPruner struct {
884
-	streams client.ImageStreamsNamespacer
885
-}
886
-
887
-var _ ImageStreamPruner = &deletingImageStreamPruner{}
888
-
889
-// NewDeletingImageStreamPruner creates a new deletingImageStreamPruner.
890
-func NewDeletingImageStreamPruner(streams client.ImageStreamsNamespacer) ImageStreamPruner {
891
-	return &deletingImageStreamPruner{
892
-		streams: streams,
893
-	}
894
-}
895
-
896
-func (p *deletingImageStreamPruner) PruneImageStream(stream *imageapi.ImageStream, image *imageapi.Image, updatedTags []string) (*imageapi.ImageStream, error) {
897
-	glog.V(4).Infof("Updating ImageStream %s/%s", stream.Namespace, stream.Name)
898
-	glog.V(5).Infof("Updated stream: %#v", stream)
899
-	return p.streams.ImageStreams(stream.Namespace).UpdateStatus(stream)
900
-}
901
-
902
-// deleteFromRegistry uses registryClient to send a DELETE request to the
903
-// provided url. It attempts an https request first; if that fails, it fails
904
-// back to http.
905
-func deleteFromRegistry(registryClient *http.Client, url string) error {
906
-	deleteFunc := func(proto, url string) error {
907
-		req, err := http.NewRequest("DELETE", url, nil)
908
-		if err != nil {
909
-			glog.Errorf("Error creating request: %v", err)
910
-			return fmt.Errorf("error creating request: %v", err)
911
-		}
912
-
913
-		glog.V(4).Infof("Sending request to registry")
914
-		resp, err := registryClient.Do(req)
915
-		if err != nil {
916
-			return fmt.Errorf("error sending request: %v", err)
917
-		}
918
-		defer resp.Body.Close()
919
-
920
-		// TODO: investigate why we're getting non-existent layers, for now we're logging
921
-		// them out and continue working
922
-		if resp.StatusCode == http.StatusNotFound {
923
-			glog.Warningf("Unable to prune layer %s, returned %v", url, resp.Status)
924
-			return nil
925
-		}
926
-		// non-2xx/3xx response doesn't cause an error, so we need to check for it
927
-		// manually and return it to caller
928
-		if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest {
929
-			return fmt.Errorf(resp.Status)
930
-		}
931
-		if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusAccepted {
932
-			glog.V(1).Infof("Unexpected status code in response: %d", resp.StatusCode)
933
-			var response errcode.Errors
934
-			decoder := json.NewDecoder(resp.Body)
935
-			if err := decoder.Decode(&response); err != nil {
936
-				return err
937
-			}
938
-			glog.V(1).Infof("Response: %#v", response)
939
-			return &response
940
-		}
941
-
942
-		return nil
943
-	}
944
-
945
-	var err error
946
-	for _, proto := range []string{"https", "http"} {
947
-		glog.V(4).Infof("Trying %s for %s", proto, url)
948
-		err = deleteFunc(proto, fmt.Sprintf("%s://%s", proto, url))
949
-		if err == nil {
950
-			return nil
951
-		}
952
-
953
-		if _, ok := err.(*errcode.Errors); ok {
954
-			// we got a response back from the registry, so return it
955
-			return err
956
-		}
957
-
958
-		// we didn't get a success or a errcode.Errors response back from the registry
959
-		glog.V(4).Infof("Error with %s for %s: %v", proto, url, err)
960
-	}
961
-	return err
962
-}
963
-
964
-// deletingLayerPruner deletes a repository layer link from the registry.
965
-type deletingLayerPruner struct {
966
-}
967
-
968
-var _ LayerPruner = &deletingLayerPruner{}
969
-
970
-// NewDeletingLayerPruner creates a new deletingLayerPruner.
971
-func NewDeletingLayerPruner() LayerPruner {
972
-	return &deletingLayerPruner{}
973
-}
974
-
975
-func (p *deletingLayerPruner) PruneLayer(registryClient *http.Client, registryURL, repoName, layer string) error {
976
-	glog.V(4).Infof("Pruning registry %q, repo %q, layer %q", registryURL, repoName, layer)
977
-	return deleteFromRegistry(registryClient, fmt.Sprintf("%s/v2/%s/blobs/%s", registryURL, repoName, layer))
978
-}
979
-
980
-// deletingBlobPruner deletes a blob from the registry.
981
-type deletingBlobPruner struct {
982
-}
983
-
984
-var _ BlobPruner = &deletingBlobPruner{}
985
-
986
-// NewDeletingLayerPruner creates a new deletingBlobPruner.
987
-func NewDeletingBlobPruner() BlobPruner {
988
-	return &deletingBlobPruner{}
989
-}
990
-
991
-func (p *deletingBlobPruner) PruneBlob(registryClient *http.Client, registryURL, blob string) error {
992
-	glog.V(4).Infof("Pruning registry %q, blob %q", registryURL, blob)
993
-	return deleteFromRegistry(registryClient, fmt.Sprintf("%s/admin/blobs/%s", registryURL, blob))
994
-}
995
-
996
-// deletingManifestPruner deletes repository manifest data from the registry.
997
-type deletingManifestPruner struct {
998
-}
999
-
1000
-var _ ManifestPruner = &deletingManifestPruner{}
1001
-
1002
-// NewDeletingManifestPruner creates a new deletingManifestPruner.
1003
-func NewDeletingManifestPruner() ManifestPruner {
1004
-	return &deletingManifestPruner{}
1005
-}
1006
-
1007
-func (p *deletingManifestPruner) PruneManifest(registryClient *http.Client, registryURL, repoName, manifest string) error {
1008
-	glog.V(4).Infof("Pruning manifest for registry %q, repo %q, manifest %q", registryURL, repoName, manifest)
1009
-	return deleteFromRegistry(registryClient, fmt.Sprintf("%s/v2/%s/manifests/%s", registryURL, repoName, manifest))
1010
-}
1011 1
deleted file mode 100644
... ...
@@ -1,915 +0,0 @@
1
-package prune
2
-
3
-import (
4
-	"bytes"
5
-	"encoding/json"
6
-	"errors"
7
-	"flag"
8
-	"fmt"
9
-	"io/ioutil"
10
-	"net/http"
11
-	"reflect"
12
-	"testing"
13
-	"time"
14
-
15
-	kapi "k8s.io/kubernetes/pkg/api"
16
-	"k8s.io/kubernetes/pkg/api/unversioned"
17
-	"k8s.io/kubernetes/pkg/client/unversioned/fake"
18
-	ktc "k8s.io/kubernetes/pkg/client/unversioned/testclient"
19
-	"k8s.io/kubernetes/pkg/runtime"
20
-	"k8s.io/kubernetes/pkg/util/sets"
21
-
22
-	buildapi "github.com/openshift/origin/pkg/build/api"
23
-	"github.com/openshift/origin/pkg/client/testclient"
24
-	deployapi "github.com/openshift/origin/pkg/deploy/api"
25
-	imageapi "github.com/openshift/origin/pkg/image/api"
26
-)
27
-
28
-type fakeRegistryPinger struct {
29
-	err      error
30
-	requests []string
31
-}
32
-
33
-func (f *fakeRegistryPinger) ping(registry string) error {
34
-	f.requests = append(f.requests, registry)
35
-	return f.err
36
-}
37
-
38
-func imageList(images ...imageapi.Image) imageapi.ImageList {
39
-	return imageapi.ImageList{
40
-		Items: images,
41
-	}
42
-}
43
-
44
-func agedImage(id, ref string, ageInMinutes int64) imageapi.Image {
45
-	image := imageWithLayers(id, ref,
46
-		"tarsum.dev+sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
47
-		"tarsum.dev+sha256:b194de3772ebbcdc8f244f663669799ac1cb141834b7cb8b69100285d357a2b0",
48
-		"tarsum.dev+sha256:c937c4bb1c1a21cc6d94340812262c6472092028972ae69b551b1a70d4276171",
49
-		"tarsum.dev+sha256:2aaacc362ac6be2b9e9ae8c6029f6f616bb50aec63746521858e47841b90fabd",
50
-		"tarsum.dev+sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
51
-	)
52
-
53
-	if ageInMinutes >= 0 {
54
-		image.CreationTimestamp = unversioned.NewTime(unversioned.Now().Add(time.Duration(-1*ageInMinutes) * time.Minute))
55
-	}
56
-
57
-	return image
58
-}
59
-
60
-func image(id, ref string) imageapi.Image {
61
-	return agedImage(id, ref, -1)
62
-}
63
-
64
-func imageWithLayers(id, ref string, layers ...string) imageapi.Image {
65
-	image := imageapi.Image{
66
-		ObjectMeta: kapi.ObjectMeta{
67
-			Name: id,
68
-			Annotations: map[string]string{
69
-				imageapi.ManagedByOpenShiftAnnotation: "true",
70
-			},
71
-		},
72
-		DockerImageReference: ref,
73
-	}
74
-
75
-	manifest := imageapi.DockerImageManifest{
76
-		FSLayers: []imageapi.DockerFSLayer{},
77
-	}
78
-
79
-	for _, layer := range layers {
80
-		manifest.FSLayers = append(manifest.FSLayers, imageapi.DockerFSLayer{DockerBlobSum: layer})
81
-	}
82
-
83
-	manifestBytes, err := json.Marshal(&manifest)
84
-	if err != nil {
85
-		panic(err)
86
-	}
87
-
88
-	image.DockerImageManifest = string(manifestBytes)
89
-
90
-	return image
91
-}
92
-
93
-func unmanagedImage(id, ref string, hasAnnotations bool, annotation, value string) imageapi.Image {
94
-	image := imageWithLayers(id, ref)
95
-	if !hasAnnotations {
96
-		image.Annotations = nil
97
-	} else {
98
-		delete(image.Annotations, imageapi.ManagedByOpenShiftAnnotation)
99
-		image.Annotations[annotation] = value
100
-	}
101
-	return image
102
-}
103
-
104
-func imageWithBadManifest(id, ref string) imageapi.Image {
105
-	image := image(id, ref)
106
-	image.DockerImageManifest = "asdf"
107
-	return image
108
-}
109
-
110
-func podList(pods ...kapi.Pod) kapi.PodList {
111
-	return kapi.PodList{
112
-		Items: pods,
113
-	}
114
-}
115
-
116
-func pod(namespace, name string, phase kapi.PodPhase, containerImages ...string) kapi.Pod {
117
-	return agedPod(namespace, name, phase, -1, containerImages...)
118
-}
119
-
120
-func agedPod(namespace, name string, phase kapi.PodPhase, ageInMinutes int64, containerImages ...string) kapi.Pod {
121
-	pod := kapi.Pod{
122
-		ObjectMeta: kapi.ObjectMeta{
123
-			Namespace: namespace,
124
-			Name:      name,
125
-		},
126
-		Spec: podSpec(containerImages...),
127
-		Status: kapi.PodStatus{
128
-			Phase: phase,
129
-		},
130
-	}
131
-
132
-	if ageInMinutes >= 0 {
133
-		pod.CreationTimestamp = unversioned.NewTime(unversioned.Now().Add(time.Duration(-1*ageInMinutes) * time.Minute))
134
-	}
135
-
136
-	return pod
137
-}
138
-
139
-func podSpec(containerImages ...string) kapi.PodSpec {
140
-	spec := kapi.PodSpec{
141
-		Containers: []kapi.Container{},
142
-	}
143
-	for _, image := range containerImages {
144
-		container := kapi.Container{
145
-			Image: image,
146
-		}
147
-		spec.Containers = append(spec.Containers, container)
148
-	}
149
-	return spec
150
-}
151
-
152
-func streamList(streams ...imageapi.ImageStream) imageapi.ImageStreamList {
153
-	return imageapi.ImageStreamList{
154
-		Items: streams,
155
-	}
156
-}
157
-
158
-func stream(registry, namespace, name string, tags map[string]imageapi.TagEventList) imageapi.ImageStream {
159
-	return agedStream(registry, namespace, name, -1, tags)
160
-}
161
-
162
-func agedStream(registry, namespace, name string, ageInMinutes int64, tags map[string]imageapi.TagEventList) imageapi.ImageStream {
163
-	stream := imageapi.ImageStream{
164
-		ObjectMeta: kapi.ObjectMeta{
165
-			Namespace: namespace,
166
-			Name:      name,
167
-		},
168
-		Status: imageapi.ImageStreamStatus{
169
-			DockerImageRepository: fmt.Sprintf("%s/%s/%s", registry, namespace, name),
170
-			Tags: tags,
171
-		},
172
-	}
173
-
174
-	if ageInMinutes >= 0 {
175
-		stream.CreationTimestamp = unversioned.NewTime(unversioned.Now().Add(time.Duration(-1*ageInMinutes) * time.Minute))
176
-	}
177
-
178
-	return stream
179
-}
180
-
181
-func streamPtr(registry, namespace, name string, tags map[string]imageapi.TagEventList) *imageapi.ImageStream {
182
-	s := stream(registry, namespace, name, tags)
183
-	return &s
184
-}
185
-
186
-func tags(list ...namedTagEventList) map[string]imageapi.TagEventList {
187
-	m := make(map[string]imageapi.TagEventList, len(list))
188
-	for _, tag := range list {
189
-		m[tag.name] = tag.events
190
-	}
191
-	return m
192
-}
193
-
194
-type namedTagEventList struct {
195
-	name   string
196
-	events imageapi.TagEventList
197
-}
198
-
199
-func tag(name string, events ...imageapi.TagEvent) namedTagEventList {
200
-	return namedTagEventList{
201
-		name: name,
202
-		events: imageapi.TagEventList{
203
-			Items: events,
204
-		},
205
-	}
206
-}
207
-
208
-func tagEvent(id, ref string) imageapi.TagEvent {
209
-	return imageapi.TagEvent{
210
-		Image:                id,
211
-		DockerImageReference: ref,
212
-	}
213
-}
214
-
215
-func rcList(rcs ...kapi.ReplicationController) kapi.ReplicationControllerList {
216
-	return kapi.ReplicationControllerList{
217
-		Items: rcs,
218
-	}
219
-}
220
-
221
-func rc(namespace, name string, containerImages ...string) kapi.ReplicationController {
222
-	return kapi.ReplicationController{
223
-		ObjectMeta: kapi.ObjectMeta{
224
-			Namespace: namespace,
225
-			Name:      name,
226
-		},
227
-		Spec: kapi.ReplicationControllerSpec{
228
-			Template: &kapi.PodTemplateSpec{
229
-				Spec: podSpec(containerImages...),
230
-			},
231
-		},
232
-	}
233
-}
234
-
235
-func dcList(dcs ...deployapi.DeploymentConfig) deployapi.DeploymentConfigList {
236
-	return deployapi.DeploymentConfigList{
237
-		Items: dcs,
238
-	}
239
-}
240
-
241
-func dc(namespace, name string, containerImages ...string) deployapi.DeploymentConfig {
242
-	return deployapi.DeploymentConfig{
243
-		ObjectMeta: kapi.ObjectMeta{
244
-			Namespace: namespace,
245
-			Name:      name,
246
-		},
247
-		Spec: deployapi.DeploymentConfigSpec{
248
-			Template: &kapi.PodTemplateSpec{
249
-				Spec: podSpec(containerImages...),
250
-			},
251
-		},
252
-	}
253
-}
254
-
255
-func bcList(bcs ...buildapi.BuildConfig) buildapi.BuildConfigList {
256
-	return buildapi.BuildConfigList{
257
-		Items: bcs,
258
-	}
259
-}
260
-
261
-func bc(namespace, name, strategyType, fromKind, fromNamespace, fromName string) buildapi.BuildConfig {
262
-	return buildapi.BuildConfig{
263
-		ObjectMeta: kapi.ObjectMeta{
264
-			Namespace: namespace,
265
-			Name:      name,
266
-		},
267
-		Spec: buildapi.BuildConfigSpec{
268
-			CommonSpec: commonSpec(strategyType, fromKind, fromNamespace, fromName),
269
-		},
270
-	}
271
-}
272
-
273
-func buildList(builds ...buildapi.Build) buildapi.BuildList {
274
-	return buildapi.BuildList{
275
-		Items: builds,
276
-	}
277
-}
278
-
279
-func build(namespace, name, strategyType, fromKind, fromNamespace, fromName string) buildapi.Build {
280
-	return buildapi.Build{
281
-		ObjectMeta: kapi.ObjectMeta{
282
-			Namespace: namespace,
283
-			Name:      name,
284
-		},
285
-		Spec: buildapi.BuildSpec{
286
-			CommonSpec: commonSpec(strategyType, fromKind, fromNamespace, fromName),
287
-		},
288
-	}
289
-}
290
-
291
-func commonSpec(strategyType, fromKind, fromNamespace, fromName string) buildapi.CommonSpec {
292
-	spec := buildapi.CommonSpec{
293
-		Strategy: buildapi.BuildStrategy{},
294
-	}
295
-	switch strategyType {
296
-	case "source":
297
-		spec.Strategy.SourceStrategy = &buildapi.SourceBuildStrategy{
298
-			From: kapi.ObjectReference{
299
-				Kind:      fromKind,
300
-				Namespace: fromNamespace,
301
-				Name:      fromName,
302
-			},
303
-		}
304
-	case "docker":
305
-		spec.Strategy.DockerStrategy = &buildapi.DockerBuildStrategy{
306
-			From: &kapi.ObjectReference{
307
-				Kind:      fromKind,
308
-				Namespace: fromNamespace,
309
-				Name:      fromName,
310
-			},
311
-		}
312
-	case "custom":
313
-		spec.Strategy.CustomStrategy = &buildapi.CustomBuildStrategy{
314
-			From: kapi.ObjectReference{
315
-				Kind:      fromKind,
316
-				Namespace: fromNamespace,
317
-				Name:      fromName,
318
-			},
319
-		}
320
-	}
321
-
322
-	return spec
323
-}
324
-
325
-type fakeImagePruner struct {
326
-	invocations sets.String
327
-	err         error
328
-}
329
-
330
-var _ ImagePruner = &fakeImagePruner{}
331
-
332
-func (p *fakeImagePruner) PruneImage(image *imageapi.Image) error {
333
-	p.invocations.Insert(image.Name)
334
-	return p.err
335
-}
336
-
337
-type fakeImageStreamPruner struct {
338
-	invocations sets.String
339
-	err         error
340
-}
341
-
342
-var _ ImageStreamPruner = &fakeImageStreamPruner{}
343
-
344
-func (p *fakeImageStreamPruner) PruneImageStream(stream *imageapi.ImageStream, image *imageapi.Image, updatedTags []string) (*imageapi.ImageStream, error) {
345
-	p.invocations.Insert(fmt.Sprintf("%s/%s|%s", stream.Namespace, stream.Name, image.Name))
346
-	return stream, p.err
347
-}
348
-
349
-type fakeBlobPruner struct {
350
-	invocations sets.String
351
-	err         error
352
-}
353
-
354
-var _ BlobPruner = &fakeBlobPruner{}
355
-
356
-func (p *fakeBlobPruner) PruneBlob(registryClient *http.Client, registryURL, blob string) error {
357
-	p.invocations.Insert(fmt.Sprintf("%s|%s", registryURL, blob))
358
-	return p.err
359
-}
360
-
361
-type fakeLayerPruner struct {
362
-	invocations sets.String
363
-	err         error
364
-}
365
-
366
-var _ LayerPruner = &fakeLayerPruner{}
367
-
368
-func (p *fakeLayerPruner) PruneLayer(registryClient *http.Client, registryURL, repo, layer string) error {
369
-	p.invocations.Insert(fmt.Sprintf("%s|%s|%s", registryURL, repo, layer))
370
-	return p.err
371
-}
372
-
373
-type fakeManifestPruner struct {
374
-	invocations sets.String
375
-	err         error
376
-}
377
-
378
-var _ ManifestPruner = &fakeManifestPruner{}
379
-
380
-func (p *fakeManifestPruner) PruneManifest(registryClient *http.Client, registryURL, repo, manifest string) error {
381
-	p.invocations.Insert(fmt.Sprintf("%s|%s|%s", registryURL, repo, manifest))
382
-	return p.err
383
-}
384
-
385
-var logLevel = flag.Int("loglevel", 0, "")
386
-var testCase = flag.String("testcase", "", "")
387
-
388
-func TestImagePruning(t *testing.T) {
389
-	flag.Lookup("v").Value.Set(fmt.Sprint(*logLevel))
390
-	registryURL := "registry"
391
-
392
-	tests := map[string]struct {
393
-		registryURLs           []string
394
-		images                 imageapi.ImageList
395
-		pods                   kapi.PodList
396
-		streams                imageapi.ImageStreamList
397
-		rcs                    kapi.ReplicationControllerList
398
-		bcs                    buildapi.BuildConfigList
399
-		builds                 buildapi.BuildList
400
-		dcs                    deployapi.DeploymentConfigList
401
-		expectedDeletions      []string
402
-		expectedUpdatedStreams []string
403
-	}{
404
-		"1 pod - phase pending - don't prune": {
405
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
406
-			pods:              podList(pod("foo", "pod1", kapi.PodPending, registryURL+"/foo/bar@id")),
407
-			expectedDeletions: []string{},
408
-		},
409
-		"3 pods - last phase pending - don't prune": {
410
-			images: imageList(image("id", registryURL+"/foo/bar@id")),
411
-			pods: podList(
412
-				pod("foo", "pod1", kapi.PodSucceeded, registryURL+"/foo/bar@id"),
413
-				pod("foo", "pod2", kapi.PodSucceeded, registryURL+"/foo/bar@id"),
414
-				pod("foo", "pod3", kapi.PodPending, registryURL+"/foo/bar@id"),
415
-			),
416
-			expectedDeletions: []string{},
417
-		},
418
-		"1 pod - phase running - don't prune": {
419
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
420
-			pods:              podList(pod("foo", "pod1", kapi.PodRunning, registryURL+"/foo/bar@id")),
421
-			expectedDeletions: []string{},
422
-		},
423
-		"3 pods - last phase running - don't prune": {
424
-			images: imageList(image("id", registryURL+"/foo/bar@id")),
425
-			pods: podList(
426
-				pod("foo", "pod1", kapi.PodSucceeded, registryURL+"/foo/bar@id"),
427
-				pod("foo", "pod2", kapi.PodSucceeded, registryURL+"/foo/bar@id"),
428
-				pod("foo", "pod3", kapi.PodRunning, registryURL+"/foo/bar@id"),
429
-			),
430
-			expectedDeletions: []string{},
431
-		},
432
-		"pod phase succeeded - prune": {
433
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
434
-			pods:              podList(pod("foo", "pod1", kapi.PodSucceeded, registryURL+"/foo/bar@id")),
435
-			expectedDeletions: []string{"id"},
436
-		},
437
-		"pod phase succeeded, pod less than min pruning age - don't prune": {
438
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
439
-			pods:              podList(agedPod("foo", "pod1", kapi.PodSucceeded, 5, registryURL+"/foo/bar@id")),
440
-			expectedDeletions: []string{},
441
-		},
442
-		"pod phase succeeded, image less than min pruning age - don't prune": {
443
-			images:            imageList(agedImage("id", registryURL+"/foo/bar@id", 5)),
444
-			pods:              podList(pod("foo", "pod1", kapi.PodSucceeded, registryURL+"/foo/bar@id")),
445
-			expectedDeletions: []string{},
446
-		},
447
-		"pod phase failed - prune": {
448
-			images: imageList(image("id", registryURL+"/foo/bar@id")),
449
-			pods: podList(
450
-				pod("foo", "pod1", kapi.PodFailed, registryURL+"/foo/bar@id"),
451
-				pod("foo", "pod2", kapi.PodFailed, registryURL+"/foo/bar@id"),
452
-				pod("foo", "pod3", kapi.PodFailed, registryURL+"/foo/bar@id"),
453
-			),
454
-			expectedDeletions: []string{"id"},
455
-		},
456
-		"pod phase unknown - prune": {
457
-			images: imageList(image("id", registryURL+"/foo/bar@id")),
458
-			pods: podList(
459
-				pod("foo", "pod1", kapi.PodUnknown, registryURL+"/foo/bar@id"),
460
-				pod("foo", "pod2", kapi.PodUnknown, registryURL+"/foo/bar@id"),
461
-				pod("foo", "pod3", kapi.PodUnknown, registryURL+"/foo/bar@id"),
462
-			),
463
-			expectedDeletions: []string{"id"},
464
-		},
465
-		"pod container image not parsable": {
466
-			images: imageList(image("id", registryURL+"/foo/bar@id")),
467
-			pods: podList(
468
-				pod("foo", "pod1", kapi.PodRunning, "a/b/c/d/e"),
469
-			),
470
-			expectedDeletions: []string{"id"},
471
-		},
472
-		"pod container image doesn't have an id": {
473
-			images: imageList(image("id", registryURL+"/foo/bar@id")),
474
-			pods: podList(
475
-				pod("foo", "pod1", kapi.PodRunning, "foo/bar:latest"),
476
-			),
477
-			expectedDeletions: []string{"id"},
478
-		},
479
-		"pod refers to image not in graph": {
480
-			images: imageList(image("id", registryURL+"/foo/bar@id")),
481
-			pods: podList(
482
-				pod("foo", "pod1", kapi.PodRunning, registryURL+"/foo/bar@otherid"),
483
-			),
484
-			expectedDeletions: []string{"id"},
485
-		},
486
-		"referenced by rc - don't prune": {
487
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
488
-			rcs:               rcList(rc("foo", "rc1", registryURL+"/foo/bar@id")),
489
-			expectedDeletions: []string{},
490
-		},
491
-		"referenced by dc - don't prune": {
492
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
493
-			dcs:               dcList(dc("foo", "rc1", registryURL+"/foo/bar@id")),
494
-			expectedDeletions: []string{},
495
-		},
496
-		"referenced by bc - sti - ImageStreamImage - don't prune": {
497
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
498
-			bcs:               bcList(bc("foo", "bc1", "source", "ImageStreamImage", "foo", "bar@id")),
499
-			expectedDeletions: []string{},
500
-		},
501
-		"referenced by bc - docker - ImageStreamImage - don't prune": {
502
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
503
-			bcs:               bcList(bc("foo", "bc1", "docker", "ImageStreamImage", "foo", "bar@id")),
504
-			expectedDeletions: []string{},
505
-		},
506
-		"referenced by bc - custom - ImageStreamImage - don't prune": {
507
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
508
-			bcs:               bcList(bc("foo", "bc1", "custom", "ImageStreamImage", "foo", "bar@id")),
509
-			expectedDeletions: []string{},
510
-		},
511
-		"referenced by bc - sti - DockerImage - don't prune": {
512
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
513
-			bcs:               bcList(bc("foo", "bc1", "source", "DockerImage", "foo", registryURL+"/foo/bar@id")),
514
-			expectedDeletions: []string{},
515
-		},
516
-		"referenced by bc - docker - DockerImage - don't prune": {
517
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
518
-			bcs:               bcList(bc("foo", "bc1", "docker", "DockerImage", "foo", registryURL+"/foo/bar@id")),
519
-			expectedDeletions: []string{},
520
-		},
521
-		"referenced by bc - custom - DockerImage - don't prune": {
522
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
523
-			bcs:               bcList(bc("foo", "bc1", "custom", "DockerImage", "foo", registryURL+"/foo/bar@id")),
524
-			expectedDeletions: []string{},
525
-		},
526
-		"referenced by build - sti - ImageStreamImage - don't prune": {
527
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
528
-			builds:            buildList(build("foo", "build1", "source", "ImageStreamImage", "foo", "bar@id")),
529
-			expectedDeletions: []string{},
530
-		},
531
-		"referenced by build - docker - ImageStreamImage - don't prune": {
532
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
533
-			builds:            buildList(build("foo", "build1", "docker", "ImageStreamImage", "foo", "bar@id")),
534
-			expectedDeletions: []string{},
535
-		},
536
-		"referenced by build - custom - ImageStreamImage - don't prune": {
537
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
538
-			builds:            buildList(build("foo", "build1", "custom", "ImageStreamImage", "foo", "bar@id")),
539
-			expectedDeletions: []string{},
540
-		},
541
-		"referenced by build - sti - DockerImage - don't prune": {
542
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
543
-			builds:            buildList(build("foo", "build1", "source", "DockerImage", "foo", registryURL+"/foo/bar@id")),
544
-			expectedDeletions: []string{},
545
-		},
546
-		"referenced by build - docker - DockerImage - don't prune": {
547
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
548
-			builds:            buildList(build("foo", "build1", "docker", "DockerImage", "foo", registryURL+"/foo/bar@id")),
549
-			expectedDeletions: []string{},
550
-		},
551
-		"referenced by build - custom - DockerImage - don't prune": {
552
-			images:            imageList(image("id", registryURL+"/foo/bar@id")),
553
-			builds:            buildList(build("foo", "build1", "custom", "DockerImage", "foo", registryURL+"/foo/bar@id")),
554
-			expectedDeletions: []string{},
555
-		},
556
-		"image stream - keep most recent n images": {
557
-			images: imageList(
558
-				unmanagedImage("id", "otherregistry/foo/bar@id", false, "", ""),
559
-				image("id2", registryURL+"/foo/bar@id2"),
560
-				image("id3", registryURL+"/foo/bar@id3"),
561
-				image("id4", registryURL+"/foo/bar@id4"),
562
-			),
563
-			streams: streamList(
564
-				stream(registryURL, "foo", "bar", tags(
565
-					tag("latest",
566
-						tagEvent("id", "otherregistry/foo/bar@id"),
567
-						tagEvent("id2", registryURL+"/foo/bar@id2"),
568
-						tagEvent("id3", registryURL+"/foo/bar@id3"),
569
-						tagEvent("id4", registryURL+"/foo/bar@id4"),
570
-					),
571
-				)),
572
-			),
573
-			expectedDeletions:      []string{"id4"},
574
-			expectedUpdatedStreams: []string{"foo/bar|id4"},
575
-		},
576
-		"image stream - same manifest listed multiple times in tag history": {
577
-			images: imageList(
578
-				image("id1", registryURL+"/foo/bar@id1"),
579
-				image("id2", registryURL+"/foo/bar@id2"),
580
-			),
581
-			streams: streamList(
582
-				stream(registryURL, "foo", "bar", tags(
583
-					tag("latest",
584
-						tagEvent("id1", registryURL+"/foo/bar@id1"),
585
-						tagEvent("id2", registryURL+"/foo/bar@id2"),
586
-						tagEvent("id1", registryURL+"/foo/bar@id1"),
587
-						tagEvent("id2", registryURL+"/foo/bar@id2"),
588
-					),
589
-				)),
590
-			),
591
-		},
592
-		"image stream age less than min pruning age - don't prune": {
593
-			images: imageList(
594
-				image("id", registryURL+"/foo/bar@id"),
595
-				image("id2", registryURL+"/foo/bar@id2"),
596
-				image("id3", registryURL+"/foo/bar@id3"),
597
-				image("id4", registryURL+"/foo/bar@id4"),
598
-			),
599
-			streams: streamList(
600
-				agedStream(registryURL, "foo", "bar", 5, tags(
601
-					tag("latest",
602
-						tagEvent("id", registryURL+"/foo/bar@id"),
603
-						tagEvent("id2", registryURL+"/foo/bar@id2"),
604
-						tagEvent("id3", registryURL+"/foo/bar@id3"),
605
-						tagEvent("id4", registryURL+"/foo/bar@id4"),
606
-					),
607
-				)),
608
-			),
609
-			expectedDeletions:      []string{},
610
-			expectedUpdatedStreams: []string{},
611
-		},
612
-		"multiple resources pointing to image - don't prune": {
613
-			images: imageList(
614
-				image("id", registryURL+"/foo/bar@id"),
615
-				image("id2", registryURL+"/foo/bar@id2"),
616
-			),
617
-			streams: streamList(
618
-				stream(registryURL, "foo", "bar", tags(
619
-					tag("latest",
620
-						tagEvent("id", registryURL+"/foo/bar@id"),
621
-						tagEvent("id2", registryURL+"/foo/bar@id2"),
622
-					),
623
-				)),
624
-			),
625
-			rcs:                    rcList(rc("foo", "rc1", registryURL+"/foo/bar@id2")),
626
-			pods:                   podList(pod("foo", "pod1", kapi.PodRunning, registryURL+"/foo/bar@id2")),
627
-			dcs:                    dcList(dc("foo", "rc1", registryURL+"/foo/bar@id")),
628
-			bcs:                    bcList(bc("foo", "bc1", "source", "DockerImage", "foo", registryURL+"/foo/bar@id")),
629
-			builds:                 buildList(build("foo", "build1", "custom", "ImageStreamImage", "foo", "bar@id")),
630
-			expectedDeletions:      []string{},
631
-			expectedUpdatedStreams: []string{},
632
-		},
633
-		"image with nil annotations": {
634
-			images: imageList(
635
-				unmanagedImage("id", "someregistry/foo/bar@id", false, "", ""),
636
-			),
637
-			expectedDeletions:      []string{},
638
-			expectedUpdatedStreams: []string{},
639
-		},
640
-		"image missing managed annotation": {
641
-			images: imageList(
642
-				unmanagedImage("id", "someregistry/foo/bar@id", true, "foo", "bar"),
643
-			),
644
-			expectedDeletions:      []string{},
645
-			expectedUpdatedStreams: []string{},
646
-		},
647
-		"image with managed annotation != true": {
648
-			images: imageList(
649
-				unmanagedImage("id", "someregistry/foo/bar@id", true, imageapi.ManagedByOpenShiftAnnotation, "false"),
650
-				unmanagedImage("id", "someregistry/foo/bar@id", true, imageapi.ManagedByOpenShiftAnnotation, "0"),
651
-				unmanagedImage("id", "someregistry/foo/bar@id", true, imageapi.ManagedByOpenShiftAnnotation, "1"),
652
-				unmanagedImage("id", "someregistry/foo/bar@id", true, imageapi.ManagedByOpenShiftAnnotation, "True"),
653
-				unmanagedImage("id", "someregistry/foo/bar@id", true, imageapi.ManagedByOpenShiftAnnotation, "yes"),
654
-				unmanagedImage("id", "someregistry/foo/bar@id", true, imageapi.ManagedByOpenShiftAnnotation, "Yes"),
655
-			),
656
-			expectedDeletions:      []string{},
657
-			expectedUpdatedStreams: []string{},
658
-		},
659
-		"image with bad manifest is pruned ok": {
660
-			images: imageList(
661
-				imageWithBadManifest("id", "someregistry/foo/bar@id"),
662
-			),
663
-			expectedDeletions:      []string{"id"},
664
-			expectedUpdatedStreams: []string{},
665
-		},
666
-	}
667
-
668
-	for name, test := range tests {
669
-		tcFilter := flag.Lookup("testcase").Value.String()
670
-		if len(tcFilter) > 0 && name != tcFilter {
671
-			continue
672
-		}
673
-
674
-		options := ImageRegistryPrunerOptions{
675
-			KeepYoungerThan:  60 * time.Minute,
676
-			KeepTagRevisions: 3,
677
-			Images:           &test.images,
678
-			Streams:          &test.streams,
679
-			Pods:             &test.pods,
680
-			RCs:              &test.rcs,
681
-			BCs:              &test.bcs,
682
-			Builds:           &test.builds,
683
-			DCs:              &test.dcs,
684
-		}
685
-		p := NewImageRegistryPruner(options)
686
-		p.(*imageRegistryPruner).registryPinger = &fakeRegistryPinger{}
687
-
688
-		imagePruner := &fakeImagePruner{invocations: sets.NewString()}
689
-		streamPruner := &fakeImageStreamPruner{invocations: sets.NewString()}
690
-		layerPruner := &fakeLayerPruner{invocations: sets.NewString()}
691
-		blobPruner := &fakeBlobPruner{invocations: sets.NewString()}
692
-		manifestPruner := &fakeManifestPruner{invocations: sets.NewString()}
693
-
694
-		p.Prune(imagePruner, streamPruner, layerPruner, blobPruner, manifestPruner)
695
-
696
-		expectedDeletions := sets.NewString(test.expectedDeletions...)
697
-		if !reflect.DeepEqual(expectedDeletions, imagePruner.invocations) {
698
-			t.Errorf("%s: expected image deletions %q, got %q", name, expectedDeletions.List(), imagePruner.invocations.List())
699
-		}
700
-
701
-		expectedUpdatedStreams := sets.NewString(test.expectedUpdatedStreams...)
702
-		if !reflect.DeepEqual(expectedUpdatedStreams, streamPruner.invocations) {
703
-			t.Errorf("%s: expected stream updates %q, got %q", name, expectedUpdatedStreams.List(), streamPruner.invocations.List())
704
-		}
705
-	}
706
-}
707
-
708
-func TestDeletingImagePruner(t *testing.T) {
709
-	flag.Lookup("v").Value.Set(fmt.Sprint(*logLevel))
710
-
711
-	tests := map[string]struct {
712
-		imageDeletionError error
713
-	}{
714
-		"no error": {},
715
-		"delete error": {
716
-			imageDeletionError: fmt.Errorf("foo"),
717
-		},
718
-	}
719
-
720
-	for name, test := range tests {
721
-		imageClient := testclient.Fake{}
722
-		imageClient.AddReactor("delete", "images", func(action ktc.Action) (handled bool, ret runtime.Object, err error) {
723
-			return true, nil, test.imageDeletionError
724
-		})
725
-		imagePruner := NewDeletingImagePruner(imageClient.Images())
726
-		err := imagePruner.PruneImage(&imageapi.Image{ObjectMeta: kapi.ObjectMeta{Name: "id2"}})
727
-		if test.imageDeletionError != nil {
728
-			if e, a := test.imageDeletionError, err; e != a {
729
-				t.Errorf("%s: err: expected %v, got %v", name, e, a)
730
-			}
731
-			continue
732
-		}
733
-
734
-		if e, a := 1, len(imageClient.Actions()); e != a {
735
-			t.Errorf("%s: expected %d actions, got %d: %#v", name, e, a, imageClient.Actions())
736
-			continue
737
-		}
738
-
739
-		if !imageClient.Actions()[0].Matches("delete", "images") {
740
-			t.Errorf("%s: expected action %s, got %v", name, "delete-images", imageClient.Actions()[0])
741
-		}
742
-	}
743
-}
744
-
745
-func TestDeletingLayerPruner(t *testing.T) {
746
-	flag.Lookup("v").Value.Set(fmt.Sprint(*logLevel))
747
-
748
-	var actions []string
749
-	client := fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) {
750
-		actions = append(actions, req.Method+":"+req.URL.String())
751
-		return &http.Response{StatusCode: http.StatusServiceUnavailable, Body: ioutil.NopCloser(bytes.NewReader([]byte{}))}, nil
752
-	})
753
-	layerPruner := NewDeletingLayerPruner()
754
-	layerPruner.PruneLayer(client, "registry1", "repo", "layer1")
755
-
756
-	if !reflect.DeepEqual(actions, []string{"DELETE:https://registry1/v2/repo/blobs/layer1",
757
-		"DELETE:http://registry1/v2/repo/blobs/layer1"}) {
758
-		t.Errorf("Unexpected actions %v", actions)
759
-	}
760
-}
761
-
762
-func TestDeletingNotFoundLayerPruner(t *testing.T) {
763
-	flag.Lookup("v").Value.Set(fmt.Sprint(*logLevel))
764
-
765
-	var actions []string
766
-	client := fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) {
767
-		actions = append(actions, req.Method+":"+req.URL.String())
768
-		return &http.Response{StatusCode: http.StatusNotFound, Body: ioutil.NopCloser(bytes.NewReader([]byte{}))}, nil
769
-	})
770
-	layerPruner := NewDeletingLayerPruner()
771
-	layerPruner.PruneLayer(client, "registry1", "repo", "layer1")
772
-
773
-	if !reflect.DeepEqual(actions, []string{"DELETE:https://registry1/v2/repo/blobs/layer1"}) {
774
-		t.Errorf("Unexpected actions %v", actions)
775
-	}
776
-}
777
-
778
-func TestRegistryPruning(t *testing.T) {
779
-	flag.Lookup("v").Value.Set(fmt.Sprint(*logLevel))
780
-
781
-	tests := map[string]struct {
782
-		images                    imageapi.ImageList
783
-		streams                   imageapi.ImageStreamList
784
-		expectedLayerDeletions    sets.String
785
-		expectedBlobDeletions     sets.String
786
-		expectedManifestDeletions sets.String
787
-		pingErr                   error
788
-	}{
789
-		"layers unique to id1 pruned": {
790
-			images: imageList(
791
-				imageWithLayers("id1", "registry1/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"),
792
-				imageWithLayers("id2", "registry1/foo/bar@id2", "layer3", "layer4", "layer5", "layer6"),
793
-			),
794
-			streams: streamList(
795
-				stream("registry1", "foo", "bar", tags(
796
-					tag("latest",
797
-						tagEvent("id2", "registry1/foo/bar@id2"),
798
-						tagEvent("id1", "registry1/foo/bar@id1"),
799
-					),
800
-				)),
801
-				stream("registry1", "foo", "other", tags(
802
-					tag("latest",
803
-						tagEvent("id2", "registry1/foo/other@id2"),
804
-					),
805
-				)),
806
-			),
807
-			expectedLayerDeletions: sets.NewString(
808
-				"registry1|foo/bar|layer1",
809
-				"registry1|foo/bar|layer2",
810
-			),
811
-			expectedBlobDeletions: sets.NewString(
812
-				"registry1|layer1",
813
-				"registry1|layer2",
814
-			),
815
-			expectedManifestDeletions: sets.NewString(
816
-				"registry1|foo/bar|id1",
817
-			),
818
-		},
819
-		"no pruning when no images are pruned": {
820
-			images: imageList(
821
-				imageWithLayers("id1", "registry1/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"),
822
-			),
823
-			streams: streamList(
824
-				stream("registry1", "foo", "bar", tags(
825
-					tag("latest",
826
-						tagEvent("id1", "registry1/foo/bar@id1"),
827
-					),
828
-				)),
829
-			),
830
-			expectedLayerDeletions:    sets.NewString(),
831
-			expectedBlobDeletions:     sets.NewString(),
832
-			expectedManifestDeletions: sets.NewString(),
833
-		},
834
-		"blobs pruned when streams have already been deleted": {
835
-			images: imageList(
836
-				imageWithLayers("id1", "registry1/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"),
837
-				imageWithLayers("id2", "registry1/foo/bar@id2", "layer3", "layer4", "layer5", "layer6"),
838
-			),
839
-			expectedLayerDeletions: sets.NewString(),
840
-			expectedBlobDeletions: sets.NewString(
841
-				"registry1|layer1",
842
-				"registry1|layer2",
843
-				"registry1|layer3",
844
-				"registry1|layer4",
845
-				"registry1|layer5",
846
-				"registry1|layer6",
847
-			),
848
-			expectedManifestDeletions: sets.NewString(),
849
-		},
850
-		"ping error": {
851
-			images: imageList(
852
-				imageWithLayers("id1", "registry1/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"),
853
-				imageWithLayers("id2", "registry1/foo/bar@id2", "layer3", "layer4", "layer5", "layer6"),
854
-			),
855
-			streams: streamList(
856
-				stream("registry1", "foo", "bar", tags(
857
-					tag("latest",
858
-						tagEvent("id2", "registry1/foo/bar@id2"),
859
-						tagEvent("id1", "registry1/foo/bar@id1"),
860
-					),
861
-				)),
862
-				stream("registry1", "foo", "other", tags(
863
-					tag("latest",
864
-						tagEvent("id2", "registry1/foo/other@id2"),
865
-					),
866
-				)),
867
-			),
868
-			expectedLayerDeletions:    sets.NewString(),
869
-			expectedBlobDeletions:     sets.NewString(),
870
-			expectedManifestDeletions: sets.NewString(),
871
-			pingErr:                   errors.New("foo"),
872
-		},
873
-	}
874
-
875
-	for name, test := range tests {
876
-		tcFilter := flag.Lookup("testcase").Value.String()
877
-		if len(tcFilter) > 0 && name != tcFilter {
878
-			continue
879
-		}
880
-
881
-		t.Logf("Running test case %s", name)
882
-
883
-		options := ImageRegistryPrunerOptions{
884
-			KeepYoungerThan:  60 * time.Minute,
885
-			KeepTagRevisions: 1,
886
-			Images:           &test.images,
887
-			Streams:          &test.streams,
888
-			Pods:             &kapi.PodList{},
889
-			RCs:              &kapi.ReplicationControllerList{},
890
-			BCs:              &buildapi.BuildConfigList{},
891
-			Builds:           &buildapi.BuildList{},
892
-			DCs:              &deployapi.DeploymentConfigList{},
893
-		}
894
-		p := NewImageRegistryPruner(options)
895
-		p.(*imageRegistryPruner).registryPinger = &fakeRegistryPinger{err: test.pingErr}
896
-
897
-		imagePruner := &fakeImagePruner{invocations: sets.NewString()}
898
-		streamPruner := &fakeImageStreamPruner{invocations: sets.NewString()}
899
-		layerPruner := &fakeLayerPruner{invocations: sets.NewString()}
900
-		blobPruner := &fakeBlobPruner{invocations: sets.NewString()}
901
-		manifestPruner := &fakeManifestPruner{invocations: sets.NewString()}
902
-
903
-		p.Prune(imagePruner, streamPruner, layerPruner, blobPruner, manifestPruner)
904
-
905
-		if !reflect.DeepEqual(test.expectedLayerDeletions, layerPruner.invocations) {
906
-			t.Errorf("%s: expected layer deletions %#v, got %#v", name, test.expectedLayerDeletions, layerPruner.invocations)
907
-		}
908
-		if !reflect.DeepEqual(test.expectedBlobDeletions, blobPruner.invocations) {
909
-			t.Errorf("%s: expected blob deletions %#v, got %#v", name, test.expectedBlobDeletions, blobPruner.invocations)
910
-		}
911
-		if !reflect.DeepEqual(test.expectedManifestDeletions, manifestPruner.invocations) {
912
-			t.Errorf("%s: expected manifest deletions %#v, got %#v", name, test.expectedManifestDeletions, manifestPruner.invocations)
913
-		}
914
-	}
915
-}
916 1
new file mode 100644
... ...
@@ -0,0 +1,1003 @@
0
+package prune
1
+
2
+import (
3
+	"encoding/json"
4
+	"fmt"
5
+	"net/http"
6
+	"time"
7
+
8
+	"github.com/docker/distribution/registry/api/errcode"
9
+	"github.com/golang/glog"
10
+	gonum "github.com/gonum/graph"
11
+
12
+	kapi "k8s.io/kubernetes/pkg/api"
13
+	"k8s.io/kubernetes/pkg/api/unversioned"
14
+	kerrors "k8s.io/kubernetes/pkg/util/errors"
15
+	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
16
+	"k8s.io/kubernetes/pkg/util/sets"
17
+
18
+	"github.com/openshift/origin/pkg/api/graph"
19
+	kubegraph "github.com/openshift/origin/pkg/api/kubegraph/nodes"
20
+	buildapi "github.com/openshift/origin/pkg/build/api"
21
+	buildgraph "github.com/openshift/origin/pkg/build/graph/nodes"
22
+	buildutil "github.com/openshift/origin/pkg/build/util"
23
+	"github.com/openshift/origin/pkg/client"
24
+	deployapi "github.com/openshift/origin/pkg/deploy/api"
25
+	deploygraph "github.com/openshift/origin/pkg/deploy/graph/nodes"
26
+	imageapi "github.com/openshift/origin/pkg/image/api"
27
+	imagegraph "github.com/openshift/origin/pkg/image/graph/nodes"
28
+)
29
+
30
+// TODO these edges should probably have an `Add***Edges` method in images/graph and be moved there
31
+const (
32
+	// ReferencedImageEdgeKind defines a "strong" edge where the tail is an
33
+	// ImageNode, with strong indicating that the ImageNode tail is not a
34
+	// candidate for pruning.
35
+	ReferencedImageEdgeKind = "ReferencedImage"
36
+	// WeakReferencedImageEdgeKind defines a "weak" edge where the tail is
37
+	// an ImageNode, with weak indicating that this particular edge does
38
+	// not keep an ImageNode from being a candidate for pruning.
39
+	WeakReferencedImageEdgeKind = "WeakReferencedImage"
40
+
41
+	// ReferencedImageLayerEdgeKind defines an edge from an ImageStreamNode or an
42
+	// ImageNode to an ImageLayerNode.
43
+	ReferencedImageLayerEdgeKind = "ReferencedImageLayer"
44
+)
45
+
46
+// pruneAlgorithm contains the various settings to use when evaluating images
47
+// and layers for pruning.
48
+type pruneAlgorithm struct {
49
+	keepYoungerThan  time.Duration
50
+	keepTagRevisions int
51
+}
52
+
53
+// ImageDeleter knows how to remove images from OpenShift.
54
+type ImageDeleter interface {
55
+	// DeleteImage removes the image from OpenShift's storage.
56
+	DeleteImage(image *imageapi.Image) error
57
+}
58
+
59
+// ImageStreamDeleter knows how to remove an image reference from an image stream.
60
+type ImageStreamDeleter interface {
61
+	// DeleteImageStream removes all references to the image from the image
62
+	// stream's status.tags. The updated image stream is returned.
63
+	DeleteImageStream(stream *imageapi.ImageStream, image *imageapi.Image, updatedTags []string) (*imageapi.ImageStream, error)
64
+}
65
+
66
+// BlobDeleter knows how to delete a blob from the Docker registry.
67
+type BlobDeleter interface {
68
+	// DeleteBlob uses registryClient to ask the registry at registryURL
69
+	// to remove the blob.
70
+	DeleteBlob(registryClient *http.Client, registryURL, blob string) error
71
+}
72
+
73
+// LayerDeleter knows how to delete a repository layer link from the Docker registry.
74
+type LayerDeleter interface {
75
+	// DeleteLayer uses registryClient to ask the registry at registryURL to
76
+	// delete the repository layer link.
77
+	DeleteLayer(registryClient *http.Client, registryURL, repo, layer string) error
78
+}
79
+
80
+// ManifestDeleter knows how to delete image manifest data for a repository from
81
+// the Docker registry.
82
+type ManifestDeleter interface {
83
+	// DeleteManifest uses registryClient to ask the registry at registryURL to
84
+	// delete the repository's image manifest data.
85
+	DeleteManifest(registryClient *http.Client, registryURL, repo, manifest string) error
86
+}
87
+
88
+// PrunerOptions contains the fields used to initialize a new Pruner.
89
+type PrunerOptions struct {
90
+	// KeepYoungerThan indicates the minimum age an Image must be to be a
91
+	// candidate for pruning.
92
+	KeepYoungerThan time.Duration
93
+	// KeepTagRevisions is the minimum number of tag revisions to preserve;
94
+	// revisions older than this value are candidates for pruning.
95
+	KeepTagRevisions int
96
+	// Images is the entire list of images in OpenShift. An image must be in this
97
+	// list to be a candidate for pruning.
98
+	Images *imageapi.ImageList
99
+	// Streams is the entire list of image streams across all namespaces in the
100
+	// cluster.
101
+	Streams *imageapi.ImageStreamList
102
+	// Pods is the entire list of pods across all namespaces in the cluster.
103
+	Pods *kapi.PodList
104
+	// RCs is the entire list of replication controllers across all namespaces in
105
+	// the cluster.
106
+	RCs *kapi.ReplicationControllerList
107
+	// BCs is the entire list of build configs across all namespaces in the
108
+	// cluster.
109
+	BCs *buildapi.BuildConfigList
110
+	// Builds is the entire list of builds across all namespaces in the cluster.
111
+	Builds *buildapi.BuildList
112
+	// DCs is the entire list of deployment configs across all namespaces in the
113
+	// cluster.
114
+	DCs *deployapi.DeploymentConfigList
115
+	// DryRun indicates that no changes will be made to the cluster and nothing
116
+	// will be removed.
117
+	DryRun bool
118
+	// RegistryClient is the http.Client to use when contacting the registry.
119
+	RegistryClient *http.Client
120
+	// RegistryURL is the URL for the registry.
121
+	RegistryURL string
122
+}
123
+
124
+// Pruner knows how to prune images and layers.
125
+type Pruner interface {
126
+	// Prune uses imagePruner, streamPruner, layerPruner, blobPruner, and
127
+	// manifestPruner to remove images that have been identified as candidates
128
+	// for pruning based on the Pruner's internal pruning algorithm.
129
+	// Please see NewPruner for details on the algorithm.
130
+	Prune(imagePruner ImageDeleter, streamPruner ImageStreamDeleter, layerPruner LayerDeleter,
131
+		blobPruner BlobDeleter, manifestPruner ManifestDeleter) error
132
+}
133
+
134
+// pruner is an object that knows how to prune a data set
135
+type pruner struct {
136
+	g              graph.Graph
137
+	algorithm      pruneAlgorithm
138
+	registryPinger registryPinger
139
+	registryClient *http.Client
140
+	registryURL    string
141
+}
142
+
143
+var _ Pruner = &pruner{}
144
+
145
+// registryPinger performs a health check against a registry.
146
+type registryPinger interface {
147
+	// ping performs a health check against registry.
148
+	ping(registry string) error
149
+}
150
+
151
+// defaultRegistryPinger implements registryPinger.
152
+type defaultRegistryPinger struct {
153
+	client *http.Client
154
+}
155
+
156
+func (drp *defaultRegistryPinger) ping(registry string) error {
157
+	healthCheck := func(proto, registry string) error {
158
+		// TODO: `/healthz` route is deprecated by `/`; remove it in future versions
159
+		healthResponse, err := drp.client.Get(fmt.Sprintf("%s://%s/healthz", proto, registry))
160
+		if err != nil {
161
+			return err
162
+		}
163
+		defer healthResponse.Body.Close()
164
+
165
+		if healthResponse.StatusCode != http.StatusOK {
166
+			return fmt.Errorf("unexpected status code %d", healthResponse.StatusCode)
167
+		}
168
+
169
+		return nil
170
+	}
171
+
172
+	var err error
173
+	for _, proto := range []string{"https", "http"} {
174
+		glog.V(4).Infof("Trying %s for %s", proto, registry)
175
+		err = healthCheck(proto, registry)
176
+		if err == nil {
177
+			break
178
+		}
179
+		glog.V(4).Infof("Error with %s for %s: %v", proto, registry, err)
180
+	}
181
+
182
+	return err
183
+}
184
+
185
+// dryRunRegistryPinger implements registryPinger.
186
+type dryRunRegistryPinger struct {
187
+}
188
+
189
+func (*dryRunRegistryPinger) ping(registry string) error {
190
+	return nil
191
+}
192
+
193
+// NewPruner creates a Pruner.
194
+//
195
+// Images younger than keepYoungerThan and images referenced by image streams
196
+// and/or pods younger than keepYoungerThan are preserved. All other images are
197
+// candidates for pruning. For example, if keepYoungerThan is 60m, and an
198
+// ImageStream is only 59 minutes old, none of the images it references are
199
+// eligible for pruning.
200
+//
201
+// keepTagRevisions is the number of revisions per tag in an image stream's
202
+// status.tags that are preserved and ineligible for pruning. Any revision older
203
+// than keepTagRevisions is eligible for pruning.
204
+//
205
+// images, streams, pods, rcs, bcs, builds, and dcs are the resources used to run
206
+// the pruning algorithm. These should be the full list for each type from the
207
+// cluster; otherwise, the pruning algorithm might result in incorrect
208
+// calculations and premature pruning.
209
+//
210
+// The ImageDeleter performs the following logic: remove any image containing the
211
+// annotation openshift.io/image.managed=true that was created at least *n*
212
+// minutes ago and is *not* currently referenced by:
213
+//
214
+// - any pod created less than *n* minutes ago
215
+// - any image stream created less than *n* minutes ago
216
+// - any running pods
217
+// - any pending pods
218
+// - any replication controllers
219
+// - any deployment configs
220
+// - any build configs
221
+// - any builds
222
+// - the n most recent tag revisions in an image stream's status.tags
223
+//
224
+// When removing an image, remove all references to the image from all
225
+// ImageStreams having a reference to the image in `status.tags`.
226
+//
227
+// Also automatically remove any image layer that is no longer referenced by any
228
+// images.
229
+func NewPruner(options PrunerOptions) Pruner {
230
+	g := graph.New()
231
+
232
+	glog.V(1).Infof("Creating image pruner with keepYoungerThan=%v, keepTagRevisions=%d", options.KeepYoungerThan, options.KeepTagRevisions)
233
+
234
+	algorithm := pruneAlgorithm{
235
+		keepYoungerThan:  options.KeepYoungerThan,
236
+		keepTagRevisions: options.KeepTagRevisions,
237
+	}
238
+
239
+	addImagesToGraph(g, options.Images, algorithm)
240
+	addImageStreamsToGraph(g, options.Streams, algorithm)
241
+	addPodsToGraph(g, options.Pods, algorithm)
242
+	addReplicationControllersToGraph(g, options.RCs)
243
+	addBuildConfigsToGraph(g, options.BCs)
244
+	addBuildsToGraph(g, options.Builds)
245
+	addDeploymentConfigsToGraph(g, options.DCs)
246
+
247
+	var rp registryPinger
248
+	if options.DryRun {
249
+		rp = &dryRunRegistryPinger{}
250
+	} else {
251
+		rp = &defaultRegistryPinger{options.RegistryClient}
252
+	}
253
+
254
+	return &pruner{
255
+		g:              g,
256
+		algorithm:      algorithm,
257
+		registryPinger: rp,
258
+		registryClient: options.RegistryClient,
259
+		registryURL:    options.RegistryURL,
260
+	}
261
+}
262
+
263
+// addImagesToGraph adds all images to the graph that belong to one of the
264
+// registries in the algorithm and are at least as old as the minimum age
265
+// threshold as specified by the algorithm. It also adds all the images' layers
266
+// to the graph.
267
+func addImagesToGraph(g graph.Graph, images *imageapi.ImageList, algorithm pruneAlgorithm) {
268
+	for i := range images.Items {
269
+		image := &images.Items[i]
270
+
271
+		glog.V(4).Infof("Examining image %q", image.Name)
272
+
273
+		if image.Annotations == nil {
274
+			glog.V(4).Infof("Image %q with DockerImageReference %q belongs to an external registry - skipping", image.Name, image.DockerImageReference)
275
+			continue
276
+		}
277
+		if value, ok := image.Annotations[imageapi.ManagedByOpenShiftAnnotation]; !ok || value != "true" {
278
+			glog.V(4).Infof("Image %q with DockerImageReference %q belongs to an external registry - skipping", image.Name, image.DockerImageReference)
279
+			continue
280
+		}
281
+
282
+		age := unversioned.Now().Sub(image.CreationTimestamp.Time)
283
+		if age < algorithm.keepYoungerThan {
284
+			glog.V(4).Infof("Image %q is younger than minimum pruning age, skipping (age=%v)", image.Name, age)
285
+			continue
286
+		}
287
+
288
+		glog.V(4).Infof("Adding image %q to graph", image.Name)
289
+		imageNode := imagegraph.EnsureImageNode(g, image)
290
+
291
+		manifest := imageapi.DockerImageManifest{}
292
+		if err := json.Unmarshal([]byte(image.DockerImageManifest), &manifest); err != nil {
293
+			utilruntime.HandleError(fmt.Errorf("unable to extract manifest from image: %v. This image's layers won't be pruned if the image is pruned now.", err))
294
+			continue
295
+		}
296
+
297
+		for _, layer := range manifest.FSLayers {
298
+			glog.V(4).Infof("Adding image layer %q to graph", layer.DockerBlobSum)
299
+			layerNode := imagegraph.EnsureImageLayerNode(g, layer.DockerBlobSum)
300
+			g.AddEdge(imageNode, layerNode, ReferencedImageLayerEdgeKind)
301
+		}
302
+	}
303
+}
304
+
305
+// addImageStreamsToGraph adds all the streams to the graph. The most recent n
306
+// image revisions for a tag will be preserved, where n is specified by the
307
+// algorithm's keepTagRevisions. Image revisions older than n are candidates
308
+// for pruning.  if the image stream's age is at least as old as the minimum
309
+// threshold in algorithm.  Otherwise, if the image stream is younger than the
310
+// threshold, all image revisions for that stream are ineligible for pruning.
311
+//
312
+// addImageStreamsToGraph also adds references from each stream to all the
313
+// layers it references (via each image a stream references).
314
+func addImageStreamsToGraph(g graph.Graph, streams *imageapi.ImageStreamList, algorithm pruneAlgorithm) {
315
+	for i := range streams.Items {
316
+		stream := &streams.Items[i]
317
+
318
+		glog.V(4).Infof("Examining ImageStream %s/%s", stream.Namespace, stream.Name)
319
+
320
+		// use a weak reference for old image revisions by default
321
+		oldImageRevisionReferenceKind := WeakReferencedImageEdgeKind
322
+
323
+		age := unversioned.Now().Sub(stream.CreationTimestamp.Time)
324
+		if age < algorithm.keepYoungerThan {
325
+			// stream's age is below threshold - use a strong reference for old image revisions instead
326
+			glog.V(4).Infof("Stream %s/%s is below age threshold - none of its images are eligible for pruning", stream.Namespace, stream.Name)
327
+			oldImageRevisionReferenceKind = ReferencedImageEdgeKind
328
+		}
329
+
330
+		glog.V(4).Infof("Adding ImageStream %s/%s to graph", stream.Namespace, stream.Name)
331
+		isNode := imagegraph.EnsureImageStreamNode(g, stream)
332
+		imageStreamNode := isNode.(*imagegraph.ImageStreamNode)
333
+
334
+		for tag, history := range stream.Status.Tags {
335
+			for i := range history.Items {
336
+				n := imagegraph.FindImage(g, history.Items[i].Image)
337
+				if n == nil {
338
+					glog.V(2).Infof("Unable to find image %q in graph (from tag=%q, revision=%d, dockerImageReference=%s)", history.Items[i].Image, tag, i, history.Items[i].DockerImageReference)
339
+					continue
340
+				}
341
+				imageNode := n.(*imagegraph.ImageNode)
342
+
343
+				var kind string
344
+				switch {
345
+				case i < algorithm.keepTagRevisions:
346
+					kind = ReferencedImageEdgeKind
347
+				default:
348
+					kind = oldImageRevisionReferenceKind
349
+				}
350
+
351
+				glog.V(4).Infof("Checking for existing strong reference from stream %s/%s to image %s", stream.Namespace, stream.Name, imageNode.Image.Name)
352
+				if edge := g.Edge(imageStreamNode, imageNode); edge != nil && g.EdgeKinds(edge).Has(ReferencedImageEdgeKind) {
353
+					glog.V(4).Infof("Strong reference found")
354
+					continue
355
+				}
356
+
357
+				glog.V(4).Infof("Adding edge (kind=%s) from %q to %q", kind, imageStreamNode.UniqueName(), imageNode.UniqueName())
358
+				g.AddEdge(imageStreamNode, imageNode, kind)
359
+
360
+				glog.V(4).Infof("Adding stream->layer references")
361
+				// add stream -> layer references so we can prune them later
362
+				for _, s := range g.From(imageNode) {
363
+					if g.Kind(s) != imagegraph.ImageLayerNodeKind {
364
+						continue
365
+					}
366
+					glog.V(4).Infof("Adding reference from stream %q to layer %q", stream.Name, s.(*imagegraph.ImageLayerNode).Layer)
367
+					g.AddEdge(imageStreamNode, s, ReferencedImageLayerEdgeKind)
368
+				}
369
+			}
370
+		}
371
+	}
372
+}
373
+
374
+// addPodsToGraph adds pods to the graph.
375
+//
376
+// A pod is only *excluded* from being added to the graph if its phase is not
377
+// pending or running and it is at least as old as the minimum age threshold
378
+// defined by algorithm.
379
+//
380
+// Edges are added to the graph from each pod to the images specified by that
381
+// pod's list of containers, as long as the image is managed by OpenShift.
382
+func addPodsToGraph(g graph.Graph, pods *kapi.PodList, algorithm pruneAlgorithm) {
383
+	for i := range pods.Items {
384
+		pod := &pods.Items[i]
385
+
386
+		glog.V(4).Infof("Examining pod %s/%s", pod.Namespace, pod.Name)
387
+
388
+		if pod.Status.Phase != kapi.PodRunning && pod.Status.Phase != kapi.PodPending {
389
+			age := unversioned.Now().Sub(pod.CreationTimestamp.Time)
390
+			if age >= algorithm.keepYoungerThan {
391
+				glog.V(4).Infof("Pod %s/%s is not running or pending and age is at least minimum pruning age - skipping", pod.Namespace, pod.Name)
392
+				// not pending or running, age is at least minimum pruning age, skip
393
+				continue
394
+			}
395
+		}
396
+
397
+		glog.V(4).Infof("Adding pod %s/%s to graph", pod.Namespace, pod.Name)
398
+		podNode := kubegraph.EnsurePodNode(g, pod)
399
+
400
+		addPodSpecToGraph(g, &pod.Spec, podNode)
401
+	}
402
+}
403
+
404
+// Edges are added to the graph from each predecessor (pod or replication
405
+// controller) to the images specified by the pod spec's list of containers, as
406
+// long as the image is managed by OpenShift.
407
+func addPodSpecToGraph(g graph.Graph, spec *kapi.PodSpec, predecessor gonum.Node) {
408
+	for j := range spec.Containers {
409
+		container := spec.Containers[j]
410
+
411
+		glog.V(4).Infof("Examining container image %q", container.Image)
412
+
413
+		ref, err := imageapi.ParseDockerImageReference(container.Image)
414
+		if err != nil {
415
+			utilruntime.HandleError(fmt.Errorf("unable to parse DockerImageReference %q: %v", container.Image, err))
416
+			continue
417
+		}
418
+
419
+		if len(ref.ID) == 0 {
420
+			glog.V(4).Infof("%q has no image ID", container.Image)
421
+			continue
422
+		}
423
+
424
+		imageNode := imagegraph.FindImage(g, ref.ID)
425
+		if imageNode == nil {
426
+			glog.Infof("Unable to find image %q in the graph", ref.ID)
427
+			continue
428
+		}
429
+
430
+		glog.V(4).Infof("Adding edge from pod to image")
431
+		g.AddEdge(predecessor, imageNode, ReferencedImageEdgeKind)
432
+	}
433
+}
434
+
435
+// addReplicationControllersToGraph adds replication controllers to the graph.
436
+//
437
+// Edges are added to the graph from each replication controller to the images
438
+// specified by its pod spec's list of containers, as long as the image is
439
+// managed by OpenShift.
440
+func addReplicationControllersToGraph(g graph.Graph, rcs *kapi.ReplicationControllerList) {
441
+	for i := range rcs.Items {
442
+		rc := &rcs.Items[i]
443
+		glog.V(4).Infof("Examining replication controller %s/%s", rc.Namespace, rc.Name)
444
+		rcNode := kubegraph.EnsureReplicationControllerNode(g, rc)
445
+		addPodSpecToGraph(g, &rc.Spec.Template.Spec, rcNode)
446
+	}
447
+}
448
+
449
+// addDeploymentConfigsToGraph adds deployment configs to the graph.
450
+//
451
+// Edges are added to the graph from each deployment config to the images
452
+// specified by its pod spec's list of containers, as long as the image is
453
+// managed by OpenShift.
454
+func addDeploymentConfigsToGraph(g graph.Graph, dcs *deployapi.DeploymentConfigList) {
455
+	for i := range dcs.Items {
456
+		dc := &dcs.Items[i]
457
+		glog.V(4).Infof("Examining DeploymentConfig %s/%s", dc.Namespace, dc.Name)
458
+		dcNode := deploygraph.EnsureDeploymentConfigNode(g, dc)
459
+		addPodSpecToGraph(g, &dc.Spec.Template.Spec, dcNode)
460
+	}
461
+}
462
+
463
+// addBuildConfigsToGraph adds build configs to the graph.
464
+//
465
+// Edges are added to the graph from each build config to the image specified by its strategy.from.
466
+func addBuildConfigsToGraph(g graph.Graph, bcs *buildapi.BuildConfigList) {
467
+	for i := range bcs.Items {
468
+		bc := &bcs.Items[i]
469
+		glog.V(4).Infof("Examining BuildConfig %s/%s", bc.Namespace, bc.Name)
470
+		bcNode := buildgraph.EnsureBuildConfigNode(g, bc)
471
+		addBuildStrategyImageReferencesToGraph(g, bc.Spec.Strategy, bcNode)
472
+	}
473
+}
474
+
475
+// addBuildsToGraph adds builds to the graph.
476
+//
477
+// Edges are added to the graph from each build to the image specified by its strategy.from.
478
+func addBuildsToGraph(g graph.Graph, builds *buildapi.BuildList) {
479
+	for i := range builds.Items {
480
+		build := &builds.Items[i]
481
+		glog.V(4).Infof("Examining build %s/%s", build.Namespace, build.Name)
482
+		buildNode := buildgraph.EnsureBuildNode(g, build)
483
+		addBuildStrategyImageReferencesToGraph(g, build.Spec.Strategy, buildNode)
484
+	}
485
+}
486
+
487
+// addBuildStrategyImageReferencesToGraph ads references from the build strategy's parent node to the image
488
+// the build strategy references.
489
+//
490
+// Edges are added to the graph from each predecessor (build or build config)
491
+// to the image specified by strategy.from, as long as the image is managed by
492
+// OpenShift.
493
+func addBuildStrategyImageReferencesToGraph(g graph.Graph, strategy buildapi.BuildStrategy, predecessor gonum.Node) {
494
+	from := buildutil.GetInputReference(strategy)
495
+	if from == nil {
496
+		glog.V(4).Infof("Unable to determine 'from' reference - skipping")
497
+		return
498
+	}
499
+
500
+	glog.V(4).Infof("Examining build strategy with from: %#v", from)
501
+
502
+	var imageID string
503
+
504
+	switch from.Kind {
505
+	case "ImageStreamImage":
506
+		_, id, err := imageapi.ParseImageStreamImageName(from.Name)
507
+		if err != nil {
508
+			glog.V(2).Infof("Error parsing ImageStreamImage name %q: %v - skipping", from.Name, err)
509
+			return
510
+		}
511
+		imageID = id
512
+	case "DockerImage":
513
+		ref, err := imageapi.ParseDockerImageReference(from.Name)
514
+		if err != nil {
515
+			glog.V(2).Infof("Error parsing DockerImage name %q: %v - skipping", from.Name, err)
516
+			return
517
+		}
518
+		imageID = ref.ID
519
+	default:
520
+		return
521
+	}
522
+
523
+	glog.V(4).Infof("Looking for image %q in graph", imageID)
524
+	imageNode := imagegraph.FindImage(g, imageID)
525
+	if imageNode == nil {
526
+		glog.V(4).Infof("Unable to find image %q in graph - skipping", imageID)
527
+		return
528
+	}
529
+
530
+	glog.V(4).Infof("Adding edge from %v to %v", predecessor, imageNode)
531
+	g.AddEdge(predecessor, imageNode, ReferencedImageEdgeKind)
532
+}
533
+
534
+// getImageNodes returns only nodes of type ImageNode.
535
+func getImageNodes(nodes []gonum.Node) []*imagegraph.ImageNode {
536
+	ret := []*imagegraph.ImageNode{}
537
+	for i := range nodes {
538
+		if node, ok := nodes[i].(*imagegraph.ImageNode); ok {
539
+			ret = append(ret, node)
540
+		}
541
+	}
542
+	return ret
543
+}
544
+
545
+// edgeKind returns true if the edge from "from" to "to" is of the desired kind.
546
+func edgeKind(g graph.Graph, from, to gonum.Node, desiredKind string) bool {
547
+	edge := g.Edge(from, to)
548
+	kinds := g.EdgeKinds(edge)
549
+	return kinds.Has(desiredKind)
550
+}
551
+
552
+// imageIsPrunable returns true iff the image node only has weak references
553
+// from its predecessors to it. A weak reference to an image is a reference
554
+// from an image stream to an image where the image is not the current image
555
+// for a tag and the image stream is at least as old as the minimum pruning
556
+// age.
557
+func imageIsPrunable(g graph.Graph, imageNode *imagegraph.ImageNode) bool {
558
+	onlyWeakReferences := true
559
+
560
+	for _, n := range g.To(imageNode) {
561
+		glog.V(4).Infof("Examining predecessor %#v", n)
562
+		if !edgeKind(g, n, imageNode, WeakReferencedImageEdgeKind) {
563
+			glog.V(4).Infof("Strong reference detected")
564
+			onlyWeakReferences = false
565
+			break
566
+		}
567
+	}
568
+
569
+	return onlyWeakReferences
570
+
571
+}
572
+
573
+// calculatePrunableImages returns the list of prunable images and a
574
+// graph.NodeSet containing the image node IDs.
575
+func calculatePrunableImages(g graph.Graph, imageNodes []*imagegraph.ImageNode) ([]*imagegraph.ImageNode, graph.NodeSet) {
576
+	prunable := []*imagegraph.ImageNode{}
577
+	ids := make(graph.NodeSet)
578
+
579
+	for _, imageNode := range imageNodes {
580
+		glog.V(4).Infof("Examining image %q", imageNode.Image.Name)
581
+
582
+		if imageIsPrunable(g, imageNode) {
583
+			glog.V(4).Infof("Image %q is prunable", imageNode.Image.Name)
584
+			prunable = append(prunable, imageNode)
585
+			ids.Add(imageNode.ID())
586
+		}
587
+	}
588
+
589
+	return prunable, ids
590
+}
591
+
592
+// subgraphWithoutPrunableImages creates a subgraph from g with prunable image
593
+// nodes excluded.
594
+func subgraphWithoutPrunableImages(g graph.Graph, prunableImageIDs graph.NodeSet) graph.Graph {
595
+	return g.Subgraph(
596
+		func(g graph.Interface, node gonum.Node) bool {
597
+			return !prunableImageIDs.Has(node.ID())
598
+		},
599
+		func(g graph.Interface, from, to gonum.Node, edgeKinds sets.String) bool {
600
+			if prunableImageIDs.Has(from.ID()) {
601
+				return false
602
+			}
603
+			if prunableImageIDs.Has(to.ID()) {
604
+				return false
605
+			}
606
+			return true
607
+		},
608
+	)
609
+}
610
+
611
+// calculatePrunableLayers returns the list of prunable layers.
612
+func calculatePrunableLayers(g graph.Graph) []*imagegraph.ImageLayerNode {
613
+	prunable := []*imagegraph.ImageLayerNode{}
614
+
615
+	nodes := g.Nodes()
616
+	for i := range nodes {
617
+		layerNode, ok := nodes[i].(*imagegraph.ImageLayerNode)
618
+		if !ok {
619
+			continue
620
+		}
621
+
622
+		glog.V(4).Infof("Examining layer %q", layerNode.Layer)
623
+
624
+		if layerIsPrunable(g, layerNode) {
625
+			glog.V(4).Infof("Layer %q is prunable", layerNode.Layer)
626
+			prunable = append(prunable, layerNode)
627
+		}
628
+	}
629
+
630
+	return prunable
631
+}
632
+
633
+// pruneStreams removes references from all image streams' status.tags entries
634
+// to prunable images, invoking streamPruner.DeleteImageStream for each updated
635
+// stream.
636
+func pruneStreams(g graph.Graph, imageNodes []*imagegraph.ImageNode, streamPruner ImageStreamDeleter) []error {
637
+	errs := []error{}
638
+
639
+	glog.V(4).Infof("Removing pruned image references from streams")
640
+	for _, imageNode := range imageNodes {
641
+		for _, n := range g.To(imageNode) {
642
+			streamNode, ok := n.(*imagegraph.ImageStreamNode)
643
+			if !ok {
644
+				continue
645
+			}
646
+
647
+			stream := streamNode.ImageStream
648
+			updatedTags := sets.NewString()
649
+
650
+			glog.V(4).Infof("Checking if ImageStream %s/%s has references to image %s in status.tags", stream.Namespace, stream.Name, imageNode.Image.Name)
651
+
652
+			for tag, history := range stream.Status.Tags {
653
+				glog.V(4).Infof("Checking tag %q", tag)
654
+
655
+				newHistory := imageapi.TagEventList{}
656
+
657
+				for i, tagEvent := range history.Items {
658
+					glog.V(4).Infof("Checking tag event %d with image %q", i, tagEvent.Image)
659
+
660
+					if tagEvent.Image != imageNode.Image.Name {
661
+						glog.V(4).Infof("Tag event doesn't match deleted image - keeping")
662
+						newHistory.Items = append(newHistory.Items, tagEvent)
663
+					} else {
664
+						glog.V(4).Infof("Tag event matches deleted image - removing reference")
665
+						updatedTags.Insert(tag)
666
+					}
667
+				}
668
+				if len(newHistory.Items) == 0 {
669
+					glog.V(4).Infof("Removing tag %q from status.tags of ImageStream %s/%s", tag, stream.Namespace, stream.Name)
670
+					delete(stream.Status.Tags, tag)
671
+				} else {
672
+					stream.Status.Tags[tag] = newHistory
673
+				}
674
+			}
675
+
676
+			updatedStream, err := streamPruner.DeleteImageStream(stream, imageNode.Image, updatedTags.List())
677
+			if err != nil {
678
+				errs = append(errs, fmt.Errorf("error pruning image from stream: %v", err))
679
+				continue
680
+			}
681
+
682
+			streamNode.ImageStream = updatedStream
683
+		}
684
+	}
685
+	glog.V(4).Infof("Done removing pruned image references from streams")
686
+	return errs
687
+}
688
+
689
+// pruneImages invokes imagePruner.DeleteImage with each image that is prunable.
690
+func pruneImages(g graph.Graph, imageNodes []*imagegraph.ImageNode, imagePruner ImageDeleter) []error {
691
+	errs := []error{}
692
+
693
+	for _, imageNode := range imageNodes {
694
+		if err := imagePruner.DeleteImage(imageNode.Image); err != nil {
695
+			errs = append(errs, fmt.Errorf("error pruning image %q: %v", imageNode.Image.Name, err))
696
+		}
697
+	}
698
+
699
+	return errs
700
+}
701
+
702
+func (p *pruner) determineRegistry(imageNodes []*imagegraph.ImageNode) (string, error) {
703
+	if len(p.registryURL) > 0 {
704
+		return p.registryURL, nil
705
+	}
706
+
707
+	// we only support a single internal registry, and all images have the same registry
708
+	// so we just take the 1st one and use it
709
+	pullSpec := imageNodes[0].Image.DockerImageReference
710
+
711
+	ref, err := imageapi.ParseDockerImageReference(pullSpec)
712
+	if err != nil {
713
+		return "", fmt.Errorf("unable to parse %q: %v", pullSpec, err)
714
+	}
715
+
716
+	if len(ref.Registry) == 0 {
717
+		return "", fmt.Errorf("%s does not include a registry", pullSpec)
718
+	}
719
+
720
+	return ref.Registry, nil
721
+}
722
+
723
+// Run identifies images eligible for pruning, invoking imagePruneFunc for each
724
+// image, and then it identifies layers eligible for pruning, invoking
725
+// layerPruneFunc for each registry URL that has layers that can be pruned.
726
+func (p *pruner) Prune(imagePruner ImageDeleter, streamPruner ImageStreamDeleter, layerPruner LayerDeleter, blobPruner BlobDeleter, manifestPruner ManifestDeleter) error {
727
+	allNodes := p.g.Nodes()
728
+
729
+	imageNodes := getImageNodes(allNodes)
730
+	if len(imageNodes) == 0 {
731
+		return nil
732
+	}
733
+
734
+	registryURL, err := p.determineRegistry(imageNodes)
735
+	if err != nil {
736
+		return fmt.Errorf("unable to determine registry: %v", err)
737
+	}
738
+	glog.V(1).Infof("Using registry: %s", registryURL)
739
+
740
+	if err := p.registryPinger.ping(registryURL); err != nil {
741
+		return fmt.Errorf("error communicating with registry: %v", err)
742
+	}
743
+
744
+	prunableImageNodes, prunableImageIDs := calculatePrunableImages(p.g, imageNodes)
745
+	graphWithoutPrunableImages := subgraphWithoutPrunableImages(p.g, prunableImageIDs)
746
+	prunableLayers := calculatePrunableLayers(graphWithoutPrunableImages)
747
+
748
+	errs := []error{}
749
+
750
+	errs = append(errs, pruneStreams(p.g, prunableImageNodes, streamPruner)...)
751
+	errs = append(errs, pruneLayers(p.g, p.registryClient, registryURL, prunableLayers, layerPruner)...)
752
+	errs = append(errs, pruneBlobs(p.g, p.registryClient, registryURL, prunableLayers, blobPruner)...)
753
+	errs = append(errs, pruneManifests(p.g, p.registryClient, registryURL, prunableImageNodes, manifestPruner)...)
754
+
755
+	if len(errs) > 0 {
756
+		// If we had any errors removing image references from image streams or deleting
757
+		// layers, blobs, or manifest data from the registry, stop here and don't
758
+		// delete any images. This way, you can rerun prune and retry things that failed.
759
+		return kerrors.NewAggregate(errs)
760
+	}
761
+
762
+	errs = append(errs, pruneImages(p.g, prunableImageNodes, imagePruner)...)
763
+	return kerrors.NewAggregate(errs)
764
+}
765
+
766
+// layerIsPrunable returns true if the layer is not referenced by any images.
767
+func layerIsPrunable(g graph.Graph, layerNode *imagegraph.ImageLayerNode) bool {
768
+	for _, predecessor := range g.To(layerNode) {
769
+		glog.V(4).Infof("Examining layer predecessor %#v", predecessor)
770
+		if g.Kind(predecessor) == imagegraph.ImageNodeKind {
771
+			glog.V(4).Infof("Layer has an image predecessor")
772
+			return false
773
+		}
774
+	}
775
+
776
+	return true
777
+}
778
+
779
+// streamLayerReferences returns a list of ImageStreamNodes that reference a
780
+// given ImageLayerNode.
781
+func streamLayerReferences(g graph.Graph, layerNode *imagegraph.ImageLayerNode) []*imagegraph.ImageStreamNode {
782
+	ret := []*imagegraph.ImageStreamNode{}
783
+
784
+	for _, predecessor := range g.To(layerNode) {
785
+		if g.Kind(predecessor) != imagegraph.ImageStreamNodeKind {
786
+			continue
787
+		}
788
+
789
+		ret = append(ret, predecessor.(*imagegraph.ImageStreamNode))
790
+	}
791
+
792
+	return ret
793
+}
794
+
795
+// pruneLayers invokes layerPruner.DeleteLayer for each repository layer link to
796
+// be deleted from the registry.
797
+func pruneLayers(g graph.Graph, registryClient *http.Client, registryURL string, layerNodes []*imagegraph.ImageLayerNode, layerPruner LayerDeleter) []error {
798
+	errs := []error{}
799
+
800
+	for _, layerNode := range layerNodes {
801
+		// get streams that reference layer
802
+		streamNodes := streamLayerReferences(g, layerNode)
803
+
804
+		for _, streamNode := range streamNodes {
805
+			stream := streamNode.ImageStream
806
+			streamName := fmt.Sprintf("%s/%s", stream.Namespace, stream.Name)
807
+
808
+			glog.V(4).Infof("Pruning registry=%q, repo=%q, layer=%q", registryURL, streamName, layerNode.Layer)
809
+			if err := layerPruner.DeleteLayer(registryClient, registryURL, streamName, layerNode.Layer); err != nil {
810
+				errs = append(errs, fmt.Errorf("error pruning repo %q layer link %q: %v", streamName, layerNode.Layer, err))
811
+			}
812
+		}
813
+	}
814
+
815
+	return errs
816
+}
817
+
818
+// pruneBlobs invokes blobPruner.DeleteBlob for each blob to be deleted from the
819
+// registry.
820
+func pruneBlobs(g graph.Graph, registryClient *http.Client, registryURL string, layerNodes []*imagegraph.ImageLayerNode, blobPruner BlobDeleter) []error {
821
+	errs := []error{}
822
+
823
+	for _, layerNode := range layerNodes {
824
+		glog.V(4).Infof("Pruning registry=%q, blob=%q", registryURL, layerNode.Layer)
825
+		if err := blobPruner.DeleteBlob(registryClient, registryURL, layerNode.Layer); err != nil {
826
+			errs = append(errs, fmt.Errorf("error pruning blob %q: %v", layerNode.Layer, err))
827
+		}
828
+	}
829
+
830
+	return errs
831
+}
832
+
833
+// pruneManifests invokes manifestPruner.DeleteManifest for each repository
834
+// manifest to be deleted from the registry.
835
+func pruneManifests(g graph.Graph, registryClient *http.Client, registryURL string, imageNodes []*imagegraph.ImageNode, manifestPruner ManifestDeleter) []error {
836
+	errs := []error{}
837
+
838
+	for _, imageNode := range imageNodes {
839
+		for _, n := range g.To(imageNode) {
840
+			streamNode, ok := n.(*imagegraph.ImageStreamNode)
841
+			if !ok {
842
+				continue
843
+			}
844
+
845
+			stream := streamNode.ImageStream
846
+			repoName := fmt.Sprintf("%s/%s", stream.Namespace, stream.Name)
847
+
848
+			glog.V(4).Infof("Pruning manifest for registry %q, repo %q, image %q", registryURL, repoName, imageNode.Image.Name)
849
+			if err := manifestPruner.DeleteManifest(registryClient, registryURL, repoName, imageNode.Image.Name); err != nil {
850
+				errs = append(errs, fmt.Errorf("error pruning manifest for registry %q, repo %q, image %q: %v", registryURL, repoName, imageNode.Image.Name, err))
851
+			}
852
+		}
853
+	}
854
+
855
+	return errs
856
+}
857
+
858
+// imageDeleter removes an image from OpenShift.
859
+type imageDeleter struct {
860
+	images client.ImageInterface
861
+}
862
+
863
+var _ ImageDeleter = &imageDeleter{}
864
+
865
+// NewImageDeleter creates a new imageDeleter.
866
+func NewImageDeleter(images client.ImageInterface) ImageDeleter {
867
+	return &imageDeleter{
868
+		images: images,
869
+	}
870
+}
871
+
872
+func (p *imageDeleter) DeleteImage(image *imageapi.Image) error {
873
+	glog.V(4).Infof("Deleting image %q", image.Name)
874
+	return p.images.Delete(image.Name)
875
+}
876
+
877
+// imageStreamDeleter updates an image stream in OpenShift.
878
+type imageStreamDeleter struct {
879
+	streams client.ImageStreamsNamespacer
880
+}
881
+
882
+var _ ImageStreamDeleter = &imageStreamDeleter{}
883
+
884
+// NewImageStreamDeleter creates a new imageStreamDeleter.
885
+func NewImageStreamDeleter(streams client.ImageStreamsNamespacer) ImageStreamDeleter {
886
+	return &imageStreamDeleter{
887
+		streams: streams,
888
+	}
889
+}
890
+
891
+func (p *imageStreamDeleter) DeleteImageStream(stream *imageapi.ImageStream, image *imageapi.Image, updatedTags []string) (*imageapi.ImageStream, error) {
892
+	glog.V(4).Infof("Updating ImageStream %s/%s", stream.Namespace, stream.Name)
893
+	glog.V(5).Infof("Updated stream: %#v", stream)
894
+	return p.streams.ImageStreams(stream.Namespace).UpdateStatus(stream)
895
+}
896
+
897
+// deleteFromRegistry uses registryClient to send a DELETE request to the
898
+// provided url. It attempts an https request first; if that fails, it fails
899
+// back to http.
900
+func deleteFromRegistry(registryClient *http.Client, url string) error {
901
+	deleteFunc := func(proto, url string) error {
902
+		req, err := http.NewRequest("DELETE", url, nil)
903
+		if err != nil {
904
+			glog.Errorf("Error creating request: %v", err)
905
+			return fmt.Errorf("error creating request: %v", err)
906
+		}
907
+
908
+		glog.V(4).Infof("Sending request to registry")
909
+		resp, err := registryClient.Do(req)
910
+		if err != nil {
911
+			return fmt.Errorf("error sending request: %v", err)
912
+		}
913
+		defer resp.Body.Close()
914
+
915
+		// TODO: investigate why we're getting non-existent layers, for now we're logging
916
+		// them out and continue working
917
+		if resp.StatusCode == http.StatusNotFound {
918
+			glog.Warningf("Unable to prune layer %s, returned %v", url, resp.Status)
919
+			return nil
920
+		}
921
+		// non-2xx/3xx response doesn't cause an error, so we need to check for it
922
+		// manually and return it to caller
923
+		if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest {
924
+			return fmt.Errorf(resp.Status)
925
+		}
926
+		if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusAccepted {
927
+			glog.V(1).Infof("Unexpected status code in response: %d", resp.StatusCode)
928
+			var response errcode.Errors
929
+			decoder := json.NewDecoder(resp.Body)
930
+			if err := decoder.Decode(&response); err != nil {
931
+				return err
932
+			}
933
+			glog.V(1).Infof("Response: %#v", response)
934
+			return &response
935
+		}
936
+
937
+		return nil
938
+	}
939
+
940
+	var err error
941
+	for _, proto := range []string{"https", "http"} {
942
+		glog.V(4).Infof("Trying %s for %s", proto, url)
943
+		err = deleteFunc(proto, fmt.Sprintf("%s://%s", proto, url))
944
+		if err == nil {
945
+			return nil
946
+		}
947
+
948
+		if _, ok := err.(*errcode.Errors); ok {
949
+			// we got a response back from the registry, so return it
950
+			return err
951
+		}
952
+
953
+		// we didn't get a success or a errcode.Errors response back from the registry
954
+		glog.V(4).Infof("Error with %s for %s: %v", proto, url, err)
955
+	}
956
+	return err
957
+}
958
+
959
+// layerDeleter removes a repository layer link from the registry.
960
+type layerDeleter struct{}
961
+
962
+var _ LayerDeleter = &layerDeleter{}
963
+
964
+// NewLayerDeleter creates a new layerDeleter.
965
+func NewLayerDeleter() LayerDeleter {
966
+	return &layerDeleter{}
967
+}
968
+
969
+func (p *layerDeleter) DeleteLayer(registryClient *http.Client, registryURL, repoName, layer string) error {
970
+	glog.V(4).Infof("Pruning registry %q, repo %q, layer %q", registryURL, repoName, layer)
971
+	return deleteFromRegistry(registryClient, fmt.Sprintf("%s/v2/%s/blobs/%s", registryURL, repoName, layer))
972
+}
973
+
974
+// blobDeleter removes a blob from the registry.
975
+type blobDeleter struct{}
976
+
977
+var _ BlobDeleter = &blobDeleter{}
978
+
979
+// NewBlobDeleter creates a new blobDeleter.
980
+func NewBlobDeleter() BlobDeleter {
981
+	return &blobDeleter{}
982
+}
983
+
984
+func (p *blobDeleter) DeleteBlob(registryClient *http.Client, registryURL, blob string) error {
985
+	glog.V(4).Infof("Pruning registry %q, blob %q", registryURL, blob)
986
+	return deleteFromRegistry(registryClient, fmt.Sprintf("%s/admin/blobs/%s", registryURL, blob))
987
+}
988
+
989
+// manifestDeleter deletes repository manifest data from the registry.
990
+type manifestDeleter struct{}
991
+
992
+var _ ManifestDeleter = &manifestDeleter{}
993
+
994
+// NewManifestDeleter creates a new manifestDeleter.
995
+func NewManifestDeleter() ManifestDeleter {
996
+	return &manifestDeleter{}
997
+}
998
+
999
+func (p *manifestDeleter) DeleteManifest(registryClient *http.Client, registryURL, repoName, manifest string) error {
1000
+	glog.V(4).Infof("Pruning manifest for registry %q, repo %q, manifest %q", registryURL, repoName, manifest)
1001
+	return deleteFromRegistry(registryClient, fmt.Sprintf("%s/v2/%s/manifests/%s", registryURL, repoName, manifest))
1002
+}
0 1003
new file mode 100644
... ...
@@ -0,0 +1,915 @@
0
+package prune
1
+
2
+import (
3
+	"bytes"
4
+	"encoding/json"
5
+	"errors"
6
+	"flag"
7
+	"fmt"
8
+	"io/ioutil"
9
+	"net/http"
10
+	"reflect"
11
+	"testing"
12
+	"time"
13
+
14
+	kapi "k8s.io/kubernetes/pkg/api"
15
+	"k8s.io/kubernetes/pkg/api/unversioned"
16
+	"k8s.io/kubernetes/pkg/client/unversioned/fake"
17
+	ktc "k8s.io/kubernetes/pkg/client/unversioned/testclient"
18
+	"k8s.io/kubernetes/pkg/runtime"
19
+	"k8s.io/kubernetes/pkg/util/sets"
20
+
21
+	buildapi "github.com/openshift/origin/pkg/build/api"
22
+	"github.com/openshift/origin/pkg/client/testclient"
23
+	deployapi "github.com/openshift/origin/pkg/deploy/api"
24
+	imageapi "github.com/openshift/origin/pkg/image/api"
25
+)
26
+
27
+type fakeRegistryPinger struct {
28
+	err      error
29
+	requests []string
30
+}
31
+
32
+func (f *fakeRegistryPinger) ping(registry string) error {
33
+	f.requests = append(f.requests, registry)
34
+	return f.err
35
+}
36
+
37
+func imageList(images ...imageapi.Image) imageapi.ImageList {
38
+	return imageapi.ImageList{
39
+		Items: images,
40
+	}
41
+}
42
+
43
+func agedImage(id, ref string, ageInMinutes int64) imageapi.Image {
44
+	image := imageWithLayers(id, ref,
45
+		"tarsum.dev+sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
46
+		"tarsum.dev+sha256:b194de3772ebbcdc8f244f663669799ac1cb141834b7cb8b69100285d357a2b0",
47
+		"tarsum.dev+sha256:c937c4bb1c1a21cc6d94340812262c6472092028972ae69b551b1a70d4276171",
48
+		"tarsum.dev+sha256:2aaacc362ac6be2b9e9ae8c6029f6f616bb50aec63746521858e47841b90fabd",
49
+		"tarsum.dev+sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
50
+	)
51
+
52
+	if ageInMinutes >= 0 {
53
+		image.CreationTimestamp = unversioned.NewTime(unversioned.Now().Add(time.Duration(-1*ageInMinutes) * time.Minute))
54
+	}
55
+
56
+	return image
57
+}
58
+
59
+func image(id, ref string) imageapi.Image {
60
+	return agedImage(id, ref, -1)
61
+}
62
+
63
+func imageWithLayers(id, ref string, layers ...string) imageapi.Image {
64
+	image := imageapi.Image{
65
+		ObjectMeta: kapi.ObjectMeta{
66
+			Name: id,
67
+			Annotations: map[string]string{
68
+				imageapi.ManagedByOpenShiftAnnotation: "true",
69
+			},
70
+		},
71
+		DockerImageReference: ref,
72
+	}
73
+
74
+	manifest := imageapi.DockerImageManifest{
75
+		FSLayers: []imageapi.DockerFSLayer{},
76
+	}
77
+
78
+	for _, layer := range layers {
79
+		manifest.FSLayers = append(manifest.FSLayers, imageapi.DockerFSLayer{DockerBlobSum: layer})
80
+	}
81
+
82
+	manifestBytes, err := json.Marshal(&manifest)
83
+	if err != nil {
84
+		panic(err)
85
+	}
86
+
87
+	image.DockerImageManifest = string(manifestBytes)
88
+
89
+	return image
90
+}
91
+
92
+func unmanagedImage(id, ref string, hasAnnotations bool, annotation, value string) imageapi.Image {
93
+	image := imageWithLayers(id, ref)
94
+	if !hasAnnotations {
95
+		image.Annotations = nil
96
+	} else {
97
+		delete(image.Annotations, imageapi.ManagedByOpenShiftAnnotation)
98
+		image.Annotations[annotation] = value
99
+	}
100
+	return image
101
+}
102
+
103
+func imageWithBadManifest(id, ref string) imageapi.Image {
104
+	image := image(id, ref)
105
+	image.DockerImageManifest = "asdf"
106
+	return image
107
+}
108
+
109
+func podList(pods ...kapi.Pod) kapi.PodList {
110
+	return kapi.PodList{
111
+		Items: pods,
112
+	}
113
+}
114
+
115
+func pod(namespace, name string, phase kapi.PodPhase, containerImages ...string) kapi.Pod {
116
+	return agedPod(namespace, name, phase, -1, containerImages...)
117
+}
118
+
119
+func agedPod(namespace, name string, phase kapi.PodPhase, ageInMinutes int64, containerImages ...string) kapi.Pod {
120
+	pod := kapi.Pod{
121
+		ObjectMeta: kapi.ObjectMeta{
122
+			Namespace: namespace,
123
+			Name:      name,
124
+		},
125
+		Spec: podSpec(containerImages...),
126
+		Status: kapi.PodStatus{
127
+			Phase: phase,
128
+		},
129
+	}
130
+
131
+	if ageInMinutes >= 0 {
132
+		pod.CreationTimestamp = unversioned.NewTime(unversioned.Now().Add(time.Duration(-1*ageInMinutes) * time.Minute))
133
+	}
134
+
135
+	return pod
136
+}
137
+
138
+func podSpec(containerImages ...string) kapi.PodSpec {
139
+	spec := kapi.PodSpec{
140
+		Containers: []kapi.Container{},
141
+	}
142
+	for _, image := range containerImages {
143
+		container := kapi.Container{
144
+			Image: image,
145
+		}
146
+		spec.Containers = append(spec.Containers, container)
147
+	}
148
+	return spec
149
+}
150
+
151
+func streamList(streams ...imageapi.ImageStream) imageapi.ImageStreamList {
152
+	return imageapi.ImageStreamList{
153
+		Items: streams,
154
+	}
155
+}
156
+
157
+func stream(registry, namespace, name string, tags map[string]imageapi.TagEventList) imageapi.ImageStream {
158
+	return agedStream(registry, namespace, name, -1, tags)
159
+}
160
+
161
+func agedStream(registry, namespace, name string, ageInMinutes int64, tags map[string]imageapi.TagEventList) imageapi.ImageStream {
162
+	stream := imageapi.ImageStream{
163
+		ObjectMeta: kapi.ObjectMeta{
164
+			Namespace: namespace,
165
+			Name:      name,
166
+		},
167
+		Status: imageapi.ImageStreamStatus{
168
+			DockerImageRepository: fmt.Sprintf("%s/%s/%s", registry, namespace, name),
169
+			Tags: tags,
170
+		},
171
+	}
172
+
173
+	if ageInMinutes >= 0 {
174
+		stream.CreationTimestamp = unversioned.NewTime(unversioned.Now().Add(time.Duration(-1*ageInMinutes) * time.Minute))
175
+	}
176
+
177
+	return stream
178
+}
179
+
180
+func streamPtr(registry, namespace, name string, tags map[string]imageapi.TagEventList) *imageapi.ImageStream {
181
+	s := stream(registry, namespace, name, tags)
182
+	return &s
183
+}
184
+
185
+func tags(list ...namedTagEventList) map[string]imageapi.TagEventList {
186
+	m := make(map[string]imageapi.TagEventList, len(list))
187
+	for _, tag := range list {
188
+		m[tag.name] = tag.events
189
+	}
190
+	return m
191
+}
192
+
193
+type namedTagEventList struct {
194
+	name   string
195
+	events imageapi.TagEventList
196
+}
197
+
198
+func tag(name string, events ...imageapi.TagEvent) namedTagEventList {
199
+	return namedTagEventList{
200
+		name: name,
201
+		events: imageapi.TagEventList{
202
+			Items: events,
203
+		},
204
+	}
205
+}
206
+
207
+func tagEvent(id, ref string) imageapi.TagEvent {
208
+	return imageapi.TagEvent{
209
+		Image:                id,
210
+		DockerImageReference: ref,
211
+	}
212
+}
213
+
214
+func rcList(rcs ...kapi.ReplicationController) kapi.ReplicationControllerList {
215
+	return kapi.ReplicationControllerList{
216
+		Items: rcs,
217
+	}
218
+}
219
+
220
+func rc(namespace, name string, containerImages ...string) kapi.ReplicationController {
221
+	return kapi.ReplicationController{
222
+		ObjectMeta: kapi.ObjectMeta{
223
+			Namespace: namespace,
224
+			Name:      name,
225
+		},
226
+		Spec: kapi.ReplicationControllerSpec{
227
+			Template: &kapi.PodTemplateSpec{
228
+				Spec: podSpec(containerImages...),
229
+			},
230
+		},
231
+	}
232
+}
233
+
234
+func dcList(dcs ...deployapi.DeploymentConfig) deployapi.DeploymentConfigList {
235
+	return deployapi.DeploymentConfigList{
236
+		Items: dcs,
237
+	}
238
+}
239
+
240
+func dc(namespace, name string, containerImages ...string) deployapi.DeploymentConfig {
241
+	return deployapi.DeploymentConfig{
242
+		ObjectMeta: kapi.ObjectMeta{
243
+			Namespace: namespace,
244
+			Name:      name,
245
+		},
246
+		Spec: deployapi.DeploymentConfigSpec{
247
+			Template: &kapi.PodTemplateSpec{
248
+				Spec: podSpec(containerImages...),
249
+			},
250
+		},
251
+	}
252
+}
253
+
254
+func bcList(bcs ...buildapi.BuildConfig) buildapi.BuildConfigList {
255
+	return buildapi.BuildConfigList{
256
+		Items: bcs,
257
+	}
258
+}
259
+
260
+func bc(namespace, name, strategyType, fromKind, fromNamespace, fromName string) buildapi.BuildConfig {
261
+	return buildapi.BuildConfig{
262
+		ObjectMeta: kapi.ObjectMeta{
263
+			Namespace: namespace,
264
+			Name:      name,
265
+		},
266
+		Spec: buildapi.BuildConfigSpec{
267
+			CommonSpec: commonSpec(strategyType, fromKind, fromNamespace, fromName),
268
+		},
269
+	}
270
+}
271
+
272
+func buildList(builds ...buildapi.Build) buildapi.BuildList {
273
+	return buildapi.BuildList{
274
+		Items: builds,
275
+	}
276
+}
277
+
278
+func build(namespace, name, strategyType, fromKind, fromNamespace, fromName string) buildapi.Build {
279
+	return buildapi.Build{
280
+		ObjectMeta: kapi.ObjectMeta{
281
+			Namespace: namespace,
282
+			Name:      name,
283
+		},
284
+		Spec: buildapi.BuildSpec{
285
+			CommonSpec: commonSpec(strategyType, fromKind, fromNamespace, fromName),
286
+		},
287
+	}
288
+}
289
+
290
+func commonSpec(strategyType, fromKind, fromNamespace, fromName string) buildapi.CommonSpec {
291
+	spec := buildapi.CommonSpec{
292
+		Strategy: buildapi.BuildStrategy{},
293
+	}
294
+	switch strategyType {
295
+	case "source":
296
+		spec.Strategy.SourceStrategy = &buildapi.SourceBuildStrategy{
297
+			From: kapi.ObjectReference{
298
+				Kind:      fromKind,
299
+				Namespace: fromNamespace,
300
+				Name:      fromName,
301
+			},
302
+		}
303
+	case "docker":
304
+		spec.Strategy.DockerStrategy = &buildapi.DockerBuildStrategy{
305
+			From: &kapi.ObjectReference{
306
+				Kind:      fromKind,
307
+				Namespace: fromNamespace,
308
+				Name:      fromName,
309
+			},
310
+		}
311
+	case "custom":
312
+		spec.Strategy.CustomStrategy = &buildapi.CustomBuildStrategy{
313
+			From: kapi.ObjectReference{
314
+				Kind:      fromKind,
315
+				Namespace: fromNamespace,
316
+				Name:      fromName,
317
+			},
318
+		}
319
+	}
320
+
321
+	return spec
322
+}
323
+
324
+type fakeImageDeleter struct {
325
+	invocations sets.String
326
+	err         error
327
+}
328
+
329
+var _ ImageDeleter = &fakeImageDeleter{}
330
+
331
+func (p *fakeImageDeleter) DeleteImage(image *imageapi.Image) error {
332
+	p.invocations.Insert(image.Name)
333
+	return p.err
334
+}
335
+
336
+type fakeImageStreamDeleter struct {
337
+	invocations sets.String
338
+	err         error
339
+}
340
+
341
+var _ ImageStreamDeleter = &fakeImageStreamDeleter{}
342
+
343
+func (p *fakeImageStreamDeleter) DeleteImageStream(stream *imageapi.ImageStream, image *imageapi.Image, updatedTags []string) (*imageapi.ImageStream, error) {
344
+	p.invocations.Insert(fmt.Sprintf("%s/%s|%s", stream.Namespace, stream.Name, image.Name))
345
+	return stream, p.err
346
+}
347
+
348
+type fakeBlobDeleter struct {
349
+	invocations sets.String
350
+	err         error
351
+}
352
+
353
+var _ BlobDeleter = &fakeBlobDeleter{}
354
+
355
+func (p *fakeBlobDeleter) DeleteBlob(registryClient *http.Client, registryURL, blob string) error {
356
+	p.invocations.Insert(fmt.Sprintf("%s|%s", registryURL, blob))
357
+	return p.err
358
+}
359
+
360
+type fakeLayerDeleter struct {
361
+	invocations sets.String
362
+	err         error
363
+}
364
+
365
+var _ LayerDeleter = &fakeLayerDeleter{}
366
+
367
+func (p *fakeLayerDeleter) DeleteLayer(registryClient *http.Client, registryURL, repo, layer string) error {
368
+	p.invocations.Insert(fmt.Sprintf("%s|%s|%s", registryURL, repo, layer))
369
+	return p.err
370
+}
371
+
372
+type fakeManifestDeleter struct {
373
+	invocations sets.String
374
+	err         error
375
+}
376
+
377
+var _ ManifestDeleter = &fakeManifestDeleter{}
378
+
379
+func (p *fakeManifestDeleter) DeleteManifest(registryClient *http.Client, registryURL, repo, manifest string) error {
380
+	p.invocations.Insert(fmt.Sprintf("%s|%s|%s", registryURL, repo, manifest))
381
+	return p.err
382
+}
383
+
384
+var logLevel = flag.Int("loglevel", 0, "")
385
+var testCase = flag.String("testcase", "", "")
386
+
387
+func TestImagePruning(t *testing.T) {
388
+	flag.Lookup("v").Value.Set(fmt.Sprint(*logLevel))
389
+	registryURL := "registry"
390
+
391
+	tests := map[string]struct {
392
+		registryURLs           []string
393
+		images                 imageapi.ImageList
394
+		pods                   kapi.PodList
395
+		streams                imageapi.ImageStreamList
396
+		rcs                    kapi.ReplicationControllerList
397
+		bcs                    buildapi.BuildConfigList
398
+		builds                 buildapi.BuildList
399
+		dcs                    deployapi.DeploymentConfigList
400
+		expectedDeletions      []string
401
+		expectedUpdatedStreams []string
402
+	}{
403
+		"1 pod - phase pending - don't prune": {
404
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
405
+			pods:              podList(pod("foo", "pod1", kapi.PodPending, registryURL+"/foo/bar@id")),
406
+			expectedDeletions: []string{},
407
+		},
408
+		"3 pods - last phase pending - don't prune": {
409
+			images: imageList(image("id", registryURL+"/foo/bar@id")),
410
+			pods: podList(
411
+				pod("foo", "pod1", kapi.PodSucceeded, registryURL+"/foo/bar@id"),
412
+				pod("foo", "pod2", kapi.PodSucceeded, registryURL+"/foo/bar@id"),
413
+				pod("foo", "pod3", kapi.PodPending, registryURL+"/foo/bar@id"),
414
+			),
415
+			expectedDeletions: []string{},
416
+		},
417
+		"1 pod - phase running - don't prune": {
418
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
419
+			pods:              podList(pod("foo", "pod1", kapi.PodRunning, registryURL+"/foo/bar@id")),
420
+			expectedDeletions: []string{},
421
+		},
422
+		"3 pods - last phase running - don't prune": {
423
+			images: imageList(image("id", registryURL+"/foo/bar@id")),
424
+			pods: podList(
425
+				pod("foo", "pod1", kapi.PodSucceeded, registryURL+"/foo/bar@id"),
426
+				pod("foo", "pod2", kapi.PodSucceeded, registryURL+"/foo/bar@id"),
427
+				pod("foo", "pod3", kapi.PodRunning, registryURL+"/foo/bar@id"),
428
+			),
429
+			expectedDeletions: []string{},
430
+		},
431
+		"pod phase succeeded - prune": {
432
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
433
+			pods:              podList(pod("foo", "pod1", kapi.PodSucceeded, registryURL+"/foo/bar@id")),
434
+			expectedDeletions: []string{"id"},
435
+		},
436
+		"pod phase succeeded, pod less than min pruning age - don't prune": {
437
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
438
+			pods:              podList(agedPod("foo", "pod1", kapi.PodSucceeded, 5, registryURL+"/foo/bar@id")),
439
+			expectedDeletions: []string{},
440
+		},
441
+		"pod phase succeeded, image less than min pruning age - don't prune": {
442
+			images:            imageList(agedImage("id", registryURL+"/foo/bar@id", 5)),
443
+			pods:              podList(pod("foo", "pod1", kapi.PodSucceeded, registryURL+"/foo/bar@id")),
444
+			expectedDeletions: []string{},
445
+		},
446
+		"pod phase failed - prune": {
447
+			images: imageList(image("id", registryURL+"/foo/bar@id")),
448
+			pods: podList(
449
+				pod("foo", "pod1", kapi.PodFailed, registryURL+"/foo/bar@id"),
450
+				pod("foo", "pod2", kapi.PodFailed, registryURL+"/foo/bar@id"),
451
+				pod("foo", "pod3", kapi.PodFailed, registryURL+"/foo/bar@id"),
452
+			),
453
+			expectedDeletions: []string{"id"},
454
+		},
455
+		"pod phase unknown - prune": {
456
+			images: imageList(image("id", registryURL+"/foo/bar@id")),
457
+			pods: podList(
458
+				pod("foo", "pod1", kapi.PodUnknown, registryURL+"/foo/bar@id"),
459
+				pod("foo", "pod2", kapi.PodUnknown, registryURL+"/foo/bar@id"),
460
+				pod("foo", "pod3", kapi.PodUnknown, registryURL+"/foo/bar@id"),
461
+			),
462
+			expectedDeletions: []string{"id"},
463
+		},
464
+		"pod container image not parsable": {
465
+			images: imageList(image("id", registryURL+"/foo/bar@id")),
466
+			pods: podList(
467
+				pod("foo", "pod1", kapi.PodRunning, "a/b/c/d/e"),
468
+			),
469
+			expectedDeletions: []string{"id"},
470
+		},
471
+		"pod container image doesn't have an id": {
472
+			images: imageList(image("id", registryURL+"/foo/bar@id")),
473
+			pods: podList(
474
+				pod("foo", "pod1", kapi.PodRunning, "foo/bar:latest"),
475
+			),
476
+			expectedDeletions: []string{"id"},
477
+		},
478
+		"pod refers to image not in graph": {
479
+			images: imageList(image("id", registryURL+"/foo/bar@id")),
480
+			pods: podList(
481
+				pod("foo", "pod1", kapi.PodRunning, registryURL+"/foo/bar@otherid"),
482
+			),
483
+			expectedDeletions: []string{"id"},
484
+		},
485
+		"referenced by rc - don't prune": {
486
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
487
+			rcs:               rcList(rc("foo", "rc1", registryURL+"/foo/bar@id")),
488
+			expectedDeletions: []string{},
489
+		},
490
+		"referenced by dc - don't prune": {
491
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
492
+			dcs:               dcList(dc("foo", "rc1", registryURL+"/foo/bar@id")),
493
+			expectedDeletions: []string{},
494
+		},
495
+		"referenced by bc - sti - ImageStreamImage - don't prune": {
496
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
497
+			bcs:               bcList(bc("foo", "bc1", "source", "ImageStreamImage", "foo", "bar@id")),
498
+			expectedDeletions: []string{},
499
+		},
500
+		"referenced by bc - docker - ImageStreamImage - don't prune": {
501
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
502
+			bcs:               bcList(bc("foo", "bc1", "docker", "ImageStreamImage", "foo", "bar@id")),
503
+			expectedDeletions: []string{},
504
+		},
505
+		"referenced by bc - custom - ImageStreamImage - don't prune": {
506
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
507
+			bcs:               bcList(bc("foo", "bc1", "custom", "ImageStreamImage", "foo", "bar@id")),
508
+			expectedDeletions: []string{},
509
+		},
510
+		"referenced by bc - sti - DockerImage - don't prune": {
511
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
512
+			bcs:               bcList(bc("foo", "bc1", "source", "DockerImage", "foo", registryURL+"/foo/bar@id")),
513
+			expectedDeletions: []string{},
514
+		},
515
+		"referenced by bc - docker - DockerImage - don't prune": {
516
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
517
+			bcs:               bcList(bc("foo", "bc1", "docker", "DockerImage", "foo", registryURL+"/foo/bar@id")),
518
+			expectedDeletions: []string{},
519
+		},
520
+		"referenced by bc - custom - DockerImage - don't prune": {
521
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
522
+			bcs:               bcList(bc("foo", "bc1", "custom", "DockerImage", "foo", registryURL+"/foo/bar@id")),
523
+			expectedDeletions: []string{},
524
+		},
525
+		"referenced by build - sti - ImageStreamImage - don't prune": {
526
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
527
+			builds:            buildList(build("foo", "build1", "source", "ImageStreamImage", "foo", "bar@id")),
528
+			expectedDeletions: []string{},
529
+		},
530
+		"referenced by build - docker - ImageStreamImage - don't prune": {
531
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
532
+			builds:            buildList(build("foo", "build1", "docker", "ImageStreamImage", "foo", "bar@id")),
533
+			expectedDeletions: []string{},
534
+		},
535
+		"referenced by build - custom - ImageStreamImage - don't prune": {
536
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
537
+			builds:            buildList(build("foo", "build1", "custom", "ImageStreamImage", "foo", "bar@id")),
538
+			expectedDeletions: []string{},
539
+		},
540
+		"referenced by build - sti - DockerImage - don't prune": {
541
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
542
+			builds:            buildList(build("foo", "build1", "source", "DockerImage", "foo", registryURL+"/foo/bar@id")),
543
+			expectedDeletions: []string{},
544
+		},
545
+		"referenced by build - docker - DockerImage - don't prune": {
546
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
547
+			builds:            buildList(build("foo", "build1", "docker", "DockerImage", "foo", registryURL+"/foo/bar@id")),
548
+			expectedDeletions: []string{},
549
+		},
550
+		"referenced by build - custom - DockerImage - don't prune": {
551
+			images:            imageList(image("id", registryURL+"/foo/bar@id")),
552
+			builds:            buildList(build("foo", "build1", "custom", "DockerImage", "foo", registryURL+"/foo/bar@id")),
553
+			expectedDeletions: []string{},
554
+		},
555
+		"image stream - keep most recent n images": {
556
+			images: imageList(
557
+				unmanagedImage("id", "otherregistry/foo/bar@id", false, "", ""),
558
+				image("id2", registryURL+"/foo/bar@id2"),
559
+				image("id3", registryURL+"/foo/bar@id3"),
560
+				image("id4", registryURL+"/foo/bar@id4"),
561
+			),
562
+			streams: streamList(
563
+				stream(registryURL, "foo", "bar", tags(
564
+					tag("latest",
565
+						tagEvent("id", "otherregistry/foo/bar@id"),
566
+						tagEvent("id2", registryURL+"/foo/bar@id2"),
567
+						tagEvent("id3", registryURL+"/foo/bar@id3"),
568
+						tagEvent("id4", registryURL+"/foo/bar@id4"),
569
+					),
570
+				)),
571
+			),
572
+			expectedDeletions:      []string{"id4"},
573
+			expectedUpdatedStreams: []string{"foo/bar|id4"},
574
+		},
575
+		"image stream - same manifest listed multiple times in tag history": {
576
+			images: imageList(
577
+				image("id1", registryURL+"/foo/bar@id1"),
578
+				image("id2", registryURL+"/foo/bar@id2"),
579
+			),
580
+			streams: streamList(
581
+				stream(registryURL, "foo", "bar", tags(
582
+					tag("latest",
583
+						tagEvent("id1", registryURL+"/foo/bar@id1"),
584
+						tagEvent("id2", registryURL+"/foo/bar@id2"),
585
+						tagEvent("id1", registryURL+"/foo/bar@id1"),
586
+						tagEvent("id2", registryURL+"/foo/bar@id2"),
587
+					),
588
+				)),
589
+			),
590
+		},
591
+		"image stream age less than min pruning age - don't prune": {
592
+			images: imageList(
593
+				image("id", registryURL+"/foo/bar@id"),
594
+				image("id2", registryURL+"/foo/bar@id2"),
595
+				image("id3", registryURL+"/foo/bar@id3"),
596
+				image("id4", registryURL+"/foo/bar@id4"),
597
+			),
598
+			streams: streamList(
599
+				agedStream(registryURL, "foo", "bar", 5, tags(
600
+					tag("latest",
601
+						tagEvent("id", registryURL+"/foo/bar@id"),
602
+						tagEvent("id2", registryURL+"/foo/bar@id2"),
603
+						tagEvent("id3", registryURL+"/foo/bar@id3"),
604
+						tagEvent("id4", registryURL+"/foo/bar@id4"),
605
+					),
606
+				)),
607
+			),
608
+			expectedDeletions:      []string{},
609
+			expectedUpdatedStreams: []string{},
610
+		},
611
+		"multiple resources pointing to image - don't prune": {
612
+			images: imageList(
613
+				image("id", registryURL+"/foo/bar@id"),
614
+				image("id2", registryURL+"/foo/bar@id2"),
615
+			),
616
+			streams: streamList(
617
+				stream(registryURL, "foo", "bar", tags(
618
+					tag("latest",
619
+						tagEvent("id", registryURL+"/foo/bar@id"),
620
+						tagEvent("id2", registryURL+"/foo/bar@id2"),
621
+					),
622
+				)),
623
+			),
624
+			rcs:                    rcList(rc("foo", "rc1", registryURL+"/foo/bar@id2")),
625
+			pods:                   podList(pod("foo", "pod1", kapi.PodRunning, registryURL+"/foo/bar@id2")),
626
+			dcs:                    dcList(dc("foo", "rc1", registryURL+"/foo/bar@id")),
627
+			bcs:                    bcList(bc("foo", "bc1", "source", "DockerImage", "foo", registryURL+"/foo/bar@id")),
628
+			builds:                 buildList(build("foo", "build1", "custom", "ImageStreamImage", "foo", "bar@id")),
629
+			expectedDeletions:      []string{},
630
+			expectedUpdatedStreams: []string{},
631
+		},
632
+		"image with nil annotations": {
633
+			images: imageList(
634
+				unmanagedImage("id", "someregistry/foo/bar@id", false, "", ""),
635
+			),
636
+			expectedDeletions:      []string{},
637
+			expectedUpdatedStreams: []string{},
638
+		},
639
+		"image missing managed annotation": {
640
+			images: imageList(
641
+				unmanagedImage("id", "someregistry/foo/bar@id", true, "foo", "bar"),
642
+			),
643
+			expectedDeletions:      []string{},
644
+			expectedUpdatedStreams: []string{},
645
+		},
646
+		"image with managed annotation != true": {
647
+			images: imageList(
648
+				unmanagedImage("id", "someregistry/foo/bar@id", true, imageapi.ManagedByOpenShiftAnnotation, "false"),
649
+				unmanagedImage("id", "someregistry/foo/bar@id", true, imageapi.ManagedByOpenShiftAnnotation, "0"),
650
+				unmanagedImage("id", "someregistry/foo/bar@id", true, imageapi.ManagedByOpenShiftAnnotation, "1"),
651
+				unmanagedImage("id", "someregistry/foo/bar@id", true, imageapi.ManagedByOpenShiftAnnotation, "True"),
652
+				unmanagedImage("id", "someregistry/foo/bar@id", true, imageapi.ManagedByOpenShiftAnnotation, "yes"),
653
+				unmanagedImage("id", "someregistry/foo/bar@id", true, imageapi.ManagedByOpenShiftAnnotation, "Yes"),
654
+			),
655
+			expectedDeletions:      []string{},
656
+			expectedUpdatedStreams: []string{},
657
+		},
658
+		"image with bad manifest is pruned ok": {
659
+			images: imageList(
660
+				imageWithBadManifest("id", "someregistry/foo/bar@id"),
661
+			),
662
+			expectedDeletions:      []string{"id"},
663
+			expectedUpdatedStreams: []string{},
664
+		},
665
+	}
666
+
667
+	for name, test := range tests {
668
+		tcFilter := flag.Lookup("testcase").Value.String()
669
+		if len(tcFilter) > 0 && name != tcFilter {
670
+			continue
671
+		}
672
+
673
+		options := PrunerOptions{
674
+			KeepYoungerThan:  60 * time.Minute,
675
+			KeepTagRevisions: 3,
676
+			Images:           &test.images,
677
+			Streams:          &test.streams,
678
+			Pods:             &test.pods,
679
+			RCs:              &test.rcs,
680
+			BCs:              &test.bcs,
681
+			Builds:           &test.builds,
682
+			DCs:              &test.dcs,
683
+		}
684
+		p := NewPruner(options)
685
+		p.(*pruner).registryPinger = &fakeRegistryPinger{}
686
+
687
+		imageDeleter := &fakeImageDeleter{invocations: sets.NewString()}
688
+		streamDeleter := &fakeImageStreamDeleter{invocations: sets.NewString()}
689
+		layerDeleter := &fakeLayerDeleter{invocations: sets.NewString()}
690
+		blobDeleter := &fakeBlobDeleter{invocations: sets.NewString()}
691
+		manifestDeleter := &fakeManifestDeleter{invocations: sets.NewString()}
692
+
693
+		p.Prune(imageDeleter, streamDeleter, layerDeleter, blobDeleter, manifestDeleter)
694
+
695
+		expectedDeletions := sets.NewString(test.expectedDeletions...)
696
+		if !reflect.DeepEqual(expectedDeletions, imageDeleter.invocations) {
697
+			t.Errorf("%s: expected image deletions %q, got %q", name, expectedDeletions.List(), imageDeleter.invocations.List())
698
+		}
699
+
700
+		expectedUpdatedStreams := sets.NewString(test.expectedUpdatedStreams...)
701
+		if !reflect.DeepEqual(expectedUpdatedStreams, streamDeleter.invocations) {
702
+			t.Errorf("%s: expected stream updates %q, got %q", name, expectedUpdatedStreams.List(), streamDeleter.invocations.List())
703
+		}
704
+	}
705
+}
706
+
707
+func TestImageDeleter(t *testing.T) {
708
+	flag.Lookup("v").Value.Set(fmt.Sprint(*logLevel))
709
+
710
+	tests := map[string]struct {
711
+		imageDeletionError error
712
+	}{
713
+		"no error": {},
714
+		"delete error": {
715
+			imageDeletionError: fmt.Errorf("foo"),
716
+		},
717
+	}
718
+
719
+	for name, test := range tests {
720
+		imageClient := testclient.Fake{}
721
+		imageClient.AddReactor("delete", "images", func(action ktc.Action) (handled bool, ret runtime.Object, err error) {
722
+			return true, nil, test.imageDeletionError
723
+		})
724
+		imageDeleter := NewImageDeleter(imageClient.Images())
725
+		err := imageDeleter.DeleteImage(&imageapi.Image{ObjectMeta: kapi.ObjectMeta{Name: "id2"}})
726
+		if test.imageDeletionError != nil {
727
+			if e, a := test.imageDeletionError, err; e != a {
728
+				t.Errorf("%s: err: expected %v, got %v", name, e, a)
729
+			}
730
+			continue
731
+		}
732
+
733
+		if e, a := 1, len(imageClient.Actions()); e != a {
734
+			t.Errorf("%s: expected %d actions, got %d: %#v", name, e, a, imageClient.Actions())
735
+			continue
736
+		}
737
+
738
+		if !imageClient.Actions()[0].Matches("delete", "images") {
739
+			t.Errorf("%s: expected action %s, got %v", name, "delete-images", imageClient.Actions()[0])
740
+		}
741
+	}
742
+}
743
+
744
+func TestLayerDeleter(t *testing.T) {
745
+	flag.Lookup("v").Value.Set(fmt.Sprint(*logLevel))
746
+
747
+	var actions []string
748
+	client := fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) {
749
+		actions = append(actions, req.Method+":"+req.URL.String())
750
+		return &http.Response{StatusCode: http.StatusServiceUnavailable, Body: ioutil.NopCloser(bytes.NewReader([]byte{}))}, nil
751
+	})
752
+	layerDeleter := NewLayerDeleter()
753
+	layerDeleter.DeleteLayer(client, "registry1", "repo", "layer1")
754
+
755
+	if !reflect.DeepEqual(actions, []string{"DELETE:https://registry1/v2/repo/blobs/layer1",
756
+		"DELETE:http://registry1/v2/repo/blobs/layer1"}) {
757
+		t.Errorf("Unexpected actions %v", actions)
758
+	}
759
+}
760
+
761
+func TestNotFoundLayerDeleter(t *testing.T) {
762
+	flag.Lookup("v").Value.Set(fmt.Sprint(*logLevel))
763
+
764
+	var actions []string
765
+	client := fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) {
766
+		actions = append(actions, req.Method+":"+req.URL.String())
767
+		return &http.Response{StatusCode: http.StatusNotFound, Body: ioutil.NopCloser(bytes.NewReader([]byte{}))}, nil
768
+	})
769
+	layerDeleter := NewLayerDeleter()
770
+	layerDeleter.DeleteLayer(client, "registry1", "repo", "layer1")
771
+
772
+	if !reflect.DeepEqual(actions, []string{"DELETE:https://registry1/v2/repo/blobs/layer1"}) {
773
+		t.Errorf("Unexpected actions %v", actions)
774
+	}
775
+}
776
+
777
+func TestRegistryPruning(t *testing.T) {
778
+	flag.Lookup("v").Value.Set(fmt.Sprint(*logLevel))
779
+
780
+	tests := map[string]struct {
781
+		images                    imageapi.ImageList
782
+		streams                   imageapi.ImageStreamList
783
+		expectedLayerDeletions    sets.String
784
+		expectedBlobDeletions     sets.String
785
+		expectedManifestDeletions sets.String
786
+		pingErr                   error
787
+	}{
788
+		"layers unique to id1 pruned": {
789
+			images: imageList(
790
+				imageWithLayers("id1", "registry1/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"),
791
+				imageWithLayers("id2", "registry1/foo/bar@id2", "layer3", "layer4", "layer5", "layer6"),
792
+			),
793
+			streams: streamList(
794
+				stream("registry1", "foo", "bar", tags(
795
+					tag("latest",
796
+						tagEvent("id2", "registry1/foo/bar@id2"),
797
+						tagEvent("id1", "registry1/foo/bar@id1"),
798
+					),
799
+				)),
800
+				stream("registry1", "foo", "other", tags(
801
+					tag("latest",
802
+						tagEvent("id2", "registry1/foo/other@id2"),
803
+					),
804
+				)),
805
+			),
806
+			expectedLayerDeletions: sets.NewString(
807
+				"registry1|foo/bar|layer1",
808
+				"registry1|foo/bar|layer2",
809
+			),
810
+			expectedBlobDeletions: sets.NewString(
811
+				"registry1|layer1",
812
+				"registry1|layer2",
813
+			),
814
+			expectedManifestDeletions: sets.NewString(
815
+				"registry1|foo/bar|id1",
816
+			),
817
+		},
818
+		"no pruning when no images are pruned": {
819
+			images: imageList(
820
+				imageWithLayers("id1", "registry1/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"),
821
+			),
822
+			streams: streamList(
823
+				stream("registry1", "foo", "bar", tags(
824
+					tag("latest",
825
+						tagEvent("id1", "registry1/foo/bar@id1"),
826
+					),
827
+				)),
828
+			),
829
+			expectedLayerDeletions:    sets.NewString(),
830
+			expectedBlobDeletions:     sets.NewString(),
831
+			expectedManifestDeletions: sets.NewString(),
832
+		},
833
+		"blobs pruned when streams have already been deleted": {
834
+			images: imageList(
835
+				imageWithLayers("id1", "registry1/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"),
836
+				imageWithLayers("id2", "registry1/foo/bar@id2", "layer3", "layer4", "layer5", "layer6"),
837
+			),
838
+			expectedLayerDeletions: sets.NewString(),
839
+			expectedBlobDeletions: sets.NewString(
840
+				"registry1|layer1",
841
+				"registry1|layer2",
842
+				"registry1|layer3",
843
+				"registry1|layer4",
844
+				"registry1|layer5",
845
+				"registry1|layer6",
846
+			),
847
+			expectedManifestDeletions: sets.NewString(),
848
+		},
849
+		"ping error": {
850
+			images: imageList(
851
+				imageWithLayers("id1", "registry1/foo/bar@id1", "layer1", "layer2", "layer3", "layer4"),
852
+				imageWithLayers("id2", "registry1/foo/bar@id2", "layer3", "layer4", "layer5", "layer6"),
853
+			),
854
+			streams: streamList(
855
+				stream("registry1", "foo", "bar", tags(
856
+					tag("latest",
857
+						tagEvent("id2", "registry1/foo/bar@id2"),
858
+						tagEvent("id1", "registry1/foo/bar@id1"),
859
+					),
860
+				)),
861
+				stream("registry1", "foo", "other", tags(
862
+					tag("latest",
863
+						tagEvent("id2", "registry1/foo/other@id2"),
864
+					),
865
+				)),
866
+			),
867
+			expectedLayerDeletions:    sets.NewString(),
868
+			expectedBlobDeletions:     sets.NewString(),
869
+			expectedManifestDeletions: sets.NewString(),
870
+			pingErr:                   errors.New("foo"),
871
+		},
872
+	}
873
+
874
+	for name, test := range tests {
875
+		tcFilter := flag.Lookup("testcase").Value.String()
876
+		if len(tcFilter) > 0 && name != tcFilter {
877
+			continue
878
+		}
879
+
880
+		t.Logf("Running test case %s", name)
881
+
882
+		options := PrunerOptions{
883
+			KeepYoungerThan:  60 * time.Minute,
884
+			KeepTagRevisions: 1,
885
+			Images:           &test.images,
886
+			Streams:          &test.streams,
887
+			Pods:             &kapi.PodList{},
888
+			RCs:              &kapi.ReplicationControllerList{},
889
+			BCs:              &buildapi.BuildConfigList{},
890
+			Builds:           &buildapi.BuildList{},
891
+			DCs:              &deployapi.DeploymentConfigList{},
892
+		}
893
+		p := NewPruner(options)
894
+		p.(*pruner).registryPinger = &fakeRegistryPinger{err: test.pingErr}
895
+
896
+		imageDeleter := &fakeImageDeleter{invocations: sets.NewString()}
897
+		streamDeleter := &fakeImageStreamDeleter{invocations: sets.NewString()}
898
+		layerDeleter := &fakeLayerDeleter{invocations: sets.NewString()}
899
+		blobDeleter := &fakeBlobDeleter{invocations: sets.NewString()}
900
+		manifestDeleter := &fakeManifestDeleter{invocations: sets.NewString()}
901
+
902
+		p.Prune(imageDeleter, streamDeleter, layerDeleter, blobDeleter, manifestDeleter)
903
+
904
+		if !reflect.DeepEqual(test.expectedLayerDeletions, layerDeleter.invocations) {
905
+			t.Errorf("%s: expected layer deletions %#v, got %#v", name, test.expectedLayerDeletions, layerDeleter.invocations)
906
+		}
907
+		if !reflect.DeepEqual(test.expectedBlobDeletions, blobDeleter.invocations) {
908
+			t.Errorf("%s: expected blob deletions %#v, got %#v", name, test.expectedBlobDeletions, blobDeleter.invocations)
909
+		}
910
+		if !reflect.DeepEqual(test.expectedManifestDeletions, manifestDeleter.invocations) {
911
+			t.Errorf("%s: expected manifest deletions %#v, got %#v", name, test.expectedManifestDeletions, manifestDeleter.invocations)
912
+		}
913
+	}
914
+}