Browse code

builder: add prune options to the API

Signed-off-by: Tibor Vass <tibor@docker.com>

Tibor Vass authored on 2018/08/16 06:24:37
Showing 9 changed files
... ...
@@ -88,7 +88,7 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
88 88
 }
89 89
 
90 90
 // PruneCache removes all cached build sources
91
-func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport, error) {
91
+func (b *Backend) PruneCache(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error) {
92 92
 	eg, ctx := errgroup.WithContext(ctx)
93 93
 
94 94
 	var fsCacheSize uint64
... ...
@@ -102,9 +102,10 @@ func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport,
102 102
 	})
103 103
 
104 104
 	var buildCacheSize int64
105
+	var cacheIDs []string
105 106
 	eg.Go(func() error {
106 107
 		var err error
107
-		buildCacheSize, err = b.buildkit.Prune(ctx)
108
+		buildCacheSize, cacheIDs, err = b.buildkit.Prune(ctx, opts)
108 109
 		if err != nil {
109 110
 			return errors.Wrap(err, "failed to prune build cache")
110 111
 		}
... ...
@@ -115,7 +116,7 @@ func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport,
115 115
 		return nil, err
116 116
 	}
117 117
 
118
-	return &types.BuildCachePruneReport{SpaceReclaimed: fsCacheSize + uint64(buildCacheSize)}, nil
118
+	return &types.BuildCachePruneReport{SpaceReclaimed: fsCacheSize + uint64(buildCacheSize), CachesDeleted: cacheIDs}, nil
119 119
 }
120 120
 
121 121
 // Cancel cancels the build by ID
... ...
@@ -14,7 +14,7 @@ type Backend interface {
14 14
 	Build(context.Context, backend.BuildConfig) (string, error)
15 15
 
16 16
 	// Prune build cache
17
-	PruneCache(context.Context) (*types.BuildCachePruneReport, error)
17
+	PruneCache(context.Context, types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error)
18 18
 
19 19
 	Cancel(context.Context, string) error
20 20
 }
... ...
@@ -18,6 +18,7 @@ import (
18 18
 	"github.com/docker/docker/api/types"
19 19
 	"github.com/docker/docker/api/types/backend"
20 20
 	"github.com/docker/docker/api/types/container"
21
+	"github.com/docker/docker/api/types/filters"
21 22
 	"github.com/docker/docker/api/types/versions"
22 23
 	"github.com/docker/docker/errdefs"
23 24
 	"github.com/docker/docker/pkg/ioutils"
... ...
@@ -161,7 +162,26 @@ func parseVersion(s string) (types.BuilderVersion, error) {
161 161
 }
162 162
 
163 163
 func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
164
-	report, err := br.backend.PruneCache(ctx)
164
+	if err := httputils.ParseForm(r); err != nil {
165
+		return err
166
+	}
167
+	filters, err := filters.FromJSON(r.Form.Get("filters"))
168
+	if err != nil {
169
+		return errors.Wrap(err, "could not parse filters")
170
+	}
171
+	ksfv := r.FormValue("keep-storage")
172
+	ks, err := strconv.Atoi(ksfv)
173
+	if err != nil {
174
+		return errors.Wrapf(err, "keep-storage is in bytes and expects an integer, got %v", ksfv)
175
+	}
176
+
177
+	opts := types.BuildCachePruneOptions{
178
+		All:         httputils.BoolValue(r, "all"),
179
+		Filters:     filters,
180
+		KeepStorage: int64(ks),
181
+	}
182
+
183
+	report, err := br.backend.PruneCache(ctx, opts)
165 184
 	if err != nil {
166 185
 		return err
167 186
 	}
... ...
@@ -1513,6 +1513,31 @@ definitions:
1513 1513
       aux:
1514 1514
         $ref: "#/definitions/ImageID"
1515 1515
 
1516
+  BuildCache:
1517
+    type: "object"
1518
+    properties:
1519
+      ID:
1520
+        type: "string"
1521
+      Parent:
1522
+        type: "string"
1523
+      Type:
1524
+        type: "string"
1525
+      Description:
1526
+        type: "string"
1527
+      InUse:
1528
+        type: "boolean"
1529
+      Shared:
1530
+        type: "boolean"
1531
+      Size:
1532
+        type: "integer"
1533
+      CreatedAt:
1534
+        type: "integer"
1535
+      LastUsedAt:
1536
+        type: "integer"
1537
+        x-nullable: true
1538
+      UsageCount:
1539
+        type: "integer"
1540
+
1516 1541
   ImageID:
1517 1542
     type: "object"
1518 1543
     description: "Image ID or Digest"
... ...
@@ -6358,6 +6383,29 @@ paths:
6358 6358
       produces:
6359 6359
         - "application/json"
6360 6360
       operationId: "BuildPrune"
6361
+      parameters:
6362
+        - name: "keep-storage"
6363
+          in: "query"
6364
+          description: "Amount of disk space in bytes to keep for cache"
6365
+          type: "integer"
6366
+          format: "int64"
6367
+        - name: "all"
6368
+          in: "query"
6369
+          type: "boolean"
6370
+          description: "Remove all types of build cache"
6371
+        - name: "filters"
6372
+          in: "query"
6373
+          type: "string"
6374
+          description: |
6375
+            A JSON encoded value of the filters (a `map[string][]string`) to process on the list of build cache objects. Available filters:
6376
+            - `unused-for=<duration>`: duration relative to daemon's time, during which build cache was not used, in Go's duration format (e.g., '24h')
6377
+            - `id=<id>`
6378
+            - `parent=<id>`
6379
+            - `type=<string>`
6380
+            - `description=<string>`
6381
+            - `inuse`
6382
+            - `shared`
6383
+            - `private`
6361 6384
       responses:
6362 6385
         200:
6363 6386
           description: "No error"
... ...
@@ -6365,6 +6413,11 @@ paths:
6365 6365
             type: "object"
6366 6366
             title: "BuildPruneResponse"
6367 6367
             properties:
6368
+              CachesDeleted:
6369
+                type: "array"
6370
+                items:
6371
+                  description: "ID of build cache object"
6372
+                  type: "string"
6368 6373
               SpaceReclaimed:
6369 6374
                 description: "Disk space reclaimed in bytes"
6370 6375
                 type: "integer"
... ...
@@ -7199,6 +7252,10 @@ paths:
7199 7199
                 type: "array"
7200 7200
                 items:
7201 7201
                   $ref: "#/definitions/Volume"
7202
+              BuildCache:
7203
+                type: "array"
7204
+                items:
7205
+                  $ref: "#/definitions/BuildCache"
7202 7206
             example:
7203 7207
               LayersSize: 1092588
7204 7208
               Images:
... ...
@@ -543,6 +543,7 @@ type ImagesPruneReport struct {
543 543
 // BuildCachePruneReport contains the response for Engine API:
544 544
 // POST "/build/prune"
545 545
 type BuildCachePruneReport struct {
546
+	CachesDeleted  []string
546 547
 	SpaceReclaimed uint64
547 548
 }
548 549
 
... ...
@@ -592,14 +593,21 @@ type BuildResult struct {
592 592
 
593 593
 // BuildCache contains information about a build cache record
594 594
 type BuildCache struct {
595
-	ID      string
596
-	Mutable bool
597
-	InUse   bool
598
-	Size    int64
599
-
595
+	ID          string
596
+	Parent      string
597
+	Type        string
598
+	Description string
599
+	InUse       bool
600
+	Shared      bool
601
+	Size        int64
600 602
 	CreatedAt   time.Time
601 603
 	LastUsedAt  *time.Time
602 604
 	UsageCount  int
603
-	Parent      string
604
-	Description string
605
+}
606
+
607
+// BuildCachePruneOptions hold parameters to prune the build cache
608
+type BuildCachePruneOptions struct {
609
+	All         bool
610
+	KeepStorage int64
611
+	Filters     filters.Args
605 612
 }
... ...
@@ -29,6 +29,21 @@ import (
29 29
 	grpcmetadata "google.golang.org/grpc/metadata"
30 30
 )
31 31
 
32
+var errMultipleFilterValues = errors.New("filters expect only one value")
33
+
34
+var cacheFields = map[string]bool{
35
+	"id":          true,
36
+	"parent":      true,
37
+	"type":        true,
38
+	"description": true,
39
+	"inuse":       true,
40
+	"shared":      true,
41
+	"private":     true,
42
+	// fields from buildkit that are not exposed
43
+	"mutable":   false,
44
+	"immutable": false,
45
+}
46
+
32 47
 func init() {
33 48
 	llbsolver.AllowNetworkHostUnstable = true
34 49
 }
... ...
@@ -87,48 +102,94 @@ func (b *Builder) DiskUsage(ctx context.Context) ([]*types.BuildCache, error) {
87 87
 	var items []*types.BuildCache
88 88
 	for _, r := range duResp.Record {
89 89
 		items = append(items, &types.BuildCache{
90
-			ID:      r.ID,
91
-			Mutable: r.Mutable,
92
-			InUse:   r.InUse,
93
-			Size:    r.Size_,
94
-
90
+			ID:          r.ID,
91
+			Parent:      r.Parent,
92
+			Type:        r.RecordType,
93
+			Description: r.Description,
94
+			InUse:       r.InUse,
95
+			Shared:      r.Shared,
96
+			Size:        r.Size_,
95 97
 			CreatedAt:   r.CreatedAt,
96 98
 			LastUsedAt:  r.LastUsedAt,
97 99
 			UsageCount:  int(r.UsageCount),
98
-			Parent:      r.Parent,
99
-			Description: r.Description,
100 100
 		})
101 101
 	}
102 102
 	return items, nil
103 103
 }
104 104
 
105 105
 // Prune clears all reclaimable build cache
106
-func (b *Builder) Prune(ctx context.Context) (int64, error) {
106
+func (b *Builder) Prune(ctx context.Context, opts types.BuildCachePruneOptions) (int64, []string, error) {
107 107
 	ch := make(chan *controlapi.UsageRecord)
108 108
 
109 109
 	eg, ctx := errgroup.WithContext(ctx)
110 110
 
111
+	validFilters := make(map[string]bool, 1+len(cacheFields))
112
+	validFilters["unused-for"] = true
113
+	for k, v := range cacheFields {
114
+		validFilters[k] = v
115
+	}
116
+	if err := opts.Filters.Validate(validFilters); err != nil {
117
+		return 0, nil, err
118
+	}
119
+
120
+	var unusedFor time.Duration
121
+	unusedForValues := opts.Filters.Get("unused-for")
122
+
123
+	switch len(unusedForValues) {
124
+	case 0:
125
+
126
+	case 1:
127
+		var err error
128
+		unusedFor, err = time.ParseDuration(unusedForValues[0])
129
+		if err != nil {
130
+			return 0, nil, errors.Wrap(err, "unused-for filter expects a duration (e.g., '24h')")
131
+		}
132
+
133
+	default:
134
+		return 0, nil, errMultipleFilterValues
135
+	}
136
+
137
+	bkFilter := make([]string, 0, opts.Filters.Len())
138
+	for cacheField := range cacheFields {
139
+		values := opts.Filters.Get(cacheField)
140
+		switch len(values) {
141
+		case 0:
142
+			bkFilter = append(bkFilter, cacheField)
143
+		case 1:
144
+			bkFilter = append(bkFilter, cacheField+"=="+values[0])
145
+		default:
146
+			return 0, nil, errMultipleFilterValues
147
+		}
148
+	}
149
+
111 150
 	eg.Go(func() error {
112 151
 		defer close(ch)
113
-		return b.controller.Prune(&controlapi.PruneRequest{}, &pruneProxy{
152
+		return b.controller.Prune(&controlapi.PruneRequest{
153
+			All:          opts.All,
154
+			KeepDuration: int64(unusedFor),
155
+			KeepBytes:    opts.KeepStorage,
156
+			Filter:       bkFilter,
157
+		}, &pruneProxy{
114 158
 			streamProxy: streamProxy{ctx: ctx},
115 159
 			ch:          ch,
116 160
 		})
117 161
 	})
118 162
 
119 163
 	var size int64
164
+	var cacheIDs []string
120 165
 	eg.Go(func() error {
121 166
 		for r := range ch {
122 167
 			size += r.Size_
168
+			cacheIDs = append(cacheIDs, r.ID)
123 169
 		}
124 170
 		return nil
125 171
 	})
126 172
 
127 173
 	if err := eg.Wait(); err != nil {
128
-		return 0, err
174
+		return 0, nil, err
129 175
 	}
130 176
 
131
-	return size, nil
177
+	return size, cacheIDs, nil
132 178
 }
133 179
 
134 180
 // Build executes a build request
... ...
@@ -4,19 +4,34 @@ import (
4 4
 	"context"
5 5
 	"encoding/json"
6 6
 	"fmt"
7
+	"net/url"
7 8
 
8 9
 	"github.com/docker/docker/api/types"
10
+	"github.com/docker/docker/api/types/filters"
11
+	"github.com/pkg/errors"
9 12
 )
10 13
 
11 14
 // BuildCachePrune requests the daemon to delete unused cache data
12
-func (cli *Client) BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error) {
15
+func (cli *Client) BuildCachePrune(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error) {
13 16
 	if err := cli.NewVersionError("1.31", "build prune"); err != nil {
14 17
 		return nil, err
15 18
 	}
16 19
 
17 20
 	report := types.BuildCachePruneReport{}
18 21
 
19
-	serverResp, err := cli.post(ctx, "/build/prune", nil, nil, nil)
22
+	query := url.Values{}
23
+	if opts.All {
24
+		query.Set("all", "1")
25
+	}
26
+	query.Set("keep-storage", fmt.Sprintf("%d", opts.KeepStorage))
27
+	filters, err := filters.ToJSON(opts.Filters)
28
+	if err != nil {
29
+		return nil, errors.Wrap(err, "prune could not marshal filters option")
30
+	}
31
+	query.Set("filters", filters)
32
+
33
+	serverResp, err := cli.post(ctx, "/build/prune", query, nil, nil)
34
+
20 35
 	if err != nil {
21 36
 		return nil, err
22 37
 	}
... ...
@@ -86,7 +86,7 @@ type DistributionAPIClient interface {
86 86
 // ImageAPIClient defines API client methods for the images
87 87
 type ImageAPIClient interface {
88 88
 	ImageBuild(ctx context.Context, context io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error)
89
-	BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error)
89
+	BuildCachePrune(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error)
90 90
 	BuildCancel(ctx context.Context, id string) error
91 91
 	ImageCreate(ctx context.Context, parentReference string, options types.ImageCreateOptions) (io.ReadCloser, error)
92 92
 	ImageHistory(ctx context.Context, image string) ([]image.HistoryResponseItem, error)
... ...
@@ -7,6 +7,7 @@ import (
7 7
 	"strings"
8 8
 	"testing"
9 9
 
10
+	"github.com/docker/docker/api/types"
10 11
 	dclient "github.com/docker/docker/client"
11 12
 	"github.com/docker/docker/internal/test/fakecontext"
12 13
 	"github.com/docker/docker/internal/test/request"
... ...
@@ -76,7 +77,7 @@ func TestBuildWithSession(t *testing.T) {
76 76
 	assert.Check(t, is.Contains(string(outBytes), "Successfully built"))
77 77
 	assert.Check(t, is.Equal(strings.Count(string(outBytes), "Using cache"), 4))
78 78
 
79
-	_, err = client.BuildCachePrune(context.TODO())
79
+	_, err = client.BuildCachePrune(context.TODO(), types.BuildCachePruneOptions{All: true})
80 80
 	assert.Check(t, err)
81 81
 
82 82
 	du, err = client.DiskUsage(context.TODO())