Browse code

builder: use buildkit's GC for build cache

This allows users to configure the buildkit GC.

The following enables the default GC:
```
{
"builder": {
"gc": {
"enabled": true
}
}
}
```

The default GC policy has a simple config:
```
{
"builder": {
"gc": {
"enabled": true,
"defaultKeepStorage": "30GB"
}
}
}
```

A custom GC policy can be used instead by specifying a list of cache prune rules:
```
{
"builder": {
"gc": {
"enabled": true,
"policy": [
{"keepStorage": "512MB", "filter": ["unused-for=1400h"]]},
{"keepStorage": "30GB", "all": true}
]
}
}
}
```

Signed-off-by: Tibor Vass <tibor@docker.com>

Tibor Vass authored on 2018/09/05 11:12:44
Showing 8 changed files
... ...
@@ -13,11 +13,13 @@ import (
13 13
 	"github.com/docker/docker/api/types"
14 14
 	"github.com/docker/docker/api/types/backend"
15 15
 	"github.com/docker/docker/builder"
16
+	"github.com/docker/docker/daemon/config"
16 17
 	"github.com/docker/docker/daemon/images"
17 18
 	"github.com/docker/docker/pkg/streamformatter"
18 19
 	"github.com/docker/docker/pkg/system"
19 20
 	"github.com/docker/libnetwork"
20 21
 	controlapi "github.com/moby/buildkit/api/services/control"
22
+	"github.com/moby/buildkit/client"
21 23
 	"github.com/moby/buildkit/control"
22 24
 	"github.com/moby/buildkit/identity"
23 25
 	"github.com/moby/buildkit/session"
... ...
@@ -57,6 +59,7 @@ type Opt struct {
57 57
 	NetworkController   libnetwork.NetworkController
58 58
 	DefaultCgroupParent string
59 59
 	ResolverOpt         resolver.ResolveOptionsFunc
60
+	BuilderConfig       config.BuilderConfig
60 61
 }
61 62
 
62 63
 // Builder can build using BuildKit backend
... ...
@@ -134,43 +137,18 @@ func (b *Builder) Prune(ctx context.Context, opts types.BuildCachePruneOptions)
134 134
 		return 0, nil, err
135 135
 	}
136 136
 
137
-	var unusedFor time.Duration
138
-	unusedForValues := opts.Filters.Get("unused-for")
139
-
140
-	switch len(unusedForValues) {
141
-	case 0:
142
-
143
-	case 1:
144
-		var err error
145
-		unusedFor, err = time.ParseDuration(unusedForValues[0])
146
-		if err != nil {
147
-			return 0, nil, errors.Wrap(err, "unused-for filter expects a duration (e.g., '24h')")
148
-		}
149
-
150
-	default:
151
-		return 0, nil, errMultipleFilterValues
152
-	}
153
-
154
-	bkFilter := make([]string, 0, opts.Filters.Len())
155
-	for cacheField := range cacheFields {
156
-		values := opts.Filters.Get(cacheField)
157
-		switch len(values) {
158
-		case 0:
159
-			bkFilter = append(bkFilter, cacheField)
160
-		case 1:
161
-			bkFilter = append(bkFilter, cacheField+"=="+values[0])
162
-		default:
163
-			return 0, nil, errMultipleFilterValues
164
-		}
137
+	pi, err := toBuildkitPruneInfo(opts)
138
+	if err != nil {
139
+		return 0, nil, err
165 140
 	}
166 141
 
167 142
 	eg.Go(func() error {
168 143
 		defer close(ch)
169 144
 		return b.controller.Prune(&controlapi.PruneRequest{
170
-			All:          opts.All,
171
-			KeepDuration: int64(unusedFor),
172
-			KeepBytes:    opts.KeepStorage,
173
-			Filter:       bkFilter,
145
+			All:          pi.All,
146
+			KeepDuration: int64(pi.KeepDuration),
147
+			KeepBytes:    pi.KeepBytes,
148
+			Filter:       pi.Filter,
174 149
 		}, &pruneProxy{
175 150
 			streamProxy: streamProxy{ctx: ctx},
176 151
 			ch:          ch,
... ...
@@ -531,3 +509,41 @@ func toBuildkitExtraHosts(inp []string) (string, error) {
531 531
 	}
532 532
 	return strings.Join(hosts, ","), nil
533 533
 }
534
+
535
+func toBuildkitPruneInfo(opts types.BuildCachePruneOptions) (client.PruneInfo, error) {
536
+	var unusedFor time.Duration
537
+	unusedForValues := opts.Filters.Get("unused-for")
538
+
539
+	switch len(unusedForValues) {
540
+	case 0:
541
+
542
+	case 1:
543
+		var err error
544
+		unusedFor, err = time.ParseDuration(unusedForValues[0])
545
+		if err != nil {
546
+			return client.PruneInfo{}, errors.Wrap(err, "unused-for filter expects a duration (e.g., '24h')")
547
+		}
548
+
549
+	default:
550
+		return client.PruneInfo{}, errMultipleFilterValues
551
+	}
552
+
553
+	bkFilter := make([]string, 0, opts.Filters.Len())
554
+	for cacheField := range cacheFields {
555
+		values := opts.Filters.Get(cacheField)
556
+		switch len(values) {
557
+		case 0:
558
+			bkFilter = append(bkFilter, cacheField)
559
+		case 1:
560
+			bkFilter = append(bkFilter, cacheField+"=="+values[0])
561
+		default:
562
+			return client.PruneInfo{}, errMultipleFilterValues
563
+		}
564
+	}
565
+	return client.PruneInfo{
566
+		All:          opts.All,
567
+		KeepDuration: unusedFor,
568
+		KeepBytes:    opts.KeepStorage,
569
+		Filter:       bkFilter,
570
+	}, nil
571
+}
... ...
@@ -6,15 +6,19 @@ import (
6 6
 	"path/filepath"
7 7
 
8 8
 	"github.com/containerd/containerd/content/local"
9
+	"github.com/docker/docker/api/types"
9 10
 	"github.com/docker/docker/builder/builder-next/adapters/containerimage"
10 11
 	"github.com/docker/docker/builder/builder-next/adapters/snapshot"
11 12
 	containerimageexp "github.com/docker/docker/builder/builder-next/exporter"
12 13
 	"github.com/docker/docker/builder/builder-next/imagerefchecker"
13 14
 	mobyworker "github.com/docker/docker/builder/builder-next/worker"
15
+	"github.com/docker/docker/daemon/config"
14 16
 	"github.com/docker/docker/daemon/graphdriver"
17
+	units "github.com/docker/go-units"
15 18
 	"github.com/moby/buildkit/cache"
16 19
 	"github.com/moby/buildkit/cache/metadata"
17 20
 	registryremotecache "github.com/moby/buildkit/cache/remotecache/registry"
21
+	"github.com/moby/buildkit/client"
18 22
 	"github.com/moby/buildkit/control"
19 23
 	"github.com/moby/buildkit/exporter"
20 24
 	"github.com/moby/buildkit/frontend"
... ...
@@ -127,12 +131,18 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
127 127
 		return nil, err
128 128
 	}
129 129
 
130
+	gcPolicy, err := getGCPolicy(opt.BuilderConfig, root)
131
+	if err != nil {
132
+		return nil, errors.Wrap(err, "could not get builder GC policy")
133
+	}
134
+
130 135
 	wopt := mobyworker.Opt{
131 136
 		ID:                "moby",
132 137
 		SessionManager:    opt.SessionManager,
133 138
 		MetadataStore:     md,
134 139
 		ContentStore:      store,
135 140
 		CacheManager:      cm,
141
+		GCPolicy:          gcPolicy,
136 142
 		Snapshotter:       snapshotter,
137 143
 		Executor:          exec,
138 144
 		ImageSource:       src,
... ...
@@ -165,3 +175,44 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
165 165
 		// TODO: set ResolveCacheExporterFunc for exporting cache
166 166
 	})
167 167
 }
168
+
169
+func getGCPolicy(conf config.BuilderConfig, root string) ([]client.PruneInfo, error) {
170
+	var gcPolicy []client.PruneInfo
171
+	if conf.GC.Enabled {
172
+		var (
173
+			defaultKeepStorage int64
174
+			err                error
175
+		)
176
+
177
+		if conf.GC.DefaultKeepStorage != "" {
178
+			defaultKeepStorage, err = units.RAMInBytes(conf.GC.DefaultKeepStorage)
179
+			if err != nil {
180
+				return nil, errors.Wrapf(err, "could not parse '%s' as Builder.GC.DefaultKeepStorage config", conf.GC.DefaultKeepStorage)
181
+			}
182
+		}
183
+
184
+		if conf.GC.Policy == nil {
185
+			gcPolicy = mobyworker.DefaultGCPolicy(root, defaultKeepStorage)
186
+		} else {
187
+			gcPolicy = make([]client.PruneInfo, len(conf.GC.Policy))
188
+			for i, p := range conf.GC.Policy {
189
+				b, err := units.RAMInBytes(p.KeepStorage)
190
+				if err != nil {
191
+					return nil, err
192
+				}
193
+				if b == 0 {
194
+					b = defaultKeepStorage
195
+				}
196
+				gcPolicy[i], err = toBuildkitPruneInfo(types.BuildCachePruneOptions{
197
+					All:         p.All,
198
+					KeepStorage: b,
199
+					Filters:     p.Filter,
200
+				})
201
+				if err != nil {
202
+					return nil, err
203
+				}
204
+			}
205
+		}
206
+	}
207
+	return gcPolicy, nil
208
+}
168 209
new file mode 100644
... ...
@@ -0,0 +1,51 @@
0
+package worker
1
+
2
+import (
3
+	"math"
4
+
5
+	"github.com/moby/buildkit/client"
6
+)
7
+
8
+const defaultCap int64 = 2e9 // 2GB
9
+
10
+// tempCachePercent represents the percentage ratio of the cache size in bytes to temporarily keep for a short period of time (couple of days)
11
+// over the total cache size in bytes. Because there is no perfect value, a mathematically pleasing one was chosen.
12
+// The value is approximately 13.8
13
+const tempCachePercent = math.E * math.Pi * math.Phi
14
+
15
+// DefaultGCPolicy returns a default builder GC policy
16
+func DefaultGCPolicy(p string, defaultKeepBytes int64) []client.PruneInfo {
17
+	keep := defaultKeepBytes
18
+	if defaultKeepBytes == 0 {
19
+		keep = detectDefaultGCCap(p)
20
+	}
21
+
22
+	tempCacheKeepBytes := int64(math.Round(float64(keep) / 100. * float64(tempCachePercent)))
23
+	const minTempCacheKeepBytes = 512 * 1e6 // 512MB
24
+	if tempCacheKeepBytes < minTempCacheKeepBytes {
25
+		tempCacheKeepBytes = minTempCacheKeepBytes
26
+	}
27
+
28
+	return []client.PruneInfo{
29
+		// if build cache uses more than 512MB delete the most easily reproducible data after it has not been used for 2 days
30
+		{
31
+			Filter:       []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"},
32
+			KeepDuration: 48 * 3600, // 48h
33
+			KeepBytes:    tempCacheKeepBytes,
34
+		},
35
+		// remove any data not used for 60 days
36
+		{
37
+			KeepDuration: 60 * 24 * 3600, // 60d
38
+			KeepBytes:    keep,
39
+		},
40
+		// keep the unshared build cache under cap
41
+		{
42
+			KeepBytes: keep,
43
+		},
44
+		// if previous policies were insufficient start deleting internal data to keep build cache under cap
45
+		{
46
+			All:       true,
47
+			KeepBytes: keep,
48
+		},
49
+	}
50
+}
0 51
new file mode 100644
... ...
@@ -0,0 +1,17 @@
0
+// +build !windows
1
+
2
+package worker
3
+
4
+import (
5
+	"syscall"
6
+)
7
+
8
+func detectDefaultGCCap(root string) int64 {
9
+	var st syscall.Statfs_t
10
+	if err := syscall.Statfs(root, &st); err != nil {
11
+		return defaultCap
12
+	}
13
+	diskSize := int64(st.Bsize) * int64(st.Blocks) // nolint unconvert
14
+	avail := diskSize / 10
15
+	return (avail/(1<<30) + 1) * 1e9 // round up
16
+}
0 17
new file mode 100644
... ...
@@ -0,0 +1,7 @@
0
+// +build windows
1
+
2
+package worker
3
+
4
+func detectDefaultGCCap(root string) int64 {
5
+	return defaultCap
6
+}
... ...
@@ -292,6 +292,7 @@ func newRouterOptions(config *config.Config, d *daemon.Daemon) (routerOptions, e
292 292
 		NetworkController:   d.NetworkController(),
293 293
 		DefaultCgroupParent: cgroupParent,
294 294
 		ResolverOpt:         d.NewResolveOptionsFunc(),
295
+		BuilderConfig:       config.Builder,
295 296
 	})
296 297
 	if err != nil {
297 298
 		return opts, err
298 299
new file mode 100644
... ...
@@ -0,0 +1,22 @@
0
+package config
1
+
2
+import "github.com/docker/docker/api/types/filters"
3
+
4
+// BuilderGCRule represents a GC rule for buildkit cache
5
+type BuilderGCRule struct {
6
+	All         bool         `json:",omitempty"`
7
+	Filter      filters.Args `json:",omitempty"`
8
+	KeepStorage string       `json:",omitempty"`
9
+}
10
+
11
+// BuilderGCConfig contains GC config for a buildkit builder
12
+type BuilderGCConfig struct {
13
+	Enabled            bool            `json:",omitempty"`
14
+	Policy             []BuilderGCRule `json:",omitempty"`
15
+	DefaultKeepStorage string          `json:",omitempty"`
16
+}
17
+
18
+// BuilderConfig contains config for the builder
19
+type BuilderConfig struct {
20
+	GC BuilderGCConfig `json:",omitempty"`
21
+}
... ...
@@ -55,6 +55,7 @@ var flatOptions = map[string]bool{
55 55
 	"runtimes":           true,
56 56
 	"default-ulimits":    true,
57 57
 	"features":           true,
58
+	"builder":            true,
58 59
 }
59 60
 
60 61
 // skipValidateOptions contains configuration keys
... ...
@@ -62,6 +63,7 @@ var flatOptions = map[string]bool{
62 62
 // for unknown flag validation.
63 63
 var skipValidateOptions = map[string]bool{
64 64
 	"features": true,
65
+	"builder":  true,
65 66
 }
66 67
 
67 68
 // skipDuplicates contains configuration keys that
... ...
@@ -225,6 +227,8 @@ type CommonConfig struct {
225 225
 	// Features contains a list of feature key value pairs indicating what features are enabled or disabled.
226 226
 	// If a certain feature doesn't appear in this list then it's unset (i.e. neither true nor false).
227 227
 	Features map[string]bool `json:"features,omitempty"`
228
+
229
+	Builder BuilderConfig `json:"builder,omitempty"`
228 230
 }
229 231
 
230 232
 // IsValueSet returns true if a configuration value