This allows users to configure the buildkit GC.
The following enables the default GC:
```
{
"builder": {
"gc": {
"enabled": true
}
}
}
```
The default GC policy has a simple config:
```
{
"builder": {
"gc": {
"enabled": true,
"defaultKeepStorage": "30GB"
}
}
}
```
A custom GC policy can be used instead by specifying a list of cache prune rules:
```
{
"builder": {
"gc": {
"enabled": true,
"policy": [
{"keepStorage": "512MB", "filter": ["unused-for=1400h"]]},
{"keepStorage": "30GB", "all": true}
]
}
}
}
```
Signed-off-by: Tibor Vass <tibor@docker.com>
| ... | ... |
@@ -13,11 +13,13 @@ import ( |
| 13 | 13 |
"github.com/docker/docker/api/types" |
| 14 | 14 |
"github.com/docker/docker/api/types/backend" |
| 15 | 15 |
"github.com/docker/docker/builder" |
| 16 |
+ "github.com/docker/docker/daemon/config" |
|
| 16 | 17 |
"github.com/docker/docker/daemon/images" |
| 17 | 18 |
"github.com/docker/docker/pkg/streamformatter" |
| 18 | 19 |
"github.com/docker/docker/pkg/system" |
| 19 | 20 |
"github.com/docker/libnetwork" |
| 20 | 21 |
controlapi "github.com/moby/buildkit/api/services/control" |
| 22 |
+ "github.com/moby/buildkit/client" |
|
| 21 | 23 |
"github.com/moby/buildkit/control" |
| 22 | 24 |
"github.com/moby/buildkit/identity" |
| 23 | 25 |
"github.com/moby/buildkit/session" |
| ... | ... |
@@ -57,6 +59,7 @@ type Opt struct {
|
| 57 | 57 |
NetworkController libnetwork.NetworkController |
| 58 | 58 |
DefaultCgroupParent string |
| 59 | 59 |
ResolverOpt resolver.ResolveOptionsFunc |
| 60 |
+ BuilderConfig config.BuilderConfig |
|
| 60 | 61 |
} |
| 61 | 62 |
|
| 62 | 63 |
// Builder can build using BuildKit backend |
| ... | ... |
@@ -134,43 +137,18 @@ func (b *Builder) Prune(ctx context.Context, opts types.BuildCachePruneOptions) |
| 134 | 134 |
return 0, nil, err |
| 135 | 135 |
} |
| 136 | 136 |
|
| 137 |
- var unusedFor time.Duration |
|
| 138 |
- unusedForValues := opts.Filters.Get("unused-for")
|
|
| 139 |
- |
|
| 140 |
- switch len(unusedForValues) {
|
|
| 141 |
- case 0: |
|
| 142 |
- |
|
| 143 |
- case 1: |
|
| 144 |
- var err error |
|
| 145 |
- unusedFor, err = time.ParseDuration(unusedForValues[0]) |
|
| 146 |
- if err != nil {
|
|
| 147 |
- return 0, nil, errors.Wrap(err, "unused-for filter expects a duration (e.g., '24h')") |
|
| 148 |
- } |
|
| 149 |
- |
|
| 150 |
- default: |
|
| 151 |
- return 0, nil, errMultipleFilterValues |
|
| 152 |
- } |
|
| 153 |
- |
|
| 154 |
- bkFilter := make([]string, 0, opts.Filters.Len()) |
|
| 155 |
- for cacheField := range cacheFields {
|
|
| 156 |
- values := opts.Filters.Get(cacheField) |
|
| 157 |
- switch len(values) {
|
|
| 158 |
- case 0: |
|
| 159 |
- bkFilter = append(bkFilter, cacheField) |
|
| 160 |
- case 1: |
|
| 161 |
- bkFilter = append(bkFilter, cacheField+"=="+values[0]) |
|
| 162 |
- default: |
|
| 163 |
- return 0, nil, errMultipleFilterValues |
|
| 164 |
- } |
|
| 137 |
+ pi, err := toBuildkitPruneInfo(opts) |
|
| 138 |
+ if err != nil {
|
|
| 139 |
+ return 0, nil, err |
|
| 165 | 140 |
} |
| 166 | 141 |
|
| 167 | 142 |
eg.Go(func() error {
|
| 168 | 143 |
defer close(ch) |
| 169 | 144 |
return b.controller.Prune(&controlapi.PruneRequest{
|
| 170 |
- All: opts.All, |
|
| 171 |
- KeepDuration: int64(unusedFor), |
|
| 172 |
- KeepBytes: opts.KeepStorage, |
|
| 173 |
- Filter: bkFilter, |
|
| 145 |
+ All: pi.All, |
|
| 146 |
+ KeepDuration: int64(pi.KeepDuration), |
|
| 147 |
+ KeepBytes: pi.KeepBytes, |
|
| 148 |
+ Filter: pi.Filter, |
|
| 174 | 149 |
}, &pruneProxy{
|
| 175 | 150 |
streamProxy: streamProxy{ctx: ctx},
|
| 176 | 151 |
ch: ch, |
| ... | ... |
@@ -531,3 +509,41 @@ func toBuildkitExtraHosts(inp []string) (string, error) {
|
| 531 | 531 |
} |
| 532 | 532 |
return strings.Join(hosts, ","), nil |
| 533 | 533 |
} |
| 534 |
+ |
|
| 535 |
+func toBuildkitPruneInfo(opts types.BuildCachePruneOptions) (client.PruneInfo, error) {
|
|
| 536 |
+ var unusedFor time.Duration |
|
| 537 |
+ unusedForValues := opts.Filters.Get("unused-for")
|
|
| 538 |
+ |
|
| 539 |
+ switch len(unusedForValues) {
|
|
| 540 |
+ case 0: |
|
| 541 |
+ |
|
| 542 |
+ case 1: |
|
| 543 |
+ var err error |
|
| 544 |
+ unusedFor, err = time.ParseDuration(unusedForValues[0]) |
|
| 545 |
+ if err != nil {
|
|
| 546 |
+ return client.PruneInfo{}, errors.Wrap(err, "unused-for filter expects a duration (e.g., '24h')")
|
|
| 547 |
+ } |
|
| 548 |
+ |
|
| 549 |
+ default: |
|
| 550 |
+ return client.PruneInfo{}, errMultipleFilterValues
|
|
| 551 |
+ } |
|
| 552 |
+ |
|
| 553 |
+ bkFilter := make([]string, 0, opts.Filters.Len()) |
|
| 554 |
+ for cacheField := range cacheFields {
|
|
| 555 |
+ values := opts.Filters.Get(cacheField) |
|
| 556 |
+ switch len(values) {
|
|
| 557 |
+ case 0: |
|
| 558 |
+ bkFilter = append(bkFilter, cacheField) |
|
| 559 |
+ case 1: |
|
| 560 |
+ bkFilter = append(bkFilter, cacheField+"=="+values[0]) |
|
| 561 |
+ default: |
|
| 562 |
+ return client.PruneInfo{}, errMultipleFilterValues
|
|
| 563 |
+ } |
|
| 564 |
+ } |
|
| 565 |
+ return client.PruneInfo{
|
|
| 566 |
+ All: opts.All, |
|
| 567 |
+ KeepDuration: unusedFor, |
|
| 568 |
+ KeepBytes: opts.KeepStorage, |
|
| 569 |
+ Filter: bkFilter, |
|
| 570 |
+ }, nil |
|
| 571 |
+} |
| ... | ... |
@@ -6,15 +6,19 @@ import ( |
| 6 | 6 |
"path/filepath" |
| 7 | 7 |
|
| 8 | 8 |
"github.com/containerd/containerd/content/local" |
| 9 |
+ "github.com/docker/docker/api/types" |
|
| 9 | 10 |
"github.com/docker/docker/builder/builder-next/adapters/containerimage" |
| 10 | 11 |
"github.com/docker/docker/builder/builder-next/adapters/snapshot" |
| 11 | 12 |
containerimageexp "github.com/docker/docker/builder/builder-next/exporter" |
| 12 | 13 |
"github.com/docker/docker/builder/builder-next/imagerefchecker" |
| 13 | 14 |
mobyworker "github.com/docker/docker/builder/builder-next/worker" |
| 15 |
+ "github.com/docker/docker/daemon/config" |
|
| 14 | 16 |
"github.com/docker/docker/daemon/graphdriver" |
| 17 |
+ units "github.com/docker/go-units" |
|
| 15 | 18 |
"github.com/moby/buildkit/cache" |
| 16 | 19 |
"github.com/moby/buildkit/cache/metadata" |
| 17 | 20 |
registryremotecache "github.com/moby/buildkit/cache/remotecache/registry" |
| 21 |
+ "github.com/moby/buildkit/client" |
|
| 18 | 22 |
"github.com/moby/buildkit/control" |
| 19 | 23 |
"github.com/moby/buildkit/exporter" |
| 20 | 24 |
"github.com/moby/buildkit/frontend" |
| ... | ... |
@@ -127,12 +131,18 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
|
| 127 | 127 |
return nil, err |
| 128 | 128 |
} |
| 129 | 129 |
|
| 130 |
+ gcPolicy, err := getGCPolicy(opt.BuilderConfig, root) |
|
| 131 |
+ if err != nil {
|
|
| 132 |
+ return nil, errors.Wrap(err, "could not get builder GC policy") |
|
| 133 |
+ } |
|
| 134 |
+ |
|
| 130 | 135 |
wopt := mobyworker.Opt{
|
| 131 | 136 |
ID: "moby", |
| 132 | 137 |
SessionManager: opt.SessionManager, |
| 133 | 138 |
MetadataStore: md, |
| 134 | 139 |
ContentStore: store, |
| 135 | 140 |
CacheManager: cm, |
| 141 |
+ GCPolicy: gcPolicy, |
|
| 136 | 142 |
Snapshotter: snapshotter, |
| 137 | 143 |
Executor: exec, |
| 138 | 144 |
ImageSource: src, |
| ... | ... |
@@ -165,3 +175,44 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
|
| 165 | 165 |
// TODO: set ResolveCacheExporterFunc for exporting cache |
| 166 | 166 |
}) |
| 167 | 167 |
} |
| 168 |
+ |
|
| 169 |
+func getGCPolicy(conf config.BuilderConfig, root string) ([]client.PruneInfo, error) {
|
|
| 170 |
+ var gcPolicy []client.PruneInfo |
|
| 171 |
+ if conf.GC.Enabled {
|
|
| 172 |
+ var ( |
|
| 173 |
+ defaultKeepStorage int64 |
|
| 174 |
+ err error |
|
| 175 |
+ ) |
|
| 176 |
+ |
|
| 177 |
+ if conf.GC.DefaultKeepStorage != "" {
|
|
| 178 |
+ defaultKeepStorage, err = units.RAMInBytes(conf.GC.DefaultKeepStorage) |
|
| 179 |
+ if err != nil {
|
|
| 180 |
+ return nil, errors.Wrapf(err, "could not parse '%s' as Builder.GC.DefaultKeepStorage config", conf.GC.DefaultKeepStorage) |
|
| 181 |
+ } |
|
| 182 |
+ } |
|
| 183 |
+ |
|
| 184 |
+ if conf.GC.Policy == nil {
|
|
| 185 |
+ gcPolicy = mobyworker.DefaultGCPolicy(root, defaultKeepStorage) |
|
| 186 |
+ } else {
|
|
| 187 |
+ gcPolicy = make([]client.PruneInfo, len(conf.GC.Policy)) |
|
| 188 |
+ for i, p := range conf.GC.Policy {
|
|
| 189 |
+ b, err := units.RAMInBytes(p.KeepStorage) |
|
| 190 |
+ if err != nil {
|
|
| 191 |
+ return nil, err |
|
| 192 |
+ } |
|
| 193 |
+ if b == 0 {
|
|
| 194 |
+ b = defaultKeepStorage |
|
| 195 |
+ } |
|
| 196 |
+ gcPolicy[i], err = toBuildkitPruneInfo(types.BuildCachePruneOptions{
|
|
| 197 |
+ All: p.All, |
|
| 198 |
+ KeepStorage: b, |
|
| 199 |
+ Filters: p.Filter, |
|
| 200 |
+ }) |
|
| 201 |
+ if err != nil {
|
|
| 202 |
+ return nil, err |
|
| 203 |
+ } |
|
| 204 |
+ } |
|
| 205 |
+ } |
|
| 206 |
+ } |
|
| 207 |
+ return gcPolicy, nil |
|
| 208 |
+} |
| 168 | 209 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,51 @@ |
| 0 |
+package worker |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "math" |
|
| 4 |
+ |
|
| 5 |
+ "github.com/moby/buildkit/client" |
|
| 6 |
+) |
|
| 7 |
+ |
|
| 8 |
+const defaultCap int64 = 2e9 // 2GB |
|
| 9 |
+ |
|
| 10 |
+// tempCachePercent represents the percentage ratio of the cache size in bytes to temporarily keep for a short period of time (couple of days) |
|
| 11 |
+// over the total cache size in bytes. Because there is no perfect value, a mathematically pleasing one was chosen. |
|
| 12 |
+// The value is approximately 13.8 |
|
| 13 |
+const tempCachePercent = math.E * math.Pi * math.Phi |
|
| 14 |
+ |
|
| 15 |
+// DefaultGCPolicy returns a default builder GC policy |
|
| 16 |
+func DefaultGCPolicy(p string, defaultKeepBytes int64) []client.PruneInfo {
|
|
| 17 |
+ keep := defaultKeepBytes |
|
| 18 |
+ if defaultKeepBytes == 0 {
|
|
| 19 |
+ keep = detectDefaultGCCap(p) |
|
| 20 |
+ } |
|
| 21 |
+ |
|
| 22 |
+ tempCacheKeepBytes := int64(math.Round(float64(keep) / 100. * float64(tempCachePercent))) |
|
| 23 |
+ const minTempCacheKeepBytes = 512 * 1e6 // 512MB |
|
| 24 |
+ if tempCacheKeepBytes < minTempCacheKeepBytes {
|
|
| 25 |
+ tempCacheKeepBytes = minTempCacheKeepBytes |
|
| 26 |
+ } |
|
| 27 |
+ |
|
| 28 |
+ return []client.PruneInfo{
|
|
| 29 |
+ // if build cache uses more than 512MB delete the most easily reproducible data after it has not been used for 2 days |
|
| 30 |
+ {
|
|
| 31 |
+ Filter: []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"},
|
|
| 32 |
+ KeepDuration: 48 * 3600, // 48h |
|
| 33 |
+ KeepBytes: tempCacheKeepBytes, |
|
| 34 |
+ }, |
|
| 35 |
+ // remove any data not used for 60 days |
|
| 36 |
+ {
|
|
| 37 |
+ KeepDuration: 60 * 24 * 3600, // 60d |
|
| 38 |
+ KeepBytes: keep, |
|
| 39 |
+ }, |
|
| 40 |
+ // keep the unshared build cache under cap |
|
| 41 |
+ {
|
|
| 42 |
+ KeepBytes: keep, |
|
| 43 |
+ }, |
|
| 44 |
+ // if previous policies were insufficient start deleting internal data to keep build cache under cap |
|
| 45 |
+ {
|
|
| 46 |
+ All: true, |
|
| 47 |
+ KeepBytes: keep, |
|
| 48 |
+ }, |
|
| 49 |
+ } |
|
| 50 |
+} |
| 0 | 51 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,17 @@ |
| 0 |
+// +build !windows |
|
| 1 |
+ |
|
| 2 |
+package worker |
|
| 3 |
+ |
|
| 4 |
+import ( |
|
| 5 |
+ "syscall" |
|
| 6 |
+) |
|
| 7 |
+ |
|
| 8 |
+func detectDefaultGCCap(root string) int64 {
|
|
| 9 |
+ var st syscall.Statfs_t |
|
| 10 |
+ if err := syscall.Statfs(root, &st); err != nil {
|
|
| 11 |
+ return defaultCap |
|
| 12 |
+ } |
|
| 13 |
+ diskSize := int64(st.Bsize) * int64(st.Blocks) // nolint unconvert |
|
| 14 |
+ avail := diskSize / 10 |
|
| 15 |
+ return (avail/(1<<30) + 1) * 1e9 // round up |
|
| 16 |
+} |
| ... | ... |
@@ -292,6 +292,7 @@ func newRouterOptions(config *config.Config, d *daemon.Daemon) (routerOptions, e |
| 292 | 292 |
NetworkController: d.NetworkController(), |
| 293 | 293 |
DefaultCgroupParent: cgroupParent, |
| 294 | 294 |
ResolverOpt: d.NewResolveOptionsFunc(), |
| 295 |
+ BuilderConfig: config.Builder, |
|
| 295 | 296 |
}) |
| 296 | 297 |
if err != nil {
|
| 297 | 298 |
return opts, err |
| 298 | 299 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,22 @@ |
| 0 |
+package config |
|
| 1 |
+ |
|
| 2 |
+import "github.com/docker/docker/api/types/filters" |
|
| 3 |
+ |
|
| 4 |
+// BuilderGCRule represents a GC rule for buildkit cache |
|
| 5 |
+type BuilderGCRule struct {
|
|
| 6 |
+ All bool `json:",omitempty"` |
|
| 7 |
+ Filter filters.Args `json:",omitempty"` |
|
| 8 |
+ KeepStorage string `json:",omitempty"` |
|
| 9 |
+} |
|
| 10 |
+ |
|
| 11 |
+// BuilderGCConfig contains GC config for a buildkit builder |
|
| 12 |
+type BuilderGCConfig struct {
|
|
| 13 |
+ Enabled bool `json:",omitempty"` |
|
| 14 |
+ Policy []BuilderGCRule `json:",omitempty"` |
|
| 15 |
+ DefaultKeepStorage string `json:",omitempty"` |
|
| 16 |
+} |
|
| 17 |
+ |
|
| 18 |
+// BuilderConfig contains config for the builder |
|
| 19 |
+type BuilderConfig struct {
|
|
| 20 |
+ GC BuilderGCConfig `json:",omitempty"` |
|
| 21 |
+} |
| ... | ... |
@@ -55,6 +55,7 @@ var flatOptions = map[string]bool{
|
| 55 | 55 |
"runtimes": true, |
| 56 | 56 |
"default-ulimits": true, |
| 57 | 57 |
"features": true, |
| 58 |
+ "builder": true, |
|
| 58 | 59 |
} |
| 59 | 60 |
|
| 60 | 61 |
// skipValidateOptions contains configuration keys |
| ... | ... |
@@ -62,6 +63,7 @@ var flatOptions = map[string]bool{
|
| 62 | 62 |
// for unknown flag validation. |
| 63 | 63 |
var skipValidateOptions = map[string]bool{
|
| 64 | 64 |
"features": true, |
| 65 |
+ "builder": true, |
|
| 65 | 66 |
} |
| 66 | 67 |
|
| 67 | 68 |
// skipDuplicates contains configuration keys that |
| ... | ... |
@@ -225,6 +227,8 @@ type CommonConfig struct {
|
| 225 | 225 |
// Features contains a list of feature key value pairs indicating what features are enabled or disabled. |
| 226 | 226 |
// If a certain feature doesn't appear in this list then it's unset (i.e. neither true nor false). |
| 227 | 227 |
Features map[string]bool `json:"features,omitempty"` |
| 228 |
+ |
|
| 229 |
+ Builder BuilderConfig `json:"builder,omitempty"` |
|
| 228 | 230 |
} |
| 229 | 231 |
|
| 230 | 232 |
// IsValueSet returns true if a configuration value |