Browse code

builder: expand prune to buildkit

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

Tonis Tiigi authored on 2018/05/16 07:12:03
Showing 4 changed files
... ...
@@ -14,6 +14,7 @@ import (
14 14
 	"github.com/docker/docker/image"
15 15
 	"github.com/docker/docker/pkg/stringid"
16 16
 	"github.com/pkg/errors"
17
+	"golang.org/x/sync/errgroup"
17 18
 )
18 19
 
19 20
 // ImageComponent provides an interface for working with images
... ...
@@ -89,11 +90,33 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
89 89
 
90 90
 // PruneCache removes all cached build sources
91 91
 func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport, error) {
92
-	size, err := b.fsCache.Prune(ctx)
93
-	if err != nil {
94
-		return nil, errors.Wrap(err, "failed to prune build cache")
92
+	eg, ctx := errgroup.WithContext(ctx)
93
+
94
+	var fsCacheSize uint64
95
+	eg.Go(func() error {
96
+		var err error
97
+		fsCacheSize, err = b.fsCache.Prune(ctx)
98
+		if err != nil {
99
+			return errors.Wrap(err, "failed to prune fscache")
100
+		}
101
+		return nil
102
+	})
103
+
104
+	var buildCacheSize int64
105
+	eg.Go(func() error {
106
+		var err error
107
+		buildCacheSize, err = b.buildkit.Prune(ctx)
108
+		if err != nil {
109
+			return errors.Wrap(err, "failed to prune build cache")
110
+		}
111
+		return nil
112
+	})
113
+
114
+	if err := eg.Wait(); err != nil {
115
+		return nil, err
95 116
 	}
96
-	return &types.BuildCachePruneReport{SpaceReclaimed: size}, nil
117
+
118
+	return &types.BuildCachePruneReport{SpaceReclaimed: fsCacheSize + uint64(buildCacheSize)}, nil
97 119
 }
98 120
 
99 121
 func (b *Backend) Cancel(ctx context.Context, id string) error {
... ...
@@ -13,7 +13,7 @@ import (
13 13
 )
14 14
 
15 15
 func (s *snapshotter) EnsureLayer(ctx context.Context, key string) ([]layer.DiffID, error) {
16
-	if l, err := s.getLayer(key); err != nil {
16
+	if l, err := s.getLayer(key, true); err != nil {
17 17
 		return nil, err
18 18
 	} else if l != nil {
19 19
 		return getDiffChain(l), nil
... ...
@@ -57,7 +57,7 @@ func (s *snapshotter) EnsureLayer(ctx context.Context, key string) ([]layer.Diff
57 57
 	eg.Go(func() error {
58 58
 		parent := ""
59 59
 		if p := info.Parent; p != "" {
60
-			if l, err := s.getLayer(p); err != nil {
60
+			if l, err := s.getLayer(p, true); err != nil {
61 61
 				return err
62 62
 			} else if l != nil {
63 63
 				parent, err = getGraphID(l)
... ...
@@ -74,7 +74,7 @@ func NewSnapshotter(opt Opt) (snapshot.SnapshotterBase, error) {
74 74
 func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) error {
75 75
 	origParent := parent
76 76
 	if parent != "" {
77
-		if l, err := s.getLayer(parent); err != nil {
77
+		if l, err := s.getLayer(parent, false); err != nil {
78 78
 			return err
79 79
 		} else if l != nil {
80 80
 			parent, err = getGraphID(l)
... ...
@@ -115,12 +115,16 @@ func (s *snapshotter) chainID(key string) (layer.ChainID, bool) {
115 115
 	return "", false
116 116
 }
117 117
 
118
-func (s *snapshotter) getLayer(key string) (layer.Layer, error) {
118
+func (s *snapshotter) getLayer(key string, withCommitted bool) (layer.Layer, error) {
119 119
 	s.mu.Lock()
120 120
 	l, ok := s.refs[key]
121 121
 	if !ok {
122 122
 		id, ok := s.chainID(key)
123 123
 		if !ok {
124
+			if !withCommitted {
125
+				s.mu.Unlock()
126
+				return nil, nil
127
+			}
124 128
 			if err := s.db.View(func(tx *bolt.Tx) error {
125 129
 				b := tx.Bucket([]byte(key))
126 130
 				if b == nil {
... ...
@@ -146,7 +150,7 @@ func (s *snapshotter) getLayer(key string) (layer.Layer, error) {
146 146
 			s.mu.Unlock()
147 147
 			return nil, err
148 148
 		}
149
-		s.refs[string(id)] = l
149
+		s.refs[key] = l
150 150
 		if err := s.db.Update(func(tx *bolt.Tx) error {
151 151
 			_, err := tx.CreateBucketIfNotExists([]byte(key))
152 152
 			return err
... ...
@@ -179,23 +183,26 @@ func (s *snapshotter) getGraphDriverID(key string) (string, bool) {
179 179
 }
180 180
 
181 181
 func (s *snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, error) {
182
-	if l, err := s.getLayer(key); err != nil {
182
+	inf := snapshots.Info{
183
+		Kind: snapshots.KindActive,
184
+	}
185
+
186
+	l, err := s.getLayer(key, false)
187
+	if err != nil {
183 188
 		return snapshots.Info{}, err
184
-	} else if l != nil {
185
-		var parentID string
189
+	}
190
+	if l != nil {
186 191
 		if p := l.Parent(); p != nil {
187
-			parentID = p.ChainID().String()
188
-		}
189
-		info := snapshots.Info{
190
-			Kind:   snapshots.KindCommitted,
191
-			Name:   key,
192
-			Parent: parentID,
192
+			inf.Parent = p.ChainID().String()
193 193
 		}
194
-		return info, nil
194
+		inf.Kind = snapshots.KindCommitted
195
+		inf.Name = string(key)
196
+		return inf, nil
195 197
 	}
196 198
 
197
-	inf := snapshots.Info{
198
-		Kind: snapshots.KindActive,
199
+	l, err = s.getLayer(key, true)
200
+	if err != nil {
201
+		return snapshots.Info{}, err
199 202
 	}
200 203
 
201 204
 	id, committed := s.getGraphDriverID(key)
... ...
@@ -205,13 +212,22 @@ func (s *snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, err
205 205
 
206 206
 	if err := s.db.View(func(tx *bolt.Tx) error {
207 207
 		b := tx.Bucket([]byte(id))
208
-		if b == nil {
209
-			return errors.Errorf("not found") // TODO: typed
208
+		if b == nil && l == nil {
209
+			return errors.Errorf("snapshot %s not found", id) // TODO: typed
210 210
 		}
211 211
 		inf.Name = string(key)
212
-		v := b.Get(keyParent)
213
-		if v != nil {
214
-			inf.Parent = string(v)
212
+		if b != nil {
213
+			v := b.Get(keyParent)
214
+			if v != nil {
215
+				inf.Parent = string(v)
216
+				return nil
217
+			}
218
+		}
219
+		if l != nil {
220
+			if p := l.Parent(); p != nil {
221
+				inf.Parent = p.ChainID().String()
222
+			}
223
+			inf.Kind = snapshots.KindCommitted
215 224
 		}
216 225
 		return nil
217 226
 	}); err != nil {
... ...
@@ -221,7 +237,7 @@ func (s *snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, err
221 221
 }
222 222
 
223 223
 func (s *snapshotter) Mounts(ctx context.Context, key string) (snapshot.Mountable, error) {
224
-	l, err := s.getLayer(key)
224
+	l, err := s.getLayer(key, true)
225 225
 	if err != nil {
226 226
 		return nil, err
227 227
 	}
... ...
@@ -269,16 +285,17 @@ func (s *snapshotter) Mounts(ctx context.Context, key string) (snapshot.Mountabl
269 269
 }
270 270
 
271 271
 func (s *snapshotter) Remove(ctx context.Context, key string) error {
272
-	l, err := s.getLayer(key)
272
+	l, err := s.getLayer(key, true)
273 273
 	if err != nil {
274 274
 		return err
275 275
 	}
276 276
 
277
+	id, _ := s.getGraphDriverID(key)
278
+
277 279
 	var found bool
278 280
 	if err := s.db.Update(func(tx *bolt.Tx) error {
279 281
 		found = tx.Bucket([]byte(key)) != nil
280 282
 		if found {
281
-			id, _ := s.getGraphDriverID(key)
282 283
 			tx.DeleteBucket([]byte(key))
283 284
 			if id != key {
284 285
 				tx.DeleteBucket([]byte(id))
... ...
@@ -298,7 +315,6 @@ func (s *snapshotter) Remove(ctx context.Context, key string) error {
298 298
 		return nil
299 299
 	}
300 300
 
301
-	id, _ := s.getGraphDriverID(key)
302 301
 	return s.opt.GraphDriver.Remove(id)
303 302
 }
304 303
 
... ...
@@ -331,9 +347,9 @@ func (s *snapshotter) Update(ctx context.Context, info snapshots.Info, fieldpath
331 331
 	return s.Stat(ctx, info.Name)
332 332
 }
333 333
 
334
-func (s *snapshotter) Usage(ctx context.Context, key string) (snapshots.Usage, error) {
334
+func (s *snapshotter) Usage(ctx context.Context, key string) (us snapshots.Usage, retErr error) {
335 335
 	usage := snapshots.Usage{}
336
-	if l, err := s.getLayer(key); err != nil {
336
+	if l, err := s.getLayer(key, true); err != nil {
337 337
 		return usage, err
338 338
 	} else if l != nil {
339 339
 		s, err := l.DiffSize()
... ...
@@ -376,7 +392,16 @@ func (s *snapshotter) Usage(ctx context.Context, key string) (snapshots.Usage, e
376 376
 	}
377 377
 	var parent string
378 378
 	if info.Parent != "" {
379
-		parent, _ = s.getGraphDriverID(info.Parent)
379
+		if l, err := s.getLayer(info.Parent, false); err != nil {
380
+			return usage, err
381
+		} else if l != nil {
382
+			parent, err = getGraphID(l)
383
+			if err != nil {
384
+				return usage, err
385
+			}
386
+		} else {
387
+			parent, _ = s.getGraphDriverID(info.Parent)
388
+		}
380 389
 	}
381 390
 
382 391
 	diffSize, err := s.opt.GraphDriver.DiffSize(id, parent)
... ...
@@ -80,6 +80,34 @@ func (b *Builder) DiskUsage(ctx context.Context) ([]*types.BuildCache, error) {
80 80
 	return items, nil
81 81
 }
82 82
 
83
+func (b *Builder) Prune(ctx context.Context) (int64, error) {
84
+	ch := make(chan *controlapi.UsageRecord)
85
+
86
+	eg, ctx := errgroup.WithContext(ctx)
87
+
88
+	eg.Go(func() error {
89
+		defer close(ch)
90
+		return b.controller.Prune(&controlapi.PruneRequest{}, &pruneProxy{
91
+			streamProxy: streamProxy{ctx: ctx},
92
+			ch:          ch,
93
+		})
94
+	})
95
+
96
+	var size int64
97
+	eg.Go(func() error {
98
+		for r := range ch {
99
+			size += r.Size_
100
+		}
101
+		return nil
102
+	})
103
+
104
+	if err := eg.Wait(); err != nil {
105
+		return 0, err
106
+	}
107
+
108
+	return size, nil
109
+}
110
+
83 111
 func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.Result, error) {
84 112
 	var out builder.Result
85 113
 
... ...
@@ -149,7 +177,7 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
149 149
 		defer close(ch)
150 150
 		return b.controller.Status(&controlapi.StatusRequest{
151 151
 			Ref: id,
152
-		}, &statusProxy{ctx: ctx, ch: ch})
152
+		}, &statusProxy{streamProxy: streamProxy{ctx: ctx}, ch: ch})
153 153
 	})
154 154
 
155 155
 	eg.Go(func() error {
... ...
@@ -188,28 +216,35 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
188 188
 	return &out, nil
189 189
 }
190 190
 
191
-type statusProxy struct {
191
+type streamProxy struct {
192 192
 	ctx context.Context
193
-	ch  chan *controlapi.StatusResponse
194 193
 }
195 194
 
196
-func (sp *statusProxy) SetHeader(_ grpcmetadata.MD) error {
195
+func (sp *streamProxy) SetHeader(_ grpcmetadata.MD) error {
197 196
 	return nil
198 197
 }
199 198
 
200
-func (sp *statusProxy) SendHeader(_ grpcmetadata.MD) error {
199
+func (sp *streamProxy) SendHeader(_ grpcmetadata.MD) error {
201 200
 	return nil
202 201
 }
203 202
 
204
-func (sp *statusProxy) SetTrailer(_ grpcmetadata.MD) {
203
+func (sp *streamProxy) SetTrailer(_ grpcmetadata.MD) {
205 204
 }
206 205
 
207
-func (sp *statusProxy) Send(resp *controlapi.StatusResponse) error {
208
-	return sp.SendMsg(resp)
206
+func (sp *streamProxy) Context() context.Context {
207
+	return sp.ctx
208
+}
209
+func (sp *streamProxy) RecvMsg(m interface{}) error {
210
+	return io.EOF
209 211
 }
210 212
 
211
-func (sp *statusProxy) Context() context.Context {
212
-	return sp.ctx
213
+type statusProxy struct {
214
+	streamProxy
215
+	ch chan *controlapi.StatusResponse
216
+}
217
+
218
+func (sp *statusProxy) Send(resp *controlapi.StatusResponse) error {
219
+	return sp.SendMsg(resp)
213 220
 }
214 221
 func (sp *statusProxy) SendMsg(m interface{}) error {
215 222
 	if sr, ok := m.(*controlapi.StatusResponse); ok {
... ...
@@ -217,8 +252,20 @@ func (sp *statusProxy) SendMsg(m interface{}) error {
217 217
 	}
218 218
 	return nil
219 219
 }
220
-func (sp *statusProxy) RecvMsg(m interface{}) error {
221
-	return io.EOF
220
+
221
+type pruneProxy struct {
222
+	streamProxy
223
+	ch chan *controlapi.UsageRecord
224
+}
225
+
226
+func (sp *pruneProxy) Send(resp *controlapi.UsageRecord) error {
227
+	return sp.SendMsg(resp)
228
+}
229
+func (sp *pruneProxy) SendMsg(m interface{}) error {
230
+	if sr, ok := m.(*controlapi.UsageRecord); ok {
231
+		sp.ch <- sr
232
+	}
233
+	return nil
222 234
 }
223 235
 
224 236
 type contentStoreNoLabels struct {