Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
| ... | ... |
@@ -14,6 +14,7 @@ import ( |
| 14 | 14 |
"github.com/docker/docker/image" |
| 15 | 15 |
"github.com/docker/docker/pkg/stringid" |
| 16 | 16 |
"github.com/pkg/errors" |
| 17 |
+ "golang.org/x/sync/errgroup" |
|
| 17 | 18 |
) |
| 18 | 19 |
|
| 19 | 20 |
// ImageComponent provides an interface for working with images |
| ... | ... |
@@ -89,11 +90,33 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string |
| 89 | 89 |
|
| 90 | 90 |
// PruneCache removes all cached build sources |
| 91 | 91 |
func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport, error) {
|
| 92 |
- size, err := b.fsCache.Prune(ctx) |
|
| 93 |
- if err != nil {
|
|
| 94 |
- return nil, errors.Wrap(err, "failed to prune build cache") |
|
| 92 |
+ eg, ctx := errgroup.WithContext(ctx) |
|
| 93 |
+ |
|
| 94 |
+ var fsCacheSize uint64 |
|
| 95 |
+ eg.Go(func() error {
|
|
| 96 |
+ var err error |
|
| 97 |
+ fsCacheSize, err = b.fsCache.Prune(ctx) |
|
| 98 |
+ if err != nil {
|
|
| 99 |
+ return errors.Wrap(err, "failed to prune fscache") |
|
| 100 |
+ } |
|
| 101 |
+ return nil |
|
| 102 |
+ }) |
|
| 103 |
+ |
|
| 104 |
+ var buildCacheSize int64 |
|
| 105 |
+ eg.Go(func() error {
|
|
| 106 |
+ var err error |
|
| 107 |
+ buildCacheSize, err = b.buildkit.Prune(ctx) |
|
| 108 |
+ if err != nil {
|
|
| 109 |
+ return errors.Wrap(err, "failed to prune build cache") |
|
| 110 |
+ } |
|
| 111 |
+ return nil |
|
| 112 |
+ }) |
|
| 113 |
+ |
|
| 114 |
+ if err := eg.Wait(); err != nil {
|
|
| 115 |
+ return nil, err |
|
| 95 | 116 |
} |
| 96 |
- return &types.BuildCachePruneReport{SpaceReclaimed: size}, nil
|
|
| 117 |
+ |
|
| 118 |
+ return &types.BuildCachePruneReport{SpaceReclaimed: fsCacheSize + uint64(buildCacheSize)}, nil
|
|
| 97 | 119 |
} |
| 98 | 120 |
|
| 99 | 121 |
func (b *Backend) Cancel(ctx context.Context, id string) error {
|
| ... | ... |
@@ -13,7 +13,7 @@ import ( |
| 13 | 13 |
) |
| 14 | 14 |
|
| 15 | 15 |
func (s *snapshotter) EnsureLayer(ctx context.Context, key string) ([]layer.DiffID, error) {
|
| 16 |
- if l, err := s.getLayer(key); err != nil {
|
|
| 16 |
+ if l, err := s.getLayer(key, true); err != nil {
|
|
| 17 | 17 |
return nil, err |
| 18 | 18 |
} else if l != nil {
|
| 19 | 19 |
return getDiffChain(l), nil |
| ... | ... |
@@ -57,7 +57,7 @@ func (s *snapshotter) EnsureLayer(ctx context.Context, key string) ([]layer.Diff |
| 57 | 57 |
eg.Go(func() error {
|
| 58 | 58 |
parent := "" |
| 59 | 59 |
if p := info.Parent; p != "" {
|
| 60 |
- if l, err := s.getLayer(p); err != nil {
|
|
| 60 |
+ if l, err := s.getLayer(p, true); err != nil {
|
|
| 61 | 61 |
return err |
| 62 | 62 |
} else if l != nil {
|
| 63 | 63 |
parent, err = getGraphID(l) |
| ... | ... |
@@ -74,7 +74,7 @@ func NewSnapshotter(opt Opt) (snapshot.SnapshotterBase, error) {
|
| 74 | 74 |
func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) error {
|
| 75 | 75 |
origParent := parent |
| 76 | 76 |
if parent != "" {
|
| 77 |
- if l, err := s.getLayer(parent); err != nil {
|
|
| 77 |
+ if l, err := s.getLayer(parent, false); err != nil {
|
|
| 78 | 78 |
return err |
| 79 | 79 |
} else if l != nil {
|
| 80 | 80 |
parent, err = getGraphID(l) |
| ... | ... |
@@ -115,12 +115,16 @@ func (s *snapshotter) chainID(key string) (layer.ChainID, bool) {
|
| 115 | 115 |
return "", false |
| 116 | 116 |
} |
| 117 | 117 |
|
| 118 |
-func (s *snapshotter) getLayer(key string) (layer.Layer, error) {
|
|
| 118 |
+func (s *snapshotter) getLayer(key string, withCommitted bool) (layer.Layer, error) {
|
|
| 119 | 119 |
s.mu.Lock() |
| 120 | 120 |
l, ok := s.refs[key] |
| 121 | 121 |
if !ok {
|
| 122 | 122 |
id, ok := s.chainID(key) |
| 123 | 123 |
if !ok {
|
| 124 |
+ if !withCommitted {
|
|
| 125 |
+ s.mu.Unlock() |
|
| 126 |
+ return nil, nil |
|
| 127 |
+ } |
|
| 124 | 128 |
if err := s.db.View(func(tx *bolt.Tx) error {
|
| 125 | 129 |
b := tx.Bucket([]byte(key)) |
| 126 | 130 |
if b == nil {
|
| ... | ... |
@@ -146,7 +150,7 @@ func (s *snapshotter) getLayer(key string) (layer.Layer, error) {
|
| 146 | 146 |
s.mu.Unlock() |
| 147 | 147 |
return nil, err |
| 148 | 148 |
} |
| 149 |
- s.refs[string(id)] = l |
|
| 149 |
+ s.refs[key] = l |
|
| 150 | 150 |
if err := s.db.Update(func(tx *bolt.Tx) error {
|
| 151 | 151 |
_, err := tx.CreateBucketIfNotExists([]byte(key)) |
| 152 | 152 |
return err |
| ... | ... |
@@ -179,23 +183,26 @@ func (s *snapshotter) getGraphDriverID(key string) (string, bool) {
|
| 179 | 179 |
} |
| 180 | 180 |
|
| 181 | 181 |
func (s *snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, error) {
|
| 182 |
- if l, err := s.getLayer(key); err != nil {
|
|
| 182 |
+ inf := snapshots.Info{
|
|
| 183 |
+ Kind: snapshots.KindActive, |
|
| 184 |
+ } |
|
| 185 |
+ |
|
| 186 |
+ l, err := s.getLayer(key, false) |
|
| 187 |
+ if err != nil {
|
|
| 183 | 188 |
return snapshots.Info{}, err
|
| 184 |
- } else if l != nil {
|
|
| 185 |
- var parentID string |
|
| 189 |
+ } |
|
| 190 |
+ if l != nil {
|
|
| 186 | 191 |
if p := l.Parent(); p != nil {
|
| 187 |
- parentID = p.ChainID().String() |
|
| 188 |
- } |
|
| 189 |
- info := snapshots.Info{
|
|
| 190 |
- Kind: snapshots.KindCommitted, |
|
| 191 |
- Name: key, |
|
| 192 |
- Parent: parentID, |
|
| 192 |
+ inf.Parent = p.ChainID().String() |
|
| 193 | 193 |
} |
| 194 |
- return info, nil |
|
| 194 |
+ inf.Kind = snapshots.KindCommitted |
|
| 195 |
+ inf.Name = string(key) |
|
| 196 |
+ return inf, nil |
|
| 195 | 197 |
} |
| 196 | 198 |
|
| 197 |
- inf := snapshots.Info{
|
|
| 198 |
- Kind: snapshots.KindActive, |
|
| 199 |
+ l, err = s.getLayer(key, true) |
|
| 200 |
+ if err != nil {
|
|
| 201 |
+ return snapshots.Info{}, err
|
|
| 199 | 202 |
} |
| 200 | 203 |
|
| 201 | 204 |
id, committed := s.getGraphDriverID(key) |
| ... | ... |
@@ -205,13 +212,22 @@ func (s *snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, err |
| 205 | 205 |
|
| 206 | 206 |
if err := s.db.View(func(tx *bolt.Tx) error {
|
| 207 | 207 |
b := tx.Bucket([]byte(id)) |
| 208 |
- if b == nil {
|
|
| 209 |
- return errors.Errorf("not found") // TODO: typed
|
|
| 208 |
+ if b == nil && l == nil {
|
|
| 209 |
+ return errors.Errorf("snapshot %s not found", id) // TODO: typed
|
|
| 210 | 210 |
} |
| 211 | 211 |
inf.Name = string(key) |
| 212 |
- v := b.Get(keyParent) |
|
| 213 |
- if v != nil {
|
|
| 214 |
- inf.Parent = string(v) |
|
| 212 |
+ if b != nil {
|
|
| 213 |
+ v := b.Get(keyParent) |
|
| 214 |
+ if v != nil {
|
|
| 215 |
+ inf.Parent = string(v) |
|
| 216 |
+ return nil |
|
| 217 |
+ } |
|
| 218 |
+ } |
|
| 219 |
+ if l != nil {
|
|
| 220 |
+ if p := l.Parent(); p != nil {
|
|
| 221 |
+ inf.Parent = p.ChainID().String() |
|
| 222 |
+ } |
|
| 223 |
+ inf.Kind = snapshots.KindCommitted |
|
| 215 | 224 |
} |
| 216 | 225 |
return nil |
| 217 | 226 |
}); err != nil {
|
| ... | ... |
@@ -221,7 +237,7 @@ func (s *snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, err |
| 221 | 221 |
} |
| 222 | 222 |
|
| 223 | 223 |
func (s *snapshotter) Mounts(ctx context.Context, key string) (snapshot.Mountable, error) {
|
| 224 |
- l, err := s.getLayer(key) |
|
| 224 |
+ l, err := s.getLayer(key, true) |
|
| 225 | 225 |
if err != nil {
|
| 226 | 226 |
return nil, err |
| 227 | 227 |
} |
| ... | ... |
@@ -269,16 +285,17 @@ func (s *snapshotter) Mounts(ctx context.Context, key string) (snapshot.Mountabl |
| 269 | 269 |
} |
| 270 | 270 |
|
| 271 | 271 |
func (s *snapshotter) Remove(ctx context.Context, key string) error {
|
| 272 |
- l, err := s.getLayer(key) |
|
| 272 |
+ l, err := s.getLayer(key, true) |
|
| 273 | 273 |
if err != nil {
|
| 274 | 274 |
return err |
| 275 | 275 |
} |
| 276 | 276 |
|
| 277 |
+ id, _ := s.getGraphDriverID(key) |
|
| 278 |
+ |
|
| 277 | 279 |
var found bool |
| 278 | 280 |
if err := s.db.Update(func(tx *bolt.Tx) error {
|
| 279 | 281 |
found = tx.Bucket([]byte(key)) != nil |
| 280 | 282 |
if found {
|
| 281 |
- id, _ := s.getGraphDriverID(key) |
|
| 282 | 283 |
tx.DeleteBucket([]byte(key)) |
| 283 | 284 |
if id != key {
|
| 284 | 285 |
tx.DeleteBucket([]byte(id)) |
| ... | ... |
@@ -298,7 +315,6 @@ func (s *snapshotter) Remove(ctx context.Context, key string) error {
|
| 298 | 298 |
return nil |
| 299 | 299 |
} |
| 300 | 300 |
|
| 301 |
- id, _ := s.getGraphDriverID(key) |
|
| 302 | 301 |
return s.opt.GraphDriver.Remove(id) |
| 303 | 302 |
} |
| 304 | 303 |
|
| ... | ... |
@@ -331,9 +347,9 @@ func (s *snapshotter) Update(ctx context.Context, info snapshots.Info, fieldpath |
| 331 | 331 |
return s.Stat(ctx, info.Name) |
| 332 | 332 |
} |
| 333 | 333 |
|
| 334 |
-func (s *snapshotter) Usage(ctx context.Context, key string) (snapshots.Usage, error) {
|
|
| 334 |
+func (s *snapshotter) Usage(ctx context.Context, key string) (us snapshots.Usage, retErr error) {
|
|
| 335 | 335 |
usage := snapshots.Usage{}
|
| 336 |
- if l, err := s.getLayer(key); err != nil {
|
|
| 336 |
+ if l, err := s.getLayer(key, true); err != nil {
|
|
| 337 | 337 |
return usage, err |
| 338 | 338 |
} else if l != nil {
|
| 339 | 339 |
s, err := l.DiffSize() |
| ... | ... |
@@ -376,7 +392,16 @@ func (s *snapshotter) Usage(ctx context.Context, key string) (snapshots.Usage, e |
| 376 | 376 |
} |
| 377 | 377 |
var parent string |
| 378 | 378 |
if info.Parent != "" {
|
| 379 |
- parent, _ = s.getGraphDriverID(info.Parent) |
|
| 379 |
+ if l, err := s.getLayer(info.Parent, false); err != nil {
|
|
| 380 |
+ return usage, err |
|
| 381 |
+ } else if l != nil {
|
|
| 382 |
+ parent, err = getGraphID(l) |
|
| 383 |
+ if err != nil {
|
|
| 384 |
+ return usage, err |
|
| 385 |
+ } |
|
| 386 |
+ } else {
|
|
| 387 |
+ parent, _ = s.getGraphDriverID(info.Parent) |
|
| 388 |
+ } |
|
| 380 | 389 |
} |
| 381 | 390 |
|
| 382 | 391 |
diffSize, err := s.opt.GraphDriver.DiffSize(id, parent) |
| ... | ... |
@@ -80,6 +80,34 @@ func (b *Builder) DiskUsage(ctx context.Context) ([]*types.BuildCache, error) {
|
| 80 | 80 |
return items, nil |
| 81 | 81 |
} |
| 82 | 82 |
|
| 83 |
+func (b *Builder) Prune(ctx context.Context) (int64, error) {
|
|
| 84 |
+ ch := make(chan *controlapi.UsageRecord) |
|
| 85 |
+ |
|
| 86 |
+ eg, ctx := errgroup.WithContext(ctx) |
|
| 87 |
+ |
|
| 88 |
+ eg.Go(func() error {
|
|
| 89 |
+ defer close(ch) |
|
| 90 |
+ return b.controller.Prune(&controlapi.PruneRequest{}, &pruneProxy{
|
|
| 91 |
+ streamProxy: streamProxy{ctx: ctx},
|
|
| 92 |
+ ch: ch, |
|
| 93 |
+ }) |
|
| 94 |
+ }) |
|
| 95 |
+ |
|
| 96 |
+ var size int64 |
|
| 97 |
+ eg.Go(func() error {
|
|
| 98 |
+ for r := range ch {
|
|
| 99 |
+ size += r.Size_ |
|
| 100 |
+ } |
|
| 101 |
+ return nil |
|
| 102 |
+ }) |
|
| 103 |
+ |
|
| 104 |
+ if err := eg.Wait(); err != nil {
|
|
| 105 |
+ return 0, err |
|
| 106 |
+ } |
|
| 107 |
+ |
|
| 108 |
+ return size, nil |
|
| 109 |
+} |
|
| 110 |
+ |
|
| 83 | 111 |
func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.Result, error) {
|
| 84 | 112 |
var out builder.Result |
| 85 | 113 |
|
| ... | ... |
@@ -149,7 +177,7 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder. |
| 149 | 149 |
defer close(ch) |
| 150 | 150 |
return b.controller.Status(&controlapi.StatusRequest{
|
| 151 | 151 |
Ref: id, |
| 152 |
- }, &statusProxy{ctx: ctx, ch: ch})
|
|
| 152 |
+ }, &statusProxy{streamProxy: streamProxy{ctx: ctx}, ch: ch})
|
|
| 153 | 153 |
}) |
| 154 | 154 |
|
| 155 | 155 |
eg.Go(func() error {
|
| ... | ... |
@@ -188,28 +216,35 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder. |
| 188 | 188 |
return &out, nil |
| 189 | 189 |
} |
| 190 | 190 |
|
| 191 |
-type statusProxy struct {
|
|
| 191 |
+type streamProxy struct {
|
|
| 192 | 192 |
ctx context.Context |
| 193 |
- ch chan *controlapi.StatusResponse |
|
| 194 | 193 |
} |
| 195 | 194 |
|
| 196 |
-func (sp *statusProxy) SetHeader(_ grpcmetadata.MD) error {
|
|
| 195 |
+func (sp *streamProxy) SetHeader(_ grpcmetadata.MD) error {
|
|
| 197 | 196 |
return nil |
| 198 | 197 |
} |
| 199 | 198 |
|
| 200 |
-func (sp *statusProxy) SendHeader(_ grpcmetadata.MD) error {
|
|
| 199 |
+func (sp *streamProxy) SendHeader(_ grpcmetadata.MD) error {
|
|
| 201 | 200 |
return nil |
| 202 | 201 |
} |
| 203 | 202 |
|
| 204 |
-func (sp *statusProxy) SetTrailer(_ grpcmetadata.MD) {
|
|
| 203 |
+func (sp *streamProxy) SetTrailer(_ grpcmetadata.MD) {
|
|
| 205 | 204 |
} |
| 206 | 205 |
|
| 207 |
-func (sp *statusProxy) Send(resp *controlapi.StatusResponse) error {
|
|
| 208 |
- return sp.SendMsg(resp) |
|
| 206 |
+func (sp *streamProxy) Context() context.Context {
|
|
| 207 |
+ return sp.ctx |
|
| 208 |
+} |
|
| 209 |
+func (sp *streamProxy) RecvMsg(m interface{}) error {
|
|
| 210 |
+ return io.EOF |
|
| 209 | 211 |
} |
| 210 | 212 |
|
| 211 |
-func (sp *statusProxy) Context() context.Context {
|
|
| 212 |
- return sp.ctx |
|
| 213 |
+type statusProxy struct {
|
|
| 214 |
+ streamProxy |
|
| 215 |
+ ch chan *controlapi.StatusResponse |
|
| 216 |
+} |
|
| 217 |
+ |
|
| 218 |
+func (sp *statusProxy) Send(resp *controlapi.StatusResponse) error {
|
|
| 219 |
+ return sp.SendMsg(resp) |
|
| 213 | 220 |
} |
| 214 | 221 |
func (sp *statusProxy) SendMsg(m interface{}) error {
|
| 215 | 222 |
if sr, ok := m.(*controlapi.StatusResponse); ok {
|
| ... | ... |
@@ -217,8 +252,20 @@ func (sp *statusProxy) SendMsg(m interface{}) error {
|
| 217 | 217 |
} |
| 218 | 218 |
return nil |
| 219 | 219 |
} |
| 220 |
-func (sp *statusProxy) RecvMsg(m interface{}) error {
|
|
| 221 |
- return io.EOF |
|
| 220 |
+ |
|
| 221 |
+type pruneProxy struct {
|
|
| 222 |
+ streamProxy |
|
| 223 |
+ ch chan *controlapi.UsageRecord |
|
| 224 |
+} |
|
| 225 |
+ |
|
| 226 |
+func (sp *pruneProxy) Send(resp *controlapi.UsageRecord) error {
|
|
| 227 |
+ return sp.SendMsg(resp) |
|
| 228 |
+} |
|
| 229 |
+func (sp *pruneProxy) SendMsg(m interface{}) error {
|
|
| 230 |
+ if sr, ok := m.(*controlapi.UsageRecord); ok {
|
|
| 231 |
+ sp.ch <- sr |
|
| 232 |
+ } |
|
| 233 |
+ return nil |
|
| 222 | 234 |
} |
| 223 | 235 |
|
| 224 | 236 |
type contentStoreNoLabels struct {
|