Browse code

vendor: update buildkit to 588c73e1e4

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
(cherry picked from commit 52ed97c5c129193342e5f411d7a6534bee1d9213)
Signed-off-by: Tibor Vass <tibor@docker.com>

Tonis Tiigi authored on 2019/08/22 08:21:19
Showing 11 changed files
... ...
@@ -27,7 +27,7 @@ github.com/imdario/mergo                            7c29201646fa3de8506f70121347
27 27
 golang.org/x/sync                                   e225da77a7e68af35c70ccbf71af2b83e6acac3c
28 28
 
29 29
 # buildkit
30
-github.com/moby/buildkit                            be0d75f074e7a4b0f5b5877c719213a3f5057e60 # v0.6.1
30
+github.com/moby/buildkit                            588c73e1e4f0f3d7d3738abaaa7cf8026064b33e
31 31
 github.com/tonistiigi/fsutil                        3bbb99cdbd76619ab717299830c60f6f2a533a6b
32 32
 github.com/grpc-ecosystem/grpc-opentracing          8e809c8a86450a29b90dcc9efbf062d0fe6d9746
33 33
 github.com/opentracing/opentracing-go               1361b9cd60be79c4c3a7fa9841b3c132e40066a7
... ...
@@ -792,7 +792,7 @@ func getFollowLinksWalk(root *iradix.Node, k []byte, follow bool, linksWalked *i
792 792
 		return k, v.(*CacheRecord), nil
793 793
 	}
794 794
 	if !follow || len(k) == 0 {
795
-		return nil, nil, nil
795
+		return k, nil, nil
796 796
 	}
797 797
 
798 798
 	dir, file := splitKey(k)
... ...
@@ -817,14 +817,13 @@ func getFollowLinksWalk(root *iradix.Node, k []byte, follow bool, linksWalked *i
817 817
 			}
818 818
 			return getFollowLinksWalk(root, append(convertPathToKey([]byte(link)), file...), follow, linksWalked)
819 819
 		}
820
-
821
-		k = append(k, file...)
822
-		v, ok = root.Get(k)
823
-		if ok {
824
-			return k, v.(*CacheRecord), nil
825
-		}
826 820
 	}
827
-	return nil, nil, nil
821
+	k = append(k, file...)
822
+	v, ok = root.Get(k)
823
+	if ok {
824
+		return k, v.(*CacheRecord), nil
825
+	}
826
+	return k, nil, nil
828 827
 }
829 828
 
830 829
 func prepareDigest(fp, p string, fi os.FileInfo) (digest.Digest, error) {
... ...
@@ -424,10 +424,10 @@ type readOnlyMounter struct {
424 424
 	snapshot.Mountable
425 425
 }
426 426
 
427
-func (m *readOnlyMounter) Mount() ([]mount.Mount, error) {
428
-	mounts, err := m.Mountable.Mount()
427
+func (m *readOnlyMounter) Mount() ([]mount.Mount, func() error, error) {
428
+	mounts, release, err := m.Mountable.Mount()
429 429
 	if err != nil {
430
-		return nil, err
430
+		return nil, nil, err
431 431
 	}
432 432
 	for i, m := range mounts {
433 433
 		if m.Type == "overlay" {
... ...
@@ -443,7 +443,7 @@ func (m *readOnlyMounter) Mount() ([]mount.Mount, error) {
443 443
 		opts = append(opts, "ro")
444 444
 		mounts[i].Options = opts
445 445
 	}
446
-	return mounts, nil
446
+	return mounts, release, nil
447 447
 }
448 448
 
449 449
 func readonlyOverlay(opt []string) []string {
... ...
@@ -136,12 +136,12 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou
136 136
 			releaseAll()
137 137
 			return nil, nil, errors.Wrapf(err, "failed to mount %s", m.Dest)
138 138
 		}
139
-		mounts, err := mountable.Mount()
139
+		mounts, release, err := mountable.Mount()
140 140
 		if err != nil {
141 141
 			releaseAll()
142 142
 			return nil, nil, errors.WithStack(err)
143 143
 		}
144
-		releasers = append(releasers, mountable.Release)
144
+		releasers = append(releasers, release)
145 145
 		for _, mount := range mounts {
146 146
 			mount, err = sm.subMount(mount, m.Selector)
147 147
 			if err != nil {
... ...
@@ -4,11 +4,9 @@ import (
4 4
 	"context"
5 5
 	"encoding/json"
6 6
 	"io"
7
-	"io/ioutil"
8 7
 	"os"
9 8
 	"os/exec"
10 9
 	"path/filepath"
11
-	"strconv"
12 10
 	"strings"
13 11
 	"syscall"
14 12
 	"time"
... ...
@@ -25,7 +23,6 @@ import (
25 25
 	"github.com/moby/buildkit/solver/pb"
26 26
 	"github.com/moby/buildkit/util/network"
27 27
 	rootlessspecconv "github.com/moby/buildkit/util/rootless/specconv"
28
-	specs "github.com/opencontainers/runtime-spec/specs-go"
29 28
 	"github.com/pkg/errors"
30 29
 	"github.com/sirupsen/logrus"
31 30
 )
... ...
@@ -42,8 +39,9 @@ type Opt struct {
42 42
 	ProcessMode     oci.ProcessMode
43 43
 	IdentityMapping *idtools.IdentityMapping
44 44
 	// runc run --no-pivot (unrecommended)
45
-	NoPivot bool
46
-	DNS     *oci.DNSConfig
45
+	NoPivot     bool
46
+	DNS         *oci.DNSConfig
47
+	OOMScoreAdj *int
47 48
 }
48 49
 
49 50
 var defaultCommandCandidates = []string{"buildkit-runc", "runc"}
... ...
@@ -59,6 +57,7 @@ type runcExecutor struct {
59 59
 	idmap            *idtools.IdentityMapping
60 60
 	noPivot          bool
61 61
 	dns              *oci.DNSConfig
62
+	oomScoreAdj      *int
62 63
 }
63 64
 
64 65
 func New(opt Opt, networkProviders map[pb.NetMode]network.Provider) (executor.Executor, error) {
... ...
@@ -118,6 +117,7 @@ func New(opt Opt, networkProviders map[pb.NetMode]network.Provider) (executor.Ex
118 118
 		idmap:            opt.IdentityMapping,
119 119
 		noPivot:          opt.NoPivot,
120 120
 		dns:              opt.DNS,
121
+		oomScoreAdj:      opt.OOMScoreAdj,
121 122
 	}
122 123
 	return w, nil
123 124
 }
... ...
@@ -155,11 +155,13 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
155 155
 		return err
156 156
 	}
157 157
 
158
-	rootMount, err := mountable.Mount()
158
+	rootMount, release, err := mountable.Mount()
159 159
 	if err != nil {
160 160
 		return err
161 161
 	}
162
-	defer mountable.Release()
162
+	if release != nil {
163
+		defer release()
164
+	}
163 165
 
164 166
 	id := identity.NewID()
165 167
 	bundle := filepath.Join(w.root, id)
... ...
@@ -242,9 +244,7 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
242 242
 		}
243 243
 	}
244 244
 
245
-	if err := setOOMScoreAdj(spec); err != nil {
246
-		return err
247
-	}
245
+	spec.Process.OOMScoreAdj = w.oomScoreAdj
248 246
 	if w.rootless {
249 247
 		if err := rootlessspecconv.ToRootless(spec); err != nil {
250 248
 			return err
... ...
@@ -336,19 +336,3 @@ func (s *forwardIO) Stdout() io.ReadCloser {
336 336
 func (s *forwardIO) Stderr() io.ReadCloser {
337 337
 	return nil
338 338
 }
339
-
340
-// setOOMScoreAdj comes from https://github.com/genuinetools/img/blob/2fabe60b7dc4623aa392b515e013bbc69ad510ab/executor/runc/executor.go#L182-L192
341
-func setOOMScoreAdj(spec *specs.Spec) error {
342
-	// Set the oom_score_adj of our children containers to that of the current process.
343
-	b, err := ioutil.ReadFile("/proc/self/oom_score_adj")
344
-	if err != nil {
345
-		return errors.Wrap(err, "failed to read /proc/self/oom_score_adj")
346
-	}
347
-	s := strings.TrimSpace(string(b))
348
-	oom, err := strconv.Atoi(s)
349
-	if err != nil {
350
-		return errors.Wrapf(err, "failed to parse %s as int", s)
351
-	}
352
-	spec.Process.OOMScoreAdj = &oom
353
-	return nil
354
-}
... ...
@@ -31,6 +31,7 @@ type localMounter struct {
31 31
 	mounts    []mount.Mount
32 32
 	mountable Mountable
33 33
 	target    string
34
+	release   func() error
34 35
 }
35 36
 
36 37
 func (lm *localMounter) Mount() (string, error) {
... ...
@@ -38,11 +39,12 @@ func (lm *localMounter) Mount() (string, error) {
38 38
 	defer lm.mu.Unlock()
39 39
 
40 40
 	if lm.mounts == nil {
41
-		mounts, err := lm.mountable.Mount()
41
+		mounts, release, err := lm.mountable.Mount()
42 42
 		if err != nil {
43 43
 			return "", err
44 44
 		}
45 45
 		lm.mounts = mounts
46
+		lm.release = release
46 47
 	}
47 48
 
48 49
 	if len(lm.mounts) == 1 && (lm.mounts[0].Type == "bind" || lm.mounts[0].Type == "rbind") {
... ...
@@ -21,8 +21,8 @@ func (lm *localMounter) Unmount() error {
21 21
 		lm.target = ""
22 22
 	}
23 23
 
24
-	if lm.mountable != nil {
25
-		return lm.mountable.Release()
24
+	if lm.release != nil {
25
+		return lm.release()
26 26
 	}
27 27
 
28 28
 	return nil
... ...
@@ -18,8 +18,8 @@ func (lm *localMounter) Unmount() error {
18 18
 		lm.target = ""
19 19
 	}
20 20
 
21
-	if lm.mountable != nil {
22
-		return lm.mountable.Release()
21
+	if lm.release != nil {
22
+		return lm.release()
23 23
 	}
24 24
 
25 25
 	return nil
... ...
@@ -2,7 +2,9 @@ package snapshot
2 2
 
3 3
 import (
4 4
 	"context"
5
+	"os"
5 6
 	"sync"
7
+	"sync/atomic"
6 8
 
7 9
 	"github.com/containerd/containerd/mount"
8 10
 	"github.com/containerd/containerd/snapshots"
... ...
@@ -12,8 +14,7 @@ import (
12 12
 
13 13
 type Mountable interface {
14 14
 	// ID() string
15
-	Mount() ([]mount.Mount, error)
16
-	Release() error
15
+	Mount() ([]mount.Mount, func() error, error)
17 16
 	IdentityMapping() *idtools.IdentityMapping
18 17
 }
19 18
 
... ...
@@ -63,7 +64,7 @@ func (s *fromContainerd) Mounts(ctx context.Context, key string) (Mountable, err
63 63
 	if err != nil {
64 64
 		return nil, err
65 65
 	}
66
-	return &staticMountable{mounts, s.idmap}, nil
66
+	return &staticMountable{mounts: mounts, idmap: s.idmap, id: key}, nil
67 67
 }
68 68
 func (s *fromContainerd) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) error {
69 69
 	_, err := s.Snapshotter.Prepare(ctx, key, parent, opts...)
... ...
@@ -74,23 +75,29 @@ func (s *fromContainerd) View(ctx context.Context, key, parent string, opts ...s
74 74
 	if err != nil {
75 75
 		return nil, err
76 76
 	}
77
-	return &staticMountable{mounts, s.idmap}, nil
77
+	return &staticMountable{mounts: mounts, idmap: s.idmap, id: key}, nil
78 78
 }
79 79
 func (s *fromContainerd) IdentityMapping() *idtools.IdentityMapping {
80 80
 	return s.idmap
81 81
 }
82 82
 
83 83
 type staticMountable struct {
84
+	count  int32
85
+	id     string
84 86
 	mounts []mount.Mount
85 87
 	idmap  *idtools.IdentityMapping
86 88
 }
87 89
 
88
-func (m *staticMountable) Mount() ([]mount.Mount, error) {
89
-	return m.mounts, nil
90
-}
91
-
92
-func (cm *staticMountable) Release() error {
93
-	return nil
90
+func (cm *staticMountable) Mount() ([]mount.Mount, func() error, error) {
91
+	atomic.AddInt32(&cm.count, 1)
92
+	return cm.mounts, func() error {
93
+		if atomic.AddInt32(&cm.count, -1) < 0 {
94
+			if v := os.Getenv("BUILDKIT_DEBUG_PANIC_ON_ERROR"); v == "1" {
95
+				panic("release of released mount " + cm.id)
96
+			}
97
+		}
98
+		return nil
99
+	}, nil
94 100
 }
95 101
 
96 102
 func (cm *staticMountable) IdentityMapping() *idtools.IdentityMapping {
... ...
@@ -122,12 +129,12 @@ func (cs *containerdSnapshotter) release() error {
122 122
 }
123 123
 
124 124
 func (cs *containerdSnapshotter) returnMounts(mf Mountable) ([]mount.Mount, error) {
125
-	mounts, err := mf.Mount()
125
+	mounts, release, err := mf.Mount()
126 126
 	if err != nil {
127 127
 		return nil, err
128 128
 	}
129 129
 	cs.mu.Lock()
130
-	cs.releasers = append(cs.releasers, mf.Release)
130
+	cs.releasers = append(cs.releasers, release)
131 131
 	cs.mu.Unlock()
132 132
 	return mounts, nil
133 133
 }
... ...
@@ -47,12 +47,12 @@ func (rm *RefManager) Commit(ctx context.Context, mount fileoptypes.Mount) (file
47 47
 	if !ok {
48 48
 		return nil, errors.Errorf("invalid mount type %T", mount)
49 49
 	}
50
-	if err := m.m.Release(); err != nil {
51
-		return nil, err
52
-	}
53 50
 	if m.mr == nil {
54 51
 		return nil, errors.Errorf("invalid mount without active ref for commit")
55 52
 	}
53
+	defer func() {
54
+		m.mr = nil
55
+	}()
56 56
 	return m.mr.Commit(ctx)
57 57
 }
58 58
 
... ...
@@ -62,7 +62,6 @@ type Mount struct {
62 62
 }
63 63
 
64 64
 func (m *Mount) Release(ctx context.Context) error {
65
-	m.m.Release()
66 65
 	if m.mr != nil {
67 66
 		return m.mr.Release(ctx)
68 67
 	}
... ...
@@ -349,12 +349,11 @@ func (sm *sshMount) Mount(ctx context.Context, readonly bool) (snapshot.Mountabl
349 349
 }
350 350
 
351 351
 type sshMountInstance struct {
352
-	sm      *sshMount
353
-	cleanup func() error
354
-	idmap   *idtools.IdentityMapping
352
+	sm    *sshMount
353
+	idmap *idtools.IdentityMapping
355 354
 }
356 355
 
357
-func (sm *sshMountInstance) Mount() ([]mount.Mount, error) {
356
+func (sm *sshMountInstance) Mount() ([]mount.Mount, func() error, error) {
358 357
 	ctx, cancel := context.WithCancel(context.TODO())
359 358
 
360 359
 	uid := int(sm.sm.mount.SSHOpt.Uid)
... ...
@@ -366,7 +365,7 @@ func (sm *sshMountInstance) Mount() ([]mount.Mount, error) {
366 366
 			GID: gid,
367 367
 		})
368 368
 		if err != nil {
369
-			return nil, err
369
+			return nil, nil, err
370 370
 		}
371 371
 		uid = identity.UID
372 372
 		gid = identity.GID
... ...
@@ -380,9 +379,9 @@ func (sm *sshMountInstance) Mount() ([]mount.Mount, error) {
380 380
 	})
381 381
 	if err != nil {
382 382
 		cancel()
383
-		return nil, err
383
+		return nil, nil, err
384 384
 	}
385
-	sm.cleanup = func() error {
385
+	release := func() error {
386 386
 		var err error
387 387
 		if cleanup != nil {
388 388
 			err = cleanup()
... ...
@@ -395,16 +394,7 @@ func (sm *sshMountInstance) Mount() ([]mount.Mount, error) {
395 395
 		Type:    "bind",
396 396
 		Source:  sock,
397 397
 		Options: []string{"rbind"},
398
-	}}, nil
399
-}
400
-
401
-func (sm *sshMountInstance) Release() error {
402
-	if sm.cleanup != nil {
403
-		if err := sm.cleanup(); err != nil {
404
-			return err
405
-		}
406
-	}
407
-	return nil
398
+	}}, release, nil
408 399
 }
409 400
 
410 401
 func (sm *sshMountInstance) IdentityMapping() *idtools.IdentityMapping {
... ...
@@ -462,14 +452,18 @@ type secretMountInstance struct {
462 462
 	idmap *idtools.IdentityMapping
463 463
 }
464 464
 
465
-func (sm *secretMountInstance) Mount() ([]mount.Mount, error) {
465
+func (sm *secretMountInstance) Mount() ([]mount.Mount, func() error, error) {
466 466
 	dir, err := ioutil.TempDir("", "buildkit-secrets")
467 467
 	if err != nil {
468
-		return nil, errors.Wrap(err, "failed to create temp dir")
468
+		return nil, nil, errors.Wrap(err, "failed to create temp dir")
469
+	}
470
+	cleanupDir := func() error {
471
+		return os.RemoveAll(dir)
469 472
 	}
470 473
 
471 474
 	if err := os.Chmod(dir, 0711); err != nil {
472
-		return nil, err
475
+		cleanupDir()
476
+		return nil, nil, err
473 477
 	}
474 478
 
475 479
 	tmpMount := mount.Mount{
... ...
@@ -483,15 +477,23 @@ func (sm *secretMountInstance) Mount() ([]mount.Mount, error) {
483 483
 	}
484 484
 
485 485
 	if err := mount.All([]mount.Mount{tmpMount}, dir); err != nil {
486
-		return nil, errors.Wrap(err, "unable to setup secret mount")
486
+		cleanupDir()
487
+		return nil, nil, errors.Wrap(err, "unable to setup secret mount")
487 488
 	}
488 489
 	sm.root = dir
489 490
 
491
+	cleanup := func() error {
492
+		if err := mount.Unmount(dir, 0); err != nil {
493
+			return err
494
+		}
495
+		return cleanupDir()
496
+	}
497
+
490 498
 	randID := identity.NewID()
491 499
 	fp := filepath.Join(dir, randID)
492 500
 	if err := ioutil.WriteFile(fp, sm.sm.data, 0600); err != nil {
493
-		sm.Release()
494
-		return nil, err
501
+		cleanup()
502
+		return nil, nil, err
495 503
 	}
496 504
 
497 505
 	uid := int(sm.sm.mount.SecretOpt.Uid)
... ...
@@ -503,35 +505,28 @@ func (sm *secretMountInstance) Mount() ([]mount.Mount, error) {
503 503
 			GID: gid,
504 504
 		})
505 505
 		if err != nil {
506
-			return nil, err
506
+			cleanup()
507
+			return nil, nil, err
507 508
 		}
508 509
 		uid = identity.UID
509 510
 		gid = identity.GID
510 511
 	}
511 512
 
512 513
 	if err := os.Chown(fp, uid, gid); err != nil {
513
-		return nil, err
514
+		cleanup()
515
+		return nil, nil, err
514 516
 	}
515 517
 
516 518
 	if err := os.Chmod(fp, os.FileMode(sm.sm.mount.SecretOpt.Mode&0777)); err != nil {
517
-		return nil, err
519
+		cleanup()
520
+		return nil, nil, err
518 521
 	}
519 522
 
520 523
 	return []mount.Mount{{
521 524
 		Type:    "bind",
522 525
 		Source:  fp,
523 526
 		Options: []string{"ro", "rbind"},
524
-	}}, nil
525
-}
526
-
527
-func (sm *secretMountInstance) Release() error {
528
-	if sm.root != "" {
529
-		if err := mount.Unmount(sm.root, 0); err != nil {
530
-			return err
531
-		}
532
-		return os.RemoveAll(sm.root)
533
-	}
534
-	return nil
527
+	}}, cleanup, nil
535 528
 }
536 529
 
537 530
 func (sm *secretMountInstance) IdentityMapping() *idtools.IdentityMapping {
... ...
@@ -767,7 +762,7 @@ type tmpfsMount struct {
767 767
 	idmap    *idtools.IdentityMapping
768 768
 }
769 769
 
770
-func (m *tmpfsMount) Mount() ([]mount.Mount, error) {
770
+func (m *tmpfsMount) Mount() ([]mount.Mount, func() error, error) {
771 771
 	opt := []string{"nosuid"}
772 772
 	if m.readonly {
773 773
 		opt = append(opt, "ro")
... ...
@@ -776,10 +771,7 @@ func (m *tmpfsMount) Mount() ([]mount.Mount, error) {
776 776
 		Type:    "tmpfs",
777 777
 		Source:  "tmpfs",
778 778
 		Options: opt,
779
-	}}, nil
780
-}
781
-func (m *tmpfsMount) Release() error {
782
-	return nil
779
+	}}, func() error { return nil }, nil
783 780
 }
784 781
 
785 782
 func (m *tmpfsMount) IdentityMapping() *idtools.IdentityMapping {