Browse code

vendor: update buildkit to f238f1ef

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
(cherry picked from commit a3cbd53ed2b3c532b8669ef0d4d578f7f4236ad6)
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

Tonis Tiigi authored on 2019/05/14 08:50:01
Showing 13 changed files
... ...
@@ -27,7 +27,7 @@ github.com/imdario/mergo                            7c29201646fa3de8506f70121347
27 27
 golang.org/x/sync                                   e225da77a7e68af35c70ccbf71af2b83e6acac3c
28 28
 
29 29
 # buildkit
30
-github.com/moby/buildkit                            8c0fa8fdec187d8f259a349d2da16dc2dc5f144a # v0.5.0
30
+github.com/moby/buildkit                            f238f1efb04f00bf0cc147141fda9ddb55c8bc49
31 31
 github.com/tonistiigi/fsutil                        3bbb99cdbd76619ab717299830c60f6f2a533a6b
32 32
 github.com/grpc-ecosystem/grpc-opentracing          8e809c8a86450a29b90dcc9efbf062d0fe6d9746
33 33
 github.com/opentracing/opentracing-go               1361b9cd60be79c4c3a7fa9841b3c132e40066a7
... ...
@@ -177,7 +177,7 @@ func (e *ExecOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata,
177 177
 		addCap(&e.constraints, pb.CapExecMetaNetwork)
178 178
 	}
179 179
 
180
-	if e.meta.Security != SecurityModeInsecure {
180
+	if e.meta.Security != SecurityModeSandbox {
181 181
 		addCap(&e.constraints, pb.CapExecMetaSecurity)
182 182
 	}
183 183
 
... ...
@@ -410,9 +410,6 @@ func parseCacheOptions(opt SolveOpt) (*cacheOptions, error) {
410 410
 			if csDir == "" {
411 411
 				return nil, errors.New("local cache importer requires src")
412 412
 			}
413
-			if err := os.MkdirAll(csDir, 0755); err != nil {
414
-				return nil, err
415
-			}
416 413
 			cs, err := contentlocal.NewStore(csDir)
417 414
 			if err != nil {
418 415
 				return nil, err
... ...
@@ -63,7 +63,9 @@ func (gwf *GatewayForwarder) lookupForwarder(ctx context.Context) (gateway.LLBBr
63 63
 
64 64
 	go func() {
65 65
 		<-ctx.Done()
66
+		gwf.mu.Lock()
66 67
 		gwf.updateCond.Broadcast()
68
+		gwf.mu.Unlock()
67 69
 	}()
68 70
 
69 71
 	gwf.mu.RLock()
... ...
@@ -95,6 +95,23 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou
95 95
 		Options:     []string{"ro", "nosuid", "noexec", "nodev"},
96 96
 	})
97 97
 
98
+	if processMode == NoProcessSandbox {
99
+		var maskedPaths []string
100
+		for _, s := range s.Linux.MaskedPaths {
101
+			if !hasPrefix(s, "/proc") {
102
+				maskedPaths = append(maskedPaths, s)
103
+			}
104
+		}
105
+		s.Linux.MaskedPaths = maskedPaths
106
+		var readonlyPaths []string
107
+		for _, s := range s.Linux.ReadonlyPaths {
108
+			if !hasPrefix(s, "/proc") {
109
+				readonlyPaths = append(readonlyPaths, s)
110
+			}
111
+		}
112
+		s.Linux.ReadonlyPaths = readonlyPaths
113
+	}
114
+
98 115
 	if meta.SecurityMode == pb.SecurityMode_INSECURE {
99 116
 		//make sysfs rw mount for insecure mode.
100 117
 		for _, m := range s.Mounts {
... ...
@@ -41,6 +41,8 @@ type Opt struct {
41 41
 	// ProcessMode
42 42
 	ProcessMode     oci.ProcessMode
43 43
 	IdentityMapping *idtools.IdentityMapping
44
+	// runc run --no-pivot (unrecommended)
45
+	NoPivot bool
44 46
 }
45 47
 
46 48
 var defaultCommandCandidates = []string{"buildkit-runc", "runc"}
... ...
@@ -54,6 +56,7 @@ type runcExecutor struct {
54 54
 	networkProviders map[pb.NetMode]network.Provider
55 55
 	processMode      oci.ProcessMode
56 56
 	idmap            *idtools.IdentityMapping
57
+	noPivot          bool
57 58
 }
58 59
 
59 60
 func New(opt Opt, networkProviders map[pb.NetMode]network.Provider) (executor.Executor, error) {
... ...
@@ -111,6 +114,7 @@ func New(opt Opt, networkProviders map[pb.NetMode]network.Provider) (executor.Ex
111 111
 		networkProviders: networkProviders,
112 112
 		processMode:      opt.ProcessMode,
113 113
 		idmap:            opt.IdentityMapping,
114
+		noPivot:          opt.NoPivot,
114 115
 	}
115 116
 	return w, nil
116 117
 }
... ...
@@ -193,6 +197,17 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
193 193
 		opts = append(opts, containerdoci.WithRootFSReadonly())
194 194
 	}
195 195
 
196
+	identity = idtools.Identity{
197
+		UID: int(uid),
198
+		GID: int(gid),
199
+	}
200
+	if w.idmap != nil {
201
+		identity, err = w.idmap.ToHost(identity)
202
+		if err != nil {
203
+			return err
204
+		}
205
+	}
206
+
196 207
 	if w.cgroupParent != "" {
197 208
 		var cgroupsPath string
198 209
 		lastSeparator := w.cgroupParent[len(w.cgroupParent)-1:]
... ...
@@ -269,7 +284,8 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
269 269
 
270 270
 	logrus.Debugf("> creating %s %v", id, meta.Args)
271 271
 	status, err := w.runc.Run(runCtx, id, bundle, &runc.CreateOpts{
272
-		IO: &forwardIO{stdin: stdin, stdout: stdout, stderr: stderr},
272
+		IO:      &forwardIO{stdin: stdin, stdout: stdout, stderr: stderr},
273
+		NoPivot: w.noPivot,
273 274
 	})
274 275
 	close(done)
275 276
 	if err != nil {
... ...
@@ -172,10 +172,6 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
172 172
 		}
173 173
 	}
174 174
 
175
-	if len(allDispatchStates.states) == 1 {
176
-		allDispatchStates.states[0].stageName = ""
177
-	}
178
-
179 175
 	var target *dispatchState
180 176
 	if opt.Target == "" {
181 177
 		target = allDispatchStates.lastTarget()
... ...
@@ -207,6 +203,14 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
207 207
 		}
208 208
 	}
209 209
 
210
+	if has, state := hasCircularDependency(allDispatchStates.states); has {
211
+		return nil, nil, fmt.Errorf("circular dependency detected on stage: %s", state.stageName)
212
+	}
213
+
214
+	if len(allDispatchStates.states) == 1 {
215
+		allDispatchStates.states[0].stageName = ""
216
+	}
217
+
210 218
 	eg, ctx := errgroup.WithContext(ctx)
211 219
 	for i, d := range allDispatchStates.states {
212 220
 		reachable := isReachable(target, d)
... ...
@@ -1130,6 +1134,41 @@ func isReachable(from, to *dispatchState) (ret bool) {
1130 1130
 	return false
1131 1131
 }
1132 1132
 
1133
+func hasCircularDependency(states []*dispatchState) (bool, *dispatchState) {
1134
+	var visit func(state *dispatchState) bool
1135
+	if states == nil {
1136
+		return false, nil
1137
+	}
1138
+	visited := make(map[*dispatchState]struct{})
1139
+	path := make(map[*dispatchState]struct{})
1140
+
1141
+	visit = func(state *dispatchState) bool {
1142
+		_, ok := visited[state]
1143
+		if ok {
1144
+			return false
1145
+		}
1146
+		visited[state] = struct{}{}
1147
+		path[state] = struct{}{}
1148
+		for dep := range state.deps {
1149
+			_, ok = path[dep]
1150
+			if ok {
1151
+				return true
1152
+			}
1153
+			if visit(dep) {
1154
+				return true
1155
+			}
1156
+		}
1157
+		delete(path, state)
1158
+		return false
1159
+	}
1160
+	for _, state := range states {
1161
+		if visit(state) {
1162
+			return true, state
1163
+		}
1164
+	}
1165
+	return false, nil
1166
+}
1167
+
1133 1168
 func parseUser(str string) (uid uint32, gid uint32, err error) {
1134 1169
 	if str == "" {
1135 1170
 		return 0, 0, nil
... ...
@@ -158,7 +158,7 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
158 158
 		rootFS = workerRef.ImmutableRef
159 159
 	}
160 160
 
161
-	lbf, err := newLLBBridgeForwarder(ctx, llbBridge, gf.workers)
161
+	lbf, ctx, err := newLLBBridgeForwarder(ctx, llbBridge, gf.workers)
162 162
 	defer lbf.conn.Close()
163 163
 	if err != nil {
164 164
 		return nil, err
... ...
@@ -210,6 +210,9 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
210 210
 	err = llbBridge.Exec(ctx, meta, rootFS, lbf.Stdin, lbf.Stdout, os.Stderr)
211 211
 
212 212
 	if err != nil {
213
+		if errors.Cause(err) == context.Canceled && lbf.isErrServerClosed {
214
+			err = errors.Errorf("frontend grpc server closed unexpectedly")
215
+		}
213 216
 		// An existing error (set via Return rpc) takes
214 217
 		// precedence over this error, which in turn takes
215 218
 		// precedence over a success reported via Return.
... ...
@@ -294,15 +297,24 @@ func NewBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridg
294 294
 	return lbf
295 295
 }
296 296
 
297
-func newLLBBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, workers frontend.WorkerInfos) (*llbBridgeForwarder, error) {
297
+func newLLBBridgeForwarder(ctx context.Context, llbBridge frontend.FrontendLLBBridge, workers frontend.WorkerInfos) (*llbBridgeForwarder, context.Context, error) {
298
+	ctx, cancel := context.WithCancel(ctx)
298 299
 	lbf := NewBridgeForwarder(ctx, llbBridge, workers)
299 300
 	server := grpc.NewServer()
300 301
 	grpc_health_v1.RegisterHealthServer(server, health.NewServer())
301 302
 	pb.RegisterLLBBridgeServer(server, lbf)
302 303
 
303
-	go serve(ctx, server, lbf.conn)
304
+	go func() {
305
+		serve(ctx, server, lbf.conn)
306
+		select {
307
+		case <-ctx.Done():
308
+		default:
309
+			lbf.isErrServerClosed = true
310
+		}
311
+		cancel()
312
+	}()
304 313
 
305
-	return lbf, nil
314
+	return lbf, ctx, nil
306 315
 }
307 316
 
308 317
 type pipe struct {
... ...
@@ -372,11 +384,12 @@ type llbBridgeForwarder struct {
372 372
 	// lastRef      solver.CachedResult
373 373
 	// lastRefs     map[string]solver.CachedResult
374 374
 	// err          error
375
-	doneCh       chan struct{} // closed when result or err become valid through a call to a Return
376
-	result       *frontend.Result
377
-	err          error
378
-	exporterAttr map[string][]byte
379
-	workers      frontend.WorkerInfos
375
+	doneCh            chan struct{} // closed when result or err become valid through a call to a Return
376
+	result            *frontend.Result
377
+	err               error
378
+	exporterAttr      map[string][]byte
379
+	workers           frontend.WorkerInfos
380
+	isErrServerClosed bool
380 381
 	*pipe
381 382
 }
382 383
 
... ...
@@ -28,6 +28,8 @@ type GrpcClient interface {
28 28
 }
29 29
 
30 30
 func New(ctx context.Context, opts map[string]string, session, product string, c pb.LLBBridgeClient, w []client.WorkerInfo) (GrpcClient, error) {
31
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
32
+	defer cancel()
31 33
 	resp, err := c.Ping(ctx, &pb.PingRequest{})
32 34
 	if err != nil {
33 35
 		return nil, err
... ...
@@ -46,6 +46,7 @@ type conn struct {
46 46
 
47 47
 	closedOnce sync.Once
48 48
 	readMu     sync.Mutex
49
+	writeMu    sync.Mutex
49 50
 	err        error
50 51
 	closeCh    chan struct{}
51 52
 }
... ...
@@ -79,6 +80,8 @@ func (c *conn) Read(b []byte) (n int, err error) {
79 79
 }
80 80
 
81 81
 func (c *conn) Write(b []byte) (int, error) {
82
+	c.writeMu.Lock()
83
+	defer c.writeMu.Unlock()
82 84
 	m := &controlapi.BytesMessage{Data: b}
83 85
 	if err := c.stream.SendMsg(m); err != nil {
84 86
 		return 0, err
... ...
@@ -93,7 +96,9 @@ func (c *conn) Close() (err error) {
93 93
 		}()
94 94
 
95 95
 		if cs, ok := c.stream.(grpc.ClientStream); ok {
96
+			c.writeMu.Lock()
96 97
 			err = cs.CloseSend()
98
+			c.writeMu.Unlock()
97 99
 			if err != nil {
98 100
 				return
99 101
 			}
... ...
@@ -106,6 +111,7 @@ func (c *conn) Close() (err error) {
106 106
 			err = c.stream.RecvMsg(m)
107 107
 			if err != nil {
108 108
 				if err != io.EOF {
109
+					c.readMu.Unlock()
109 110
 					return
110 111
 				}
111 112
 				err = nil
... ...
@@ -162,7 +162,9 @@ func (sm *Manager) Get(ctx context.Context, id string) (Caller, error) {
162 162
 	go func() {
163 163
 		select {
164 164
 		case <-ctx.Done():
165
+			sm.mu.Lock()
165 166
 			sm.updateCondition.Broadcast()
167
+			sm.mu.Unlock()
166 168
 		}
167 169
 	}()
168 170
 
... ...
@@ -404,7 +404,9 @@ func (jl *Solver) Get(id string) (*Job, error) {
404 404
 
405 405
 	go func() {
406 406
 		<-ctx.Done()
407
+		jl.mu.Lock()
407 408
 		jl.updateCond.Broadcast()
409
+		jl.mu.Unlock()
408 410
 	}()
409 411
 
410 412
 	jl.mu.RLock()
... ...
@@ -101,7 +101,9 @@ func (pr *progressReader) Read(ctx context.Context) ([]*Progress, error) {
101 101
 		select {
102 102
 		case <-done:
103 103
 		case <-ctx.Done():
104
+			pr.mu.Lock()
104 105
 			pr.cond.Broadcast()
106
+			pr.mu.Unlock()
105 107
 		}
106 108
 	}()
107 109
 	pr.mu.Lock()
... ...
@@ -163,7 +165,9 @@ func pipe() (*progressReader, *progressWriter, func()) {
163 163
 	pr.cond = sync.NewCond(&pr.mu)
164 164
 	go func() {
165 165
 		<-ctx.Done()
166
+		pr.mu.Lock()
166 167
 		pr.cond.Broadcast()
168
+		pr.mu.Unlock()
167 169
 	}()
168 170
 	pw := &progressWriter{
169 171
 		reader: pr,