1. docker
  2. Commit b7424599f696ed30d6e29630e3324c2915732f2e
Browse code

builder: adapter update after vendor update

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

Tonis Tiigi authored on 2018/05/15 03:05:49
Showing 11 changed files
builder/builder-next/adapters/containerimage/pull.go
History View file @ b742459
1 1 
new file mode 100644
... ... 
@@ -0,0 +1,656 @@
0 
+package containerimage
1 
+
2 
+import (
3 
+	"context"
4 
+	"encoding/json"
5 
+	"fmt"
6 
+	"io"
7 
+	"io/ioutil"
8 
+	"runtime"
9 
+	"sync"
10 
+	"time"
11 
+
12 
+	"github.com/containerd/containerd/content"
13 
+	"github.com/containerd/containerd/errdefs"
14 
+	"github.com/containerd/containerd/images"
15 
+	"github.com/containerd/containerd/platforms"
16 
+	ctdreference "github.com/containerd/containerd/reference"
17 
+	"github.com/containerd/containerd/remotes"
18 
+	"github.com/containerd/containerd/remotes/docker"
19 
+	"github.com/containerd/containerd/remotes/docker/schema1"
20 
+	distreference "github.com/docker/distribution/reference"
21 
+	"github.com/docker/docker/distribution"
22 
+	"github.com/docker/docker/distribution/metadata"
23 
+	"github.com/docker/docker/distribution/xfer"
24 
+	"github.com/docker/docker/image"
25 
+	"github.com/docker/docker/layer"
26 
+	pkgprogress "github.com/docker/docker/pkg/progress"
27 
+	"github.com/docker/docker/reference"
28 
+	"github.com/moby/buildkit/cache"
29 
+	"github.com/moby/buildkit/session"
30 
+	"github.com/moby/buildkit/session/auth"
31 
+	"github.com/moby/buildkit/source"
32 
+	"github.com/moby/buildkit/util/flightcontrol"
33 
+	"github.com/moby/buildkit/util/imageutil"
34 
+	"github.com/moby/buildkit/util/progress"
35 
+	"github.com/moby/buildkit/util/tracing"
36 
+	digest "github.com/opencontainers/go-digest"
37 
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
38 
+	"github.com/pkg/errors"
39 
+	netcontext "golang.org/x/net/context"
40 
+	"golang.org/x/time/rate"
41 
+)
42 
+
43 
+const preferLocal = true // FIXME: make this optional from the op
44 
+
45 
+type SourceOpt struct {
46 
+	SessionManager  *session.Manager
47 
+	ContentStore    content.Store
48 
+	CacheAccessor   cache.Accessor
49 
+	ReferenceStore  reference.Store
50 
+	DownloadManager distribution.RootFSDownloadManager
51 
+	MetadataStore   metadata.V2MetadataService
52 
+	ImageStore      image.Store
53 
+}
54 
+
55 
+type imageSource struct {
56 
+	SourceOpt
57 
+	g flightcontrol.Group
58 
+}
59 
+
60 
+func NewSource(opt SourceOpt) (source.Source, error) {
61 
+	is := &imageSource{
62 
+		SourceOpt: opt,
63 
+	}
64 
+
65 
+	return is, nil
66 
+}
67 
+
68 
+func (is *imageSource) ID() string {
69 
+	return source.DockerImageScheme
70 
+}
71 
+
72 
+func (is *imageSource) getResolver(ctx context.Context) remotes.Resolver {
73 
+	return docker.NewResolver(docker.ResolverOptions{
74 
+		Client:      tracing.DefaultClient,
75 
+		Credentials: is.getCredentialsFromSession(ctx),
76 
+	})
77 
+}
78 
+
79 
+func (is *imageSource) getCredentialsFromSession(ctx context.Context) func(string) (string, string, error) {
80 
+	id := session.FromContext(ctx)
81 
+	if id == "" {
82 
+		return nil
83 
+	}
84 
+	return func(host string) (string, string, error) {
85 
+		timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
86 
+		defer cancel()
87 
+
88 
+		caller, err := is.SessionManager.Get(timeoutCtx, id)
89 
+		if err != nil {
90 
+			return "", "", err
91 
+		}
92 
+
93 
+		return auth.CredentialsFunc(tracing.ContextWithSpanFromContext(context.TODO(), ctx), caller)(host)
94 
+	}
95 
+}
96 
+
97 
+func (is *imageSource) resolveLocal(refStr string) ([]byte, error) {
98 
+	ref, err := distreference.ParseNormalizedNamed(refStr)
99 
+	if err != nil {
100 
+		return nil, err
101 
+	}
102 
+	dgst, err := is.ReferenceStore.Get(ref)
103 
+	if err != nil {
104 
+		return nil, err
105 
+	}
106 
+	img, err := is.ImageStore.Get(image.ID(dgst))
107 
+	if err != nil {
108 
+		return nil, err
109 
+	}
110 
+	return img.RawJSON(), nil
111 
+}
112 
+
113 
+func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string) (digest.Digest, []byte, error) {
114 
+	if preferLocal {
115 
+		dt, err := is.resolveLocal(ref)
116 
+		if err == nil {
117 
+			return "", dt, nil
118 
+		}
119 
+	}
120 
+
121 
+	type t struct {
122 
+		dgst digest.Digest
123 
+		dt   []byte
124 
+	}
125 
+	res, err := is.g.Do(ctx, ref, func(ctx context.Context) (interface{}, error) {
126 
+		dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx), is.ContentStore, "")
127 
+		if err != nil {
128 
+			return nil, err
129 
+		}
130 
+		return &t{dgst: dgst, dt: dt}, nil
131 
+	})
132 
+	if err != nil {
133 
+		return "", nil, err
134 
+	}
135 
+	typed := res.(*t)
136 
+	return typed.dgst, typed.dt, nil
137 
+}
138 
+
139 
+func (is *imageSource) Resolve(ctx context.Context, id source.Identifier) (source.SourceInstance, error) {
140 
+	imageIdentifier, ok := id.(*source.ImageIdentifier)
141 
+	if !ok {
142 
+		return nil, errors.Errorf("invalid image identifier %v", id)
143 
+	}
144 
+
145 
+	p := &puller{
146 
+		src:      imageIdentifier,
147 
+		is:       is,
148 
+		resolver: is.getResolver(ctx),
149 
+	}
150 
+	return p, nil
151 
+}
152 
+
153 
+type puller struct {
154 
+	is          *imageSource
155 
+	resolveOnce sync.Once
156 
+	src         *source.ImageIdentifier
157 
+	desc        ocispec.Descriptor
158 
+	ref         string
159 
+	resolveErr  error
160 
+	resolver    remotes.Resolver
161 
+	imageID     image.ID
162 
+	cacheKey    digest.Digest
163 
+}
164 
+
165 
+func (p *puller) resolve(ctx context.Context) error {
166 
+	p.resolveOnce.Do(func() {
167 
+		resolveProgressDone := oneOffProgress(ctx, "resolve "+p.src.Reference.String())
168 
+
169 
+		// dgst := p.src.Reference.Digest()
170 
+		// if dgst != "" {
171 
+		// 	info, err := p.is.ContentStore.Info(ctx, dgst)
172 
+		// 	if err == nil {
173 
+		// 		p.ref = p.src.Reference.String()
174 
+		// 		ra, err := p.is.ContentStore.ReaderAt(ctx, dgst)
175 
+		// 		if err == nil {
176 
+		// 			mt, err := imageutil.DetectManifestMediaType(ra)
177 
+		// 			if err == nil {
178 
+		// 				p.desc = ocispec.Descriptor{
179 
+		// 					Size:      info.Size,
180 
+		// 					Digest:    dgst,
181 
+		// 					MediaType: mt,
182 
+		// 				}
183 
+		// 				resolveProgressDone(nil)
184 
+		// 				return
185 
+		// 			}
186 
+		// 		}
187 
+		// 	}
188 
+		// }
189 
+
190 
+		// ref, desc, err := p.resolver.Resolve(ctx, p.src.Reference.String())
191 
+		// if err != nil {
192 
+		// 	p.resolveErr = err
193 
+		// 	resolveProgressDone(err)
194 
+		// 	return
195 
+		// }
196 
+
197 
+		if preferLocal {
198 
+			dt, err := p.is.resolveLocal(p.src.Reference.String())
199 
+			if err == nil {
200 
+				dgst := digest.FromBytes(dt)
201 
+				p.imageID = image.ID(dgst)
202 
+				p.cacheKey = dgst
203 
+				resolveProgressDone(nil)
204 
+				return
205 
+			}
206 
+
207 
+		}
208 
+
209 
+		ref, err := distreference.ParseNormalizedNamed(p.src.Reference.String())
210 
+		if err != nil {
211 
+			p.resolveErr = err
212 
+			resolveProgressDone(err)
213 
+			return
214 
+		}
215 
+
216 
+		outRef, desc, err := p.resolver.Resolve(ctx, p.src.Reference.String())
217 
+		if err != nil {
218 
+			p.resolveErr = err
219 
+			resolveProgressDone(err)
220 
+			return
221 
+		}
222 
+
223 
+		ref, err = distreference.WithDigest(ref, desc.Digest)
224 
+		if err != nil {
225 
+			p.resolveErr = err
226 
+			resolveProgressDone(err)
227 
+			return
228 
+		}
229 
+
230 
+		_, dt, err := p.is.ResolveImageConfig(ctx, ref.String())
231 
+		if err != nil {
232 
+			p.resolveErr = err
233 
+			resolveProgressDone(err)
234 
+			return
235 
+		}
236 
+		p.desc = desc
237 
+		p.cacheKey = digest.FromBytes(dt)
238 
+		p.ref = outRef
239 
+		resolveProgressDone(nil)
240 
+	})
241 
+	return p.resolveErr
242 
+}
243 
+
244 
+func (p *puller) CacheKey(ctx context.Context, index int) (string, bool, error) {
245 
+	if err := p.resolve(ctx); err != nil {
246 
+		return "", false, err
247 
+	}
248 
+	return p.cacheKey.String(), true, nil
249 
+}
250 
+
251 
+func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
252 
+	if err := p.resolve(ctx); err != nil {
253 
+		return nil, err
254 
+	}
255 
+
256 
+	if p.imageID != "" {
257 
+		img, err := p.is.ImageStore.Get(p.imageID)
258 
+		if err != nil {
259 
+			return nil, err
260 
+		}
261 
+		ref, err := p.is.CacheAccessor.Get(ctx, string(img.RootFS.ChainID()), cache.WithDescription(fmt.Sprintf("from local %s", p.ref)))
262 
+		if err != nil {
263 
+			return nil, err
264 
+		}
265 
+		return ref, nil
266 
+	}
267 
+
268 
+	ongoing := newJobs(p.ref)
269 
+
270 
+	pctx, stopProgress := context.WithCancel(ctx)
271 
+
272 
+	pw, _, ctx := progress.FromContext(ctx)
273 
+	defer pw.Close()
274 
+
275 
+	progressDone := make(chan struct{})
276 
+	go func() {
277 
+		showProgress(pctx, ongoing, p.is.ContentStore, pw)
278 
+		close(progressDone)
279 
+	}()
280 
+	defer func() {
281 
+		<-progressDone
282 
+	}()
283 
+
284 
+	fetcher, err := p.resolver.Fetcher(ctx, p.ref)
285 
+	if err != nil {
286 
+		stopProgress()
287 
+		return nil, err
288 
+	}
289 
+
290 
+	// TODO: need a wrapper snapshot interface that combines content
291 
+	// and snapshots as 1) buildkit shouldn't have a dependency on contentstore
292 
+	// or 2) cachemanager should manage the contentstore
293 
+	handlers := []images.Handler{
294 
+		images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
295 
+			switch desc.MediaType {
296 
+			case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest,
297 
+				images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex,
298 
+				images.MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig:
299 
+			default:
300 
+				return nil, images.ErrSkipDesc
301 
+			}
302 
+			ongoing.add(desc)
303 
+			return nil, nil
304 
+		}),
305 
+	}
306 
+	// var schema1Converter *schema1.Converter
307 
+	// if p.desc.MediaType == images.MediaTypeDockerSchema1Manifest {
308 
+	// 	schema1Converter = schema1.NewConverter(p.is.ContentStore, fetcher)
309 
+	// 	handlers = append(handlers, schema1Converter)
310 
+	// } else {
311 
+	// 	handlers = append(handlers,
312 
+	// 		remotes.FetchHandler(p.is.ContentStore, fetcher),
313 
+	//
314 
+	// 		images.ChildrenHandler(p.is.ContentStore),
315 
+	// 	)
316 
+	// }
317 
+	//
318 
+	var schema1Converter *schema1.Converter
319 
+	if p.desc.MediaType == images.MediaTypeDockerSchema1Manifest {
320 
+		schema1Converter = schema1.NewConverter(p.is.ContentStore, fetcher)
321 
+		handlers = append(handlers, schema1Converter)
322 
+	} else {
323 
+		// Get all the children for a descriptor
324 
+		childrenHandler := images.ChildrenHandler(p.is.ContentStore)
325 
+		// Set any children labels for that content
326 
+		childrenHandler = images.SetChildrenLabels(p.is.ContentStore, childrenHandler)
327 
+		// Filter the childen by the platform
328 
+		childrenHandler = images.FilterPlatforms(childrenHandler, platforms.Default())
329 
+
330 
+		handlers = append(handlers,
331 
+			remotes.FetchHandler(p.is.ContentStore, fetcher),
332 
+			childrenHandler,
333 
+		)
334 
+	}
335 
+
336 
+	if err := images.Dispatch(ctx, images.Handlers(handlers...), p.desc); err != nil {
337 
+		stopProgress()
338 
+		return nil, err
339 
+	}
340 
+	defer stopProgress()
341 
+
342 
+	mfst, err := images.Manifest(ctx, p.is.ContentStore, p.desc, platforms.Default())
343 
+	if err != nil {
344 
+		return nil, err
345 
+	}
346 
+
347 
+	config, err := images.Config(ctx, p.is.ContentStore, p.desc, platforms.Default())
348 
+	if err != nil {
349 
+		return nil, err
350 
+	}
351 
+
352 
+	dt, err := content.ReadBlob(ctx, p.is.ContentStore, config.Digest)
353 
+	if err != nil {
354 
+		return nil, err
355 
+	}
356 
+
357 
+	var img ocispec.Image
358 
+	if err := json.Unmarshal(dt, &img); err != nil {
359 
+		return nil, err
360 
+	}
361 
+
362 
+	if len(mfst.Layers) != len(img.RootFS.DiffIDs) {
363 
+		return nil, errors.Errorf("invalid config for manifest")
364 
+	}
365 
+
366 
+	pchan := make(chan pkgprogress.Progress, 10)
367 
+	defer close(pchan)
368 
+
369 
+	go func() {
370 
+		m := map[string]struct {
371 
+			st      time.Time
372 
+			limiter *rate.Limiter
373 
+		}{}
374 
+		for p := range pchan {
375 
+			if p.Action == "Extracting" {
376 
+				st, ok := m[p.ID]
377 
+				if !ok {
378 
+					st.st = time.Now()
379 
+					st.limiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 1)
380 
+					m[p.ID] = st
381 
+				}
382 
+				var end *time.Time
383 
+				if p.LastUpdate || st.limiter.Allow() {
384 
+					if p.LastUpdate {
385 
+						tm := time.Now()
386 
+						end = &tm
387 
+					}
388 
+					pw.Write("extracting "+p.ID, progress.Status{
389 
+						Action:    "extract",
390 
+						Started:   &st.st,
391 
+						Completed: end,
392 
+					})
393 
+				}
394 
+			}
395 
+		}
396 
+	}()
397 
+
398 
+	layers := make([]xfer.DownloadDescriptor, 0, len(mfst.Layers))
399 
+
400 
+	for i, desc := range mfst.Layers {
401 
+		ongoing.add(desc)
402 
+		layers = append(layers, &layerDescriptor{
403 
+			desc:    desc,
404 
+			diffID:  layer.DiffID(img.RootFS.DiffIDs[i]),
405 
+			fetcher: fetcher,
406 
+			ref:     p.src.Reference,
407 
+			is:      p.is,
408 
+		})
409 
+	}
410 
+
411 
+	defer func() {
412 
+		<-progressDone
413 
+		for _, desc := range mfst.Layers {
414 
+			p.is.ContentStore.Delete(context.TODO(), desc.Digest)
415 
+		}
416 
+	}()
417 
+
418 
+	r := image.NewRootFS()
419 
+	rootFS, release, err := p.is.DownloadManager.Download(ctx, *r, runtime.GOOS, layers, pkgprogress.ChanOutput(pchan))
420 
+	if err != nil {
421 
+		return nil, err
422 
+	}
423 
+	stopProgress()
424 
+
425 
+	ref, err := p.is.CacheAccessor.Get(ctx, string(rootFS.ChainID()), cache.WithDescription(fmt.Sprintf("pulled from %s", p.ref)))
426 
+	release()
427 
+	if err != nil {
428 
+		return nil, err
429 
+	}
430 
+
431 
+	return ref, nil
432 
+}
433 
+
434 
+// Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error)
435 
+type layerDescriptor struct {
436 
+	is      *imageSource
437 
+	fetcher remotes.Fetcher
438 
+	desc    ocispec.Descriptor
439 
+	diffID  layer.DiffID
440 
+	ref     ctdreference.Spec
441 
+}
442 
+
443 
+func (ld *layerDescriptor) Key() string {
444 
+	return "v2:" + ld.desc.Digest.String()
445 
+}
446 
+
447 
+func (ld *layerDescriptor) ID() string {
448 
+	return ld.desc.Digest.String()
449 
+}
450 
+
451 
+func (ld *layerDescriptor) DiffID() (layer.DiffID, error) {
452 
+	return ld.diffID, nil
453 
+}
454 
+
455 
+func (ld *layerDescriptor) Download(ctx netcontext.Context, progressOutput pkgprogress.Output) (io.ReadCloser, int64, error) {
456 
+	rc, err := ld.fetcher.Fetch(ctx, ld.desc)
457 
+	if err != nil {
458 
+		return nil, 0, err
459 
+	}
460 
+	defer rc.Close()
461 
+
462 
+	refKey := remotes.MakeRefKey(ctx, ld.desc)
463 
+
464 
+	if err := content.WriteBlob(ctx, ld.is.ContentStore, refKey, rc, ld.desc.Size, ld.desc.Digest); err != nil {
465 
+		return nil, 0, err
466 
+	}
467 
+
468 
+	ra, err := ld.is.ContentStore.ReaderAt(ctx, ld.desc.Digest)
469 
+	if err != nil {
470 
+		return nil, 0, err
471 
+	}
472 
+
473 
+	return ioutil.NopCloser(content.NewReader(ra)), ld.desc.Size, nil
474 
+}
475 
+
476 
+func (ld *layerDescriptor) Close() {
477 
+	// ld.is.ContentStore.Delete(context.TODO(), ld.desc.Digest))
478 
+}
479 
+
480 
+func (ld *layerDescriptor) Registered(diffID layer.DiffID) {
481 
+	// Cache mapping from this layer's DiffID to the blobsum
482 
+	ld.is.MetadataStore.Add(diffID, metadata.V2Metadata{Digest: ld.desc.Digest, SourceRepository: ld.ref.Locator})
483 
+}
484 
+
485 
+func showProgress(ctx context.Context, ongoing *jobs, cs content.Store, pw progress.Writer) {
486 
+	var (
487 
+		ticker   = time.NewTicker(100 * time.Millisecond)
488 
+		statuses = map[string]statusInfo{}
489 
+		done     bool
490 
+	)
491 
+	defer ticker.Stop()
492 
+
493 
+	for {
494 
+		select {
495 
+		case <-ticker.C:
496 
+		case <-ctx.Done():
497 
+			done = true
498 
+		}
499 
+
500 
+		resolved := "resolved"
501 
+		if !ongoing.isResolved() {
502 
+			resolved = "resolving"
503 
+		}
504 
+		statuses[ongoing.name] = statusInfo{
505 
+			Ref:    ongoing.name,
506 
+			Status: resolved,
507 
+		}
508 
+
509 
+		actives := make(map[string]statusInfo)
510 
+
511 
+		if !done {
512 
+			active, err := cs.ListStatuses(ctx)
513 
+			if err != nil {
514 
+				// log.G(ctx).WithError(err).Error("active check failed")
515 
+				continue
516 
+			}
517 
+			// update status of active entries!
518 
+			for _, active := range active {
519 
+				actives[active.Ref] = statusInfo{
520 
+					Ref:       active.Ref,
521 
+					Status:    "downloading",
522 
+					Offset:    active.Offset,
523 
+					Total:     active.Total,
524 
+					StartedAt: active.StartedAt,
525 
+					UpdatedAt: active.UpdatedAt,
526 
+				}
527 
+			}
528 
+		}
529 
+
530 
+		// now, update the items in jobs that are not in active
531 
+		for _, j := range ongoing.jobs() {
532 
+			refKey := remotes.MakeRefKey(ctx, j.Descriptor)
533 
+			if a, ok := actives[refKey]; ok {
534 
+				started := j.started
535 
+				pw.Write(j.Digest.String(), progress.Status{
536 
+					Action:  a.Status,
537 
+					Total:   int(a.Total),
538 
+					Current: int(a.Offset),
539 
+					Started: &started,
540 
+				})
541 
+				continue
542 
+			}
543 
+
544 
+			if !j.done {
545 
+				info, err := cs.Info(context.TODO(), j.Digest)
546 
+				if err != nil {
547 
+					if errdefs.IsNotFound(err) {
548 
+						// pw.Write(j.Digest.String(), progress.Status{
549 
+						// 	Action: "waiting",
550 
+						// })
551 
+						continue
552 
+					}
553 
+				} else {
554 
+					j.done = true
555 
+				}
556 
+
557 
+				if done || j.done {
558 
+					started := j.started
559 
+					createdAt := info.CreatedAt
560 
+					pw.Write(j.Digest.String(), progress.Status{
561 
+						Action:    "done",
562 
+						Current:   int(info.Size),
563 
+						Total:     int(info.Size),
564 
+						Completed: &createdAt,
565 
+						Started:   &started,
566 
+					})
567 
+				}
568 
+			}
569 
+		}
570 
+		if done {
571 
+			return
572 
+		}
573 
+	}
574 
+}
575 
+
576 
+// jobs provides a way of identifying the download keys for a particular task
577 
+// encountering during the pull walk.
578 
+//
579 
+// This is very minimal and will probably be replaced with something more
580 
+// featured.
581 
+type jobs struct {
582 
+	name     string
583 
+	added    map[digest.Digest]job
584 
+	mu       sync.Mutex
585 
+	resolved bool
586 
+}
587 
+
588 
+type job struct {
589 
+	ocispec.Descriptor
590 
+	done    bool
591 
+	started time.Time
592 
+}
593 
+
594 
+func newJobs(name string) *jobs {
595 
+	return &jobs{
596 
+		name:  name,
597 
+		added: make(map[digest.Digest]job),
598 
+	}
599 
+}
600 
+
601 
+func (j *jobs) add(desc ocispec.Descriptor) {
602 
+	j.mu.Lock()
603 
+	defer j.mu.Unlock()
604 
+
605 
+	if _, ok := j.added[desc.Digest]; ok {
606 
+		return
607 
+	}
608 
+	j.added[desc.Digest] = job{
609 
+		Descriptor: desc,
610 
+		started:    time.Now(),
611 
+	}
612 
+}
613 
+
614 
+func (j *jobs) jobs() []job {
615 
+	j.mu.Lock()
616 
+	defer j.mu.Unlock()
617 
+
618 
+	descs := make([]job, 0, len(j.added))
619 
+	for _, j := range j.added {
620 
+		descs = append(descs, j)
621 
+	}
622 
+	return descs
623 
+}
624 
+
625 
+func (j *jobs) isResolved() bool {
626 
+	j.mu.Lock()
627 
+	defer j.mu.Unlock()
628 
+	return j.resolved
629 
+}
630 
+
631 
+type statusInfo struct {
632 
+	Ref       string
633 
+	Status    string
634 
+	Offset    int64
635 
+	Total     int64
636 
+	StartedAt time.Time
637 
+	UpdatedAt time.Time
638 
+}
639 
+
640 
+func oneOffProgress(ctx context.Context, id string) func(err error) error {
641 
+	pw, _, _ := progress.FromContext(ctx)
642 
+	now := time.Now()
643 
+	st := progress.Status{
644 
+		Started: &now,
645 
+	}
646 
+	pw.Write(id, st)
647 
+	return func(err error) error {
648 
+		// TODO: set error on status
649 
+		now := time.Now()
650 
+		st.Completed = &now
651 
+		pw.Write(id, st)
652 
+		pw.Close()
653 
+		return err
654 
+	}
655 
+}
builder/builder-next/adapters/snapshot/layer.go
History View file @ b742459
0 656 
new file mode 100644
... ... 
@@ -0,0 +1,116 @@
0 
+package snapshot
1 
+
2 
+import (
3 
+	"context"
4 
+	"os"
5 
+	"path/filepath"
6 
+
7 
+	"github.com/boltdb/bolt"
8 
+	"github.com/docker/docker/layer"
9 
+	"github.com/docker/docker/pkg/ioutils"
10 
+	"github.com/pkg/errors"
11 
+	"golang.org/x/sync/errgroup"
12 
+)
13 
+
14 
+func (s *snapshotter) EnsureLayer(ctx context.Context, key string) ([]layer.DiffID, error) {
15 
+	if l, err := s.getLayer(key); err != nil {
16 
+		return nil, err
17 
+	} else if l != nil {
18 
+		return getDiffChain(l), nil
19 
+	}
20 
+
21 
+	id, committed := s.getGraphDriverID(key)
22 
+	if !committed {
23 
+		return nil, errors.Errorf("can not convert active %s to layer", key)
24 
+	}
25 
+
26 
+	info, err := s.Stat(ctx, key)
27 
+	if err != nil {
28 
+		return nil, err
29 
+	}
30 
+
31 
+	eg, gctx := errgroup.WithContext(ctx)
32 
+
33 
+	// TODO: add flightcontrol
34 
+
35 
+	var parentChainID layer.ChainID
36 
+	if info.Parent != "" {
37 
+		eg.Go(func() error {
38 
+			diffIDs, err := s.EnsureLayer(gctx, info.Parent)
39 
+			if err != nil {
40 
+				return err
41 
+			}
42 
+			parentChainID = layer.CreateChainID(diffIDs)
43 
+			return nil
44 
+		})
45 
+	}
46 
+
47 
+	tmpDir, err := ioutils.TempDir("", "docker-tarsplit")
48 
+	if err != nil {
49 
+		return nil, err
50 
+	}
51 
+	defer os.RemoveAll(tmpDir)
52 
+	tarSplitPath := filepath.Join(tmpDir, "tar-split")
53 
+
54 
+	var diffID layer.DiffID
55 
+	var size int64
56 
+	eg.Go(func() error {
57 
+		parent := ""
58 
+		if p := info.Parent; p != "" {
59 
+			if l, err := s.getLayer(p); err != nil {
60 
+				return err
61 
+			} else if l != nil {
62 
+				parent, err = getGraphID(l)
63 
+				if err != nil {
64 
+					return err
65 
+				}
66 
+			} else {
67 
+				parent, _ = s.getGraphDriverID(info.Parent)
68 
+			}
69 
+		}
70 
+		diffID, size, err = s.reg.ChecksumForGraphID(id, parent, "", tarSplitPath)
71 
+		if err != nil {
72 
+			return err
73 
+		}
74 
+		return nil
75 
+	})
76 
+
77 
+	if err := eg.Wait(); err != nil {
78 
+		return nil, err
79 
+	}
80 
+
81 
+	l, err := s.reg.RegisterByGraphID(id, parentChainID, diffID, tarSplitPath, size)
82 
+	if err != nil {
83 
+		return nil, err
84 
+	}
85 
+
86 
+	if err := s.db.Update(func(tx *bolt.Tx) error {
87 
+		b := tx.Bucket([]byte(key))
88 
+		b.Put(keyChainID, []byte(l.ChainID()))
89 
+		return nil
90 
+	}); err != nil {
91 
+		return nil, err
92 
+	}
93 
+
94 
+	s.mu.Lock()
95 
+	s.refs[key] = l
96 
+	s.mu.Unlock()
97 
+
98 
+	return getDiffChain(l), nil
99 
+}
100 
+
101 
+func getDiffChain(l layer.Layer) []layer.DiffID {
102 
+	if p := l.Parent(); p != nil {
103 
+		return append(getDiffChain(p), l.DiffID())
104 
+	}
105 
+	return []layer.DiffID{l.DiffID()}
106 
+}
107 
+
108 
+func getGraphID(l layer.Layer) (string, error) {
109 
+	if l, ok := l.(interface {
110 
+		CacheID() string
111 
+	}); ok {
112 
+		return l.CacheID(), nil
113 
+	}
114 
+	return "", errors.Errorf("couldn't access cacheID for %s", l.ChainID())
115 
+}
builder/builder-next/adapters/snapshot/snapshot.go
History View file @ b742459
0 116 
new file mode 100644
... ... 
@@ -0,0 +1,355 @@
0 
+package snapshot
1 
+
2 
+import (
3 
+	"context"
4 
+	"path/filepath"
5 
+	"strings"
6 
+	"sync"
7 
+
8 
+	"github.com/boltdb/bolt"
9 
+	"github.com/containerd/containerd/mount"
10 
+	"github.com/containerd/containerd/snapshots"
11 
+	"github.com/docker/docker/daemon/graphdriver"
12 
+	"github.com/docker/docker/layer"
13 
+	"github.com/moby/buildkit/identity"
14 
+	"github.com/moby/buildkit/snapshot"
15 
+	digest "github.com/opencontainers/go-digest"
16 
+	"github.com/pkg/errors"
17 
+)
18 
+
19 
+var keyParent = []byte("parent")
20 
+var keyCommitted = []byte("committed")
21 
+var keyChainID = []byte("chainid")
22 
+var keySize = []byte("size")
23 
+
24 
+type Opt struct {
25 
+	GraphDriver graphdriver.Driver
26 
+	LayerStore  layer.Store
27 
+	Root        string
28 
+}
29 
+
30 
+type graphIDRegistrar interface {
31 
+	RegisterByGraphID(string, layer.ChainID, layer.DiffID, string, int64) (layer.Layer, error)
32 
+	Release(layer.Layer) ([]layer.Metadata, error)
33 
+	checksumCalculator
34 
+}
35 
+
36 
+type checksumCalculator interface {
37 
+	ChecksumForGraphID(id, parent, oldTarDataPath, newTarDataPath string) (diffID layer.DiffID, size int64, err error)
38 
+}
39 
+
40 
+type snapshotter struct {
41 
+	opt Opt
42 
+
43 
+	refs map[string]layer.Layer
44 
+	db   *bolt.DB
45 
+	mu   sync.Mutex
46 
+	reg  graphIDRegistrar
47 
+}
48 
+
49 
+var _ snapshot.SnapshotterBase = &snapshotter{}
50 
+
51 
+func NewSnapshotter(opt Opt) (snapshot.SnapshotterBase, error) {
52 
+	dbPath := filepath.Join(opt.Root, "snapshots.db")
53 
+	db, err := bolt.Open(dbPath, 0600, nil)
54 
+	if err != nil {
55 
+		return nil, errors.Wrapf(err, "failed to open database file %s", dbPath)
56 
+	}
57 
+
58 
+	reg, ok := opt.LayerStore.(graphIDRegistrar)
59 
+	if !ok {
60 
+		return nil, errors.Errorf("layerstore doesn't support graphID registration")
61 
+	}
62 
+
63 
+	s := &snapshotter{
64 
+		opt:  opt,
65 
+		db:   db,
66 
+		refs: map[string]layer.Layer{},
67 
+		reg:  reg,
68 
+	}
69 
+	return s, nil
70 
+}
71 
+
72 
+func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) error {
73 
+	origParent := parent
74 
+	if parent != "" {
75 
+		if l, err := s.getLayer(parent); err != nil {
76 
+			return err
77 
+		} else if l != nil {
78 
+			parent, err = getGraphID(l)
79 
+			if err != nil {
80 
+				return err
81 
+			}
82 
+		} else {
83 
+			parent, _ = s.getGraphDriverID(parent)
84 
+		}
85 
+	}
86 
+	if err := s.opt.GraphDriver.Create(key, parent, nil); err != nil {
87 
+		return err
88 
+	}
89 
+	if err := s.db.Update(func(tx *bolt.Tx) error {
90 
+		b, err := tx.CreateBucketIfNotExists([]byte(key))
91 
+		if err != nil {
92 
+			return err
93 
+		}
94 
+
95 
+		if err := b.Put(keyParent, []byte(origParent)); err != nil {
96 
+			return err
97 
+		}
98 
+		return nil
99 
+	}); err != nil {
100 
+		return err
101 
+	}
102 
+	return nil
103 
+}
104 
+
105 
+func (s *snapshotter) chainID(key string) (layer.ChainID, bool) {
106 
+	if strings.HasPrefix(key, "sha256:") {
107 
+		dgst, err := digest.Parse(key)
108 
+		if err != nil {
109 
+			return "", false
110 
+		}
111 
+		return layer.ChainID(dgst), true
112 
+	}
113 
+	return "", false
114 
+}
115 
+
116 
+func (s *snapshotter) getLayer(key string) (layer.Layer, error) {
117 
+	s.mu.Lock()
118 
+	l, ok := s.refs[key]
119 
+	if !ok {
120 
+		id, ok := s.chainID(key)
121 
+		if !ok {
122 
+			if err := s.db.View(func(tx *bolt.Tx) error {
123 
+				b := tx.Bucket([]byte(key))
124 
+				if b == nil {
125 
+					return nil
126 
+				}
127 
+				v := b.Get(keyChainID)
128 
+				if v != nil {
129 
+					id = layer.ChainID(v)
130 
+				}
131 
+				return nil
132 
+			}); err != nil {
133 
+				s.mu.Unlock()
134 
+				return nil, err
135 
+			}
136 
+			if id == "" {
137 
+				s.mu.Unlock()
138 
+				return nil, nil
139 
+			}
140 
+		}
141 
+		var err error
142 
+		l, err = s.opt.LayerStore.Get(id)
143 
+		if err != nil {
144 
+			s.mu.Unlock()
145 
+			return nil, err
146 
+		}
147 
+		s.refs[string(id)] = l
148 
+		if err := s.db.Update(func(tx *bolt.Tx) error {
149 
+			_, err := tx.CreateBucketIfNotExists([]byte(key))
150 
+			return err
151 
+		}); err != nil {
152 
+			s.mu.Unlock()
153 
+			return nil, err
154 
+		}
155 
+	}
156 
+	s.mu.Unlock()
157 
+
158 
+	return l, nil
159 
+}
160 
+
161 
+func (s *snapshotter) getGraphDriverID(key string) (string, bool) {
162 
+	var gdID string
163 
+	if err := s.db.View(func(tx *bolt.Tx) error {
164 
+		b := tx.Bucket([]byte(key))
165 
+		if b == nil {
166 
+			return errors.Errorf("not found") // TODO: typed
167 
+		}
168 
+		v := b.Get(keyCommitted)
169 
+		if v != nil {
170 
+			gdID = string(v)
171 
+		}
172 
+		return nil
173 
+	}); err != nil || gdID == "" {
174 
+		return key, false
175 
+	}
176 
+	return gdID, true
177 
+}
178 
+
179 
+func (s *snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, error) {
180 
+	if l, err := s.getLayer(key); err != nil {
181 
+		return snapshots.Info{}, err
182 
+	} else if l != nil {
183 
+		var parentID string
184 
+		if p := l.Parent(); p != nil {
185 
+			parentID = p.ChainID().String()
186 
+		}
187 
+		info := snapshots.Info{
188 
+			Kind:   snapshots.KindCommitted,
189 
+			Name:   key,
190 
+			Parent: parentID,
191 
+		}
192 
+		return info, nil
193 
+	}
194 
+
195 
+	inf := snapshots.Info{
196 
+		Kind: snapshots.KindActive,
197 
+	}
198 
+
199 
+	id, committed := s.getGraphDriverID(key)
200 
+	if committed {
201 
+		inf.Kind = snapshots.KindCommitted
202 
+	}
203 
+
204 
+	if err := s.db.View(func(tx *bolt.Tx) error {
205 
+		b := tx.Bucket([]byte(id))
206 
+		if b == nil {
207 
+			return errors.Errorf("not found") // TODO: typed
208 
+		}
209 
+		inf.Name = string(key)
210 
+		v := b.Get(keyParent)
211 
+		if v != nil {
212 
+			inf.Parent = string(v)
213 
+		}
214 
+		return nil
215 
+	}); err != nil {
216 
+		return snapshots.Info{}, err
217 
+	}
218 
+	return inf, nil
219 
+}
220 
+
221 
+func (s *snapshotter) Mounts(ctx context.Context, key string) (snapshot.Mountable, error) {
222 
+	l, err := s.getLayer(key)
223 
+	if err != nil {
224 
+		return nil, err
225 
+	}
226 
+	if l != nil {
227 
+		id := identity.NewID()
228 
+		rwlayer, err := s.opt.LayerStore.CreateRWLayer(id, l.ChainID(), nil)
229 
+		if err != nil {
230 
+			return nil, err
231 
+		}
232 
+		rootfs, err := rwlayer.Mount("")
233 
+		if err != nil {
234 
+			return nil, err
235 
+		}
236 
+		mnt := []mount.Mount{{
237 
+			Source:  rootfs.Path(),
238 
+			Type:    "bind",
239 
+			Options: []string{"rbind"},
240 
+		}}
241 
+		return &constMountable{
242 
+			mounts: mnt,
243 
+			release: func() error {
244 
+				_, err := s.opt.LayerStore.ReleaseRWLayer(rwlayer)
245 
+				return err
246 
+			},
247 
+		}, nil
248 
+	}
249 
+
250 
+	id, _ := s.getGraphDriverID(key)
251 
+
252 
+	rootfs, err := s.opt.GraphDriver.Get(id, "")
253 
+	if err != nil {
254 
+		return nil, err
255 
+	}
256 
+	mnt := []mount.Mount{{
257 
+		Source:  rootfs.Path(),
258 
+		Type:    "bind",
259 
+		Options: []string{"rbind"},
260 
+	}}
261 
+	return &constMountable{
262 
+		mounts: mnt,
263 
+		release: func() error {
264 
+			return s.opt.GraphDriver.Put(id)
265 
+		},
266 
+	}, nil
267 
+}
268 
+
269 
+func (s *snapshotter) Remove(ctx context.Context, key string) error {
270 
+	l, err := s.getLayer(key)
271 
+	if err != nil {
272 
+		return err
273 
+	}
274 
+
275 
+	var found bool
276 
+	if err := s.db.Update(func(tx *bolt.Tx) error {
277 
+		found = tx.Bucket([]byte(key)) != nil
278 
+		if found {
279 
+			id, _ := s.getGraphDriverID(key)
280 
+			tx.DeleteBucket([]byte(key))
281 
+			if id != key {
282 
+				tx.DeleteBucket([]byte(id))
283 
+			}
284 
+		}
285 
+		return nil
286 
+	}); err != nil {
287 
+		return err
288 
+	}
289 
+
290 
+	if l != nil {
291 
+		_, err := s.opt.LayerStore.Release(l)
292 
+		return err
293 
+	}
294 
+
295 
+	if !found { // this happens when removing views
296 
+		return nil
297 
+	}
298 
+
299 
+	id, _ := s.getGraphDriverID(key)
300 
+	return s.opt.GraphDriver.Remove(id)
301 
+}
302 
+
303 
+func (s *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) error {
304 
+	if err := s.db.Update(func(tx *bolt.Tx) error {
305 
+		b, err := tx.CreateBucketIfNotExists([]byte(name))
306 
+		if err != nil {
307 
+			return err
308 
+		}
309 
+		if err := b.Put(keyCommitted, []byte(key)); err != nil {
310 
+			return err
311 
+		}
312 
+		return nil
313 
+	}); err != nil {
314 
+		return err
315 
+	}
316 
+	return nil
317 
+}
318 
+
319 
+func (s *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) (snapshot.Mountable, error) {
320 
+	return s.Mounts(ctx, parent)
321 
+}
322 
+
323 
+func (s *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error) error {
324 
+	return errors.Errorf("not-implemented")
325 
+}
326 
+
327 
+func (s *snapshotter) Update(ctx context.Context, info snapshots.Info, fieldpaths ...string) (snapshots.Info, error) {
328 
+	// not implemented
329 
+	return s.Stat(ctx, info.Name)
330 
+}
331 
+
332 
+func (s *snapshotter) Usage(ctx context.Context, key string) (snapshots.Usage, error) {
333 
+	return snapshots.Usage{}, nil
334 
+}
335 
+
336 
+func (s *snapshotter) Close() error {
337 
+	return s.db.Close()
338 
+}
339 
+
340 
+type constMountable struct {
341 
+	mounts  []mount.Mount
342 
+	release func() error
343 
+}
344 
+
345 
+func (m *constMountable) Mount() ([]mount.Mount, error) {
346 
+	return m.mounts, nil
347 
+}
348 
+
349 
+func (m *constMountable) Release() error {
350 
+	if m.release == nil {
351 
+		return nil
352 
+	}
353 
+	return m.release()
354 
+}
builder/builder-next/builder.go
History View file @ b742459
... ... 
@@ -4,53 +4,24 @@ import (
4 4 
 	"context"
5 5 
 	"encoding/json"
6 6 
 	"io"
7 
-	"os"
8 
-	"path/filepath"
7 
+	"strings"
9 8 
 	"sync"
10 9
11 10 
 	"github.com/containerd/containerd/content"
12 
-	"github.com/containerd/containerd/content/local"
13 11 
 	"github.com/docker/docker/api/types/backend"
14 12 
 	"github.com/docker/docker/builder"
15 
-	"github.com/docker/docker/builder/builder-next/containerimage"
16 
-	containerimageexp "github.com/docker/docker/builder/builder-next/exporter"
17 
-	"github.com/docker/docker/builder/builder-next/snapshot"
18 
-	mobyworker "github.com/docker/docker/builder/builder-next/worker"
19 
-	"github.com/docker/docker/daemon/graphdriver"
20 13 
 	"github.com/docker/docker/daemon/images"
21 14 
 	"github.com/docker/docker/pkg/jsonmessage"
22 15 
 	controlapi "github.com/moby/buildkit/api/services/control"
23 
-	"github.com/moby/buildkit/cache"
24 
-	"github.com/moby/buildkit/cache/cacheimport"
25 
-	"github.com/moby/buildkit/cache/metadata"
26 16 
 	"github.com/moby/buildkit/control"
27 
-	"github.com/moby/buildkit/executor/runcexecutor"
28 
-	"github.com/moby/buildkit/exporter"
29 
-	"github.com/moby/buildkit/frontend"
30 
-	"github.com/moby/buildkit/frontend/dockerfile"
31 17 
 	"github.com/moby/buildkit/identity"
32 18 
 	"github.com/moby/buildkit/session"
33 
-	"github.com/moby/buildkit/snapshot/blobmapping"
34 
-	"github.com/moby/buildkit/solver-next/boltdbcachestorage"
35 
-	"github.com/moby/buildkit/worker"
36 19 
 	"github.com/pkg/errors"
37 
-	"github.com/sirupsen/logrus"
38 20 
 	netcontext "golang.org/x/net/context"
39 21 
 	"golang.org/x/sync/errgroup"
40 22 
 	grpcmetadata "google.golang.org/grpc/metadata"
41 23 
 )
42 24
43 
-// Builder defines interface for running a build
44 
-// type Builder interface {
45 
-// 	Build(context.Context, backend.BuildConfig) (*builder.Result, error)
46 
-// }
47 
-
48 
-// Result is the output produced by a Builder
49 
-// type Result struct {
50 
-// 	ImageID string
51 
-// 	// FromImage Image
52 
-// }
53 
-
54 25 
 type Opt struct {
55 26 
 	SessionManager *session.Manager
56 27 
 	Root           string
... ... 
@@ -59,22 +30,18 @@ type Opt struct {
59 59
60 60 
 type Builder struct {
61 61 
 	controller *control.Controller
62 
-	results    *results
63 62
64 63 
 	mu   sync.Mutex
65 64 
 	jobs map[string]func()
66 65 
 }
67 66
68 67 
 func New(opt Opt) (*Builder, error) {
69 
-	results := newResultsGetter()
70 
-
71 
-	c, err := newController(opt, results.ch)
68 
+	c, err := newController(opt)
72 69 
 	if err != nil {
73 70 
 		return nil, err
74 71 
 	}
75 72 
 	b := &Builder{
76 73 
 		controller: c,
77 
-		results:    results,
78 74 
 		jobs:       map[string]func(){},
79 75 
 	}
80 76 
 	return b, nil
... ... 
@@ -90,21 +57,10 @@ func (b *Builder) Cancel(ctx context.Context, id string) error {
90 90 
 }
91 91
92 92 
 func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.Result, error) {
93 
-	if buildID := opt.Options.BuildID; buildID != "" {
94 
-		b.mu.Lock()
95 
-		ctx, b.jobs[buildID] = context.WithCancel(ctx)
96 
-		b.mu.Unlock()
97 
-		defer func() {
98 
-			delete(b.jobs, buildID)
99 
-		}()
100 
-	}
93 
+	var out builder.Result
101 94
102 95 
 	id := identity.NewID()
103 96
104 
-	attrs := map[string]string{
105 
-		"ref": id,
106 
-	}
107 
-
108 97 
 	frontendAttrs := map[string]string{}
109 98
110 99 
 	if opt.Options.Target != "" {
... ... 
@@ -119,11 +75,11 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
119 119 
 		frontendAttrs["context"] = opt.Options.RemoteContext
120 120 
 	}
121 121
122 
-	if len(opt.Options.CacheFrom) > 0 {
123 
-		frontendAttrs["cache-from"] = opt.Options.CacheFrom[0]
122 
+	var cacheFrom []string
123 
+	for _, v := range opt.Options.CacheFrom {
124 
+		cacheFrom = append(cacheFrom, v)
124 125 
 	}
125 
-
126 
-	logrus.Debugf("frontend: %+v", frontendAttrs)
126 
+	frontendAttrs["cache-from"] = strings.Join(cacheFrom, ",")
127 127
128 128 
 	for k, v := range opt.Options.BuildArgs {
129 129 
 		if v == nil {
... ... 
@@ -132,10 +88,17 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
132 132 
 		frontendAttrs["build-arg:"+k] = *v
133 133 
 	}
134 134
135 
+	for k, v := range opt.Options.Labels {
136 
+		frontendAttrs["label:"+k] = v
137 
+	}
138 
+
139 
+	if opt.Options.NoCache {
140 
+		frontendAttrs["no-cache"] = ""
141 
+	}
142 
+
135 143 
 	req := &controlapi.SolveRequest{
136 144 
 		Ref:           id,
137 
-		Exporter:      "image",
138 
-		ExporterAttrs: attrs,
145 
+		Exporter:      "moby",
139 146 
 		Frontend:      "dockerfile.v0",
140 147 
 		FrontendAttrs: frontendAttrs,
141 148 
 		Session:       opt.Options.SessionID,
... ... 
@@ -144,8 +107,16 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
144 144 
 	eg, ctx := errgroup.WithContext(ctx)
145 145
146 146 
 	eg.Go(func() error {
147 
-		_, err := b.controller.Solve(ctx, req)
148 
-		return err
147 
+		resp, err := b.controller.Solve(ctx, req)
148 
+		if err != nil {
149 
+			return err
150 
+		}
151 
+		id, ok := resp.ExporterResponse["containerimage.digest"]
152 
+		if !ok {
153 
+			return errors.Errorf("missing image id")
154 
+		}
155 
+		out.ImageID = id
156 
+		return nil
149 157 
 	})
150 158
151 159 
 	ch := make(chan *controlapi.StatusResponse)
... ... 
@@ -186,173 +157,11 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
186 186 
 		return nil
187 187 
 	})
188 188
189 
-	out := &builder.Result{}
190 
-	eg.Go(func() error {
191 
-		res, err := b.results.wait(ctx, id)
192 
-		if err != nil {
193 
-			return err
194 
-		}
195 
-		out.ImageID = string(res.ID)
196 
-		return nil
197 
-	})
198 
-
199 189 
 	if err := eg.Wait(); err != nil {
200 190 
 		return nil, err
201 191 
 	}
202 192
203 
-	return out, nil
204 
-}
205 
-
206 
-func newController(opt Opt, reporter chan containerimageexp.Result) (*control.Controller, error) {
207 
-	if err := os.MkdirAll(opt.Root, 0700); err != nil {
208 
-		return nil, err
209 
-	}
210 
-
211 
-	dist := opt.Dist
212 
-	root := opt.Root
213 
-
214 
-	var driver graphdriver.Driver
215 
-	if ls, ok := dist.LayerStore.(interface {
216 
-		Driver() graphdriver.Driver
217 
-	}); ok {
218 
-		driver = ls.Driver()
219 
-	} else {
220 
-		return nil, errors.Errorf("could not access graphdriver")
221 
-	}
222 
-
223 
-	sbase, err := snapshot.NewSnapshotter(snapshot.Opt{
224 
-		GraphDriver: driver,
225 
-		LayerStore:  dist.LayerStore,
226 
-		Root:        root,
227 
-	})
228 
-	if err != nil {
229 
-		return nil, err
230 
-	}
231 
-
232 
-	store, err := local.NewStore(filepath.Join(root, "content"))
233 
-	if err != nil {
234 
-		return nil, err
235 
-	}
236 
-	store = &contentStoreNoLabels{store}
237 
-
238 
-	md, err := metadata.NewStore(filepath.Join(root, "metadata.db"))
239 
-	if err != nil {
240 
-		return nil, err
241 
-	}
242 
-
243 
-	snapshotter := blobmapping.NewSnapshotter(blobmapping.Opt{
244 
-		Content:       store,
245 
-		Snapshotter:   sbase,
246 
-		MetadataStore: md,
247 
-	})
248 
-
249 
-	cm, err := cache.NewManager(cache.ManagerOpt{
250 
-		Snapshotter:   snapshotter,
251 
-		MetadataStore: md,
252 
-	})
253 
-	if err != nil {
254 
-		return nil, err
255 
-	}
256 
-
257 
-	src, err := containerimage.NewSource(containerimage.SourceOpt{
258 
-		SessionManager:  opt.SessionManager,
259 
-		CacheAccessor:   cm,
260 
-		ContentStore:    store,
261 
-		DownloadManager: dist.DownloadManager,
262 
-		MetadataStore:   dist.V2MetadataService,
263 
-		ImageStore:      dist.ImageStore,
264 
-		ReferenceStore:  dist.ReferenceStore,
265 
-	})
266 
-	if err != nil {
267 
-		return nil, err
268 
-	}
269 
-
270 
-	exec, err := runcexecutor.New(runcexecutor.Opt{
271 
-		Root:              filepath.Join(root, "executor"),
272 
-		CommandCandidates: []string{"docker-runc", "runc"},
273 
-	})
274 
-	if err != nil {
275 
-		return nil, err
276 
-	}
277 
-
278 
-	differ, ok := sbase.(containerimageexp.Differ)
279 
-	if !ok {
280 
-		return nil, errors.Errorf("snapshotter doesn't support differ")
281 
-	}
282 
-
283 
-	exp, err := containerimageexp.New(containerimageexp.Opt{
284 
-		ImageStore:     dist.ImageStore,
285 
-		ReferenceStore: dist.ReferenceStore,
286 
-		Differ:         differ,
287 
-		Reporter:       reporter,
288 
-	})
289 
-	if err != nil {
290 
-		return nil, err
291 
-	}
292 
-
293 
-	cacheStorage, err := boltdbcachestorage.NewStore(filepath.Join(opt.Root, "cache.db"))
294 
-	if err != nil {
295 
-		return nil, err
296 
-	}
297 
-
298 
-	frontends := map[string]frontend.Frontend{}
299 
-	frontends["dockerfile.v0"] = dockerfile.NewDockerfileFrontend()
300 
-	// frontends["gateway.v0"] = gateway.NewGatewayFrontend()
301 
-
302 
-	// mdb := ctdmetadata.NewDB(db, c, map[string]ctdsnapshot.Snapshotter{
303 
-	// 	"moby": s,
304 
-	// })
305 
-	// if err := mdb.Init(context.TODO()); err != nil {
306 
-	// 	return opt, err
307 
-	// }
308 
-	//
309 
-	// throttledGC := throttle.Throttle(time.Second, func() {
310 
-	// 	if _, err := mdb.GarbageCollect(context.TODO()); err != nil {
311 
-	// 		logrus.Errorf("GC error: %+v", err)
312 
-	// 	}
313 
-	// })
314 
-	//
315 
-	// gc := func(ctx context.Context) error {
316 
-	// 	throttledGC()
317 
-	// 	return nil
318 
-	// }
319 
-
320 
-	wopt := mobyworker.WorkerOpt{
321 
-		ID:                "moby",
322 
-		SessionManager:    opt.SessionManager,
323 
-		MetadataStore:     md,
324 
-		ContentStore:      store,
325 
-		CacheManager:      cm,
326 
-		Snapshotter:       snapshotter,
327 
-		Executor:          exec,
328 
-		ImageSource:       src,
329 
-		DownloadManager:   dist.DownloadManager,
330 
-		V2MetadataService: dist.V2MetadataService,
331 
-		Exporters: map[string]exporter.Exporter{
332 
-			"image": exp,
333 
-		},
334 
-	}
335 
-
336 
-	wc := &worker.Controller{}
337 
-	w, err := mobyworker.NewWorker(wopt)
338 
-	if err != nil {
339 
-		return nil, err
340 
-	}
341 
-	wc.Add(w)
342 
-
343 
-	ci := cacheimport.NewCacheImporter(cacheimport.ImportOpt{
344 
-		Worker:         w,
345 
-		SessionManager: opt.SessionManager,
346 
-	})
347 
-
348 
-	return control.NewController(control.Opt{
349 
-		SessionManager:   opt.SessionManager,
350 
-		WorkerController: wc,
351 
-		Frontends:        frontends,
352 
-		CacheKeyStorage:  cacheStorage,
353 
-		// CacheExporter:    ce,
354 
-		CacheImporter: ci,
355 
-	})
193 
+	return &out, nil
356 194 
 }
357 195
358 196 
 type statusProxy struct {
... ... 
@@ -388,61 +197,6 @@ func (sp *statusProxy) RecvMsg(m interface{}) error {
388 388 
 	return io.EOF
389 389 
 }
390 390
391 
-type results struct {
392 
-	ch   chan containerimageexp.Result
393 
-	res  map[string]containerimageexp.Result
394 
-	mu   sync.Mutex
395 
-	cond *sync.Cond
396 
-}
397 
-
398 
-func newResultsGetter() *results {
399 
-	r := &results{
400 
-		ch:  make(chan containerimageexp.Result),
401 
-		res: map[string]containerimageexp.Result{},
402 
-	}
403 
-	r.cond = sync.NewCond(&r.mu)
404 
-
405 
-	go func() {
406 
-		for res := range r.ch {
407 
-			r.mu.Lock()
408 
-			r.res[res.Ref] = res
409 
-			r.cond.Broadcast()
410 
-			r.mu.Unlock()
411 
-		}
412 
-	}()
413 
-	return r
414 
-}
415 
-
416 
-func (r *results) wait(ctx context.Context, ref string) (*containerimageexp.Result, error) {
417 
-	done := make(chan struct{})
418 
-	defer close(done)
419 
-	go func() {
420 
-		select {
421 
-		case <-ctx.Done():
422 
-			r.mu.Lock()
423 
-			r.cond.Broadcast()
424 
-			r.mu.Unlock()
425 
-		case <-done:
426 
-		}
427 
-	}()
428 
-
429 
-	r.mu.Lock()
430 
-	for {
431 
-		select {
432 
-		case <-ctx.Done():
433 
-			r.mu.Unlock()
434 
-			return nil, ctx.Err()
435 
-		default:
436 
-		}
437 
-		res, ok := r.res[ref]
438 
-		if ok {
439 
-			r.mu.Unlock()
440 
-			return &res, nil
441 
-		}
442 
-		r.cond.Wait()
443 
-	}
444 
-}
445 
-
446 391 
 type contentStoreNoLabels struct {
447 392 
 	content.Store
448 393 
 }
builder/builder-next/containerimage/pull.go
History View file @ b742459
449 394 
deleted file mode 100644
... ... 
@@ -1,656 +0,0 @@
1 
-package containerimage
2 
-
3 
-import (
4 
-	"context"
5 
-	"encoding/json"
6 
-	"fmt"
7 
-	"io"
8 
-	"io/ioutil"
9 
-	"runtime"
10 
-	"sync"
11 
-	"time"
12 
-
13 
-	"github.com/containerd/containerd/content"
14 
-	"github.com/containerd/containerd/errdefs"
15 
-	"github.com/containerd/containerd/images"
16 
-	"github.com/containerd/containerd/platforms"
17 
-	ctdreference "github.com/containerd/containerd/reference"
18 
-	"github.com/containerd/containerd/remotes"
19 
-	"github.com/containerd/containerd/remotes/docker"
20 
-	"github.com/containerd/containerd/remotes/docker/schema1"
21 
-	distreference "github.com/docker/distribution/reference"
22 
-	"github.com/docker/docker/distribution"
23 
-	"github.com/docker/docker/distribution/metadata"
24 
-	"github.com/docker/docker/distribution/xfer"
25 
-	"github.com/docker/docker/image"
26 
-	"github.com/docker/docker/layer"
27 
-	pkgprogress "github.com/docker/docker/pkg/progress"
28 
-	"github.com/docker/docker/reference"
29 
-	"github.com/moby/buildkit/cache"
30 
-	"github.com/moby/buildkit/session"
31 
-	"github.com/moby/buildkit/session/auth"
32 
-	"github.com/moby/buildkit/source"
33 
-	"github.com/moby/buildkit/util/flightcontrol"
34 
-	"github.com/moby/buildkit/util/imageutil"
35 
-	"github.com/moby/buildkit/util/progress"
36 
-	"github.com/moby/buildkit/util/tracing"
37 
-	digest "github.com/opencontainers/go-digest"
38 
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
39 
-	"github.com/pkg/errors"
40 
-	netcontext "golang.org/x/net/context"
41 
-	"golang.org/x/time/rate"
42 
-)
43 
-
44 
-const preferLocal = true // FIXME: make this optional from the op
45 
-
46 
-type SourceOpt struct {
47 
-	SessionManager  *session.Manager
48 
-	ContentStore    content.Store
49 
-	CacheAccessor   cache.Accessor
50 
-	ReferenceStore  reference.Store
51 
-	DownloadManager distribution.RootFSDownloadManager
52 
-	MetadataStore   metadata.V2MetadataService
53 
-	ImageStore      image.Store
54 
-}
55 
-
56 
-type imageSource struct {
57 
-	SourceOpt
58 
-	g flightcontrol.Group
59 
-}
60 
-
61 
-func NewSource(opt SourceOpt) (source.Source, error) {
62 
-	is := &imageSource{
63 
-		SourceOpt: opt,
64 
-	}
65 
-
66 
-	return is, nil
67 
-}
68 
-
69 
-func (is *imageSource) ID() string {
70 
-	return source.DockerImageScheme
71 
-}
72 
-
73 
-func (is *imageSource) getResolver(ctx context.Context) remotes.Resolver {
74 
-	return docker.NewResolver(docker.ResolverOptions{
75 
-		Client:      tracing.DefaultClient,
76 
-		Credentials: is.getCredentialsFromSession(ctx),
77 
-	})
78 
-}
79 
-
80 
-func (is *imageSource) getCredentialsFromSession(ctx context.Context) func(string) (string, string, error) {
81 
-	id := session.FromContext(ctx)
82 
-	if id == "" {
83 
-		return nil
84 
-	}
85 
-	return func(host string) (string, string, error) {
86 
-		timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
87 
-		defer cancel()
88 
-
89 
-		caller, err := is.SessionManager.Get(timeoutCtx, id)
90 
-		if err != nil {
91 
-			return "", "", err
92 
-		}
93 
-
94 
-		return auth.CredentialsFunc(tracing.ContextWithSpanFromContext(context.TODO(), ctx), caller)(host)
95 
-	}
96 
-}
97 
-
98 
-func (is *imageSource) resolveLocal(refStr string) ([]byte, error) {
99 
-	ref, err := distreference.ParseNormalizedNamed(refStr)
100 
-	if err != nil {
101 
-		return nil, err
102 
-	}
103 
-	dgst, err := is.ReferenceStore.Get(ref)
104 
-	if err != nil {
105 
-		return nil, err
106 
-	}
107 
-	img, err := is.ImageStore.Get(image.ID(dgst))
108 
-	if err != nil {
109 
-		return nil, err
110 
-	}
111 
-	return img.RawJSON(), nil
112 
-}
113 
-
114 
-func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string) (digest.Digest, []byte, error) {
115 
-	if preferLocal {
116 
-		dt, err := is.resolveLocal(ref)
117 
-		if err == nil {
118 
-			return "", dt, nil
119 
-		}
120 
-	}
121 
-
122 
-	type t struct {
123 
-		dgst digest.Digest
124 
-		dt   []byte
125 
-	}
126 
-	res, err := is.g.Do(ctx, ref, func(ctx context.Context) (interface{}, error) {
127 
-		dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx), is.ContentStore, "")
128 
-		if err != nil {
129 
-			return nil, err
130 
-		}
131 
-		return &t{dgst: dgst, dt: dt}, nil
132 
-	})
133 
-	if err != nil {
134 
-		return "", nil, err
135 
-	}
136 
-	typed := res.(*t)
137 
-	return typed.dgst, typed.dt, nil
138 
-}
139 
-
140 
-func (is *imageSource) Resolve(ctx context.Context, id source.Identifier) (source.SourceInstance, error) {
141 
-	imageIdentifier, ok := id.(*source.ImageIdentifier)
142 
-	if !ok {
143 
-		return nil, errors.Errorf("invalid image identifier %v", id)
144 
-	}
145 
-
146 
-	p := &puller{
147 
-		src:      imageIdentifier,
148 
-		is:       is,
149 
-		resolver: is.getResolver(ctx),
150 
-	}
151 
-	return p, nil
152 
-}
153 
-
154 
-type puller struct {
155 
-	is          *imageSource
156 
-	resolveOnce sync.Once
157 
-	src         *source.ImageIdentifier
158 
-	desc        ocispec.Descriptor
159 
-	ref         string
160 
-	resolveErr  error
161 
-	resolver    remotes.Resolver
162 
-	imageID     image.ID
163 
-	cacheKey    digest.Digest
164 
-}
165 
-
166 
-func (p *puller) resolve(ctx context.Context) error {
167 
-	p.resolveOnce.Do(func() {
168 
-		resolveProgressDone := oneOffProgress(ctx, "resolve "+p.src.Reference.String())
169 
-
170 
-		// dgst := p.src.Reference.Digest()
171 
-		// if dgst != "" {
172 
-		// 	info, err := p.is.ContentStore.Info(ctx, dgst)
173 
-		// 	if err == nil {
174 
-		// 		p.ref = p.src.Reference.String()
175 
-		// 		ra, err := p.is.ContentStore.ReaderAt(ctx, dgst)
176 
-		// 		if err == nil {
177 
-		// 			mt, err := imageutil.DetectManifestMediaType(ra)
178 
-		// 			if err == nil {
179 
-		// 				p.desc = ocispec.Descriptor{
180 
-		// 					Size:      info.Size,
181 
-		// 					Digest:    dgst,
182 
-		// 					MediaType: mt,
183 
-		// 				}
184 
-		// 				resolveProgressDone(nil)
185 
-		// 				return
186 
-		// 			}
187 
-		// 		}
188 
-		// 	}
189 
-		// }
190 
-
191 
-		// ref, desc, err := p.resolver.Resolve(ctx, p.src.Reference.String())
192 
-		// if err != nil {
193 
-		// 	p.resolveErr = err
194 
-		// 	resolveProgressDone(err)
195 
-		// 	return
196 
-		// }
197 
-
198 
-		if preferLocal {
199 
-			dt, err := p.is.resolveLocal(p.src.Reference.String())
200 
-			if err == nil {
201 
-				dgst := digest.FromBytes(dt)
202 
-				p.imageID = image.ID(dgst)
203 
-				p.cacheKey = dgst
204 
-				resolveProgressDone(nil)
205 
-				return
206 
-			}
207 
-
208 
-		}
209 
-
210 
-		ref, err := distreference.ParseNormalizedNamed(p.src.Reference.String())
211 
-		if err != nil {
212 
-			p.resolveErr = err
213 
-			resolveProgressDone(err)
214 
-			return
215 
-		}
216 
-
217 
-		outRef, desc, err := p.resolver.Resolve(ctx, p.src.Reference.String())
218 
-		if err != nil {
219 
-			p.resolveErr = err
220 
-			resolveProgressDone(err)
221 
-			return
222 
-		}
223 
-
224 
-		ref, err = distreference.WithDigest(ref, desc.Digest)
225 
-		if err != nil {
226 
-			p.resolveErr = err
227 
-			resolveProgressDone(err)
228 
-			return
229 
-		}
230 
-
231 
-		_, dt, err := p.is.ResolveImageConfig(ctx, ref.String())
232 
-		if err != nil {
233 
-			p.resolveErr = err
234 
-			resolveProgressDone(err)
235 
-			return
236 
-		}
237 
-		p.desc = desc
238 
-		p.cacheKey = digest.FromBytes(dt)
239 
-		p.ref = outRef
240 
-		resolveProgressDone(nil)
241 
-	})
242 
-	return p.resolveErr
243 
-}
244 
-
245 
-func (p *puller) CacheKey(ctx context.Context) (string, error) {
246 
-	if err := p.resolve(ctx); err != nil {
247 
-		return "", err
248 
-	}
249 
-	return p.cacheKey.String(), nil
250 
-}
251 
-
252 
-func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
253 
-	if err := p.resolve(ctx); err != nil {
254 
-		return nil, err
255 
-	}
256 
-
257 
-	if p.imageID != "" {
258 
-		img, err := p.is.ImageStore.Get(p.imageID)
259 
-		if err != nil {
260 
-			return nil, err
261 
-		}
262 
-		ref, err := p.is.CacheAccessor.Get(ctx, string(img.RootFS.ChainID()), cache.WithDescription(fmt.Sprintf("from local %s", p.ref)))
263 
-		if err != nil {
264 
-			return nil, err
265 
-		}
266 
-		return ref, nil
267 
-	}
268 
-
269 
-	ongoing := newJobs(p.ref)
270 
-
271 
-	pctx, stopProgress := context.WithCancel(ctx)
272 
-
273 
-	pw, _, ctx := progress.FromContext(ctx)
274 
-	defer pw.Close()
275 
-
276 
-	progressDone := make(chan struct{})
277 
-	go func() {
278 
-		showProgress(pctx, ongoing, p.is.ContentStore, pw)
279 
-		close(progressDone)
280 
-	}()
281 
-	defer func() {
282 
-		<-progressDone
283 
-	}()
284 
-
285 
-	fetcher, err := p.resolver.Fetcher(ctx, p.ref)
286 
-	if err != nil {
287 
-		stopProgress()
288 
-		return nil, err
289 
-	}
290 
-
291 
-	// TODO: need a wrapper snapshot interface that combines content
292 
-	// and snapshots as 1) buildkit shouldn't have a dependency on contentstore
293 
-	// or 2) cachemanager should manage the contentstore
294 
-	handlers := []images.Handler{
295 
-		images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
296 
-			switch desc.MediaType {
297 
-			case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest,
298 
-				images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex,
299 
-				images.MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig:
300 
-			default:
301 
-				return nil, images.ErrSkipDesc
302 
-			}
303 
-			ongoing.add(desc)
304 
-			return nil, nil
305 
-		}),
306 
-	}
307 
-	// var schema1Converter *schema1.Converter
308 
-	// if p.desc.MediaType == images.MediaTypeDockerSchema1Manifest {
309 
-	// 	schema1Converter = schema1.NewConverter(p.is.ContentStore, fetcher)
310 
-	// 	handlers = append(handlers, schema1Converter)
311 
-	// } else {
312 
-	// 	handlers = append(handlers,
313 
-	// 		remotes.FetchHandler(p.is.ContentStore, fetcher),
314 
-	//
315 
-	// 		images.ChildrenHandler(p.is.ContentStore),
316 
-	// 	)
317 
-	// }
318 
-	//
319 
-	var schema1Converter *schema1.Converter
320 
-	if p.desc.MediaType == images.MediaTypeDockerSchema1Manifest {
321 
-		schema1Converter = schema1.NewConverter(p.is.ContentStore, fetcher)
322 
-		handlers = append(handlers, schema1Converter)
323 
-	} else {
324 
-		// Get all the children for a descriptor
325 
-		childrenHandler := images.ChildrenHandler(p.is.ContentStore)
326 
-		// Set any children labels for that content
327 
-		childrenHandler = images.SetChildrenLabels(p.is.ContentStore, childrenHandler)
328 
-		// Filter the childen by the platform
329 
-		childrenHandler = images.FilterPlatforms(childrenHandler, platforms.Default())
330 
-
331 
-		handlers = append(handlers,
332 
-			remotes.FetchHandler(p.is.ContentStore, fetcher),
333 
-			childrenHandler,
334 
-		)
335 
-	}
336 
-
337 
-	if err := images.Dispatch(ctx, images.Handlers(handlers...), p.desc); err != nil {
338 
-		stopProgress()
339 
-		return nil, err
340 
-	}
341 
-	defer stopProgress()
342 
-
343 
-	mfst, err := images.Manifest(ctx, p.is.ContentStore, p.desc, platforms.Default())
344 
-	if err != nil {
345 
-		return nil, err
346 
-	}
347 
-
348 
-	config, err := images.Config(ctx, p.is.ContentStore, p.desc, platforms.Default())
349 
-	if err != nil {
350 
-		return nil, err
351 
-	}
352 
-
353 
-	dt, err := content.ReadBlob(ctx, p.is.ContentStore, config.Digest)
354 
-	if err != nil {
355 
-		return nil, err
356 
-	}
357 
-
358 
-	var img ocispec.Image
359 
-	if err := json.Unmarshal(dt, &img); err != nil {
360 
-		return nil, err
361 
-	}
362 
-
363 
-	if len(mfst.Layers) != len(img.RootFS.DiffIDs) {
364 
-		return nil, errors.Errorf("invalid config for manifest")
365 
-	}
366 
-
367 
-	pchan := make(chan pkgprogress.Progress, 10)
368 
-	defer close(pchan)
369 
-
370 
-	go func() {
371 
-		m := map[string]struct {
372 
-			st      time.Time
373 
-			limiter *rate.Limiter
374 
-		}{}
375 
-		for p := range pchan {
376 
-			if p.Action == "Extracting" {
377 
-				st, ok := m[p.ID]
378 
-				if !ok {
379 
-					st.st = time.Now()
380 
-					st.limiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 1)
381 
-					m[p.ID] = st
382 
-				}
383 
-				var end *time.Time
384 
-				if p.LastUpdate || st.limiter.Allow() {
385 
-					if p.LastUpdate {
386 
-						tm := time.Now()
387 
-						end = &tm
388 
-					}
389 
-					pw.Write("extracting "+p.ID, progress.Status{
390 
-						Action:    "extract",
391 
-						Started:   &st.st,
392 
-						Completed: end,
393 
-					})
394 
-				}
395 
-			}
396 
-		}
397 
-	}()
398 
-
399 
-	layers := make([]xfer.DownloadDescriptor, 0, len(mfst.Layers))
400 
-
401 
-	for i, desc := range mfst.Layers {
402 
-		ongoing.add(desc)
403 
-		layers = append(layers, &layerDescriptor{
404 
-			desc:    desc,
405 
-			diffID:  layer.DiffID(img.RootFS.DiffIDs[i]),
406 
-			fetcher: fetcher,
407 
-			ref:     p.src.Reference,
408 
-			is:      p.is,
409 
-		})
410 
-	}
411 
-
412 
-	defer func() {
413 
-		<-progressDone
414 
-		for _, desc := range mfst.Layers {
415 
-			p.is.ContentStore.Delete(context.TODO(), desc.Digest)
416 
-		}
417 
-	}()
418 
-
419 
-	r := image.NewRootFS()
420 
-	rootFS, release, err := p.is.DownloadManager.Download(ctx, *r, runtime.GOOS, layers, pkgprogress.ChanOutput(pchan))
421 
-	if err != nil {
422 
-		return nil, err
423 
-	}
424 
-	stopProgress()
425 
-
426 
-	ref, err := p.is.CacheAccessor.Get(ctx, string(rootFS.ChainID()), cache.WithDescription(fmt.Sprintf("pulled from %s", p.ref)))
427 
-	release()
428 
-	if err != nil {
429 
-		return nil, err
430 
-	}
431 
-
432 
-	return ref, nil
433 
-}
434 
-
435 
-// Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error)
436 
-type layerDescriptor struct {
437 
-	is      *imageSource
438 
-	fetcher remotes.Fetcher
439 
-	desc    ocispec.Descriptor
440 
-	diffID  layer.DiffID
441 
-	ref     ctdreference.Spec
442 
-}
443 
-
444 
-func (ld *layerDescriptor) Key() string {
445 
-	return "v2:" + ld.desc.Digest.String()
446 
-}
447 
-
448 
-func (ld *layerDescriptor) ID() string {
449 
-	return ld.desc.Digest.String()
450 
-}
451 
-
452 
-func (ld *layerDescriptor) DiffID() (layer.DiffID, error) {
453 
-	return ld.diffID, nil
454 
-}
455 
-
456 
-func (ld *layerDescriptor) Download(ctx netcontext.Context, progressOutput pkgprogress.Output) (io.ReadCloser, int64, error) {
457 
-	rc, err := ld.fetcher.Fetch(ctx, ld.desc)
458 
-	if err != nil {
459 
-		return nil, 0, err
460 
-	}
461 
-	defer rc.Close()
462 
-
463 
-	refKey := remotes.MakeRefKey(ctx, ld.desc)
464 
-
465 
-	if err := content.WriteBlob(ctx, ld.is.ContentStore, refKey, rc, ld.desc.Size, ld.desc.Digest); err != nil {
466 
-		return nil, 0, err
467 
-	}
468 
-
469 
-	ra, err := ld.is.ContentStore.ReaderAt(ctx, ld.desc.Digest)
470 
-	if err != nil {
471 
-		return nil, 0, err
472 
-	}
473 
-
474 
-	return ioutil.NopCloser(content.NewReader(ra)), ld.desc.Size, nil
475 
-}
476 
-
477 
-func (ld *layerDescriptor) Close() {
478 
-	// ld.is.ContentStore.Delete(context.TODO(), ld.desc.Digest))
479 
-}
480 
-
481 
-func (ld *layerDescriptor) Registered(diffID layer.DiffID) {
482 
-	// Cache mapping from this layer's DiffID to the blobsum
483 
-	ld.is.MetadataStore.Add(diffID, metadata.V2Metadata{Digest: ld.desc.Digest, SourceRepository: ld.ref.Locator})
484 
-}
485 
-
486 
-func showProgress(ctx context.Context, ongoing *jobs, cs content.Store, pw progress.Writer) {
487 
-	var (
488 
-		ticker   = time.NewTicker(100 * time.Millisecond)
489 
-		statuses = map[string]statusInfo{}
490 
-		done     bool
491 
-	)
492 
-	defer ticker.Stop()
493 
-
494 
-	for {
495 
-		select {
496 
-		case <-ticker.C:
497 
-		case <-ctx.Done():
498 
-			done = true
499 
-		}
500 
-
501 
-		resolved := "resolved"
502 
-		if !ongoing.isResolved() {
503 
-			resolved = "resolving"
504 
-		}
505 
-		statuses[ongoing.name] = statusInfo{
506 
-			Ref:    ongoing.name,
507 
-			Status: resolved,
508 
-		}
509 
-
510 
-		actives := make(map[string]statusInfo)
511 
-
512 
-		if !done {
513 
-			active, err := cs.ListStatuses(ctx)
514 
-			if err != nil {
515 
-				// log.G(ctx).WithError(err).Error("active check failed")
516 
-				continue
517 
-			}
518 
-			// update status of active entries!
519 
-			for _, active := range active {
520 
-				actives[active.Ref] = statusInfo{
521 
-					Ref:       active.Ref,
522 
-					Status:    "downloading",
523 
-					Offset:    active.Offset,
524 
-					Total:     active.Total,
525 
-					StartedAt: active.StartedAt,
526 
-					UpdatedAt: active.UpdatedAt,
527 
-				}
528 
-			}
529 
-		}
530 
-
531 
-		// now, update the items in jobs that are not in active
532 
-		for _, j := range ongoing.jobs() {
533 
-			refKey := remotes.MakeRefKey(ctx, j.Descriptor)
534 
-			if a, ok := actives[refKey]; ok {
535 
-				started := j.started
536 
-				pw.Write(j.Digest.String(), progress.Status{
537 
-					Action:  a.Status,
538 
-					Total:   int(a.Total),
539 
-					Current: int(a.Offset),
540 
-					Started: &started,
541 
-				})
542 
-				continue
543 
-			}
544 
-
545 
-			if !j.done {
546 
-				info, err := cs.Info(context.TODO(), j.Digest)
547 
-				if err != nil {
548 
-					if errdefs.IsNotFound(err) {
549 
-						// pw.Write(j.Digest.String(), progress.Status{
550 
-						// 	Action: "waiting",
551 
-						// })
552 
-						continue
553 
-					}
554 
-				} else {
555 
-					j.done = true
556 
-				}
557 
-
558 
-				if done || j.done {
559 
-					started := j.started
560 
-					createdAt := info.CreatedAt
561 
-					pw.Write(j.Digest.String(), progress.Status{
562 
-						Action:    "done",
563 
-						Current:   int(info.Size),
564 
-						Total:     int(info.Size),
565 
-						Completed: &createdAt,
566 
-						Started:   &started,
567 
-					})
568 
-				}
569 
-			}
570 
-		}
571 
-		if done {
572 
-			return
573 
-		}
574 
-	}
575 
-}
576 
-
577 
-// jobs provides a way of identifying the download keys for a particular task
578 
-// encountering during the pull walk.
579 
-//
580 
-// This is very minimal and will probably be replaced with something more
581 
-// featured.
582 
-type jobs struct {
583 
-	name     string
584 
-	added    map[digest.Digest]job
585 
-	mu       sync.Mutex
586 
-	resolved bool
587 
-}
588 
-
589 
-type job struct {
590 
-	ocispec.Descriptor
591 
-	done    bool
592 
-	started time.Time
593 
-}
594 
-
595 
-func newJobs(name string) *jobs {
596 
-	return &jobs{
597 
-		name:  name,
598 
-		added: make(map[digest.Digest]job),
599 
-	}
600 
-}
601 
-
602 
-func (j *jobs) add(desc ocispec.Descriptor) {
603 
-	j.mu.Lock()
604 
-	defer j.mu.Unlock()
605 
-
606 
-	if _, ok := j.added[desc.Digest]; ok {
607 
-		return
608 
-	}
609 
-	j.added[desc.Digest] = job{
610 
-		Descriptor: desc,
611 
-		started:    time.Now(),
612 
-	}
613 
-}
614 
-
615 
-func (j *jobs) jobs() []job {
616 
-	j.mu.Lock()
617 
-	defer j.mu.Unlock()
618 
-
619 
-	descs := make([]job, 0, len(j.added))
620 
-	for _, j := range j.added {
621 
-		descs = append(descs, j)
622 
-	}
623 
-	return descs
624 
-}
625 
-
626 
-func (j *jobs) isResolved() bool {
627 
-	j.mu.Lock()
628 
-	defer j.mu.Unlock()
629 
-	return j.resolved
630 
-}
631 
-
632 
-type statusInfo struct {
633 
-	Ref       string
634 
-	Status    string
635 
-	Offset    int64
636 
-	Total     int64
637 
-	StartedAt time.Time
638 
-	UpdatedAt time.Time
639 
-}
640 
-
641 
-func oneOffProgress(ctx context.Context, id string) func(err error) error {
642 
-	pw, _, _ := progress.FromContext(ctx)
643 
-	now := time.Now()
644 
-	st := progress.Status{
645 
-		Started: &now,
646 
-	}
647 
-	pw.Write(id, st)
648 
-	return func(err error) error {
649 
-		// TODO: set error on status
650 
-		now := time.Now()
651 
-		st.Completed = &now
652 
-		pw.Write(id, st)
653 
-		pw.Close()
654 
-		return err
655 
-	}
656 
-}
builder/builder-next/controller.go
History View file @ b742459
657 1 
new file mode 100644
... ... 
@@ -0,0 +1,158 @@
0 
+package buildkit
1 
+
2 
+import (
3 
+	"os"
4 
+	"path/filepath"
5 
+
6 
+	"github.com/containerd/containerd/content/local"
7 
+	"github.com/docker/docker/builder/builder-next/adapters/containerimage"
8 
+	"github.com/docker/docker/builder/builder-next/adapters/snapshot"
9 
+	containerimageexp "github.com/docker/docker/builder/builder-next/exporter"
10 
+	mobyworker "github.com/docker/docker/builder/builder-next/worker"
11 
+	"github.com/docker/docker/daemon/graphdriver"
12 
+	"github.com/moby/buildkit/cache"
13 
+	"github.com/moby/buildkit/cache/metadata"
14 
+	"github.com/moby/buildkit/cache/remotecache"
15 
+	"github.com/moby/buildkit/control"
16 
+	"github.com/moby/buildkit/executor/runcexecutor"
17 
+	"github.com/moby/buildkit/exporter"
18 
+	"github.com/moby/buildkit/frontend"
19 
+	"github.com/moby/buildkit/frontend/dockerfile"
20 
+	"github.com/moby/buildkit/snapshot/blobmapping"
21 
+	"github.com/moby/buildkit/solver/boltdbcachestorage"
22 
+	"github.com/moby/buildkit/worker"
23 
+	"github.com/pkg/errors"
24 
+)
25 
+
26 
+func newController(opt Opt) (*control.Controller, error) {
27 
+	if err := os.MkdirAll(opt.Root, 0700); err != nil {
28 
+		return nil, err
29 
+	}
30 
+
31 
+	dist := opt.Dist
32 
+	root := opt.Root
33 
+
34 
+	var driver graphdriver.Driver
35 
+	if ls, ok := dist.LayerStore.(interface {
36 
+		Driver() graphdriver.Driver
37 
+	}); ok {
38 
+		driver = ls.Driver()
39 
+	} else {
40 
+		return nil, errors.Errorf("could not access graphdriver")
41 
+	}
42 
+
43 
+	sbase, err := snapshot.NewSnapshotter(snapshot.Opt{
44 
+		GraphDriver: driver,
45 
+		LayerStore:  dist.LayerStore,
46 
+		Root:        root,
47 
+	})
48 
+	if err != nil {
49 
+		return nil, err
50 
+	}
51 
+
52 
+	store, err := local.NewStore(filepath.Join(root, "content"))
53 
+	if err != nil {
54 
+		return nil, err
55 
+	}
56 
+	store = &contentStoreNoLabels{store}
57 
+
58 
+	md, err := metadata.NewStore(filepath.Join(root, "metadata.db"))
59 
+	if err != nil {
60 
+		return nil, err
61 
+	}
62 
+
63 
+	snapshotter := blobmapping.NewSnapshotter(blobmapping.Opt{
64 
+		Content:       store,
65 
+		Snapshotter:   sbase,
66 
+		MetadataStore: md,
67 
+	})
68 
+
69 
+	cm, err := cache.NewManager(cache.ManagerOpt{
70 
+		Snapshotter:   snapshotter,
71 
+		MetadataStore: md,
72 
+	})
73 
+	if err != nil {
74 
+		return nil, err
75 
+	}
76 
+
77 
+	src, err := containerimage.NewSource(containerimage.SourceOpt{
78 
+		SessionManager:  opt.SessionManager,
79 
+		CacheAccessor:   cm,
80 
+		ContentStore:    store,
81 
+		DownloadManager: dist.DownloadManager,
82 
+		MetadataStore:   dist.V2MetadataService,
83 
+		ImageStore:      dist.ImageStore,
84 
+		ReferenceStore:  dist.ReferenceStore,
85 
+	})
86 
+	if err != nil {
87 
+		return nil, err
88 
+	}
89 
+
90 
+	exec, err := runcexecutor.New(runcexecutor.Opt{
91 
+		Root:              filepath.Join(root, "executor"),
92 
+		CommandCandidates: []string{"docker-runc", "runc"},
93 
+	})
94 
+	if err != nil {
95 
+		return nil, err
96 
+	}
97 
+
98 
+	differ, ok := sbase.(containerimageexp.Differ)
99 
+	if !ok {
100 
+		return nil, errors.Errorf("snapshotter doesn't support differ")
101 
+	}
102 
+
103 
+	exp, err := containerimageexp.New(containerimageexp.Opt{
104 
+		ImageStore:     dist.ImageStore,
105 
+		ReferenceStore: dist.ReferenceStore,
106 
+		Differ:         differ,
107 
+	})
108 
+	if err != nil {
109 
+		return nil, err
110 
+	}
111 
+
112 
+	cacheStorage, err := boltdbcachestorage.NewStore(filepath.Join(opt.Root, "cache.db"))
113 
+	if err != nil {
114 
+		return nil, err
115 
+	}
116 
+
117 
+	frontends := map[string]frontend.Frontend{}
118 
+	frontends["dockerfile.v0"] = dockerfile.NewDockerfileFrontend()
119 
+	// frontends["gateway.v0"] = gateway.NewGatewayFrontend()
120 
+
121 
+	wopt := mobyworker.WorkerOpt{
122 
+		ID:                "moby",
123 
+		SessionManager:    opt.SessionManager,
124 
+		MetadataStore:     md,
125 
+		ContentStore:      store,
126 
+		CacheManager:      cm,
127 
+		Snapshotter:       snapshotter,
128 
+		Executor:          exec,
129 
+		ImageSource:       src,
130 
+		DownloadManager:   dist.DownloadManager,
131 
+		V2MetadataService: dist.V2MetadataService,
132 
+		Exporters: map[string]exporter.Exporter{
133 
+			"moby": exp,
134 
+		},
135 
+	}
136 
+
137 
+	wc := &worker.Controller{}
138 
+	w, err := mobyworker.NewWorker(wopt)
139 
+	if err != nil {
140 
+		return nil, err
141 
+	}
142 
+	wc.Add(w)
143 
+
144 
+	ci := remotecache.NewCacheImporter(remotecache.ImportOpt{
145 
+		Worker:         w,
146 
+		SessionManager: opt.SessionManager,
147 
+	})
148 
+
149 
+	return control.NewController(control.Opt{
150 
+		SessionManager:   opt.SessionManager,
151 
+		WorkerController: wc,
152 
+		Frontends:        frontends,
153 
+		CacheKeyStorage:  cacheStorage,
154 
+		// CacheExporter:    ce,
155 
+		CacheImporter: ci,
156 
+	})
157 
+}
builder/builder-next/exporter/export.go
History View file @ b742459
... ... 
@@ -23,17 +23,10 @@ type Differ interface {
23 23 
 	EnsureLayer(ctx context.Context, key string) ([]layer.DiffID, error)
24 24 
 }
25 25
26 
-// TODO: this needs to be handled differently (return from solve)
27 
-type Result struct {
28 
-	Ref string
29 
-	ID  image.ID
30 
-}
31 
-
32 26 
 type Opt struct {
33 27 
 	ImageStore     image.Store
34 28 
 	ReferenceStore reference.Store
35 29 
 	Differ         Differ
36 
-	Reporter       chan Result
37 30 
 }
38 31
39 32 
 type imageExporter struct {
... ... 
@@ -57,8 +50,6 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
57 57 
 			i.targetName = ref
58 58 
 		case exporterImageConfig:
59 59 
 			i.config = []byte(v)
60 
-		case "ref":
61 
-			i.ref = v
62 60 
 		default:
63 61 
 			logrus.Warnf("image exporter: unknown option %s", k)
64 62 
 		}
... ... 
@@ -70,14 +61,13 @@ type imageExporterInstance struct {
70 70 
 	*imageExporter
71 71 
 	targetName distref.Named
72 72 
 	config     []byte
73 
-	ref        string
74 73 
 }
75 74
76 75 
 func (e *imageExporterInstance) Name() string {
77 76 
 	return "exporting to image"
78 77 
 }
79 78
80 
-func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableRef, opt map[string][]byte) error {
79 
+func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableRef, opt map[string][]byte) (map[string]string, error) {
81 80 
 	if config, ok := opt[exporterImageConfig]; ok {
82 81 
 		e.config = config
83 82 
 	}
... ... 
@@ -86,12 +76,12 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
86 86 
 	layersDone := oneOffProgress(ctx, "exporting layers")
87 87
88 88 
 	if err := ref.Finalize(ctx); err != nil {
89 
-		return err
89 
+		return nil, err
90 90 
 	}
91 91
92 92 
 	diffIDs, err := e.opt.Differ.EnsureLayer(ctx, ref.ID())
93 93 
 	if err != nil {
94 
-		return err
94 
+		return nil, err
95 95 
 	}
96 96
97 97 
 	diffs := make([]digest.Digest, len(diffIDs))
... ... 
@@ -105,20 +95,20 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
105 105 
 		var err error
106 106 
 		config, err = emptyImageConfig()
107 107 
 		if err != nil {
108 
-			return err
108 
+			return nil, err
109 109 
 		}
110 110 
 	}
111 111
112 112 
 	history, err := parseHistoryFromConfig(config)
113 113 
 	if err != nil {
114 
-		return err
114 
+		return nil, err
115 115 
 	}
116 116
117 117 
 	diffs, history = normalizeLayersAndHistory(diffs, history, ref)
118 118
119 119 
 	config, err = patchImageConfig(config, diffs, history)
120 120 
 	if err != nil {
121 
-		return err
121 
+		return nil, err
122 122 
 	}
123 123
124 124 
 	configDigest := digest.FromBytes(config)
... ... 
@@ -126,7 +116,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
126 126 
 	configDone := oneOffProgress(ctx, fmt.Sprintf("writing image %s", configDigest))
127 127 
 	id, err := e.opt.ImageStore.Create(config)
128 128 
 	if err != nil {
129 
-		return configDone(err)
129 
+		return nil, configDone(err)
130 130 
 	}
131 131 
 	configDone(nil)
132 132
... ... 
@@ -135,15 +125,13 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
135 135 
 			tagDone := oneOffProgress(ctx, "naming to "+e.targetName.String())
136 136
137 137 
 			if err := e.opt.ReferenceStore.AddTag(e.targetName, digest.Digest(id), true); err != nil {
138 
-				return tagDone(err)
138 
+				return nil, tagDone(err)
139 139 
 			}
140 140 
 			tagDone(nil)
141 141 
 		}
142 142 
 	}
143 143
144 
-	if e.opt.Reporter != nil {
145 
-		e.opt.Reporter <- Result{ID: id, Ref: e.ref}
146 
-	}
147 
-
148 
-	return nil
144 
+	return map[string]string{
145 
+		"containerimage.digest": id.String(),
146 
+	}, nil
149 147 
 }
builder/builder-next/exporter/writer.go
History View file @ b742459
... ... 
@@ -4,7 +4,6 @@ import (
4 4 
 	"context"
5 5 
 	"encoding/json"
6 6 
 	"runtime"
7 
-	"strings"
8 7 
 	"time"
9 8
10 9 
 	"github.com/moby/buildkit/cache"
... ... 
@@ -16,9 +15,9 @@ import (
16 16 
 	"github.com/sirupsen/logrus"
17 17 
 )
18 18
19 
-const (
20 
-	emptyGZLayer = digest.Digest("sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1")
21 
-)
19 
+// const (
20 
+// 	emptyGZLayer = digest.Digest("sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1")
21 
+// )
22 22
23 23 
 func emptyImageConfig() ([]byte, error) {
24 24 
 	img := ocispec.Image{
... ... 
@@ -65,18 +64,26 @@ func patchImageConfig(dt []byte, dps []digest.Digest, history []ocispec.History)
65 65 
 	}
66 66 
 	m["history"] = dt
67 67
68 
-	// now := time.Now()
69 
-	// dt, err = json.Marshal(&now)
70 
-	// if err != nil {
71 
-	// 	return nil, errors.Wrap(err, "failed to marshal creation time")
72 
-	// }
73 
-	// m["created"] = dt
68 
+	if _, ok := m["created"]; !ok {
69 
+		var tm *time.Time
70 
+		for _, h := range history {
71 
+			if h.Created != nil {
72 
+				tm = h.Created
73 
+			}
74 
+		}
75 
+		dt, err = json.Marshal(&tm)
76 
+		if err != nil {
77 
+			return nil, errors.Wrap(err, "failed to marshal creation time")
78 
+		}
79 
+		m["created"] = dt
80 
+	}
74 81
75 82 
 	dt, err = json.Marshal(m)
76 83 
 	return dt, errors.Wrap(err, "failed to marshal config after patch")
77 84 
 }
78 85
79 86 
 func normalizeLayersAndHistory(diffs []digest.Digest, history []ocispec.History, ref cache.ImmutableRef) ([]digest.Digest, []ocispec.History) {
87 
+	refMeta := getRefMetadata(ref, len(diffs))
80 88 
 	var historyLayers int
81 89 
 	for _, h := range history {
82 90 
 		if !h.EmptyLayer {
... ... 
@@ -103,11 +110,10 @@ func normalizeLayersAndHistory(diffs []digest.Digest, history []ocispec.History,
103 103
104 104 
 	if len(diffs) > historyLayers {
105 105 
 		// some history items are missing. add them based on the ref metadata
106 
-		for _, msg := range getRefDesciptions(ref, len(diffs)-historyLayers) {
107 
-			// tm := time.Now().UTC()
106 
+		for _, md := range refMeta[historyLayers:] {
108 107 
 			history = append(history, ocispec.History{
109 
-				// Created:   &tm,
110 
-				CreatedBy: msg,
108 
+				Created:   &md.createdAt,
109 
+				CreatedBy: md.description,
111 110 
 				Comment:   "buildkit.exporter.image.v0",
112 111 
 			})
113 112 
 		}
... ... 
@@ -129,23 +135,31 @@ func normalizeLayersAndHistory(diffs []digest.Digest, history []ocispec.History,
129 129 
 	return diffs, history
130 130 
 }
131 131
132 
-func getRefDesciptions(ref cache.ImmutableRef, limit int) []string {
132 
+type refMetadata struct {
133 
+	description string
134 
+	createdAt   time.Time
135 
+}
136 
+
137 
+func getRefMetadata(ref cache.ImmutableRef, limit int) []refMetadata {
133 138 
 	if limit <= 0 {
134 139 
 		return nil
135 140 
 	}
136 
-	defaultMsg := "created by buildkit" // shouldn't happen but don't fail build
141 
+	meta := refMetadata{
142 
+		description: "created by buildkit", // shouldn't be shown but don't fail build
143 
+		createdAt:   time.Now(),
144 
+	}
137 145 
 	if ref == nil {
138 
-		strings.Repeat(defaultMsg, limit)
146 
+		return append(getRefMetadata(nil, limit-1), meta)
139 147 
 	}
140 
-	descr := cache.GetDescription(ref.Metadata())
141 
-	if descr == "" {
142 
-		descr = defaultMsg
148 
+	if descr := cache.GetDescription(ref.Metadata()); descr != "" {
149 
+		meta.description = descr
143 150 
 	}
151 
+	meta.createdAt = cache.GetCreatedAt(ref.Metadata())
144 152 
 	p := ref.Parent()
145 153 
 	if p != nil {
146 154 
 		defer p.Release(context.TODO())
147 155 
 	}
148 
-	return append(getRefDesciptions(p, limit-1), descr)
156 
+	return append(getRefMetadata(p, limit-1), meta)
149 157 
 }
150 158
151 159 
 func oneOffProgress(ctx context.Context, id string) func(err error) error {
builder/builder-next/snapshot/layer.go
History View file @ b742459
152 160 
deleted file mode 100644
... ... 
@@ -1,116 +0,0 @@
1 
-package snapshot
2 
-
3 
-import (
4 
-	"context"
5 
-	"os"
6 
-	"path/filepath"
7 
-
8 
-	"github.com/boltdb/bolt"
9 
-	"github.com/docker/docker/layer"
10 
-	"github.com/docker/docker/pkg/ioutils"
11 
-	"github.com/pkg/errors"
12 
-	"golang.org/x/sync/errgroup"
13 
-)
14 
-
15 
-func (s *snapshotter) EnsureLayer(ctx context.Context, key string) ([]layer.DiffID, error) {
16 
-	if l, err := s.getLayer(key); err != nil {
17 
-		return nil, err
18 
-	} else if l != nil {
19 
-		return getDiffChain(l), nil
20 
-	}
21 
-
22 
-	id, committed := s.getGraphDriverID(key)
23 
-	if !committed {
24 
-		return nil, errors.Errorf("can not convert active %s to layer", key)
25 
-	}
26 
-
27 
-	info, err := s.Stat(ctx, key)
28 
-	if err != nil {
29 
-		return nil, err
30 
-	}
31 
-
32 
-	eg, gctx := errgroup.WithContext(ctx)
33 
-
34 
-	// TODO: add flightcontrol
35 
-
36 
-	var parentChainID layer.ChainID
37 
-	if info.Parent != "" {
38 
-		eg.Go(func() error {
39 
-			diffIDs, err := s.EnsureLayer(gctx, info.Parent)
40 
-			if err != nil {
41 
-				return err
42 
-			}
43 
-			parentChainID = layer.CreateChainID(diffIDs)
44 
-			return nil
45 
-		})
46 
-	}
47 
-
48 
-	tmpDir, err := ioutils.TempDir("", "docker-tarsplit")
49 
-	if err != nil {
50 
-		return nil, err
51 
-	}
52 
-	defer os.RemoveAll(tmpDir)
53 
-	tarSplitPath := filepath.Join(tmpDir, "tar-split")
54 
-
55 
-	var diffID layer.DiffID
56 
-	var size int64
57 
-	eg.Go(func() error {
58 
-		parent := ""
59 
-		if p := info.Parent; p != "" {
60 
-			if l, err := s.getLayer(p); err != nil {
61 
-				return err
62 
-			} else if l != nil {
63 
-				parent, err = getGraphID(l)
64 
-				if err != nil {
65 
-					return err
66 
-				}
67 
-			} else {
68 
-				parent, _ = s.getGraphDriverID(info.Parent)
69 
-			}
70 
-		}
71 
-		diffID, size, err = s.reg.ChecksumForGraphID(id, parent, "", tarSplitPath)
72 
-		if err != nil {
73 
-			return err
74 
-		}
75 
-		return nil
76 
-	})
77 
-
78 
-	if err := eg.Wait(); err != nil {
79 
-		return nil, err
80 
-	}
81 
-
82 
-	l, err := s.reg.RegisterByGraphID(id, parentChainID, diffID, tarSplitPath, size)
83 
-	if err != nil {
84 
-		return nil, err
85 
-	}
86 
-
87 
-	if err := s.db.Update(func(tx *bolt.Tx) error {
88 
-		b := tx.Bucket([]byte(key))
89 
-		b.Put(keyChainID, []byte(l.ChainID()))
90 
-		return nil
91 
-	}); err != nil {
92 
-		return nil, err
93 
-	}
94 
-
95 
-	s.mu.Lock()
96 
-	s.refs[key] = l
97 
-	s.mu.Unlock()
98 
-
99 
-	return getDiffChain(l), nil
100 
-}
101 
-
102 
-func getDiffChain(l layer.Layer) []layer.DiffID {
103 
-	if p := l.Parent(); p != nil {
104 
-		return append(getDiffChain(p), l.DiffID())
105 
-	}
106 
-	return []layer.DiffID{l.DiffID()}
107 
-}
108 
-
109 
-func getGraphID(l layer.Layer) (string, error) {
110 
-	if l, ok := l.(interface {
111 
-		CacheID() string
112 
-	}); ok {
113 
-		return l.CacheID(), nil
114 
-	}
115 
-	return "", errors.Errorf("couldn't access cacheID for %s", l.ChainID())
116 
-}
builder/builder-next/snapshot/snapshot.go
History View file @ b742459
117 1 
deleted file mode 100644
... ... 
@@ -1,408 +0,0 @@
1 
-package snapshot
2 
-
3 
-import (
4 
-	"context"
5 
-	"path/filepath"
6 
-	"strings"
7 
-	"sync"
8 
-
9 
-	"github.com/boltdb/bolt"
10 
-	"github.com/containerd/containerd/mount"
11 
-	"github.com/containerd/containerd/snapshots"
12 
-	"github.com/docker/docker/daemon/graphdriver"
13 
-	"github.com/docker/docker/layer"
14 
-	"github.com/moby/buildkit/identity"
15 
-	"github.com/moby/buildkit/snapshot"
16 
-	digest "github.com/opencontainers/go-digest"
17 
-	"github.com/pkg/errors"
18 
-)
19 
-
20 
-var keyParent = []byte("parent")
21 
-var keyCommitted = []byte("committed")
22 
-var keyChainID = []byte("chainid")
23 
-var keySize = []byte("size")
24 
-
25 
-type Opt struct {
26 
-	GraphDriver graphdriver.Driver
27 
-	LayerStore  layer.Store
28 
-	Root        string
29 
-}
30 
-
31 
-type graphIDRegistrar interface {
32 
-	RegisterByGraphID(string, layer.ChainID, layer.DiffID, string, int64) (layer.Layer, error)
33 
-	Release(layer.Layer) ([]layer.Metadata, error)
34 
-	checksumCalculator
35 
-}
36 
-
37 
-type checksumCalculator interface {
38 
-	ChecksumForGraphID(id, parent, oldTarDataPath, newTarDataPath string) (diffID layer.DiffID, size int64, err error)
39 
-}
40 
-
41 
-type snapshotter struct {
42 
-	opt Opt
43 
-
44 
-	refs map[string]layer.Layer
45 
-	db   *bolt.DB
46 
-	mu   sync.Mutex
47 
-	reg  graphIDRegistrar
48 
-}
49 
-
50 
-var _ snapshot.SnapshotterBase = &snapshotter{}
51 
-
52 
-func NewSnapshotter(opt Opt) (snapshot.SnapshotterBase, error) {
53 
-	dbPath := filepath.Join(opt.Root, "snapshots.db")
54 
-	db, err := bolt.Open(dbPath, 0600, nil)
55 
-	if err != nil {
56 
-		return nil, errors.Wrapf(err, "failed to open database file %s", dbPath)
57 
-	}
58 
-
59 
-	reg, ok := opt.LayerStore.(graphIDRegistrar)
60 
-	if !ok {
61 
-		return nil, errors.Errorf("layerstore doesn't support graphID registration")
62 
-	}
63 
-
64 
-	s := &snapshotter{
65 
-		opt:  opt,
66 
-		db:   db,
67 
-		refs: map[string]layer.Layer{},
68 
-		reg:  reg,
69 
-	}
70 
-	return s, nil
71 
-}
72 
-
73 
-func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) error {
74 
-	origParent := parent
75 
-	if parent != "" {
76 
-		if l, err := s.getLayer(parent); err != nil {
77 
-			return err
78 
-		} else if l != nil {
79 
-			parent, err = getGraphID(l)
80 
-			if err != nil {
81 
-				return err
82 
-			}
83 
-		} else {
84 
-			parent, _ = s.getGraphDriverID(parent)
85 
-		}
86 
-	}
87 
-	if err := s.opt.GraphDriver.Create(key, parent, nil); err != nil {
88 
-		return err
89 
-	}
90 
-	if err := s.db.Update(func(tx *bolt.Tx) error {
91 
-		b, err := tx.CreateBucketIfNotExists([]byte(key))
92 
-		if err != nil {
93 
-			return err
94 
-		}
95 
-
96 
-		if err := b.Put(keyParent, []byte(origParent)); err != nil {
97 
-			return err
98 
-		}
99 
-		return nil
100 
-	}); err != nil {
101 
-		return err
102 
-	}
103 
-	return nil
104 
-}
105 
-
106 
-func (s *snapshotter) chainID(key string) (layer.ChainID, bool) {
107 
-	if strings.HasPrefix(key, "sha256:") {
108 
-		dgst, err := digest.Parse(key)
109 
-		if err != nil {
110 
-			return "", false
111 
-		}
112 
-		return layer.ChainID(dgst), true
113 
-	}
114 
-	return "", false
115 
-}
116 
-
117 
-func (s *snapshotter) getLayer(key string) (layer.Layer, error) {
118 
-	s.mu.Lock()
119 
-	l, ok := s.refs[key]
120 
-	if !ok {
121 
-		id, ok := s.chainID(key)
122 
-		if !ok {
123 
-			if err := s.db.View(func(tx *bolt.Tx) error {
124 
-				b := tx.Bucket([]byte(key))
125 
-				if b == nil {
126 
-					return nil
127 
-				}
128 
-				v := b.Get(keyChainID)
129 
-				if v != nil {
130 
-					id = layer.ChainID(v)
131 
-				}
132 
-				return nil
133 
-			}); err != nil {
134 
-				s.mu.Unlock()
135 
-				return nil, err
136 
-			}
137 
-			if id == "" {
138 
-				s.mu.Unlock()
139 
-				return nil, nil
140 
-			}
141 
-		}
142 
-		var err error
143 
-		l, err = s.opt.LayerStore.Get(id)
144 
-		if err != nil {
145 
-			s.mu.Unlock()
146 
-			return nil, err
147 
-		}
148 
-		s.refs[string(id)] = l
149 
-		if err := s.db.Update(func(tx *bolt.Tx) error {
150 
-			_, err := tx.CreateBucketIfNotExists([]byte(key))
151 
-			return err
152 
-		}); err != nil {
153 
-			s.mu.Unlock()
154 
-			return nil, err
155 
-		}
156 
-	}
157 
-	s.mu.Unlock()
158 
-
159 
-	return l, nil
160 
-}
161 
-
162 
-func (s *snapshotter) getGraphDriverID(key string) (string, bool) {
163 
-	var gdID string
164 
-	if err := s.db.View(func(tx *bolt.Tx) error {
165 
-		b := tx.Bucket([]byte(key))
166 
-		if b == nil {
167 
-			return errors.Errorf("not found") // TODO: typed
168 
-		}
169 
-		v := b.Get(keyCommitted)
170 
-		if v != nil {
171 
-			gdID = string(v)
172 
-		}
173 
-		return nil
174 
-	}); err != nil || gdID == "" {
175 
-		return key, false
176 
-	}
177 
-	return gdID, true
178 
-}
179 
-
180 
-func (s *snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, error) {
181 
-	if l, err := s.getLayer(key); err != nil {
182 
-		return snapshots.Info{}, err
183 
-	} else if l != nil {
184 
-		var parentID string
185 
-		if p := l.Parent(); p != nil {
186 
-			parentID = p.ChainID().String()
187 
-		}
188 
-		info := snapshots.Info{
189 
-			Kind:   snapshots.KindCommitted,
190 
-			Name:   key,
191 
-			Parent: parentID,
192 
-		}
193 
-		return info, nil
194 
-	}
195 
-
196 
-	inf := snapshots.Info{
197 
-		Kind: snapshots.KindActive,
198 
-	}
199 
-
200 
-	id, committed := s.getGraphDriverID(key)
201 
-	if committed {
202 
-		inf.Kind = snapshots.KindCommitted
203 
-	}
204 
-
205 
-	if err := s.db.View(func(tx *bolt.Tx) error {
206 
-		b := tx.Bucket([]byte(id))
207 
-		if b == nil {
208 
-			return errors.Errorf("not found") // TODO: typed
209 
-		}
210 
-		inf.Name = string(key)
211 
-		v := b.Get(keyParent)
212 
-		if v != nil {
213 
-			inf.Parent = string(v)
214 
-		}
215 
-		return nil
216 
-	}); err != nil {
217 
-		return snapshots.Info{}, err
218 
-	}
219 
-	return inf, nil
220 
-}
221 
-
222 
-func (s *snapshotter) Mounts(ctx context.Context, key string) (snapshot.MountFactory, error) {
223 
-	l, err := s.getLayer(key)
224 
-	if err != nil {
225 
-		return nil, err
226 
-	}
227 
-	if l != nil {
228 
-		id := identity.NewID()
229 
-		rwlayer, err := s.opt.LayerStore.CreateRWLayer(id, l.ChainID(), nil)
230 
-		if err != nil {
231 
-			return nil, err
232 
-		}
233 
-		rootfs, err := rwlayer.Mount("")
234 
-		if err != nil {
235 
-			return nil, err
236 
-		}
237 
-		mnt := []mount.Mount{{
238 
-			Source:  rootfs.Path(),
239 
-			Type:    "bind",
240 
-			Options: []string{"rbind"},
241 
-		}}
242 
-		return &constMountFactory{
243 
-			mounts: mnt,
244 
-			release: func() error {
245 
-				_, err := s.opt.LayerStore.ReleaseRWLayer(rwlayer)
246 
-				return err
247 
-			},
248 
-		}, nil
249 
-	}
250 
-
251 
-	id, _ := s.getGraphDriverID(key)
252 
-
253 
-	rootfs, err := s.opt.GraphDriver.Get(id, "")
254 
-	if err != nil {
255 
-		return nil, err
256 
-	}
257 
-	mnt := []mount.Mount{{
258 
-		Source:  rootfs.Path(),
259 
-		Type:    "bind",
260 
-		Options: []string{"rbind"},
261 
-	}}
262 
-	return &constMountFactory{
263 
-		mounts: mnt,
264 
-		release: func() error {
265 
-			return s.opt.GraphDriver.Put(id)
266 
-		},
267 
-	}, nil
268 
-}
269 
-
270 
-func (s *snapshotter) Remove(ctx context.Context, key string) error {
271 
-	l, err := s.getLayer(key)
272 
-	if err != nil {
273 
-		return err
274 
-	}
275 
-
276 
-	var found bool
277 
-	if err := s.db.Update(func(tx *bolt.Tx) error {
278 
-		found = tx.Bucket([]byte(key)) != nil
279 
-		if found {
280 
-			id, _ := s.getGraphDriverID(key)
281 
-			tx.DeleteBucket([]byte(key))
282 
-			if id != key {
283 
-				tx.DeleteBucket([]byte(id))
284 
-			}
285 
-		}
286 
-		return nil
287 
-	}); err != nil {
288 
-		return err
289 
-	}
290 
-
291 
-	if l != nil {
292 
-		_, err := s.opt.LayerStore.Release(l)
293 
-		return err
294 
-	}
295 
-
296 
-	if !found { // this happens when removing views
297 
-		return nil
298 
-	}
299 
-
300 
-	id, _ := s.getGraphDriverID(key)
301 
-	return s.opt.GraphDriver.Remove(id)
302 
-}
303 
-
304 
-func (s *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) error {
305 
-	if err := s.db.Update(func(tx *bolt.Tx) error {
306 
-		b, err := tx.CreateBucketIfNotExists([]byte(name))
307 
-		if err != nil {
308 
-			return err
309 
-		}
310 
-		if err := b.Put(keyCommitted, []byte(key)); err != nil {
311 
-			return err
312 
-		}
313 
-		return nil
314 
-	}); err != nil {
315 
-		return err
316 
-	}
317 
-	// logrus.Debugf("committed %s as %s", name, key));
318 
-	return nil
319 
-}
320 
-
321 
-func (s *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) (snapshot.MountFactory, error) {
322 
-	return s.Mounts(ctx, parent)
323 
-}
324 
-
325 
-func (s *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error) error {
326 
-	// allKeys := map[string]struct{}{}
327 
-	// commitedIDs := map[string]string{}
328 
-	// chainIDs := map[string]layer.ChainID{}
329 
-	//
330 
-	// if err := s.db.View(func(tx *bolt.Tx) error {
331 
-	// 	tx.ForEach(func(name []byte, b *bolt.Bucket) error {
332 
-	// 		allKeys[string(name)] = struct{}{}
333 
-	// 		v := b.Get(keyCommitted)
334 
-	// 		if v != nil {
335 
-	// 			commitedIDs[string(v)] = string(name)
336 
-	// 		}
337 
-	//
338 
-	// 		v = b.Get(keyChainID)
339 
-	// 		if v != nil {
340 
-	// 			logrus.Debugf("loaded layer %s %s", name, v)
341 
-	// 			chainIDs[string(name)] = layer.ChainID(v)
342 
-	// 		}
343 
-	// 		return nil
344 
-	// 	})
345 
-	// 	return nil
346 
-	// }); err != nil {
347 
-	// 	return err
348 
-	// }
349 
-	//
350 
-	// for k := range allKeys {
351 
-	// 	if chainID, ok := chainIDs[k]; ok {
352 
-	// 		s.mu.Lock()
353 
-	// 		if _, ok := s.refs[k]; !ok {
354 
-	// 			l, err := s.opt.LayerStore.Get(chainID)
355 
-	// 			if err != nil {
356 
-	// 				s.mu.Unlock()
357 
-	// 				return err
358 
-	// 			}
359 
-	// 			s.refs[k] = l
360 
-	// 		}
361 
-	// 		s.mu.Unlock()
362 
-	// 	}
363 
-	// 	if _, ok := commitedIDs[k]; ok {
364 
-	// 		continue
365 
-	// 	}
366 
-	//
367 
-	// 	if _, err := s.getLayer(k); err != nil {
368 
-	// 		s.Remove(ctx, k)
369 
-	// 		continue
370 
-	// 	}
371 
-	// 	info, err := s.Stat(ctx, k)
372 
-	// 	if err != nil {
373 
-	// 		s.Remove(ctx, k)
374 
-	// 		continue
375 
-	// 	}
376 
-	// 	if err := fn(ctx, info); err != nil {
377 
-	// 		return err
378 
-	// 	}
379 
-	// }
380 
-
381 
-	return errors.Errorf("not-implemented")
382 
-}
383 
-
384 
-func (s *snapshotter) Update(ctx context.Context, info snapshots.Info, fieldpaths ...string) (snapshots.Info, error) {
385 
-	// not implemented
386 
-	return s.Stat(ctx, info.Name)
387 
-}
388 
-
389 
-func (s *snapshotter) Usage(ctx context.Context, key string) (snapshots.Usage, error) {
390 
-	return snapshots.Usage{}, nil
391 
-}
392 
-
393 
-func (s *snapshotter) Close() error {
394 
-	return s.db.Close()
395 
-}
396 
-
397 
-type constMountFactory struct {
398 
-	mounts  []mount.Mount
399 
-	release func() error
400 
-}
401 
-
402 
-func (mf *constMountFactory) Mount() ([]mount.Mount, func() error, error) {
403 
-	release := mf.release
404 
-	if release == nil {
405 
-		release = func() error { return nil }
406 
-	}
407 
-	return mf.mounts, release, nil
408 
-}
builder/builder-next/worker/worker.go
History View file @ b742459
... ... 
@@ -24,8 +24,8 @@ import (
24 24 
 	"github.com/moby/buildkit/frontend"
25 25 
 	"github.com/moby/buildkit/session"
26 26 
 	"github.com/moby/buildkit/snapshot"
27 
-	"github.com/moby/buildkit/solver-next"
28 
-	"github.com/moby/buildkit/solver-next/llbsolver/ops"
27 
+	"github.com/moby/buildkit/solver"
28 
+	"github.com/moby/buildkit/solver/llbsolver/ops"
29 29 
 	"github.com/moby/buildkit/solver/pb"
30 30 
 	"github.com/moby/buildkit/source"
31 31 
 	"github.com/moby/buildkit/source/git"
... ... 
@@ -39,8 +39,6 @@ import (
39 39 
 	netcontext "golang.org/x/net/context"
40 40 
 )
41 41
42 
-// TODO: this file should be removed. containerd defines ContainerdWorker, oci defines OCIWorker. There is no base worker.
43 
-
44 42 
 // WorkerOpt is specific to a worker.
45 43 
 // See also CommonOpt.
46 44 
 type WorkerOpt struct {
... ... 
@@ -56,7 +54,6 @@ type WorkerOpt struct {
56 56 
 	Exporters         map[string]exporter.Exporter
57 57 
 	DownloadManager   distribution.RootFSDownloadManager
58 58 
 	V2MetadataService distmetadata.V2MetadataService
59 
-	// ImageStore     images.Store // optional
60 59 
 }
61 60
62 61 
 // Worker is a local worker instance with dedicated snapshotter, cache, and so on.
... ... 
@@ -64,8 +61,6 @@ type WorkerOpt struct {
64 64 
 type Worker struct {
65 65 
 	WorkerOpt
66 66 
 	SourceManager *source.Manager
67 
-	// Exporters     map[string]exporter.Exporter
68 
-	// ImageSource source.Source
69 67 
 }
70 68
71 69 
 // NewWorker instantiates a local worker
... ... 
@@ -177,36 +172,7 @@ func (w *Worker) Exporter(name string) (exporter.Exporter, error) {
177 177 
 	return exp, nil
178 178 
 }
179 179
180 
-func (w *Worker) GetRemote(ctx context.Context, ref cache.ImmutableRef) (*solver.Remote, error) {
181 
-	// diffPairs, err := blobs.GetDiffPairs(ctx, w.ContentStore, w.Snapshotter, w.Differ, ref)
182 
-	// if err != nil {
183 
-	// 	return nil, errors.Wrap(err, "failed calculaing diff pairs for exported snapshot")
184 
-	// }
185 
-	// if len(diffPairs) == 0 {
186 
-	// 	return nil, nil
187 
-	// }
188 
-	//
189 
-	// descs := make([]ocispec.Descriptor, len(diffPairs))
190 
-	//
191 
-	// for i, dp := range diffPairs {
192 
-	// 	info, err := w.ContentStore.Info(ctx, dp.Blobsum)
193 
-	// 	if err != nil {
194 
-	// 		return nil, err
195 
-	// 	}
196 
-	// 	descs[i] = ocispec.Descriptor{
197 
-	// 		Digest:    dp.Blobsum,
198 
-	// 		Size:      info.Size,
199 
-	// 		MediaType: schema2.MediaTypeLayer,
200 
-	// 		Annotations: map[string]string{
201 
-	// 			"containerd.io/uncompressed": dp.DiffID.String(),
202 
-	// 		},
203 
-	// 	}
204 
-	// }
205 
-	//
206 
-	// return &solver.Remote{
207 
-	// 	Descriptors: descs,
208 
-	// 	Provider:    w.ContentStore,
209 
-	// }, nil
180 
+func (w *Worker) GetRemote(ctx context.Context, ref cache.ImmutableRef, createIfNeeded bool) (*solver.Remote, error) {
210 181 
 	return nil, errors.Errorf("getremote not implemented")
211 182 
 }
212 183
... ... 
@@ -226,7 +192,6 @@ func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.I
226 226 
 			provider: remote.Provider,
227 227 
 			w:        w,
228 228 
 			pctx:     ctx,
229 
-			// ref:      l.Blob.Digest.String(),
230 229 
 		})
231 230 
 	}
232 231
... ... 
@@ -243,73 +208,13 @@ func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.I
243 243 
 	}
244 244 
 	defer release()
245 245
246 
-	ref, err := w.CacheManager.Get(ctx, string(rootFS.ChainID()), cache.WithDescription(fmt.Sprintf("imported %s", remote.Descriptors[len(remote.Descriptors)-1].Digest)))
246 
+	ref, err := w.CacheManager.GetFromSnapshotter(ctx, string(rootFS.ChainID()), cache.WithDescription(fmt.Sprintf("imported %s", remote.Descriptors[len(remote.Descriptors)-1].Digest)))
247 247 
 	if err != nil {
248 248 
 		return nil, err
249 249 
 	}
250 
-
251 
-	// eg, gctx := errgroup.WithContext(ctx)
252 
-	// for _, desc := range remote.Descriptors {
253 
-	// 	func(desc ocispec.Descriptor) {
254 
-	// 		eg.Go(func() error {
255 
-	// 			done := oneOffProgress(ctx, fmt.Sprintf("pulling %s", desc.Digest))
256 
-	// 			return done(contentutil.Copy(gctx, w.ContentStore, remote.Provider, desc))
257 
-	// 		})
258 
-	// 	}(desc)
259 
-	// }
260 
-	//
261 
-	// if err := eg.Wait(); err != nil {
262 
-	// 	return nil, err
263 
-	// }
264 
-	//
265 
-	// csh, release := snapshot.NewCompatibilitySnapshotter(w.Snapshotter)
266 
-	// defer release()
267 
-	//
268 
-	// unpackProgressDone := oneOffProgress(ctx, "unpacking")
269 
-	// chainID, err := w.unpack(ctx, remote.Descriptors, csh)
270 
-	// if err != nil {
271 
-	// 	return nil, unpackProgressDone(err)
272 
-	// }
273 
-	// unpackProgressDone(nil)
274 
-	//
275 
-	// return w.CacheManager.Get(ctx, chainID, cache.WithDescription(fmt.Sprintf("imported %s", remote.Descriptors[len(remote.Descriptors)-1].Digest)))
276 
-	// return nil, errors.Errorf("fromremote not implemented")
277 250 
 	return ref, nil
278 251 
 }
279 252
280 
-// utility function. could be moved to the constructor logic?
281 
-// func Labels(executor, snapshotter string) map[string]string {
282 
-// 	hostname, err := os.Hostname()
283 
-// 	if err != nil {
284 
-// 		hostname = "unknown"
285 
-// 	}
286 
-// 	labels := map[string]string{
287 
-// 		worker.LabelOS:          runtime.GOOS,
288 
-// 		worker.LabelArch:        runtime.GOOSARCH,
289 
-// 		worker.LabelExecutor:    executor,
290 
-// 		worker.LabelSnapshotter: snapshotter,
291 
-// 		worker.LabelHostname:    hostname,
292 
-// 	}
293 
-// 	return labels
294 
-// }
295 
-//
296 
-// // ID reads the worker id from the `workerid` file.
297 
-// // If not exist, it creates a random one,
298 
-// func ID(root string) (string, error) {
299 
-// 	f := filepath.Join(root, "workerid")
300 
-// 	b, err := ioutil.ReadFile(f)
301 
-// 	if err != nil {
302 
-// 		if os.IsNotExist(err) {
303 
-// 			id := identity.NewID()
304 
-// 			err := ioutil.WriteFile(f, []byte(id), 0400)
305 
-// 			return id, err
306 
-// 		} else {
307 
-// 			return "", err
308 
-// 		}
309 
-// 	}
310 
-// 	return string(b), nil
311 
-// }
312 
-
313 253 
 type discardProgress struct{}
314 254
315 255 
 func (_ *discardProgress) WriteProgress(_ pkgprogress.Progress) error {