Browse code

builder: add support for building from tarball

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

Tonis Tiigi authored on 2018/05/18 14:47:34
Showing 8 changed files
... ...
@@ -3,7 +3,6 @@ package build // import "github.com/docker/docker/api/server/backend/build"
3 3
 import (
4 4
 	"context"
5 5
 	"fmt"
6
-	"strings"
7 6
 
8 7
 	"github.com/docker/distribution/reference"
9 8
 	"github.com/docker/docker/api/types"
... ...
@@ -44,11 +43,7 @@ func NewBackend(components ImageComponent, builder Builder, fsCache *fscache.FSC
44 44
 // Build builds an image from a Source
45 45
 func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string, error) {
46 46
 	options := config.Options
47
-	useBuildKit := false
48
-	if strings.HasPrefix(options.SessionID, "buildkit:") {
49
-		useBuildKit = true
50
-		options.SessionID = strings.TrimPrefix(options.SessionID, "buildkit:")
51
-	}
47
+	useBuildKit := options.Version == types.BuilderBuildKit
52 48
 
53 49
 	tagger, err := NewTagger(b.imageComponent, config.ProgressWriter.StdoutFormatter, options.Tags)
54 50
 	if err != nil {
... ...
@@ -146,10 +146,25 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
146 146
 	}
147 147
 	options.SessionID = r.FormValue("session")
148 148
 	options.BuildID = r.FormValue("buildid")
149
+	builderVersion, err := parseVersion(r.FormValue("version"))
150
+	if err != nil {
151
+		return nil, err
152
+	}
153
+	options.Version = builderVersion
149 154
 
150 155
 	return options, nil
151 156
 }
152 157
 
158
+func parseVersion(s string) (types.BuilderVersion, error) {
159
+	if s == "" || s == string(types.BuilderV1) {
160
+		return types.BuilderV1, nil
161
+	}
162
+	if s == string(types.BuilderBuildKit) {
163
+		return types.BuilderBuildKit, nil
164
+	}
165
+	return "", errors.Errorf("invalid version %s", s)
166
+}
167
+
153 168
 func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
154 169
 	report, err := br.backend.PruneCache(ctx)
155 170
 	if err != nil {
... ...
@@ -181,9 +181,18 @@ type ImageBuildOptions struct {
181 181
 	Target      string
182 182
 	SessionID   string
183 183
 	Platform    string
184
+	Version     BuilderVersion
184 185
 	BuildID     string
185 186
 }
186 187
 
188
+// BuilderVersion sets the version of underlying builder to use
189
+type BuilderVersion string
190
+
191
+const (
192
+	BuilderV1       BuilderVersion = "1"
193
+	BuilderBuildKit                = "2"
194
+)
195
+
187 196
 // ImageBuildResponse holds information
188 197
 // returned by a server after building
189 198
 // an image.
... ...
@@ -17,6 +17,7 @@ import (
17 17
 	"github.com/moby/buildkit/control"
18 18
 	"github.com/moby/buildkit/identity"
19 19
 	"github.com/moby/buildkit/session"
20
+	"github.com/moby/buildkit/util/tracing"
20 21
 	"github.com/pkg/errors"
21 22
 	"golang.org/x/sync/errgroup"
22 23
 	grpcmetadata "google.golang.org/grpc/metadata"
... ...
@@ -29,20 +30,24 @@ type Opt struct {
29 29
 }
30 30
 
31 31
 type Builder struct {
32
-	controller *control.Controller
32
+	controller     *control.Controller
33
+	reqBodyHandler *reqBodyHandler
33 34
 
34 35
 	mu   sync.Mutex
35 36
 	jobs map[string]func()
36 37
 }
37 38
 
38 39
 func New(opt Opt) (*Builder, error) {
39
-	c, err := newController(opt)
40
+	reqHandler := newReqBodyHandler(tracing.DefaultTransport)
41
+
42
+	c, err := newController(reqHandler, opt)
40 43
 	if err != nil {
41 44
 		return nil, err
42 45
 	}
43 46
 	b := &Builder{
44
-		controller: c,
45
-		jobs:       map[string]func(){},
47
+		controller:     c,
48
+		reqBodyHandler: reqHandler,
49
+		jobs:           map[string]func(){},
46 50
 	}
47 51
 	return b, nil
48 52
 }
... ...
@@ -133,7 +138,13 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
133 133
 	}
134 134
 
135 135
 	if opt.Options.RemoteContext != "" {
136
-		frontendAttrs["context"] = opt.Options.RemoteContext
136
+		if opt.Options.RemoteContext != "client-session" {
137
+			frontendAttrs["context"] = opt.Options.RemoteContext
138
+		}
139
+	} else {
140
+		url, cancel := b.reqBodyHandler.newRequest(opt.Source)
141
+		defer cancel()
142
+		frontendAttrs["context"] = url
137 143
 	}
138 144
 
139 145
 	var cacheFrom []string
... ...
@@ -1,6 +1,7 @@
1 1
 package buildkit
2 2
 
3 3
 import (
4
+	"net/http"
4 5
 	"os"
5 6
 	"path/filepath"
6 7
 
... ...
@@ -24,7 +25,7 @@ import (
24 24
 	"github.com/pkg/errors"
25 25
 )
26 26
 
27
-func newController(opt Opt) (*control.Controller, error) {
27
+func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
28 28
 	if err := os.MkdirAll(opt.Root, 0700); err != nil {
29 29
 		return nil, err
30 30
 	}
... ...
@@ -133,6 +134,7 @@ func newController(opt Opt) (*control.Controller, error) {
133 133
 		Exporters: map[string]exporter.Exporter{
134 134
 			"moby": exp,
135 135
 		},
136
+		Transport: rt,
136 137
 	}
137 138
 
138 139
 	wc := &worker.Controller{}
139 140
new file mode 100644
... ...
@@ -0,0 +1,74 @@
0
+package buildkit
1
+
2
+import (
3
+	"bufio"
4
+	"io"
5
+	"net/http"
6
+	"strings"
7
+	"sync"
8
+
9
+	"github.com/moby/buildkit/identity"
10
+	"github.com/pkg/errors"
11
+)
12
+
13
+const urlPrefix = "build-context-"
14
+
15
+type reqBodyHandler struct {
16
+	mu sync.Mutex
17
+	rt http.RoundTripper
18
+
19
+	requests map[string]io.ReadCloser
20
+}
21
+
22
+func newReqBodyHandler(rt http.RoundTripper) *reqBodyHandler {
23
+	return &reqBodyHandler{
24
+		rt:       rt,
25
+		requests: map[string]io.ReadCloser{},
26
+	}
27
+}
28
+
29
+func (h *reqBodyHandler) newRequest(rc io.ReadCloser) (string, func()) {
30
+	// handle expect-continue vs chunked output
31
+	r := bufio.NewReader(rc)
32
+	r.Peek(1)
33
+	id := identity.NewID()
34
+	h.mu.Lock()
35
+	h.requests[id] = &readCloser{Reader: r, Closer: rc}
36
+	h.mu.Unlock()
37
+	return "http://" + urlPrefix + id, func() {
38
+		h.mu.Lock()
39
+		delete(h.requests, id)
40
+		h.mu.Unlock()
41
+	}
42
+}
43
+
44
+func (h *reqBodyHandler) RoundTrip(req *http.Request) (*http.Response, error) {
45
+	host := req.URL.Host
46
+	if strings.HasPrefix(host, urlPrefix) {
47
+		if req.Method != "GET" {
48
+			return nil, errors.Errorf("invalid request")
49
+		}
50
+		id := strings.TrimPrefix(host, urlPrefix)
51
+		h.mu.Lock()
52
+		rc, ok := h.requests[id]
53
+		delete(h.requests, id)
54
+		h.mu.Unlock()
55
+
56
+		if !ok {
57
+			return nil, errors.Errorf("context not found")
58
+		}
59
+
60
+		return &http.Response{
61
+			Status:        "200 OK",
62
+			StatusCode:    200,
63
+			Body:          rc,
64
+			ContentLength: -1,
65
+		}, nil
66
+	}
67
+	return h.rt.RoundTrip(req)
68
+}
69
+
70
+type readCloser struct {
71
+	io.Reader
72
+	io.Closer
73
+}
... ...
@@ -5,6 +5,7 @@ import (
5 5
 	"fmt"
6 6
 	"io"
7 7
 	"io/ioutil"
8
+	nethttp "net/http"
8 9
 	"runtime"
9 10
 	"time"
10 11
 
... ...
@@ -53,6 +54,7 @@ type WorkerOpt struct {
53 53
 	Exporters         map[string]exporter.Exporter
54 54
 	DownloadManager   distribution.RootFSDownloadManager
55 55
 	V2MetadataService distmetadata.V2MetadataService
56
+	Transport         nethttp.RoundTripper
56 57
 }
57 58
 
58 59
 // Worker is a local worker instance with dedicated snapshotter, cache, and so on.
... ...
@@ -85,6 +87,7 @@ func NewWorker(opt WorkerOpt) (*Worker, error) {
85 85
 	hs, err := http.NewSource(http.Opt{
86 86
 		CacheAccessor: cm,
87 87
 		MetadataStore: opt.MetadataStore,
88
+		Transport:     opt.Transport,
88 89
 	})
89 90
 	if err != nil {
90 91
 		return nil, err
... ...
@@ -125,7 +128,7 @@ func (w *Worker) ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge) (solve
125 125
 	case *pb.Op_Source:
126 126
 		return ops.NewSourceOp(v, op, w.SourceManager, w)
127 127
 	case *pb.Op_Exec:
128
-		return ops.NewExecOp(v, op, w.CacheManager, w.Executor, w)
128
+		return ops.NewExecOp(v, op, w.CacheManager, w.MetadataStore, w.Executor, w)
129 129
 	case *pb.Op_Build:
130 130
 		return ops.NewBuildOp(v, op, s, w)
131 131
 	default:
... ...
@@ -136,5 +136,6 @@ func (cli *Client) imageBuildOptionsToQuery(options types.ImageBuildOptions) (ur
136 136
 	if options.BuildID != "" {
137 137
 		query.Set("buildid", options.BuildID)
138 138
 	}
139
+	query.Set("version", string(options.Version))
139 140
 	return query, nil
140 141
 }