Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
| ... | ... |
@@ -31,13 +31,11 @@ import ( |
| 31 | 31 |
"github.com/moby/buildkit/cache" |
| 32 | 32 |
"github.com/moby/buildkit/client/llb" |
| 33 | 33 |
"github.com/moby/buildkit/session" |
| 34 |
- "github.com/moby/buildkit/session/auth" |
|
| 35 | 34 |
"github.com/moby/buildkit/source" |
| 36 | 35 |
"github.com/moby/buildkit/util/flightcontrol" |
| 37 | 36 |
"github.com/moby/buildkit/util/imageutil" |
| 38 | 37 |
"github.com/moby/buildkit/util/progress" |
| 39 | 38 |
"github.com/moby/buildkit/util/resolver" |
| 40 |
- "github.com/moby/buildkit/util/tracing" |
|
| 41 | 39 |
digest "github.com/opencontainers/go-digest" |
| 42 | 40 |
"github.com/opencontainers/image-spec/identity" |
| 43 | 41 |
ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
| ... | ... |
@@ -54,7 +52,7 @@ type SourceOpt struct {
|
| 54 | 54 |
DownloadManager distribution.RootFSDownloadManager |
| 55 | 55 |
MetadataStore metadata.V2MetadataService |
| 56 | 56 |
ImageStore image.Store |
| 57 |
- ResolverOpt resolver.ResolveOptionsFunc |
|
| 57 |
+ RegistryHosts docker.RegistryHosts |
|
| 58 | 58 |
LayerStore layer.Store |
| 59 | 59 |
} |
| 60 | 60 |
|
| ... | ... |
@@ -78,44 +76,15 @@ func (is *imageSource) ID() string {
|
| 78 | 78 |
return source.DockerImageScheme |
| 79 | 79 |
} |
| 80 | 80 |
|
| 81 |
-func (is *imageSource) getResolver(ctx context.Context, rfn resolver.ResolveOptionsFunc, ref string, sm *session.Manager) remotes.Resolver {
|
|
| 81 |
+func (is *imageSource) getResolver(ctx context.Context, hosts docker.RegistryHosts, ref string, sm *session.Manager) remotes.Resolver {
|
|
| 82 | 82 |
if res := is.resolverCache.Get(ctx, ref); res != nil {
|
| 83 | 83 |
return res |
| 84 | 84 |
} |
| 85 |
- |
|
| 86 |
- opt := docker.ResolverOptions{
|
|
| 87 |
- Client: tracing.DefaultClient, |
|
| 88 |
- } |
|
| 89 |
- if rfn != nil {
|
|
| 90 |
- opt = rfn(ref) |
|
| 91 |
- } |
|
| 92 |
- opt.Credentials = is.getCredentialsFromSession(ctx, sm) |
|
| 93 |
- r := docker.NewResolver(opt) |
|
| 85 |
+ r := resolver.New(ctx, hosts, sm) |
|
| 94 | 86 |
r = is.resolverCache.Add(ctx, ref, r) |
| 95 | 87 |
return r |
| 96 | 88 |
} |
| 97 | 89 |
|
| 98 |
-func (is *imageSource) getCredentialsFromSession(ctx context.Context, sm *session.Manager) func(string) (string, string, error) {
|
|
| 99 |
- id := session.FromContext(ctx) |
|
| 100 |
- if id == "" {
|
|
| 101 |
- // can be removed after containerd/containerd#2812 |
|
| 102 |
- return func(string) (string, string, error) {
|
|
| 103 |
- return "", "", nil |
|
| 104 |
- } |
|
| 105 |
- } |
|
| 106 |
- return func(host string) (string, string, error) {
|
|
| 107 |
- timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) |
|
| 108 |
- defer cancel() |
|
| 109 |
- |
|
| 110 |
- caller, err := sm.Get(timeoutCtx, id) |
|
| 111 |
- if err != nil {
|
|
| 112 |
- return "", "", err |
|
| 113 |
- } |
|
| 114 |
- |
|
| 115 |
- return auth.CredentialsFunc(tracing.ContextWithSpanFromContext(context.TODO(), ctx), caller)(host) |
|
| 116 |
- } |
|
| 117 |
-} |
|
| 118 |
- |
|
| 119 | 90 |
func (is *imageSource) resolveLocal(refStr string) (*image.Image, error) {
|
| 120 | 91 |
ref, err := distreference.ParseNormalizedNamed(refStr) |
| 121 | 92 |
if err != nil {
|
| ... | ... |
@@ -138,7 +107,7 @@ func (is *imageSource) resolveRemote(ctx context.Context, ref string, platform * |
| 138 | 138 |
dt []byte |
| 139 | 139 |
} |
| 140 | 140 |
res, err := is.g.Do(ctx, ref, func(ctx context.Context) (interface{}, error) {
|
| 141 |
- dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref, sm), is.ContentStore, nil, platform) |
|
| 141 |
+ dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.RegistryHosts, ref, sm), is.ContentStore, nil, platform) |
|
| 142 | 142 |
if err != nil {
|
| 143 | 143 |
return nil, err |
| 144 | 144 |
} |
| ... | ... |
@@ -208,7 +177,7 @@ func (is *imageSource) Resolve(ctx context.Context, id source.Identifier, sm *se |
| 208 | 208 |
p := &puller{
|
| 209 | 209 |
src: imageIdentifier, |
| 210 | 210 |
is: is, |
| 211 |
- resolver: is.getResolver(ctx, is.ResolverOpt, imageIdentifier.Reference.String(), sm), |
|
| 211 |
+ resolver: is.getResolver(ctx, is.RegistryHosts, imageIdentifier.Reference.String(), sm), |
|
| 212 | 212 |
platform: platform, |
| 213 | 213 |
sm: sm, |
| 214 | 214 |
} |
| ... | ... |
@@ -7,6 +7,7 @@ import ( |
| 7 | 7 |
|
| 8 | 8 |
"github.com/containerd/containerd/content" |
| 9 | 9 |
"github.com/containerd/containerd/images" |
| 10 |
+ "github.com/containerd/containerd/remotes/docker" |
|
| 10 | 11 |
distreference "github.com/docker/distribution/reference" |
| 11 | 12 |
imagestore "github.com/docker/docker/image" |
| 12 | 13 |
"github.com/docker/docker/reference" |
| ... | ... |
@@ -15,7 +16,6 @@ import ( |
| 15 | 15 |
v1 "github.com/moby/buildkit/cache/remotecache/v1" |
| 16 | 16 |
"github.com/moby/buildkit/session" |
| 17 | 17 |
"github.com/moby/buildkit/solver" |
| 18 |
- "github.com/moby/buildkit/util/resolver" |
|
| 19 | 18 |
"github.com/moby/buildkit/worker" |
| 20 | 19 |
digest "github.com/opencontainers/go-digest" |
| 21 | 20 |
specs "github.com/opencontainers/image-spec/specs-go/v1" |
| ... | ... |
@@ -23,9 +23,9 @@ import ( |
| 23 | 23 |
) |
| 24 | 24 |
|
| 25 | 25 |
// ResolveCacheImporterFunc returns a resolver function for local inline cache |
| 26 |
-func ResolveCacheImporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc, cs content.Store, rs reference.Store, is imagestore.Store) remotecache.ResolveCacheImporterFunc {
|
|
| 26 |
+func ResolveCacheImporterFunc(sm *session.Manager, resolverFunc docker.RegistryHosts, cs content.Store, rs reference.Store, is imagestore.Store) remotecache.ResolveCacheImporterFunc {
|
|
| 27 | 27 |
|
| 28 |
- upstream := registryremotecache.ResolveCacheImporterFunc(sm, cs, resolverOpt) |
|
| 28 |
+ upstream := registryremotecache.ResolveCacheImporterFunc(sm, cs, resolverFunc) |
|
| 29 | 29 |
|
| 30 | 30 |
return func(ctx context.Context, attrs map[string]string) (remotecache.Importer, specs.Descriptor, error) {
|
| 31 | 31 |
if dt, err := tryImportLocal(rs, is, attrs["ref"]); err == nil {
|
| ... | ... |
@@ -11,6 +11,7 @@ import ( |
| 11 | 11 |
"time" |
| 12 | 12 |
|
| 13 | 13 |
"github.com/containerd/containerd/platforms" |
| 14 |
+ "github.com/containerd/containerd/remotes/docker" |
|
| 14 | 15 |
"github.com/docker/docker/api/types" |
| 15 | 16 |
"github.com/docker/docker/api/types/backend" |
| 16 | 17 |
"github.com/docker/docker/builder" |
| ... | ... |
@@ -26,7 +27,6 @@ import ( |
| 26 | 26 |
"github.com/moby/buildkit/identity" |
| 27 | 27 |
"github.com/moby/buildkit/session" |
| 28 | 28 |
"github.com/moby/buildkit/util/entitlements" |
| 29 |
- "github.com/moby/buildkit/util/resolver" |
|
| 30 | 29 |
"github.com/moby/buildkit/util/tracing" |
| 31 | 30 |
"github.com/pkg/errors" |
| 32 | 31 |
"golang.org/x/sync/errgroup" |
| ... | ... |
@@ -70,7 +70,7 @@ type Opt struct {
|
| 70 | 70 |
Dist images.DistributionServices |
| 71 | 71 |
NetworkController libnetwork.NetworkController |
| 72 | 72 |
DefaultCgroupParent string |
| 73 |
- ResolverOpt resolver.ResolveOptionsFunc |
|
| 73 |
+ RegistryHosts docker.RegistryHosts |
|
| 74 | 74 |
BuilderConfig config.BuilderConfig |
| 75 | 75 |
Rootless bool |
| 76 | 76 |
IdentityMapping *idtools.IdentityMapping |
| ... | ... |
@@ -119,7 +119,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
|
| 119 | 119 |
MetadataStore: dist.V2MetadataService, |
| 120 | 120 |
ImageStore: dist.ImageStore, |
| 121 | 121 |
ReferenceStore: dist.ReferenceStore, |
| 122 |
- ResolverOpt: opt.ResolverOpt, |
|
| 122 |
+ RegistryHosts: opt.RegistryHosts, |
|
| 123 | 123 |
LayerStore: dist.LayerStore, |
| 124 | 124 |
}) |
| 125 | 125 |
if err != nil {
|
| ... | ... |
@@ -210,7 +210,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
|
| 210 | 210 |
Frontends: frontends, |
| 211 | 211 |
CacheKeyStorage: cacheStorage, |
| 212 | 212 |
ResolveCacheImporterFuncs: map[string]remotecache.ResolveCacheImporterFunc{
|
| 213 |
- "registry": localinlinecache.ResolveCacheImporterFunc(opt.SessionManager, opt.ResolverOpt, store, dist.ReferenceStore, dist.ImageStore), |
|
| 213 |
+ "registry": localinlinecache.ResolveCacheImporterFunc(opt.SessionManager, opt.RegistryHosts, store, dist.ReferenceStore, dist.ImageStore), |
|
| 214 | 214 |
"local": localremotecache.ResolveCacheImporterFunc(opt.SessionManager), |
| 215 | 215 |
}, |
| 216 | 216 |
ResolveCacheExporterFuncs: map[string]remotecache.ResolveCacheExporterFunc{
|
| ... | ... |
@@ -292,7 +292,7 @@ func newRouterOptions(config *config.Config, d *daemon.Daemon) (routerOptions, e |
| 292 | 292 |
Dist: d.DistributionServices(), |
| 293 | 293 |
NetworkController: d.NetworkController(), |
| 294 | 294 |
DefaultCgroupParent: cgroupParent, |
| 295 |
- ResolverOpt: d.NewResolveOptionsFunc(), |
|
| 295 |
+ RegistryHosts: d.RegistryHosts(), |
|
| 296 | 296 |
BuilderConfig: config.Builder, |
| 297 | 297 |
Rootless: d.Rootless(), |
| 298 | 298 |
IdentityMapping: d.IdentityMapping(), |
| ... | ... |
@@ -9,7 +9,6 @@ import ( |
| 9 | 9 |
"context" |
| 10 | 10 |
"fmt" |
| 11 | 11 |
"io/ioutil" |
| 12 |
- "math/rand" |
|
| 13 | 12 |
"net" |
| 14 | 13 |
"net/url" |
| 15 | 14 |
"os" |
| ... | ... |
@@ -28,7 +27,6 @@ import ( |
| 28 | 28 |
"github.com/containerd/containerd/defaults" |
| 29 | 29 |
"github.com/containerd/containerd/pkg/dialer" |
| 30 | 30 |
"github.com/containerd/containerd/remotes/docker" |
| 31 |
- "github.com/docker/distribution/reference" |
|
| 32 | 31 |
"github.com/docker/docker/api/types" |
| 33 | 32 |
containertypes "github.com/docker/docker/api/types/container" |
| 34 | 33 |
"github.com/docker/docker/api/types/swarm" |
| ... | ... |
@@ -42,8 +40,8 @@ import ( |
| 42 | 42 |
"github.com/docker/docker/daemon/logger" |
| 43 | 43 |
"github.com/docker/docker/daemon/network" |
| 44 | 44 |
"github.com/docker/docker/errdefs" |
| 45 |
+ bkconfig "github.com/moby/buildkit/cmd/buildkitd/config" |
|
| 45 | 46 |
"github.com/moby/buildkit/util/resolver" |
| 46 |
- "github.com/moby/buildkit/util/tracing" |
|
| 47 | 47 |
"github.com/sirupsen/logrus" |
| 48 | 48 |
|
| 49 | 49 |
// register graph drivers |
| ... | ... |
@@ -153,63 +151,46 @@ func (daemon *Daemon) Features() *map[string]bool {
|
| 153 | 153 |
return &daemon.configStore.Features |
| 154 | 154 |
} |
| 155 | 155 |
|
| 156 |
-// NewResolveOptionsFunc returns a call back function to resolve "registry-mirrors" and |
|
| 157 |
-// "insecure-registries" for buildkit |
|
| 158 |
-func (daemon *Daemon) NewResolveOptionsFunc() resolver.ResolveOptionsFunc {
|
|
| 159 |
- return func(ref string) docker.ResolverOptions {
|
|
| 160 |
- var ( |
|
| 161 |
- registryKey = "docker.io" |
|
| 162 |
- mirrors = make([]string, len(daemon.configStore.Mirrors)) |
|
| 163 |
- m = map[string]resolver.RegistryConf{}
|
|
| 164 |
- ) |
|
| 165 |
- // must trim "https://" or "http://" prefix |
|
| 166 |
- for i, v := range daemon.configStore.Mirrors {
|
|
| 167 |
- if uri, err := url.Parse(v); err == nil {
|
|
| 168 |
- v = uri.Host |
|
| 169 |
- } |
|
| 170 |
- mirrors[i] = v |
|
| 171 |
- } |
|
| 172 |
- // set "registry-mirrors" |
|
| 173 |
- m[registryKey] = resolver.RegistryConf{Mirrors: mirrors}
|
|
| 174 |
- // set "insecure-registries" |
|
| 175 |
- for _, v := range daemon.configStore.InsecureRegistries {
|
|
| 176 |
- if uri, err := url.Parse(v); err == nil {
|
|
| 177 |
- v = uri.Host |
|
| 178 |
- } |
|
| 179 |
- plainHTTP := true |
|
| 180 |
- m[v] = resolver.RegistryConf{
|
|
| 181 |
- PlainHTTP: &plainHTTP, |
|
| 182 |
- } |
|
| 183 |
- } |
|
| 184 |
- def := docker.ResolverOptions{
|
|
| 185 |
- Client: tracing.DefaultClient, |
|
| 186 |
- } |
|
| 187 |
- |
|
| 188 |
- parsed, err := reference.ParseNormalizedNamed(ref) |
|
| 189 |
- if err != nil {
|
|
| 190 |
- return def |
|
| 156 |
+// RegistryHosts returns registry configuration in containerd resolvers format |
|
| 157 |
+func (daemon *Daemon) RegistryHosts() docker.RegistryHosts {
|
|
| 158 |
+ var ( |
|
| 159 |
+ registryKey = "docker.io" |
|
| 160 |
+ mirrors = make([]string, len(daemon.configStore.Mirrors)) |
|
| 161 |
+ m = map[string]bkconfig.RegistryConfig{}
|
|
| 162 |
+ ) |
|
| 163 |
+ // must trim "https://" or "http://" prefix |
|
| 164 |
+ for i, v := range daemon.configStore.Mirrors {
|
|
| 165 |
+ if uri, err := url.Parse(v); err == nil {
|
|
| 166 |
+ v = uri.Host |
|
| 191 | 167 |
} |
| 192 |
- host := reference.Domain(parsed) |
|
| 193 |
- |
|
| 194 |
- c, ok := m[host] |
|
| 195 |
- if !ok {
|
|
| 196 |
- return def |
|
| 197 |
- } |
|
| 198 |
- |
|
| 199 |
- if len(c.Mirrors) > 0 {
|
|
| 200 |
- // TODO ResolverOptions.Host is deprecated; ResolverOptions.Hosts should be used |
|
| 201 |
- def.Host = func(string) (string, error) {
|
|
| 202 |
- return c.Mirrors[rand.Intn(len(c.Mirrors))], nil |
|
| 168 |
+ mirrors[i] = v |
|
| 169 |
+ } |
|
| 170 |
+ // set mirrors for default registry |
|
| 171 |
+ m[registryKey] = bkconfig.RegistryConfig{Mirrors: mirrors}
|
|
| 172 |
+ |
|
| 173 |
+ for _, v := range daemon.configStore.InsecureRegistries {
|
|
| 174 |
+ u, err := url.Parse(v) |
|
| 175 |
+ c := bkconfig.RegistryConfig{}
|
|
| 176 |
+ if err == nil {
|
|
| 177 |
+ v = u.Host |
|
| 178 |
+ t := true |
|
| 179 |
+ if u.Scheme == "http" {
|
|
| 180 |
+ c.PlainHTTP = &t |
|
| 181 |
+ } else {
|
|
| 182 |
+ c.Insecure = &t |
|
| 203 | 183 |
} |
| 204 | 184 |
} |
| 185 |
+ m[v] = c |
|
| 186 |
+ } |
|
| 205 | 187 |
|
| 206 |
- // TODO ResolverOptions.PlainHTTP is deprecated; ResolverOptions.Hosts should be used |
|
| 207 |
- if c.PlainHTTP != nil {
|
|
| 208 |
- def.PlainHTTP = *c.PlainHTTP |
|
| 188 |
+ for k, v := range m {
|
|
| 189 |
+ if d, err := registry.HostCertsDir(k); err == nil {
|
|
| 190 |
+ v.TLSConfigDir = []string{d}
|
|
| 191 |
+ m[k] = v |
|
| 209 | 192 |
} |
| 210 |
- |
|
| 211 |
- return def |
|
| 212 | 193 |
} |
| 194 |
+ |
|
| 195 |
+ return resolver.NewRegistryConfig(m) |
|
| 213 | 196 |
} |
| 214 | 197 |
|
| 215 | 198 |
func (daemon *Daemon) restore() error {
|
| ... | ... |
@@ -26,6 +26,24 @@ var ( |
| 26 | 26 |
ErrAlreadyExists = errors.New("Image already exists")
|
| 27 | 27 |
) |
| 28 | 28 |
|
| 29 |
+// HostCertsDir returns the config directory for a specific host |
|
| 30 |
+func HostCertsDir(hostname string) (string, error) {
|
|
| 31 |
+ certsDir := CertsDir |
|
| 32 |
+ |
|
| 33 |
+ if rootless.RunningWithRootlessKit() {
|
|
| 34 |
+ configHome, err := homedir.GetConfigHome() |
|
| 35 |
+ if err != nil {
|
|
| 36 |
+ return "", err |
|
| 37 |
+ } |
|
| 38 |
+ |
|
| 39 |
+ certsDir = filepath.Join(configHome, "docker/certs.d") |
|
| 40 |
+ } |
|
| 41 |
+ |
|
| 42 |
+ hostDir := filepath.Join(certsDir, cleanPath(hostname)) |
|
| 43 |
+ |
|
| 44 |
+ return hostDir, nil |
|
| 45 |
+} |
|
| 46 |
+ |
|
| 29 | 47 |
func newTLSConfig(hostname string, isSecure bool) (*tls.Config, error) {
|
| 30 | 48 |
// PreferredServerCipherSuites should have no effect |
| 31 | 49 |
tlsConfig := tlsconfig.ServerDefault() |
| ... | ... |
@@ -33,19 +51,11 @@ func newTLSConfig(hostname string, isSecure bool) (*tls.Config, error) {
|
| 33 | 33 |
tlsConfig.InsecureSkipVerify = !isSecure |
| 34 | 34 |
|
| 35 | 35 |
if isSecure && CertsDir != "" {
|
| 36 |
- certsDir := CertsDir |
|
| 37 |
- |
|
| 38 |
- if rootless.RunningWithRootlessKit() {
|
|
| 39 |
- configHome, err := homedir.GetConfigHome() |
|
| 40 |
- if err != nil {
|
|
| 41 |
- return nil, err |
|
| 42 |
- } |
|
| 43 |
- |
|
| 44 |
- certsDir = filepath.Join(configHome, "docker/certs.d") |
|
| 36 |
+ hostDir, err := HostCertsDir(hostname) |
|
| 37 |
+ if err != nil {
|
|
| 38 |
+ return nil, err |
|
| 45 | 39 |
} |
| 46 | 40 |
|
| 47 |
- hostDir := filepath.Join(certsDir, cleanPath(hostname)) |
|
| 48 |
- |
|
| 49 | 41 |
logrus.Debugf("hostDir: %s", hostDir)
|
| 50 | 42 |
if err := ReadCertsDirectory(tlsConfig, hostDir); err != nil {
|
| 51 | 43 |
return nil, err |
| ... | ... |
@@ -27,8 +27,8 @@ github.com/imdario/mergo 1afb36080aec31e0d1528973ebe6 |
| 27 | 27 |
golang.org/x/sync cd5d95a43a6e21273425c7ae415d3df9ea832eeb |
| 28 | 28 |
|
| 29 | 29 |
# buildkit |
| 30 |
-github.com/moby/buildkit 4d8d91bf49c769b8458e9aa84746c842b4a0e39a |
|
| 31 |
-github.com/tonistiigi/fsutil 013a9fe6aee2d1658457075bf9e688bc8c0be2e0 |
|
| 30 |
+github.com/moby/buildkit ae7ff7174f73bcb4df89b97e1623b3fb0bfb0a0c |
|
| 31 |
+github.com/tonistiigi/fsutil c2c7d7b0e1441705cd802e5699c0a10b1dfe39fd |
|
| 32 | 32 |
github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746 |
| 33 | 33 |
github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7 |
| 34 | 34 |
github.com/google/shlex e7afc7fbc51079733e9468cdfd1efcd7d196cd1d |
| ... | ... |
@@ -128,7 +128,7 @@ See [Expose BuildKit as a TCP service](#expose-buildkit-as-a-tcp-service). |
| 128 | 128 |
|
| 129 | 129 |
:information_source: Notice to Fedora 31 users: |
| 130 | 130 |
|
| 131 |
-* As runc still does not work on cgroup v2 environment like Fedora 31, you need to substitute runc with crun. Run `rm -f $(which buildkit-runc) && ln -s $(which crun) /usr/local/bin/buildkit-runc` . |
|
| 131 |
+* As runc still does not work on cgroup v2 environment like Fedora 31, you need to substitute runc with crun. Run `buildkitd` with `--oci-worker-binary=crun`. |
|
| 132 | 132 |
* If you want to use runc, you need to configure the system to use cgroup v1. Run `sudo grubby --update-kernel=ALL --args="systemd.unified_cgroup_hierarchy=0"` and reboot. |
| 133 | 133 |
|
| 134 | 134 |
### Exploring LLB |
| ... | ... |
@@ -205,8 +205,8 @@ By default, the build result and intermediate cache will only remain internally |
| 205 | 205 |
buildctl build ... --output type=image,name=docker.io/username/image,push=true |
| 206 | 206 |
``` |
| 207 | 207 |
|
| 208 |
-To export and import the cache along with the image, you need to specify `--export-cache type=inline` and `--import-cache type=registry,ref=...`. |
|
| 209 |
-See [Export cache](#export-cache). |
|
| 208 |
+To export the cache embed with the image and pushing them to registry together, type `registry` is required to import the cache, you should specify `--export-cache type=inline` and `--import-cache type=registry,ref=...`. To export the cache to a local directy, you should specify `--export-cache type=local`. |
|
| 209 |
+Details in [Export cache](#export-cache). |
|
| 210 | 210 |
|
| 211 | 211 |
```bash |
| 212 | 212 |
buildctl build ...\ |
| ... | ... |
@@ -357,7 +357,8 @@ The directory layout conforms to OCI Image Spec v1.0. |
| 357 | 357 |
- `ref=docker.io/user/image:tag`: reference for `registry` cache importer |
| 358 | 358 |
- `src=path/to/input-dir`: directory for `local` cache importer |
| 359 | 359 |
- `digest=sha256:deadbeef`: digest of the manifest list to import for `local` cache importer. |
| 360 |
- Defaults to the digest of "latest" tag in `index.json` |
|
| 360 |
+- `tag=customtag`: custom tag of image for `local` cache importer. |
|
| 361 |
+ Defaults to the digest of "latest" tag in `index.json` is for digest, not for tag |
|
| 361 | 362 |
|
| 362 | 363 |
### Consistent hashing |
| 363 | 364 |
|
| ... | ... |
@@ -5,6 +5,7 @@ import ( |
| 5 | 5 |
"io" |
| 6 | 6 |
"sort" |
| 7 | 7 |
"strconv" |
| 8 |
+ "strings" |
|
| 8 | 9 |
) |
| 9 | 10 |
|
| 10 | 11 |
// WriteV1TarsumHeaders writes a tar header to a writer in V1 tarsum format. |
| ... | ... |
@@ -38,7 +39,9 @@ func v1TarHeaderSelect(h *tar.Header) (orderedHeaders [][2]string) {
|
| 38 | 38 |
// Get extended attributes. |
| 39 | 39 |
xAttrKeys := make([]string, len(h.Xattrs)) |
| 40 | 40 |
for k := range h.Xattrs {
|
| 41 |
- xAttrKeys = append(xAttrKeys, k) |
|
| 41 |
+ if !strings.HasPrefix(k, "security.") && !strings.HasPrefix(k, "system.") {
|
|
| 42 |
+ xAttrKeys = append(xAttrKeys, k) |
|
| 43 |
+ } |
|
| 42 | 44 |
} |
| 43 | 45 |
sort.Strings(xAttrKeys) |
| 44 | 46 |
|
| ... | ... |
@@ -2,15 +2,12 @@ package registry |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 | 4 |
"context" |
| 5 |
- "time" |
|
| 6 | 5 |
|
| 7 | 6 |
"github.com/containerd/containerd/content" |
| 8 |
- "github.com/containerd/containerd/remotes" |
|
| 9 | 7 |
"github.com/containerd/containerd/remotes/docker" |
| 10 | 8 |
"github.com/docker/distribution/reference" |
| 11 | 9 |
"github.com/moby/buildkit/cache/remotecache" |
| 12 | 10 |
"github.com/moby/buildkit/session" |
| 13 |
- "github.com/moby/buildkit/session/auth" |
|
| 14 | 11 |
"github.com/moby/buildkit/util/contentutil" |
| 15 | 12 |
"github.com/moby/buildkit/util/resolver" |
| 16 | 13 |
"github.com/opencontainers/go-digest" |
| ... | ... |
@@ -34,13 +31,13 @@ const ( |
| 34 | 34 |
attrRef = "ref" |
| 35 | 35 |
) |
| 36 | 36 |
|
| 37 |
-func ResolveCacheExporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheExporterFunc {
|
|
| 37 |
+func ResolveCacheExporterFunc(sm *session.Manager, hosts docker.RegistryHosts) remotecache.ResolveCacheExporterFunc {
|
|
| 38 | 38 |
return func(ctx context.Context, attrs map[string]string) (remotecache.Exporter, error) {
|
| 39 | 39 |
ref, err := canonicalizeRef(attrs[attrRef]) |
| 40 | 40 |
if err != nil {
|
| 41 | 41 |
return nil, err |
| 42 | 42 |
} |
| 43 |
- remote := newRemoteResolver(ctx, resolverOpt, sm, ref) |
|
| 43 |
+ remote := resolver.New(ctx, hosts, sm) |
|
| 44 | 44 |
pusher, err := remote.Pusher(ctx, ref) |
| 45 | 45 |
if err != nil {
|
| 46 | 46 |
return nil, err |
| ... | ... |
@@ -49,13 +46,13 @@ func ResolveCacheExporterFunc(sm *session.Manager, resolverOpt resolver.ResolveO |
| 49 | 49 |
} |
| 50 | 50 |
} |
| 51 | 51 |
|
| 52 |
-func ResolveCacheImporterFunc(sm *session.Manager, cs content.Store, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheImporterFunc {
|
|
| 52 |
+func ResolveCacheImporterFunc(sm *session.Manager, cs content.Store, hosts docker.RegistryHosts) remotecache.ResolveCacheImporterFunc {
|
|
| 53 | 53 |
return func(ctx context.Context, attrs map[string]string) (remotecache.Importer, specs.Descriptor, error) {
|
| 54 | 54 |
ref, err := canonicalizeRef(attrs[attrRef]) |
| 55 | 55 |
if err != nil {
|
| 56 | 56 |
return nil, specs.Descriptor{}, err
|
| 57 | 57 |
} |
| 58 |
- remote := newRemoteResolver(ctx, resolverOpt, sm, ref) |
|
| 58 |
+ remote := resolver.New(ctx, hosts, sm) |
|
| 59 | 59 |
xref, desc, err := remote.Resolve(ctx, ref) |
| 60 | 60 |
if err != nil {
|
| 61 | 61 |
return nil, specs.Descriptor{}, err
|
| ... | ... |
@@ -97,27 +94,3 @@ func (dsl *withDistributionSourceLabel) SetDistributionSourceAnnotation(desc oci |
| 97 | 97 |
desc.Annotations["containerd.io/distribution.source.ref"] = dsl.ref |
| 98 | 98 |
return desc |
| 99 | 99 |
} |
| 100 |
- |
|
| 101 |
-func newRemoteResolver(ctx context.Context, resolverOpt resolver.ResolveOptionsFunc, sm *session.Manager, ref string) remotes.Resolver {
|
|
| 102 |
- opt := resolverOpt(ref) |
|
| 103 |
- opt.Credentials = getCredentialsFunc(ctx, sm) |
|
| 104 |
- return docker.NewResolver(opt) |
|
| 105 |
-} |
|
| 106 |
- |
|
| 107 |
-func getCredentialsFunc(ctx context.Context, sm *session.Manager) func(string) (string, string, error) {
|
|
| 108 |
- id := session.FromContext(ctx) |
|
| 109 |
- if id == "" {
|
|
| 110 |
- return nil |
|
| 111 |
- } |
|
| 112 |
- return func(host string) (string, string, error) {
|
|
| 113 |
- timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) |
|
| 114 |
- defer cancel() |
|
| 115 |
- |
|
| 116 |
- caller, err := sm.Get(timeoutCtx, id) |
|
| 117 |
- if err != nil {
|
|
| 118 |
- return "", "", err |
|
| 119 |
- } |
|
| 120 |
- |
|
| 121 |
- return auth.CredentialsFunc(context.TODO(), caller)(host) |
|
| 122 |
- } |
|
| 123 |
-} |
| ... | ... |
@@ -11,6 +11,8 @@ import ( |
| 11 | 11 |
"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" |
| 12 | 12 |
controlapi "github.com/moby/buildkit/api/services/control" |
| 13 | 13 |
"github.com/moby/buildkit/client/connhelper" |
| 14 |
+ "github.com/moby/buildkit/session" |
|
| 15 |
+ "github.com/moby/buildkit/session/grpchijack" |
|
| 14 | 16 |
"github.com/moby/buildkit/util/appdefaults" |
| 15 | 17 |
opentracing "github.com/opentracing/opentracing-go" |
| 16 | 18 |
"github.com/pkg/errors" |
| ... | ... |
@@ -80,6 +82,10 @@ func (c *Client) controlClient() controlapi.ControlClient {
|
| 80 | 80 |
return controlapi.NewControlClient(c.conn) |
| 81 | 81 |
} |
| 82 | 82 |
|
| 83 |
+func (c *Client) Dialer() session.Dialer {
|
|
| 84 |
+ return grpchijack.Dialer(c.controlClient()) |
|
| 85 |
+} |
|
| 86 |
+ |
|
| 83 | 87 |
func (c *Client) Close() error {
|
| 84 | 88 |
return c.conn.Close() |
| 85 | 89 |
} |
| 86 | 90 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,98 @@ |
| 0 |
+package llb |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "context" |
|
| 4 |
+ |
|
| 5 |
+ "github.com/moby/buildkit/solver/pb" |
|
| 6 |
+ "github.com/moby/buildkit/util/flightcontrol" |
|
| 7 |
+ digest "github.com/opencontainers/go-digest" |
|
| 8 |
+ "github.com/pkg/errors" |
|
| 9 |
+) |
|
| 10 |
+ |
|
| 11 |
+type asyncState struct {
|
|
| 12 |
+ f func(context.Context, State) (State, error) |
|
| 13 |
+ prev State |
|
| 14 |
+ target State |
|
| 15 |
+ set bool |
|
| 16 |
+ err error |
|
| 17 |
+ g flightcontrol.Group |
|
| 18 |
+} |
|
| 19 |
+ |
|
| 20 |
+func (as *asyncState) Output() Output {
|
|
| 21 |
+ return as |
|
| 22 |
+} |
|
| 23 |
+ |
|
| 24 |
+func (as *asyncState) Vertex(ctx context.Context) Vertex {
|
|
| 25 |
+ err := as.Do(ctx) |
|
| 26 |
+ if err != nil {
|
|
| 27 |
+ return &errVertex{err}
|
|
| 28 |
+ } |
|
| 29 |
+ if as.set {
|
|
| 30 |
+ out := as.target.Output() |
|
| 31 |
+ if out == nil {
|
|
| 32 |
+ return nil |
|
| 33 |
+ } |
|
| 34 |
+ return out.Vertex(ctx) |
|
| 35 |
+ } |
|
| 36 |
+ return nil |
|
| 37 |
+} |
|
| 38 |
+ |
|
| 39 |
+func (as *asyncState) ToInput(ctx context.Context, c *Constraints) (*pb.Input, error) {
|
|
| 40 |
+ err := as.Do(ctx) |
|
| 41 |
+ if err != nil {
|
|
| 42 |
+ return nil, err |
|
| 43 |
+ } |
|
| 44 |
+ if as.set {
|
|
| 45 |
+ out := as.target.Output() |
|
| 46 |
+ if out == nil {
|
|
| 47 |
+ return nil, nil |
|
| 48 |
+ } |
|
| 49 |
+ return out.ToInput(ctx, c) |
|
| 50 |
+ } |
|
| 51 |
+ return nil, nil |
|
| 52 |
+} |
|
| 53 |
+ |
|
| 54 |
+func (as *asyncState) Do(ctx context.Context) error {
|
|
| 55 |
+ _, err := as.g.Do(ctx, "", func(ctx context.Context) (interface{}, error) {
|
|
| 56 |
+ if as.set {
|
|
| 57 |
+ return as.target, as.err |
|
| 58 |
+ } |
|
| 59 |
+ res, err := as.f(ctx, as.prev) |
|
| 60 |
+ if err != nil {
|
|
| 61 |
+ select {
|
|
| 62 |
+ case <-ctx.Done(): |
|
| 63 |
+ if errors.Cause(err) == ctx.Err() {
|
|
| 64 |
+ return res, err |
|
| 65 |
+ } |
|
| 66 |
+ default: |
|
| 67 |
+ } |
|
| 68 |
+ } |
|
| 69 |
+ as.target = res |
|
| 70 |
+ as.err = err |
|
| 71 |
+ as.set = true |
|
| 72 |
+ return res, err |
|
| 73 |
+ }) |
|
| 74 |
+ if err != nil {
|
|
| 75 |
+ return err |
|
| 76 |
+ } |
|
| 77 |
+ return as.err |
|
| 78 |
+} |
|
| 79 |
+ |
|
| 80 |
+type errVertex struct {
|
|
| 81 |
+ err error |
|
| 82 |
+} |
|
| 83 |
+ |
|
| 84 |
+func (v *errVertex) Validate(context.Context) error {
|
|
| 85 |
+ return v.err |
|
| 86 |
+} |
|
| 87 |
+func (v *errVertex) Marshal(context.Context, *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
|
|
| 88 |
+ return "", nil, nil, v.err |
|
| 89 |
+} |
|
| 90 |
+func (v *errVertex) Output() Output {
|
|
| 91 |
+ return nil |
|
| 92 |
+} |
|
| 93 |
+func (v *errVertex) Inputs() []Output {
|
|
| 94 |
+ return nil |
|
| 95 |
+} |
|
| 96 |
+ |
|
| 97 |
+var _ Vertex = &errVertex{}
|
| ... | ... |
@@ -1,6 +1,9 @@ |
| 1 | 1 |
package llb |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 |
+ "context" |
|
| 5 |
+ "sync" |
|
| 6 |
+ |
|
| 4 | 7 |
"github.com/moby/buildkit/solver/pb" |
| 5 | 8 |
digest "github.com/opencontainers/go-digest" |
| 6 | 9 |
specs "github.com/opencontainers/image-spec/specs-go/v1" |
| ... | ... |
@@ -13,6 +16,7 @@ import ( |
| 13 | 13 |
// LLB state can be reconstructed from the definition. |
| 14 | 14 |
type DefinitionOp struct {
|
| 15 | 15 |
MarshalCache |
| 16 |
+ mu sync.Mutex |
|
| 16 | 17 |
ops map[digest.Digest]*pb.Op |
| 17 | 18 |
defs map[digest.Digest][]byte |
| 18 | 19 |
metas map[digest.Digest]pb.OpMetadata |
| ... | ... |
@@ -25,6 +29,7 @@ type DefinitionOp struct {
|
| 25 | 25 |
func NewDefinitionOp(def *pb.Definition) (*DefinitionOp, error) {
|
| 26 | 26 |
ops := make(map[digest.Digest]*pb.Op) |
| 27 | 27 |
defs := make(map[digest.Digest][]byte) |
| 28 |
+ platforms := make(map[digest.Digest]*specs.Platform) |
|
| 28 | 29 |
|
| 29 | 30 |
var dgst digest.Digest |
| 30 | 31 |
for _, dt := range def.Def {
|
| ... | ... |
@@ -35,6 +40,13 @@ func NewDefinitionOp(def *pb.Definition) (*DefinitionOp, error) {
|
| 35 | 35 |
dgst = digest.FromBytes(dt) |
| 36 | 36 |
ops[dgst] = &op |
| 37 | 37 |
defs[dgst] = dt |
| 38 |
+ |
|
| 39 |
+ var platform *specs.Platform |
|
| 40 |
+ if op.Platform != nil {
|
|
| 41 |
+ spec := op.Platform.Spec() |
|
| 42 |
+ platform = &spec |
|
| 43 |
+ } |
|
| 44 |
+ platforms[dgst] = platform |
|
| 38 | 45 |
} |
| 39 | 46 |
|
| 40 | 47 |
var index pb.OutputIndex |
| ... | ... |
@@ -47,26 +59,29 @@ func NewDefinitionOp(def *pb.Definition) (*DefinitionOp, error) {
|
| 47 | 47 |
ops: ops, |
| 48 | 48 |
defs: defs, |
| 49 | 49 |
metas: def.Metadata, |
| 50 |
- platforms: make(map[digest.Digest]*specs.Platform), |
|
| 50 |
+ platforms: platforms, |
|
| 51 | 51 |
dgst: dgst, |
| 52 | 52 |
index: index, |
| 53 | 53 |
}, nil |
| 54 | 54 |
} |
| 55 | 55 |
|
| 56 |
-func (d *DefinitionOp) ToInput(c *Constraints) (*pb.Input, error) {
|
|
| 57 |
- return d.Output().ToInput(c) |
|
| 56 |
+func (d *DefinitionOp) ToInput(ctx context.Context, c *Constraints) (*pb.Input, error) {
|
|
| 57 |
+ return d.Output().ToInput(ctx, c) |
|
| 58 | 58 |
} |
| 59 | 59 |
|
| 60 |
-func (d *DefinitionOp) Vertex() Vertex {
|
|
| 60 |
+func (d *DefinitionOp) Vertex(context.Context) Vertex {
|
|
| 61 | 61 |
return d |
| 62 | 62 |
} |
| 63 | 63 |
|
| 64 |
-func (d *DefinitionOp) Validate() error {
|
|
| 64 |
+func (d *DefinitionOp) Validate(context.Context) error {
|
|
| 65 | 65 |
// Scratch state has no digest, ops or metas. |
| 66 | 66 |
if d.dgst == "" {
|
| 67 | 67 |
return nil |
| 68 | 68 |
} |
| 69 | 69 |
|
| 70 |
+ d.mu.Lock() |
|
| 71 |
+ defer d.mu.Unlock() |
|
| 72 |
+ |
|
| 70 | 73 |
if len(d.ops) == 0 || len(d.defs) == 0 || len(d.metas) == 0 {
|
| 71 | 74 |
return errors.Errorf("invalid definition op with no ops %d %d", len(d.ops), len(d.metas))
|
| 72 | 75 |
} |
| ... | ... |
@@ -95,15 +110,18 @@ func (d *DefinitionOp) Validate() error {
|
| 95 | 95 |
return nil |
| 96 | 96 |
} |
| 97 | 97 |
|
| 98 |
-func (d *DefinitionOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
|
|
| 98 |
+func (d *DefinitionOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
|
|
| 99 | 99 |
if d.dgst == "" {
|
| 100 | 100 |
return "", nil, nil, errors.Errorf("cannot marshal empty definition op")
|
| 101 | 101 |
} |
| 102 | 102 |
|
| 103 |
- if err := d.Validate(); err != nil {
|
|
| 103 |
+ if err := d.Validate(ctx); err != nil {
|
|
| 104 | 104 |
return "", nil, nil, err |
| 105 | 105 |
} |
| 106 | 106 |
|
| 107 |
+ d.mu.Lock() |
|
| 108 |
+ defer d.mu.Unlock() |
|
| 109 |
+ |
|
| 107 | 110 |
meta := d.metas[d.dgst] |
| 108 | 111 |
return d.dgst, d.defs[d.dgst], &meta, nil |
| 109 | 112 |
|
| ... | ... |
@@ -114,7 +132,11 @@ func (d *DefinitionOp) Output() Output {
|
| 114 | 114 |
return nil |
| 115 | 115 |
} |
| 116 | 116 |
|
| 117 |
- return &output{vertex: d, platform: d.platform(), getIndex: func() (pb.OutputIndex, error) {
|
|
| 117 |
+ d.mu.Lock() |
|
| 118 |
+ platform := d.platforms[d.dgst] |
|
| 119 |
+ d.mu.Unlock() |
|
| 120 |
+ |
|
| 121 |
+ return &output{vertex: d, platform: platform, getIndex: func() (pb.OutputIndex, error) {
|
|
| 118 | 122 |
return d.index, nil |
| 119 | 123 |
}} |
| 120 | 124 |
} |
| ... | ... |
@@ -126,7 +148,11 @@ func (d *DefinitionOp) Inputs() []Output {
|
| 126 | 126 |
|
| 127 | 127 |
var inputs []Output |
| 128 | 128 |
|
| 129 |
+ d.mu.Lock() |
|
| 129 | 130 |
op := d.ops[d.dgst] |
| 131 |
+ platform := d.platforms[d.dgst] |
|
| 132 |
+ d.mu.Unlock() |
|
| 133 |
+ |
|
| 130 | 134 |
for _, input := range op.Inputs {
|
| 131 | 135 |
vtx := &DefinitionOp{
|
| 132 | 136 |
ops: d.ops, |
| ... | ... |
@@ -136,26 +162,10 @@ func (d *DefinitionOp) Inputs() []Output {
|
| 136 | 136 |
dgst: input.Digest, |
| 137 | 137 |
index: input.Index, |
| 138 | 138 |
} |
| 139 |
- inputs = append(inputs, &output{vertex: vtx, platform: d.platform(), getIndex: func() (pb.OutputIndex, error) {
|
|
| 139 |
+ inputs = append(inputs, &output{vertex: vtx, platform: platform, getIndex: func() (pb.OutputIndex, error) {
|
|
| 140 | 140 |
return pb.OutputIndex(vtx.index), nil |
| 141 | 141 |
}}) |
| 142 | 142 |
} |
| 143 | 143 |
|
| 144 | 144 |
return inputs |
| 145 | 145 |
} |
| 146 |
- |
|
| 147 |
-func (d *DefinitionOp) platform() *specs.Platform {
|
|
| 148 |
- platform, ok := d.platforms[d.dgst] |
|
| 149 |
- if ok {
|
|
| 150 |
- return platform |
|
| 151 |
- } |
|
| 152 |
- |
|
| 153 |
- op := d.ops[d.dgst] |
|
| 154 |
- if op.Platform != nil {
|
|
| 155 |
- spec := op.Platform.Spec() |
|
| 156 |
- platform = &spec |
|
| 157 |
- } |
|
| 158 |
- |
|
| 159 |
- d.platforms[d.dgst] = platform |
|
| 160 |
- return platform |
|
| 161 |
-} |
| ... | ... |
@@ -1,6 +1,7 @@ |
| 1 | 1 |
package llb |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 |
+ "context" |
|
| 4 | 5 |
_ "crypto/sha256" |
| 5 | 6 |
"fmt" |
| 6 | 7 |
"net" |
| ... | ... |
@@ -12,19 +13,9 @@ import ( |
| 12 | 12 |
"github.com/pkg/errors" |
| 13 | 13 |
) |
| 14 | 14 |
|
| 15 |
-type Meta struct {
|
|
| 16 |
- Args []string |
|
| 17 |
- Env EnvList |
|
| 18 |
- Cwd string |
|
| 19 |
- User string |
|
| 20 |
- ProxyEnv *ProxyEnv |
|
| 21 |
- ExtraHosts []HostIP |
|
| 22 |
- Network pb.NetMode |
|
| 23 |
- Security pb.SecurityMode |
|
| 24 |
-} |
|
| 25 |
- |
|
| 26 |
-func NewExecOp(root Output, meta Meta, readOnly bool, c Constraints) *ExecOp {
|
|
| 27 |
- e := &ExecOp{meta: meta, constraints: c}
|
|
| 15 |
+func NewExecOp(base State, proxyEnv *ProxyEnv, readOnly bool, c Constraints) *ExecOp {
|
|
| 16 |
+ e := &ExecOp{base: base, constraints: c, proxyEnv: proxyEnv}
|
|
| 17 |
+ root := base.Output() |
|
| 28 | 18 |
rootMount := &mount{
|
| 29 | 19 |
target: pb.RootMount, |
| 30 | 20 |
source: root, |
| ... | ... |
@@ -58,9 +49,10 @@ type mount struct {
|
| 58 | 58 |
|
| 59 | 59 |
type ExecOp struct {
|
| 60 | 60 |
MarshalCache |
| 61 |
+ proxyEnv *ProxyEnv |
|
| 61 | 62 |
root Output |
| 62 | 63 |
mounts []*mount |
| 63 |
- meta Meta |
|
| 64 |
+ base State |
|
| 64 | 65 |
constraints Constraints |
| 65 | 66 |
isValidated bool |
| 66 | 67 |
secrets []SecretInfo |
| ... | ... |
@@ -103,19 +95,27 @@ func (e *ExecOp) GetMount(target string) Output {
|
| 103 | 103 |
return nil |
| 104 | 104 |
} |
| 105 | 105 |
|
| 106 |
-func (e *ExecOp) Validate() error {
|
|
| 106 |
+func (e *ExecOp) Validate(ctx context.Context) error {
|
|
| 107 | 107 |
if e.isValidated {
|
| 108 | 108 |
return nil |
| 109 | 109 |
} |
| 110 |
- if len(e.meta.Args) == 0 {
|
|
| 110 |
+ args, err := getArgs(e.base)(ctx) |
|
| 111 |
+ if err != nil {
|
|
| 112 |
+ return err |
|
| 113 |
+ } |
|
| 114 |
+ if len(args) == 0 {
|
|
| 111 | 115 |
return errors.Errorf("arguments are required")
|
| 112 | 116 |
} |
| 113 |
- if e.meta.Cwd == "" {
|
|
| 117 |
+ cwd, err := getDir(e.base)(ctx) |
|
| 118 |
+ if err != nil {
|
|
| 119 |
+ return err |
|
| 120 |
+ } |
|
| 121 |
+ if cwd == "" {
|
|
| 114 | 122 |
return errors.Errorf("working directory is required")
|
| 115 | 123 |
} |
| 116 | 124 |
for _, m := range e.mounts {
|
| 117 | 125 |
if m.source != nil {
|
| 118 |
- if err := m.source.Vertex().Validate(); err != nil {
|
|
| 126 |
+ if err := m.source.Vertex(ctx).Validate(ctx); err != nil {
|
|
| 119 | 127 |
return err |
| 120 | 128 |
} |
| 121 | 129 |
} |
| ... | ... |
@@ -124,11 +124,11 @@ func (e *ExecOp) Validate() error {
|
| 124 | 124 |
return nil |
| 125 | 125 |
} |
| 126 | 126 |
|
| 127 |
-func (e *ExecOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
|
|
| 127 |
+func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
|
|
| 128 | 128 |
if e.Cached(c) {
|
| 129 | 129 |
return e.Load() |
| 130 | 130 |
} |
| 131 |
- if err := e.Validate(); err != nil {
|
|
| 131 |
+ if err := e.Validate(ctx); err != nil {
|
|
| 132 | 132 |
return "", nil, nil, err |
| 133 | 133 |
} |
| 134 | 134 |
// make sure mounts are sorted |
| ... | ... |
@@ -136,52 +136,86 @@ func (e *ExecOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, |
| 136 | 136 |
return e.mounts[i].target < e.mounts[j].target |
| 137 | 137 |
}) |
| 138 | 138 |
|
| 139 |
+ env, err := getEnv(e.base)(ctx) |
|
| 140 |
+ if err != nil {
|
|
| 141 |
+ return "", nil, nil, err |
|
| 142 |
+ } |
|
| 143 |
+ |
|
| 139 | 144 |
if len(e.ssh) > 0 {
|
| 140 | 145 |
for i, s := range e.ssh {
|
| 141 | 146 |
if s.Target == "" {
|
| 142 | 147 |
e.ssh[i].Target = fmt.Sprintf("/run/buildkit/ssh_agent.%d", i)
|
| 143 | 148 |
} |
| 144 | 149 |
} |
| 145 |
- if _, ok := e.meta.Env.Get("SSH_AUTH_SOCK"); !ok {
|
|
| 146 |
- e.meta.Env = e.meta.Env.AddOrReplace("SSH_AUTH_SOCK", e.ssh[0].Target)
|
|
| 150 |
+ if _, ok := env.Get("SSH_AUTH_SOCK"); !ok {
|
|
| 151 |
+ env = env.AddOrReplace("SSH_AUTH_SOCK", e.ssh[0].Target)
|
|
| 147 | 152 |
} |
| 148 | 153 |
} |
| 149 | 154 |
if c.Caps != nil {
|
| 150 | 155 |
if err := c.Caps.Supports(pb.CapExecMetaSetsDefaultPath); err != nil {
|
| 151 |
- e.meta.Env = e.meta.Env.SetDefault("PATH", system.DefaultPathEnv)
|
|
| 156 |
+ env = env.SetDefault("PATH", system.DefaultPathEnv)
|
|
| 152 | 157 |
} else {
|
| 153 | 158 |
addCap(&e.constraints, pb.CapExecMetaSetsDefaultPath) |
| 154 | 159 |
} |
| 155 | 160 |
} |
| 156 | 161 |
|
| 162 |
+ args, err := getArgs(e.base)(ctx) |
|
| 163 |
+ if err != nil {
|
|
| 164 |
+ return "", nil, nil, err |
|
| 165 |
+ } |
|
| 166 |
+ |
|
| 167 |
+ cwd, err := getDir(e.base)(ctx) |
|
| 168 |
+ if err != nil {
|
|
| 169 |
+ return "", nil, nil, err |
|
| 170 |
+ } |
|
| 171 |
+ |
|
| 172 |
+ user, err := getUser(e.base)(ctx) |
|
| 173 |
+ if err != nil {
|
|
| 174 |
+ return "", nil, nil, err |
|
| 175 |
+ } |
|
| 176 |
+ |
|
| 157 | 177 |
meta := &pb.Meta{
|
| 158 |
- Args: e.meta.Args, |
|
| 159 |
- Env: e.meta.Env.ToArray(), |
|
| 160 |
- Cwd: e.meta.Cwd, |
|
| 161 |
- User: e.meta.User, |
|
| 162 |
- } |
|
| 163 |
- if len(e.meta.ExtraHosts) > 0 {
|
|
| 164 |
- hosts := make([]*pb.HostIP, len(e.meta.ExtraHosts)) |
|
| 165 |
- for i, h := range e.meta.ExtraHosts {
|
|
| 178 |
+ Args: args, |
|
| 179 |
+ Env: env.ToArray(), |
|
| 180 |
+ Cwd: cwd, |
|
| 181 |
+ User: user, |
|
| 182 |
+ } |
|
| 183 |
+ extraHosts, err := getExtraHosts(e.base)(ctx) |
|
| 184 |
+ if err != nil {
|
|
| 185 |
+ return "", nil, nil, err |
|
| 186 |
+ } |
|
| 187 |
+ if len(extraHosts) > 0 {
|
|
| 188 |
+ hosts := make([]*pb.HostIP, len(extraHosts)) |
|
| 189 |
+ for i, h := range extraHosts {
|
|
| 166 | 190 |
hosts[i] = &pb.HostIP{Host: h.Host, IP: h.IP.String()}
|
| 167 | 191 |
} |
| 168 | 192 |
meta.ExtraHosts = hosts |
| 169 | 193 |
} |
| 170 | 194 |
|
| 195 |
+ network, err := getNetwork(e.base)(ctx) |
|
| 196 |
+ if err != nil {
|
|
| 197 |
+ return "", nil, nil, err |
|
| 198 |
+ } |
|
| 199 |
+ |
|
| 200 |
+ security, err := getSecurity(e.base)(ctx) |
|
| 201 |
+ if err != nil {
|
|
| 202 |
+ return "", nil, nil, err |
|
| 203 |
+ } |
|
| 204 |
+ |
|
| 171 | 205 |
peo := &pb.ExecOp{
|
| 172 | 206 |
Meta: meta, |
| 173 |
- Network: e.meta.Network, |
|
| 174 |
- Security: e.meta.Security, |
|
| 207 |
+ Network: network, |
|
| 208 |
+ Security: security, |
|
| 175 | 209 |
} |
| 176 |
- if e.meta.Network != NetModeSandbox {
|
|
| 210 |
+ if network != NetModeSandbox {
|
|
| 177 | 211 |
addCap(&e.constraints, pb.CapExecMetaNetwork) |
| 178 | 212 |
} |
| 179 | 213 |
|
| 180 |
- if e.meta.Security != SecurityModeSandbox {
|
|
| 214 |
+ if security != SecurityModeSandbox {
|
|
| 181 | 215 |
addCap(&e.constraints, pb.CapExecMetaSecurity) |
| 182 | 216 |
} |
| 183 | 217 |
|
| 184 |
- if p := e.meta.ProxyEnv; p != nil {
|
|
| 218 |
+ if p := e.proxyEnv; p != nil {
|
|
| 185 | 219 |
peo.Meta.ProxyEnv = &pb.ProxyEnv{
|
| 186 | 220 |
HttpProxy: p.HttpProxy, |
| 187 | 221 |
HttpsProxy: p.HttpsProxy, |
| ... | ... |
@@ -215,6 +249,14 @@ func (e *ExecOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, |
| 215 | 215 |
addCap(&e.constraints, pb.CapExecMountSSH) |
| 216 | 216 |
} |
| 217 | 217 |
|
| 218 |
+ if e.constraints.Platform == nil {
|
|
| 219 |
+ p, err := getPlatform(e.base)(ctx) |
|
| 220 |
+ if err != nil {
|
|
| 221 |
+ return "", nil, nil, err |
|
| 222 |
+ } |
|
| 223 |
+ e.constraints.Platform = p |
|
| 224 |
+ } |
|
| 225 |
+ |
|
| 218 | 226 |
pop, md := MarshalConstraints(c, &e.constraints) |
| 219 | 227 |
pop.Op = &pb.Op_Exec{
|
| 220 | 228 |
Exec: peo, |
| ... | ... |
@@ -227,7 +269,7 @@ func (e *ExecOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, |
| 227 | 227 |
if m.tmpfs {
|
| 228 | 228 |
return "", nil, nil, errors.Errorf("tmpfs mounts must use scratch")
|
| 229 | 229 |
} |
| 230 |
- inp, err := m.source.ToInput(c) |
|
| 230 |
+ inp, err := m.source.ToInput(ctx, c) |
|
| 231 | 231 |
if err != nil {
|
| 232 | 232 |
return "", nil, nil, err |
| 233 | 233 |
} |
| ... | ... |
@@ -414,17 +456,11 @@ func (fn runOptionFunc) SetRunOption(ei *ExecInfo) {
|
| 414 | 414 |
fn(ei) |
| 415 | 415 |
} |
| 416 | 416 |
|
| 417 |
-func Network(n pb.NetMode) RunOption {
|
|
| 418 |
- return runOptionFunc(func(ei *ExecInfo) {
|
|
| 419 |
- ei.State = network(n)(ei.State) |
|
| 420 |
- }) |
|
| 417 |
+func (fn StateOption) SetRunOption(ei *ExecInfo) {
|
|
| 418 |
+ ei.State = ei.State.With(fn) |
|
| 421 | 419 |
} |
| 422 | 420 |
|
| 423 |
-func Security(s pb.SecurityMode) RunOption {
|
|
| 424 |
- return runOptionFunc(func(ei *ExecInfo) {
|
|
| 425 |
- ei.State = security(s)(ei.State) |
|
| 426 |
- }) |
|
| 427 |
-} |
|
| 421 |
+var _ RunOption = StateOption(func(_ State) State { return State{} })
|
|
| 428 | 422 |
|
| 429 | 423 |
func Shlex(str string) RunOption {
|
| 430 | 424 |
return runOptionFunc(func(ei *ExecInfo) {
|
| ... | ... |
@@ -443,47 +479,12 @@ func Args(a []string) RunOption {
|
| 443 | 443 |
}) |
| 444 | 444 |
} |
| 445 | 445 |
|
| 446 |
-func AddEnv(key, value string) RunOption {
|
|
| 447 |
- return runOptionFunc(func(ei *ExecInfo) {
|
|
| 448 |
- ei.State = ei.State.AddEnv(key, value) |
|
| 449 |
- }) |
|
| 450 |
-} |
|
| 451 |
- |
|
| 452 |
-func AddEnvf(key, value string, v ...interface{}) RunOption {
|
|
| 453 |
- return runOptionFunc(func(ei *ExecInfo) {
|
|
| 454 |
- ei.State = ei.State.AddEnvf(key, value, v...) |
|
| 455 |
- }) |
|
| 456 |
-} |
|
| 457 |
- |
|
| 458 |
-func User(str string) RunOption {
|
|
| 459 |
- return runOptionFunc(func(ei *ExecInfo) {
|
|
| 460 |
- ei.State = ei.State.User(str) |
|
| 461 |
- }) |
|
| 462 |
-} |
|
| 463 |
- |
|
| 464 |
-func Dir(str string) RunOption {
|
|
| 465 |
- return runOptionFunc(func(ei *ExecInfo) {
|
|
| 466 |
- ei.State = ei.State.Dir(str) |
|
| 467 |
- }) |
|
| 468 |
-} |
|
| 469 |
-func Dirf(str string, v ...interface{}) RunOption {
|
|
| 470 |
- return runOptionFunc(func(ei *ExecInfo) {
|
|
| 471 |
- ei.State = ei.State.Dirf(str, v...) |
|
| 472 |
- }) |
|
| 473 |
-} |
|
| 474 |
- |
|
| 475 | 446 |
func AddExtraHost(host string, ip net.IP) RunOption {
|
| 476 | 447 |
return runOptionFunc(func(ei *ExecInfo) {
|
| 477 | 448 |
ei.State = ei.State.AddExtraHost(host, ip) |
| 478 | 449 |
}) |
| 479 | 450 |
} |
| 480 | 451 |
|
| 481 |
-func Reset(s State) RunOption {
|
|
| 482 |
- return runOptionFunc(func(ei *ExecInfo) {
|
|
| 483 |
- ei.State = ei.State.Reset(s) |
|
| 484 |
- }) |
|
| 485 |
-} |
|
| 486 |
- |
|
| 487 | 452 |
func With(so ...StateOption) RunOption {
|
| 488 | 453 |
return runOptionFunc(func(ei *ExecInfo) {
|
| 489 | 454 |
ei.State = ei.State.With(so...) |
| ... | ... |
@@ -1,6 +1,7 @@ |
| 1 | 1 |
package llb |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 |
+ "context" |
|
| 4 | 5 |
_ "crypto/sha256" |
| 5 | 6 |
"os" |
| 6 | 7 |
"path" |
| ... | ... |
@@ -52,7 +53,7 @@ type CopyInput interface {
|
| 52 | 52 |
} |
| 53 | 53 |
|
| 54 | 54 |
type subAction interface {
|
| 55 |
- toProtoAction(string, pb.InputIndex) pb.IsFileAction |
|
| 55 |
+ toProtoAction(context.Context, string, pb.InputIndex) (pb.IsFileAction, error) |
|
| 56 | 56 |
} |
| 57 | 57 |
|
| 58 | 58 |
type FileAction struct {
|
| ... | ... |
@@ -146,7 +147,7 @@ type fileActionMkdir struct {
|
| 146 | 146 |
info MkdirInfo |
| 147 | 147 |
} |
| 148 | 148 |
|
| 149 |
-func (a *fileActionMkdir) toProtoAction(parent string, base pb.InputIndex) pb.IsFileAction {
|
|
| 149 |
+func (a *fileActionMkdir) toProtoAction(ctx context.Context, parent string, base pb.InputIndex) (pb.IsFileAction, error) {
|
|
| 150 | 150 |
return &pb.FileAction_Mkdir{
|
| 151 | 151 |
Mkdir: &pb.FileActionMkDir{
|
| 152 | 152 |
Path: normalizePath(parent, a.file, false), |
| ... | ... |
@@ -155,7 +156,7 @@ func (a *fileActionMkdir) toProtoAction(parent string, base pb.InputIndex) pb.Is |
| 155 | 155 |
Owner: a.info.ChownOpt.marshal(base), |
| 156 | 156 |
Timestamp: marshalTime(a.info.CreatedTime), |
| 157 | 157 |
}, |
| 158 |
- } |
|
| 158 |
+ }, nil |
|
| 159 | 159 |
} |
| 160 | 160 |
|
| 161 | 161 |
type MkdirOption interface {
|
| ... | ... |
@@ -315,7 +316,7 @@ type fileActionMkfile struct {
|
| 315 | 315 |
info MkfileInfo |
| 316 | 316 |
} |
| 317 | 317 |
|
| 318 |
-func (a *fileActionMkfile) toProtoAction(parent string, base pb.InputIndex) pb.IsFileAction {
|
|
| 318 |
+func (a *fileActionMkfile) toProtoAction(ctx context.Context, parent string, base pb.InputIndex) (pb.IsFileAction, error) {
|
|
| 319 | 319 |
return &pb.FileAction_Mkfile{
|
| 320 | 320 |
Mkfile: &pb.FileActionMkFile{
|
| 321 | 321 |
Path: normalizePath(parent, a.file, false), |
| ... | ... |
@@ -324,7 +325,7 @@ func (a *fileActionMkfile) toProtoAction(parent string, base pb.InputIndex) pb.I |
| 324 | 324 |
Owner: a.info.ChownOpt.marshal(base), |
| 325 | 325 |
Timestamp: marshalTime(a.info.CreatedTime), |
| 326 | 326 |
}, |
| 327 |
- } |
|
| 327 |
+ }, nil |
|
| 328 | 328 |
} |
| 329 | 329 |
|
| 330 | 330 |
func Rm(p string, opts ...RmOption) *FileAction {
|
| ... | ... |
@@ -379,14 +380,14 @@ type fileActionRm struct {
|
| 379 | 379 |
info RmInfo |
| 380 | 380 |
} |
| 381 | 381 |
|
| 382 |
-func (a *fileActionRm) toProtoAction(parent string, base pb.InputIndex) pb.IsFileAction {
|
|
| 382 |
+func (a *fileActionRm) toProtoAction(ctx context.Context, parent string, base pb.InputIndex) (pb.IsFileAction, error) {
|
|
| 383 | 383 |
return &pb.FileAction_Rm{
|
| 384 | 384 |
Rm: &pb.FileActionRm{
|
| 385 | 385 |
Path: normalizePath(parent, a.file, false), |
| 386 | 386 |
AllowNotFound: a.info.AllowNotFound, |
| 387 | 387 |
AllowWildcard: a.info.AllowWildcard, |
| 388 | 388 |
}, |
| 389 |
- } |
|
| 389 |
+ }, nil |
|
| 390 | 390 |
} |
| 391 | 391 |
|
| 392 | 392 |
func Copy(input CopyInput, src, dest string, opts ...CopyOption) *FileAction {
|
| ... | ... |
@@ -448,9 +449,13 @@ type fileActionCopy struct {
|
| 448 | 448 |
info CopyInfo |
| 449 | 449 |
} |
| 450 | 450 |
|
| 451 |
-func (a *fileActionCopy) toProtoAction(parent string, base pb.InputIndex) pb.IsFileAction {
|
|
| 451 |
+func (a *fileActionCopy) toProtoAction(ctx context.Context, parent string, base pb.InputIndex) (pb.IsFileAction, error) {
|
|
| 452 |
+ src, err := a.sourcePath(ctx) |
|
| 453 |
+ if err != nil {
|
|
| 454 |
+ return nil, err |
|
| 455 |
+ } |
|
| 452 | 456 |
c := &pb.FileActionCopy{
|
| 453 |
- Src: a.sourcePath(), |
|
| 457 |
+ Src: src, |
|
| 454 | 458 |
Dest: normalizePath(parent, a.dest, true), |
| 455 | 459 |
Owner: a.info.ChownOpt.marshal(base), |
| 456 | 460 |
AllowWildcard: a.info.AllowWildcard, |
| ... | ... |
@@ -468,19 +473,27 @@ func (a *fileActionCopy) toProtoAction(parent string, base pb.InputIndex) pb.IsF |
| 468 | 468 |
} |
| 469 | 469 |
return &pb.FileAction_Copy{
|
| 470 | 470 |
Copy: c, |
| 471 |
- } |
|
| 471 |
+ }, nil |
|
| 472 | 472 |
} |
| 473 | 473 |
|
| 474 |
-func (c *fileActionCopy) sourcePath() string {
|
|
| 474 |
+func (c *fileActionCopy) sourcePath(ctx context.Context) (string, error) {
|
|
| 475 | 475 |
p := path.Clean(c.src) |
| 476 | 476 |
if !path.IsAbs(p) {
|
| 477 | 477 |
if c.state != nil {
|
| 478 |
- p = path.Join("/", c.state.GetDir(), p)
|
|
| 478 |
+ dir, err := c.state.GetDir(ctx) |
|
| 479 |
+ if err != nil {
|
|
| 480 |
+ return "", err |
|
| 481 |
+ } |
|
| 482 |
+ p = path.Join("/", dir, p)
|
|
| 479 | 483 |
} else if c.fas != nil {
|
| 480 |
- p = path.Join("/", c.fas.state.GetDir(), p)
|
|
| 484 |
+ dir, err := c.fas.state.GetDir(ctx) |
|
| 485 |
+ if err != nil {
|
|
| 486 |
+ return "", err |
|
| 487 |
+ } |
|
| 488 |
+ p = path.Join("/", dir, p)
|
|
| 481 | 489 |
} |
| 482 | 490 |
} |
| 483 |
- return p |
|
| 491 |
+ return p, nil |
|
| 484 | 492 |
} |
| 485 | 493 |
|
| 486 | 494 |
type CreatedTime time.Time |
| ... | ... |
@@ -517,7 +530,7 @@ type FileOp struct {
|
| 517 | 517 |
isValidated bool |
| 518 | 518 |
} |
| 519 | 519 |
|
| 520 |
-func (f *FileOp) Validate() error {
|
|
| 520 |
+func (f *FileOp) Validate(context.Context) error {
|
|
| 521 | 521 |
if f.isValidated {
|
| 522 | 522 |
return nil |
| 523 | 523 |
} |
| ... | ... |
@@ -529,14 +542,16 @@ func (f *FileOp) Validate() error {
|
| 529 | 529 |
} |
| 530 | 530 |
|
| 531 | 531 |
type marshalState struct {
|
| 532 |
+ ctx context.Context |
|
| 532 | 533 |
visited map[*FileAction]*fileActionState |
| 533 | 534 |
inputs []*pb.Input |
| 534 | 535 |
actions []*fileActionState |
| 535 | 536 |
} |
| 536 | 537 |
|
| 537 |
-func newMarshalState() *marshalState {
|
|
| 538 |
+func newMarshalState(ctx context.Context) *marshalState {
|
|
| 538 | 539 |
return &marshalState{
|
| 539 | 540 |
visited: map[*FileAction]*fileActionState{},
|
| 541 |
+ ctx: ctx, |
|
| 540 | 542 |
} |
| 541 | 543 |
} |
| 542 | 544 |
|
| ... | ... |
@@ -552,7 +567,7 @@ type fileActionState struct {
|
| 552 | 552 |
} |
| 553 | 553 |
|
| 554 | 554 |
func (ms *marshalState) addInput(st *fileActionState, c *Constraints, o Output) (pb.InputIndex, error) {
|
| 555 |
- inp, err := o.ToInput(c) |
|
| 555 |
+ inp, err := o.ToInput(ms.ctx, c) |
|
| 556 | 556 |
if err != nil {
|
| 557 | 557 |
return 0, err |
| 558 | 558 |
} |
| ... | ... |
@@ -634,11 +649,11 @@ func (ms *marshalState) add(fa *FileAction, c *Constraints) (*fileActionState, e |
| 634 | 634 |
return st, nil |
| 635 | 635 |
} |
| 636 | 636 |
|
| 637 |
-func (f *FileOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
|
|
| 637 |
+func (f *FileOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
|
|
| 638 | 638 |
if f.Cached(c) {
|
| 639 | 639 |
return f.Load() |
| 640 | 640 |
} |
| 641 |
- if err := f.Validate(); err != nil {
|
|
| 641 |
+ if err := f.Validate(ctx); err != nil {
|
|
| 642 | 642 |
return "", nil, nil, err |
| 643 | 643 |
} |
| 644 | 644 |
|
| ... | ... |
@@ -651,7 +666,7 @@ func (f *FileOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, |
| 651 | 651 |
File: pfo, |
| 652 | 652 |
} |
| 653 | 653 |
|
| 654 |
- state := newMarshalState() |
|
| 654 |
+ state := newMarshalState(ctx) |
|
| 655 | 655 |
_, err := state.add(f.action, c) |
| 656 | 656 |
if err != nil {
|
| 657 | 657 |
return "", nil, nil, err |
| ... | ... |
@@ -666,14 +681,22 @@ func (f *FileOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, |
| 666 | 666 |
|
| 667 | 667 |
var parent string |
| 668 | 668 |
if st.fa.state != nil {
|
| 669 |
- parent = st.fa.state.GetDir() |
|
| 669 |
+ parent, err = st.fa.state.GetDir(ctx) |
|
| 670 |
+ if err != nil {
|
|
| 671 |
+ return "", nil, nil, err |
|
| 672 |
+ } |
|
| 673 |
+ } |
|
| 674 |
+ |
|
| 675 |
+ action, err := st.action.toProtoAction(ctx, parent, st.base) |
|
| 676 |
+ if err != nil {
|
|
| 677 |
+ return "", nil, nil, err |
|
| 670 | 678 |
} |
| 671 | 679 |
|
| 672 | 680 |
pfo.Actions = append(pfo.Actions, &pb.FileAction{
|
| 673 | 681 |
Input: getIndex(st.input, len(state.inputs), st.inputRelative), |
| 674 | 682 |
SecondaryInput: getIndex(st.input2, len(state.inputs), st.input2Relative), |
| 675 | 683 |
Output: output, |
| 676 |
- Action: st.action.toProtoAction(parent, st.base), |
|
| 684 |
+ Action: action, |
|
| 677 | 685 |
}) |
| 678 | 686 |
} |
| 679 | 687 |
|
| ... | ... |
@@ -1,6 +1,7 @@ |
| 1 | 1 |
package llb |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 |
+ "context" |
|
| 4 | 5 |
"fmt" |
| 5 | 6 |
"net" |
| 6 | 7 |
"path" |
| ... | ... |
@@ -24,79 +25,122 @@ var ( |
| 24 | 24 |
keySecurity = contextKeyT("llb.security")
|
| 25 | 25 |
) |
| 26 | 26 |
|
| 27 |
+func AddEnvf(key, value string, v ...interface{}) StateOption {
|
|
| 28 |
+ return addEnvf(key, value, true, v...) |
|
| 29 |
+} |
|
| 30 |
+ |
|
| 31 |
+func AddEnv(key, value string) StateOption {
|
|
| 32 |
+ return addEnvf(key, value, false) |
|
| 33 |
+} |
|
| 34 |
+ |
|
| 27 | 35 |
func addEnvf(key, value string, replace bool, v ...interface{}) StateOption {
|
| 28 | 36 |
if replace {
|
| 29 | 37 |
value = fmt.Sprintf(value, v...) |
| 30 | 38 |
} |
| 31 | 39 |
return func(s State) State {
|
| 32 |
- return s.WithValue(keyEnv, getEnv(s).AddOrReplace(key, value)) |
|
| 40 |
+ return s.withValue(keyEnv, func(ctx context.Context) (interface{}, error) {
|
|
| 41 |
+ env, err := getEnv(s)(ctx) |
|
| 42 |
+ if err != nil {
|
|
| 43 |
+ return nil, err |
|
| 44 |
+ } |
|
| 45 |
+ return env.AddOrReplace(key, value), nil |
|
| 46 |
+ }) |
|
| 33 | 47 |
} |
| 34 | 48 |
} |
| 35 | 49 |
|
| 36 |
-func dir(str string) StateOption {
|
|
| 50 |
+func Dir(str string) StateOption {
|
|
| 37 | 51 |
return dirf(str, false) |
| 38 | 52 |
} |
| 39 | 53 |
|
| 54 |
+func Dirf(str string, v ...interface{}) StateOption {
|
|
| 55 |
+ return dirf(str, true, v...) |
|
| 56 |
+} |
|
| 57 |
+ |
|
| 40 | 58 |
func dirf(value string, replace bool, v ...interface{}) StateOption {
|
| 41 | 59 |
if replace {
|
| 42 | 60 |
value = fmt.Sprintf(value, v...) |
| 43 | 61 |
} |
| 44 | 62 |
return func(s State) State {
|
| 45 |
- if !path.IsAbs(value) {
|
|
| 46 |
- prev := getDir(s) |
|
| 47 |
- if prev == "" {
|
|
| 48 |
- prev = "/" |
|
| 63 |
+ return s.withValue(keyDir, func(ctx context.Context) (interface{}, error) {
|
|
| 64 |
+ if !path.IsAbs(value) {
|
|
| 65 |
+ prev, err := getDir(s)(ctx) |
|
| 66 |
+ if err != nil {
|
|
| 67 |
+ return nil, err |
|
| 68 |
+ } |
|
| 69 |
+ if prev == "" {
|
|
| 70 |
+ prev = "/" |
|
| 71 |
+ } |
|
| 72 |
+ value = path.Join(prev, value) |
|
| 49 | 73 |
} |
| 50 |
- value = path.Join(prev, value) |
|
| 51 |
- } |
|
| 52 |
- return s.WithValue(keyDir, value) |
|
| 74 |
+ return value, nil |
|
| 75 |
+ }) |
|
| 53 | 76 |
} |
| 54 | 77 |
} |
| 55 | 78 |
|
| 56 |
-func user(str string) StateOption {
|
|
| 79 |
+func User(str string) StateOption {
|
|
| 57 | 80 |
return func(s State) State {
|
| 58 | 81 |
return s.WithValue(keyUser, str) |
| 59 | 82 |
} |
| 60 | 83 |
} |
| 61 | 84 |
|
| 62 |
-func reset(s_ State) StateOption {
|
|
| 85 |
+func Reset(other State) StateOption {
|
|
| 63 | 86 |
return func(s State) State {
|
| 64 | 87 |
s = NewState(s.Output()) |
| 65 |
- s.ctx = s_.ctx |
|
| 88 |
+ s.prev = &other |
|
| 66 | 89 |
return s |
| 67 | 90 |
} |
| 68 | 91 |
} |
| 69 | 92 |
|
| 70 |
-func getEnv(s State) EnvList {
|
|
| 71 |
- v := s.Value(keyEnv) |
|
| 72 |
- if v != nil {
|
|
| 73 |
- return v.(EnvList) |
|
| 93 |
+func getEnv(s State) func(context.Context) (EnvList, error) {
|
|
| 94 |
+ return func(ctx context.Context) (EnvList, error) {
|
|
| 95 |
+ v, err := s.getValue(keyEnv)(ctx) |
|
| 96 |
+ if err != nil {
|
|
| 97 |
+ return nil, err |
|
| 98 |
+ } |
|
| 99 |
+ if v != nil {
|
|
| 100 |
+ return v.(EnvList), nil |
|
| 101 |
+ } |
|
| 102 |
+ return EnvList{}, nil
|
|
| 74 | 103 |
} |
| 75 |
- return EnvList{}
|
|
| 76 | 104 |
} |
| 77 | 105 |
|
| 78 |
-func getDir(s State) string {
|
|
| 79 |
- v := s.Value(keyDir) |
|
| 80 |
- if v != nil {
|
|
| 81 |
- return v.(string) |
|
| 106 |
+func getDir(s State) func(context.Context) (string, error) {
|
|
| 107 |
+ return func(ctx context.Context) (string, error) {
|
|
| 108 |
+ v, err := s.getValue(keyDir)(ctx) |
|
| 109 |
+ if err != nil {
|
|
| 110 |
+ return "", err |
|
| 111 |
+ } |
|
| 112 |
+ if v != nil {
|
|
| 113 |
+ return v.(string), nil |
|
| 114 |
+ } |
|
| 115 |
+ return "", nil |
|
| 82 | 116 |
} |
| 83 |
- return "" |
|
| 84 | 117 |
} |
| 85 | 118 |
|
| 86 |
-func getArgs(s State) []string {
|
|
| 87 |
- v := s.Value(keyArgs) |
|
| 88 |
- if v != nil {
|
|
| 89 |
- return v.([]string) |
|
| 119 |
+func getArgs(s State) func(context.Context) ([]string, error) {
|
|
| 120 |
+ return func(ctx context.Context) ([]string, error) {
|
|
| 121 |
+ v, err := s.getValue(keyArgs)(ctx) |
|
| 122 |
+ if err != nil {
|
|
| 123 |
+ return nil, err |
|
| 124 |
+ } |
|
| 125 |
+ if v != nil {
|
|
| 126 |
+ return v.([]string), nil |
|
| 127 |
+ } |
|
| 128 |
+ return nil, nil |
|
| 90 | 129 |
} |
| 91 |
- return nil |
|
| 92 | 130 |
} |
| 93 | 131 |
|
| 94 |
-func getUser(s State) string {
|
|
| 95 |
- v := s.Value(keyUser) |
|
| 96 |
- if v != nil {
|
|
| 97 |
- return v.(string) |
|
| 132 |
+func getUser(s State) func(context.Context) (string, error) {
|
|
| 133 |
+ return func(ctx context.Context) (string, error) {
|
|
| 134 |
+ v, err := s.getValue(keyUser)(ctx) |
|
| 135 |
+ if err != nil {
|
|
| 136 |
+ return "", err |
|
| 137 |
+ } |
|
| 138 |
+ if v != nil {
|
|
| 139 |
+ return v.(string), nil |
|
| 140 |
+ } |
|
| 141 |
+ return "", nil |
|
| 98 | 142 |
} |
| 99 |
- return "" |
|
| 100 | 143 |
} |
| 101 | 144 |
|
| 102 | 145 |
func args(args ...string) StateOption {
|
| ... | ... |
@@ -124,27 +168,43 @@ func platform(p specs.Platform) StateOption {
|
| 124 | 124 |
} |
| 125 | 125 |
} |
| 126 | 126 |
|
| 127 |
-func getPlatform(s State) *specs.Platform {
|
|
| 128 |
- v := s.Value(keyPlatform) |
|
| 129 |
- if v != nil {
|
|
| 130 |
- p := v.(specs.Platform) |
|
| 131 |
- return &p |
|
| 127 |
+func getPlatform(s State) func(context.Context) (*specs.Platform, error) {
|
|
| 128 |
+ return func(ctx context.Context) (*specs.Platform, error) {
|
|
| 129 |
+ v, err := s.getValue(keyPlatform)(ctx) |
|
| 130 |
+ if err != nil {
|
|
| 131 |
+ return nil, err |
|
| 132 |
+ } |
|
| 133 |
+ if v != nil {
|
|
| 134 |
+ p := v.(specs.Platform) |
|
| 135 |
+ return &p, nil |
|
| 136 |
+ } |
|
| 137 |
+ return nil, nil |
|
| 132 | 138 |
} |
| 133 |
- return nil |
|
| 134 | 139 |
} |
| 135 | 140 |
|
| 136 | 141 |
func extraHost(host string, ip net.IP) StateOption {
|
| 137 | 142 |
return func(s State) State {
|
| 138 |
- return s.WithValue(keyExtraHost, append(getExtraHosts(s), HostIP{Host: host, IP: ip}))
|
|
| 143 |
+ return s.withValue(keyExtraHost, func(ctx context.Context) (interface{}, error) {
|
|
| 144 |
+ v, err := getExtraHosts(s)(ctx) |
|
| 145 |
+ if err != nil {
|
|
| 146 |
+ return nil, err |
|
| 147 |
+ } |
|
| 148 |
+ return append(v, HostIP{Host: host, IP: ip}), nil
|
|
| 149 |
+ }) |
|
| 139 | 150 |
} |
| 140 | 151 |
} |
| 141 | 152 |
|
| 142 |
-func getExtraHosts(s State) []HostIP {
|
|
| 143 |
- v := s.Value(keyExtraHost) |
|
| 144 |
- if v != nil {
|
|
| 145 |
- return v.([]HostIP) |
|
| 153 |
+func getExtraHosts(s State) func(context.Context) ([]HostIP, error) {
|
|
| 154 |
+ return func(ctx context.Context) ([]HostIP, error) {
|
|
| 155 |
+ v, err := s.getValue(keyExtraHost)(ctx) |
|
| 156 |
+ if err != nil {
|
|
| 157 |
+ return nil, err |
|
| 158 |
+ } |
|
| 159 |
+ if v != nil {
|
|
| 160 |
+ return v.([]HostIP), nil |
|
| 161 |
+ } |
|
| 162 |
+ return nil, nil |
|
| 146 | 163 |
} |
| 147 |
- return nil |
|
| 148 | 164 |
} |
| 149 | 165 |
|
| 150 | 166 |
type HostIP struct {
|
| ... | ... |
@@ -152,32 +212,42 @@ type HostIP struct {
|
| 152 | 152 |
IP net.IP |
| 153 | 153 |
} |
| 154 | 154 |
|
| 155 |
-func network(v pb.NetMode) StateOption {
|
|
| 155 |
+func Network(v pb.NetMode) StateOption {
|
|
| 156 | 156 |
return func(s State) State {
|
| 157 | 157 |
return s.WithValue(keyNetwork, v) |
| 158 | 158 |
} |
| 159 | 159 |
} |
| 160 |
-func getNetwork(s State) pb.NetMode {
|
|
| 161 |
- v := s.Value(keyNetwork) |
|
| 162 |
- if v != nil {
|
|
| 163 |
- n := v.(pb.NetMode) |
|
| 164 |
- return n |
|
| 160 |
+func getNetwork(s State) func(context.Context) (pb.NetMode, error) {
|
|
| 161 |
+ return func(ctx context.Context) (pb.NetMode, error) {
|
|
| 162 |
+ v, err := s.getValue(keyNetwork)(ctx) |
|
| 163 |
+ if err != nil {
|
|
| 164 |
+ return 0, err |
|
| 165 |
+ } |
|
| 166 |
+ if v != nil {
|
|
| 167 |
+ n := v.(pb.NetMode) |
|
| 168 |
+ return n, nil |
|
| 169 |
+ } |
|
| 170 |
+ return NetModeSandbox, nil |
|
| 165 | 171 |
} |
| 166 |
- return NetModeSandbox |
|
| 167 | 172 |
} |
| 168 | 173 |
|
| 169 |
-func security(v pb.SecurityMode) StateOption {
|
|
| 174 |
+func Security(v pb.SecurityMode) StateOption {
|
|
| 170 | 175 |
return func(s State) State {
|
| 171 | 176 |
return s.WithValue(keySecurity, v) |
| 172 | 177 |
} |
| 173 | 178 |
} |
| 174 |
-func getSecurity(s State) pb.SecurityMode {
|
|
| 175 |
- v := s.Value(keySecurity) |
|
| 176 |
- if v != nil {
|
|
| 177 |
- n := v.(pb.SecurityMode) |
|
| 178 |
- return n |
|
| 179 |
+func getSecurity(s State) func(context.Context) (pb.SecurityMode, error) {
|
|
| 180 |
+ return func(ctx context.Context) (pb.SecurityMode, error) {
|
|
| 181 |
+ v, err := s.getValue(keySecurity)(ctx) |
|
| 182 |
+ if err != nil {
|
|
| 183 |
+ return 0, err |
|
| 184 |
+ } |
|
| 185 |
+ if v != nil {
|
|
| 186 |
+ n := v.(pb.SecurityMode) |
|
| 187 |
+ return n, nil |
|
| 188 |
+ } |
|
| 189 |
+ return SecurityModeSandbox, nil |
|
| 179 | 190 |
} |
| 180 |
- return SecurityModeSandbox |
|
| 181 | 191 |
} |
| 182 | 192 |
|
| 183 | 193 |
type EnvList []KeyValue |
| ... | ... |
@@ -14,6 +14,15 @@ func WithMetaResolver(mr ImageMetaResolver) ImageOption {
|
| 14 | 14 |
}) |
| 15 | 15 |
} |
| 16 | 16 |
|
| 17 |
+// ResolveDigest uses the meta resolver to update the ref of image with full digest before marshaling. |
|
| 18 |
+// This makes image ref immutable and is recommended if you want to make sure meta resolver data |
|
| 19 |
+// matches the image used during the build. |
|
| 20 |
+func ResolveDigest(v bool) ImageOption {
|
|
| 21 |
+ return imageOptionFunc(func(ii *ImageInfo) {
|
|
| 22 |
+ ii.resolveDigest = v |
|
| 23 |
+ }) |
|
| 24 |
+} |
|
| 25 |
+ |
|
| 17 | 26 |
// ImageMetaResolver can resolve image config metadata from a reference |
| 18 | 27 |
type ImageMetaResolver interface {
|
| 19 | 28 |
ResolveImageConfig(ctx context.Context, ref string, opt ResolveImageConfigOpt) (digest.Digest, []byte, error) |
| ... | ... |
@@ -34,7 +34,7 @@ func NewSource(id string, attrs map[string]string, c Constraints) *SourceOp {
|
| 34 | 34 |
return s |
| 35 | 35 |
} |
| 36 | 36 |
|
| 37 |
-func (s *SourceOp) Validate() error {
|
|
| 37 |
+func (s *SourceOp) Validate(ctx context.Context) error {
|
|
| 38 | 38 |
if s.err != nil {
|
| 39 | 39 |
return s.err |
| 40 | 40 |
} |
| ... | ... |
@@ -44,11 +44,11 @@ func (s *SourceOp) Validate() error {
|
| 44 | 44 |
return nil |
| 45 | 45 |
} |
| 46 | 46 |
|
| 47 |
-func (s *SourceOp) Marshal(constraints *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
|
|
| 47 |
+func (s *SourceOp) Marshal(ctx context.Context, constraints *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) {
|
|
| 48 | 48 |
if s.Cached(constraints) {
|
| 49 | 49 |
return s.Load() |
| 50 | 50 |
} |
| 51 |
- if err := s.Validate(); err != nil {
|
|
| 51 |
+ if err := s.Validate(ctx); err != nil {
|
|
| 52 | 52 |
return "", nil, nil, err |
| 53 | 53 |
} |
| 54 | 54 |
|
| ... | ... |
@@ -92,7 +92,8 @@ func (s *SourceOp) Inputs() []Output {
|
| 92 | 92 |
func Image(ref string, opts ...ImageOption) State {
|
| 93 | 93 |
r, err := reference.ParseNormalizedNamed(ref) |
| 94 | 94 |
if err == nil {
|
| 95 |
- ref = reference.TagNameOnly(r).String() |
|
| 95 |
+ r = reference.TagNameOnly(r) |
|
| 96 |
+ ref = r.String() |
|
| 96 | 97 |
} |
| 97 | 98 |
var info ImageInfo |
| 98 | 99 |
for _, opt := range opts {
|
| ... | ... |
@@ -116,21 +117,35 @@ func Image(ref string, opts ...ImageOption) State {
|
| 116 | 116 |
src := NewSource("docker-image://"+ref, attrs, info.Constraints) // controversial
|
| 117 | 117 |
if err != nil {
|
| 118 | 118 |
src.err = err |
| 119 |
- } |
|
| 120 |
- if info.metaResolver != nil {
|
|
| 121 |
- _, dt, err := info.metaResolver.ResolveImageConfig(context.TODO(), ref, ResolveImageConfigOpt{
|
|
| 122 |
- Platform: info.Constraints.Platform, |
|
| 123 |
- ResolveMode: info.resolveMode.String(), |
|
| 124 |
- }) |
|
| 125 |
- if err != nil {
|
|
| 126 |
- src.err = err |
|
| 127 |
- } else {
|
|
| 128 |
- st, err := NewState(src.Output()).WithImageConfig(dt) |
|
| 129 |
- if err == nil {
|
|
| 130 |
- return st |
|
| 131 |
- } |
|
| 132 |
- src.err = err |
|
| 119 |
+ } else if info.metaResolver != nil {
|
|
| 120 |
+ if _, ok := r.(reference.Digested); ok || !info.resolveDigest {
|
|
| 121 |
+ return NewState(src.Output()).Async(func(ctx context.Context, st State) (State, error) {
|
|
| 122 |
+ _, dt, err := info.metaResolver.ResolveImageConfig(ctx, ref, ResolveImageConfigOpt{
|
|
| 123 |
+ Platform: info.Constraints.Platform, |
|
| 124 |
+ ResolveMode: info.resolveMode.String(), |
|
| 125 |
+ }) |
|
| 126 |
+ if err != nil {
|
|
| 127 |
+ return State{}, err
|
|
| 128 |
+ } |
|
| 129 |
+ return st.WithImageConfig(dt) |
|
| 130 |
+ }) |
|
| 133 | 131 |
} |
| 132 |
+ return Scratch().Async(func(ctx context.Context, _ State) (State, error) {
|
|
| 133 |
+ dgst, dt, err := info.metaResolver.ResolveImageConfig(context.TODO(), ref, ResolveImageConfigOpt{
|
|
| 134 |
+ Platform: info.Constraints.Platform, |
|
| 135 |
+ ResolveMode: info.resolveMode.String(), |
|
| 136 |
+ }) |
|
| 137 |
+ if err != nil {
|
|
| 138 |
+ return State{}, err
|
|
| 139 |
+ } |
|
| 140 |
+ if dgst != "" {
|
|
| 141 |
+ r, err = reference.WithDigest(r, dgst) |
|
| 142 |
+ if err != nil {
|
|
| 143 |
+ return State{}, err
|
|
| 144 |
+ } |
|
| 145 |
+ } |
|
| 146 |
+ return NewState(NewSource("docker-image://"+r.String(), attrs, info.Constraints).Output()).WithImageConfig(dt)
|
|
| 147 |
+ }) |
|
| 134 | 148 |
} |
| 135 | 149 |
return NewState(src.Output()) |
| 136 | 150 |
} |
| ... | ... |
@@ -176,9 +191,10 @@ func (r ResolveMode) String() string {
|
| 176 | 176 |
|
| 177 | 177 |
type ImageInfo struct {
|
| 178 | 178 |
constraintsWrapper |
| 179 |
- metaResolver ImageMetaResolver |
|
| 180 |
- resolveMode ResolveMode |
|
| 181 |
- RecordType string |
|
| 179 |
+ metaResolver ImageMetaResolver |
|
| 180 |
+ resolveDigest bool |
|
| 181 |
+ resolveMode ResolveMode |
|
| 182 |
+ RecordType string |
|
| 182 | 183 |
} |
| 183 | 184 |
|
| 184 | 185 |
func Git(remote, ref string, opts ...GitOption) State {
|
| ... | ... |
@@ -18,13 +18,13 @@ import ( |
| 18 | 18 |
type StateOption func(State) State |
| 19 | 19 |
|
| 20 | 20 |
type Output interface {
|
| 21 |
- ToInput(*Constraints) (*pb.Input, error) |
|
| 22 |
- Vertex() Vertex |
|
| 21 |
+ ToInput(context.Context, *Constraints) (*pb.Input, error) |
|
| 22 |
+ Vertex(context.Context) Vertex |
|
| 23 | 23 |
} |
| 24 | 24 |
|
| 25 | 25 |
type Vertex interface {
|
| 26 |
- Validate() error |
|
| 27 |
- Marshal(*Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) |
|
| 26 |
+ Validate(context.Context) error |
|
| 27 |
+ Marshal(context.Context, *Constraints) (digest.Digest, []byte, *pb.OpMetadata, error) |
|
| 28 | 28 |
Output() Output |
| 29 | 29 |
Inputs() []Output |
| 30 | 30 |
} |
| ... | ... |
@@ -32,17 +32,18 @@ type Vertex interface {
|
| 32 | 32 |
func NewState(o Output) State {
|
| 33 | 33 |
s := State{
|
| 34 | 34 |
out: o, |
| 35 |
- ctx: context.Background(), |
|
| 36 |
- } |
|
| 37 |
- s = dir("/")(s)
|
|
| 35 |
+ }.Dir("/")
|
|
| 38 | 36 |
s = s.ensurePlatform() |
| 39 | 37 |
return s |
| 40 | 38 |
} |
| 41 | 39 |
|
| 42 | 40 |
type State struct {
|
| 43 |
- out Output |
|
| 44 |
- ctx context.Context |
|
| 45 |
- opts []ConstraintsOpt |
|
| 41 |
+ out Output |
|
| 42 |
+ prev *State |
|
| 43 |
+ key interface{}
|
|
| 44 |
+ value func(context.Context) (interface{}, error)
|
|
| 45 |
+ opts []ConstraintsOpt |
|
| 46 |
+ async *asyncState |
|
| 46 | 47 |
} |
| 47 | 48 |
|
| 48 | 49 |
func (s State) ensurePlatform() State {
|
| ... | ... |
@@ -57,14 +58,48 @@ func (s State) ensurePlatform() State {
|
| 57 | 57 |
} |
| 58 | 58 |
|
| 59 | 59 |
func (s State) WithValue(k, v interface{}) State {
|
| 60 |
+ return s.withValue(k, func(context.Context) (interface{}, error) {
|
|
| 61 |
+ return v, nil |
|
| 62 |
+ }) |
|
| 63 |
+} |
|
| 64 |
+ |
|
| 65 |
+func (s State) withValue(k interface{}, v func(context.Context) (interface{}, error)) State {
|
|
| 60 | 66 |
return State{
|
| 61 |
- out: s.out, |
|
| 62 |
- ctx: context.WithValue(s.ctx, k, v), |
|
| 67 |
+ out: s.Output(), |
|
| 68 |
+ prev: &s, // doesn't need to be original pointer |
|
| 69 |
+ key: k, |
|
| 70 |
+ value: v, |
|
| 71 |
+ } |
|
| 72 |
+} |
|
| 73 |
+ |
|
| 74 |
+func (s State) Value(ctx context.Context, k interface{}) (interface{}, error) {
|
|
| 75 |
+ return s.getValue(k)(ctx) |
|
| 76 |
+} |
|
| 77 |
+ |
|
| 78 |
+func (s State) getValue(k interface{}) func(context.Context) (interface{}, error) {
|
|
| 79 |
+ if s.key == k {
|
|
| 80 |
+ return s.value |
|
| 81 |
+ } |
|
| 82 |
+ if s.async != nil {
|
|
| 83 |
+ return func(ctx context.Context) (interface{}, error) {
|
|
| 84 |
+ err := s.async.Do(ctx) |
|
| 85 |
+ if err != nil {
|
|
| 86 |
+ return nil, err |
|
| 87 |
+ } |
|
| 88 |
+ return s.async.target.getValue(k)(ctx) |
|
| 89 |
+ } |
|
| 90 |
+ } |
|
| 91 |
+ if s.prev == nil {
|
|
| 92 |
+ return nilValue |
|
| 63 | 93 |
} |
| 94 |
+ return s.prev.getValue(k) |
|
| 64 | 95 |
} |
| 65 | 96 |
|
| 66 |
-func (s State) Value(k interface{}) interface{} {
|
|
| 67 |
- return s.ctx.Value(k) |
|
| 97 |
+func (s State) Async(f func(context.Context, State) (State, error)) State {
|
|
| 98 |
+ s2 := State{
|
|
| 99 |
+ async: &asyncState{f: f, prev: s},
|
|
| 100 |
+ } |
|
| 101 |
+ return s2 |
|
| 68 | 102 |
} |
| 69 | 103 |
|
| 70 | 104 |
func (s State) SetMarshalDefaults(co ...ConstraintsOpt) State {
|
| ... | ... |
@@ -72,11 +107,11 @@ func (s State) SetMarshalDefaults(co ...ConstraintsOpt) State {
|
| 72 | 72 |
return s |
| 73 | 73 |
} |
| 74 | 74 |
|
| 75 |
-func (s State) Marshal(co ...ConstraintsOpt) (*Definition, error) {
|
|
| 75 |
+func (s State) Marshal(ctx context.Context, co ...ConstraintsOpt) (*Definition, error) {
|
|
| 76 | 76 |
def := &Definition{
|
| 77 | 77 |
Metadata: make(map[digest.Digest]pb.OpMetadata, 0), |
| 78 | 78 |
} |
| 79 |
- if s.Output() == nil {
|
|
| 79 |
+ if s.Output() == nil || s.Output().Vertex(ctx) == nil {
|
|
| 80 | 80 |
return def, nil |
| 81 | 81 |
} |
| 82 | 82 |
|
| ... | ... |
@@ -89,11 +124,11 @@ func (s State) Marshal(co ...ConstraintsOpt) (*Definition, error) {
|
| 89 | 89 |
o.SetConstraintsOption(c) |
| 90 | 90 |
} |
| 91 | 91 |
|
| 92 |
- def, err := marshal(s.Output().Vertex(), def, map[digest.Digest]struct{}{}, map[Vertex]struct{}{}, c)
|
|
| 92 |
+ def, err := marshal(ctx, s.Output().Vertex(ctx), def, map[digest.Digest]struct{}{}, map[Vertex]struct{}{}, c)
|
|
| 93 | 93 |
if err != nil {
|
| 94 | 94 |
return def, err |
| 95 | 95 |
} |
| 96 |
- inp, err := s.Output().ToInput(c) |
|
| 96 |
+ inp, err := s.Output().ToInput(ctx, c) |
|
| 97 | 97 |
if err != nil {
|
| 98 | 98 |
return def, err |
| 99 | 99 |
} |
| ... | ... |
@@ -128,19 +163,19 @@ func (s State) Marshal(co ...ConstraintsOpt) (*Definition, error) {
|
| 128 | 128 |
return def, nil |
| 129 | 129 |
} |
| 130 | 130 |
|
| 131 |
-func marshal(v Vertex, def *Definition, cache map[digest.Digest]struct{}, vertexCache map[Vertex]struct{}, c *Constraints) (*Definition, error) {
|
|
| 131 |
+func marshal(ctx context.Context, v Vertex, def *Definition, cache map[digest.Digest]struct{}, vertexCache map[Vertex]struct{}, c *Constraints) (*Definition, error) {
|
|
| 132 | 132 |
if _, ok := vertexCache[v]; ok {
|
| 133 | 133 |
return def, nil |
| 134 | 134 |
} |
| 135 | 135 |
for _, inp := range v.Inputs() {
|
| 136 | 136 |
var err error |
| 137 |
- def, err = marshal(inp.Vertex(), def, cache, vertexCache, c) |
|
| 137 |
+ def, err = marshal(ctx, inp.Vertex(ctx), def, cache, vertexCache, c) |
|
| 138 | 138 |
if err != nil {
|
| 139 | 139 |
return def, err |
| 140 | 140 |
} |
| 141 | 141 |
} |
| 142 | 142 |
|
| 143 |
- dgst, dt, opMeta, err := v.Marshal(c) |
|
| 143 |
+ dgst, dt, opMeta, err := v.Marshal(ctx, c) |
|
| 144 | 144 |
if err != nil {
|
| 145 | 145 |
return def, err |
| 146 | 146 |
} |
| ... | ... |
@@ -156,18 +191,22 @@ func marshal(v Vertex, def *Definition, cache map[digest.Digest]struct{}, vertex
|
| 156 | 156 |
return def, nil |
| 157 | 157 |
} |
| 158 | 158 |
|
| 159 |
-func (s State) Validate() error {
|
|
| 160 |
- return s.Output().Vertex().Validate() |
|
| 159 |
+func (s State) Validate(ctx context.Context) error {
|
|
| 160 |
+ return s.Output().Vertex(ctx).Validate(ctx) |
|
| 161 | 161 |
} |
| 162 | 162 |
|
| 163 | 163 |
func (s State) Output() Output {
|
| 164 |
+ if s.async != nil {
|
|
| 165 |
+ return s.async.Output() |
|
| 166 |
+ } |
|
| 164 | 167 |
return s.out |
| 165 | 168 |
} |
| 166 | 169 |
|
| 167 | 170 |
func (s State) WithOutput(o Output) State {
|
| 171 |
+ prev := s |
|
| 168 | 172 |
s = State{
|
| 169 |
- out: o, |
|
| 170 |
- ctx: s.ctx, |
|
| 173 |
+ out: o, |
|
| 174 |
+ prev: &prev, |
|
| 171 | 175 |
} |
| 172 | 176 |
s = s.ensurePlatform() |
| 173 | 177 |
return s |
| ... | ... |
@@ -200,24 +239,10 @@ func (s State) WithImageConfig(c []byte) (State, error) {
|
| 200 | 200 |
|
| 201 | 201 |
func (s State) Run(ro ...RunOption) ExecState {
|
| 202 | 202 |
ei := &ExecInfo{State: s}
|
| 203 |
- if p := s.GetPlatform(); p != nil {
|
|
| 204 |
- ei.Constraints.Platform = p |
|
| 205 |
- } |
|
| 206 | 203 |
for _, o := range ro {
|
| 207 | 204 |
o.SetRunOption(ei) |
| 208 | 205 |
} |
| 209 |
- meta := Meta{
|
|
| 210 |
- Args: getArgs(ei.State), |
|
| 211 |
- Cwd: getDir(ei.State), |
|
| 212 |
- Env: getEnv(ei.State), |
|
| 213 |
- User: getUser(ei.State), |
|
| 214 |
- ProxyEnv: ei.ProxyEnv, |
|
| 215 |
- ExtraHosts: getExtraHosts(ei.State), |
|
| 216 |
- Network: getNetwork(ei.State), |
|
| 217 |
- Security: getSecurity(ei.State), |
|
| 218 |
- } |
|
| 219 |
- |
|
| 220 |
- exec := NewExecOp(s.Output(), meta, ei.ReadonlyRootFS, ei.Constraints) |
|
| 206 |
+ exec := NewExecOp(ei.State, ei.ProxyEnv, ei.ReadonlyRootFS, ei.Constraints) |
|
| 221 | 207 |
for _, m := range ei.Mounts {
|
| 222 | 208 |
exec.AddMount(m.Target, m.Source, m.Opts...) |
| 223 | 209 |
} |
| ... | ... |
@@ -240,65 +265,74 @@ func (s State) File(a *FileAction, opts ...ConstraintsOpt) State {
|
| 240 | 240 |
} |
| 241 | 241 |
|
| 242 | 242 |
func (s State) AddEnv(key, value string) State {
|
| 243 |
- return addEnvf(key, value, false)(s) |
|
| 243 |
+ return AddEnv(key, value)(s) |
|
| 244 | 244 |
} |
| 245 | 245 |
|
| 246 | 246 |
func (s State) AddEnvf(key, value string, v ...interface{}) State {
|
| 247 |
- return addEnvf(key, value, true, v...)(s) |
|
| 247 |
+ return AddEnvf(key, value, v...)(s) |
|
| 248 | 248 |
} |
| 249 | 249 |
|
| 250 | 250 |
func (s State) Dir(str string) State {
|
| 251 |
- return dirf(str, false)(s) |
|
| 251 |
+ return Dir(str)(s) |
|
| 252 | 252 |
} |
| 253 | 253 |
func (s State) Dirf(str string, v ...interface{}) State {
|
| 254 |
- return dirf(str, true, v...)(s) |
|
| 254 |
+ return Dirf(str, v...)(s) |
|
| 255 | 255 |
} |
| 256 | 256 |
|
| 257 |
-func (s State) GetEnv(key string) (string, bool) {
|
|
| 258 |
- return getEnv(s).Get(key) |
|
| 257 |
+func (s State) GetEnv(ctx context.Context, key string) (string, bool, error) {
|
|
| 258 |
+ env, err := getEnv(s)(ctx) |
|
| 259 |
+ if err != nil {
|
|
| 260 |
+ return "", false, err |
|
| 261 |
+ } |
|
| 262 |
+ v, ok := env.Get(key) |
|
| 263 |
+ return v, ok, nil |
|
| 259 | 264 |
} |
| 260 | 265 |
|
| 261 |
-func (s State) Env() []string {
|
|
| 262 |
- return getEnv(s).ToArray() |
|
| 266 |
+func (s State) Env(ctx context.Context) ([]string, error) {
|
|
| 267 |
+ env, err := getEnv(s)(ctx) |
|
| 268 |
+ if err != nil {
|
|
| 269 |
+ return nil, err |
|
| 270 |
+ } |
|
| 271 |
+ return env.ToArray(), nil |
|
| 263 | 272 |
} |
| 264 | 273 |
|
| 265 |
-func (s State) GetDir() string {
|
|
| 266 |
- return getDir(s) |
|
| 274 |
+func (s State) GetDir(ctx context.Context) (string, error) {
|
|
| 275 |
+ return getDir(s)(ctx) |
|
| 267 | 276 |
} |
| 268 | 277 |
|
| 269 |
-func (s State) GetArgs() []string {
|
|
| 270 |
- return getArgs(s) |
|
| 278 |
+func (s State) GetArgs(ctx context.Context) ([]string, error) {
|
|
| 279 |
+ return getArgs(s)(ctx) |
|
| 271 | 280 |
} |
| 272 | 281 |
|
| 273 | 282 |
func (s State) Reset(s2 State) State {
|
| 274 |
- return reset(s2)(s) |
|
| 283 |
+ return Reset(s2)(s) |
|
| 275 | 284 |
} |
| 276 | 285 |
|
| 277 | 286 |
func (s State) User(v string) State {
|
| 278 |
- return user(v)(s) |
|
| 287 |
+ return User(v)(s) |
|
| 279 | 288 |
} |
| 280 | 289 |
|
| 281 | 290 |
func (s State) Platform(p specs.Platform) State {
|
| 282 | 291 |
return platform(p)(s) |
| 283 | 292 |
} |
| 284 | 293 |
|
| 285 |
-func (s State) GetPlatform() *specs.Platform {
|
|
| 286 |
- return getPlatform(s) |
|
| 294 |
+func (s State) GetPlatform(ctx context.Context) (*specs.Platform, error) {
|
|
| 295 |
+ return getPlatform(s)(ctx) |
|
| 287 | 296 |
} |
| 288 | 297 |
|
| 289 | 298 |
func (s State) Network(n pb.NetMode) State {
|
| 290 |
- return network(n)(s) |
|
| 299 |
+ return Network(n)(s) |
|
| 291 | 300 |
} |
| 292 | 301 |
|
| 293 |
-func (s State) GetNetwork() pb.NetMode {
|
|
| 294 |
- return getNetwork(s) |
|
| 302 |
+func (s State) GetNetwork(ctx context.Context) (pb.NetMode, error) {
|
|
| 303 |
+ return getNetwork(s)(ctx) |
|
| 295 | 304 |
} |
| 296 | 305 |
func (s State) Security(n pb.SecurityMode) State {
|
| 297 |
- return security(n)(s) |
|
| 306 |
+ return Security(n)(s) |
|
| 298 | 307 |
} |
| 299 | 308 |
|
| 300 |
-func (s State) GetSecurity() pb.SecurityMode {
|
|
| 301 |
- return getSecurity(s) |
|
| 309 |
+func (s State) GetSecurity(ctx context.Context) (pb.SecurityMode, error) {
|
|
| 310 |
+ return getSecurity(s)(ctx) |
|
| 302 | 311 |
} |
| 303 | 312 |
|
| 304 | 313 |
func (s State) With(so ...StateOption) State {
|
| ... | ... |
@@ -321,7 +355,7 @@ type output struct {
|
| 321 | 321 |
platform *specs.Platform |
| 322 | 322 |
} |
| 323 | 323 |
|
| 324 |
-func (o *output) ToInput(c *Constraints) (*pb.Input, error) {
|
|
| 324 |
+func (o *output) ToInput(ctx context.Context, c *Constraints) (*pb.Input, error) {
|
|
| 325 | 325 |
if o.err != nil {
|
| 326 | 326 |
return nil, o.err |
| 327 | 327 |
} |
| ... | ... |
@@ -333,14 +367,14 @@ func (o *output) ToInput(c *Constraints) (*pb.Input, error) {
|
| 333 | 333 |
return nil, err |
| 334 | 334 |
} |
| 335 | 335 |
} |
| 336 |
- dgst, _, _, err := o.vertex.Marshal(c) |
|
| 336 |
+ dgst, _, _, err := o.vertex.Marshal(ctx, c) |
|
| 337 | 337 |
if err != nil {
|
| 338 | 338 |
return nil, err |
| 339 | 339 |
} |
| 340 | 340 |
return &pb.Input{Digest: dgst, Index: index}, nil
|
| 341 | 341 |
} |
| 342 | 342 |
|
| 343 |
-func (o *output) Vertex() Vertex {
|
|
| 343 |
+func (o *output) Vertex(context.Context) Vertex {
|
|
| 344 | 344 |
return o.vertex |
| 345 | 345 |
} |
| 346 | 346 |
|
| ... | ... |
@@ -513,3 +547,7 @@ func Require(filters ...string) ConstraintsOpt {
|
| 513 | 513 |
} |
| 514 | 514 |
}) |
| 515 | 515 |
} |
| 516 |
+ |
|
| 517 |
+func nilValue(context.Context) (interface{}, error) {
|
|
| 518 |
+ return nil, nil |
|
| 519 |
+} |
| ... | ... |
@@ -115,6 +115,12 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG |
| 115 | 115 |
} |
| 116 | 116 |
|
| 117 | 117 |
var ex ExportEntry |
| 118 |
+ if len(opt.Exports) > 1 {
|
|
| 119 |
+ return nil, errors.New("currently only single Exports can be specified")
|
|
| 120 |
+ } |
|
| 121 |
+ if len(opt.Exports) == 1 {
|
|
| 122 |
+ ex = opt.Exports[0] |
|
| 123 |
+ } |
|
| 118 | 124 |
|
| 119 | 125 |
if !opt.SessionPreInitialized {
|
| 120 | 126 |
if len(syncedDirs) > 0 {
|
| ... | ... |
@@ -125,13 +131,6 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG |
| 125 | 125 |
s.Allow(a) |
| 126 | 126 |
} |
| 127 | 127 |
|
| 128 |
- if len(opt.Exports) > 1 {
|
|
| 129 |
- return nil, errors.New("currently only single Exports can be specified")
|
|
| 130 |
- } |
|
| 131 |
- if len(opt.Exports) == 1 {
|
|
| 132 |
- ex = opt.Exports[0] |
|
| 133 |
- } |
|
| 134 |
- |
|
| 135 | 128 |
switch ex.Type {
|
| 136 | 129 |
case ExporterLocal: |
| 137 | 130 |
if ex.Output != nil {
|
| ... | ... |
@@ -192,7 +191,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG |
| 192 | 192 |
|
| 193 | 193 |
frontendInputs := make(map[string]*pb.Definition) |
| 194 | 194 |
for key, st := range opt.FrontendInputs {
|
| 195 |
- def, err := st.Marshal() |
|
| 195 |
+ def, err := st.Marshal(ctx) |
|
| 196 | 196 |
if err != nil {
|
| 197 | 197 |
return err |
| 198 | 198 |
} |
| ... | ... |
@@ -435,13 +434,13 @@ func parseCacheOptions(opt SolveOpt) (*cacheOptions, error) {
|
| 435 | 435 |
continue |
| 436 | 436 |
} |
| 437 | 437 |
for _, m := range idx.Manifests {
|
| 438 |
- if m.Annotations[ocispec.AnnotationRefName] == "latest" {
|
|
| 438 |
+ if (m.Annotations[ocispec.AnnotationRefName] == "latest" && attrs["tag"] == "") || (attrs["tag"] != "" && m.Annotations[ocispec.AnnotationRefName] == attrs["tag"]) {
|
|
| 439 | 439 |
attrs["digest"] = string(m.Digest) |
| 440 | 440 |
break |
| 441 | 441 |
} |
| 442 | 442 |
} |
| 443 | 443 |
if attrs["digest"] == "" {
|
| 444 |
- return nil, errors.New("local cache importer requires either explicit digest or \"latest\" tag on index.json")
|
|
| 444 |
+ return nil, errors.New("local cache importer requires either explicit digest, \"latest\" tag or custom tag on index.json")
|
|
| 445 | 445 |
} |
| 446 | 446 |
} |
| 447 | 447 |
contentStores["local:"+csDir] = cs |
| 448 | 448 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,105 @@ |
| 0 |
+package config |
|
| 1 |
+ |
|
| 2 |
+// Config provides containerd configuration data for the server |
|
| 3 |
+type Config struct {
|
|
| 4 |
+ Debug bool `toml:"debug"` |
|
| 5 |
+ |
|
| 6 |
+ // Root is the path to a directory where buildkit will store persistent data |
|
| 7 |
+ Root string `toml:"root"` |
|
| 8 |
+ |
|
| 9 |
+ //Entitlements e.g. security.insecure, network.host |
|
| 10 |
+ Entitlements []string `toml:"insecure-entitlements"` |
|
| 11 |
+ // GRPC configuration settings |
|
| 12 |
+ GRPC GRPCConfig `toml:"grpc"` |
|
| 13 |
+ |
|
| 14 |
+ Workers struct {
|
|
| 15 |
+ OCI OCIConfig `toml:"oci"` |
|
| 16 |
+ Containerd ContainerdConfig `toml:"containerd"` |
|
| 17 |
+ } `toml:"worker"` |
|
| 18 |
+ |
|
| 19 |
+ Registries map[string]RegistryConfig `toml:"registry"` |
|
| 20 |
+ |
|
| 21 |
+ DNS *DNSConfig `toml:"dns"` |
|
| 22 |
+} |
|
| 23 |
+ |
|
| 24 |
+type GRPCConfig struct {
|
|
| 25 |
+ Address []string `toml:"address"` |
|
| 26 |
+ DebugAddress string `toml:"debugAddress"` |
|
| 27 |
+ UID int `toml:"uid"` |
|
| 28 |
+ GID int `toml:"gid"` |
|
| 29 |
+ |
|
| 30 |
+ TLS TLSConfig `toml:"tls"` |
|
| 31 |
+ // MaxRecvMsgSize int `toml:"max_recv_message_size"` |
|
| 32 |
+ // MaxSendMsgSize int `toml:"max_send_message_size"` |
|
| 33 |
+} |
|
| 34 |
+ |
|
| 35 |
+type RegistryConfig struct {
|
|
| 36 |
+ Mirrors []string `toml:"mirrors"` |
|
| 37 |
+ PlainHTTP *bool `toml:"http"` |
|
| 38 |
+ Insecure *bool `toml:"insecure"` |
|
| 39 |
+ RootCAs []string `toml:"ca"` |
|
| 40 |
+ KeyPairs []TLSKeyPair `toml:"keypair"` |
|
| 41 |
+ TLSConfigDir []string `toml:"tlsconfigdir"` |
|
| 42 |
+} |
|
| 43 |
+ |
|
| 44 |
+type TLSKeyPair struct {
|
|
| 45 |
+ Key string `toml:"key"` |
|
| 46 |
+ Certificate string `toml:"cert"` |
|
| 47 |
+} |
|
| 48 |
+ |
|
| 49 |
+type TLSConfig struct {
|
|
| 50 |
+ Cert string `toml:"cert"` |
|
| 51 |
+ Key string `toml:"key"` |
|
| 52 |
+ CA string `toml:"ca"` |
|
| 53 |
+} |
|
| 54 |
+ |
|
| 55 |
+type GCConfig struct {
|
|
| 56 |
+ GC *bool `toml:"gc"` |
|
| 57 |
+ GCKeepStorage int64 `toml:"gckeepstorage"` |
|
| 58 |
+ GCPolicy []GCPolicy `toml:"gcpolicy"` |
|
| 59 |
+} |
|
| 60 |
+ |
|
| 61 |
+type NetworkConfig struct {
|
|
| 62 |
+ Mode string `toml:"networkMode"` |
|
| 63 |
+ CNIConfigPath string `toml:"cniConfigPath"` |
|
| 64 |
+ CNIBinaryPath string `toml:"cniBinaryPath"` |
|
| 65 |
+} |
|
| 66 |
+ |
|
| 67 |
+type OCIConfig struct {
|
|
| 68 |
+ Enabled *bool `toml:"enabled"` |
|
| 69 |
+ Labels map[string]string `toml:"labels"` |
|
| 70 |
+ Platforms []string `toml:"platforms"` |
|
| 71 |
+ Snapshotter string `toml:"snapshotter"` |
|
| 72 |
+ Rootless bool `toml:"rootless"` |
|
| 73 |
+ NoProcessSandbox bool `toml:"noProcessSandbox"` |
|
| 74 |
+ GCConfig |
|
| 75 |
+ NetworkConfig |
|
| 76 |
+ // UserRemapUnsupported is unsupported key for testing. The feature is |
|
| 77 |
+ // incomplete and the intention is to make it default without config. |
|
| 78 |
+ UserRemapUnsupported string `toml:"userRemapUnsupported"` |
|
| 79 |
+ // For use in storing the OCI worker binary name that will replace buildkit-runc |
|
| 80 |
+ Binary string `toml:"binary"` |
|
| 81 |
+} |
|
| 82 |
+ |
|
| 83 |
+type ContainerdConfig struct {
|
|
| 84 |
+ Address string `toml:"address"` |
|
| 85 |
+ Enabled *bool `toml:"enabled"` |
|
| 86 |
+ Labels map[string]string `toml:"labels"` |
|
| 87 |
+ Platforms []string `toml:"platforms"` |
|
| 88 |
+ Namespace string `toml:"namespace"` |
|
| 89 |
+ GCConfig |
|
| 90 |
+ NetworkConfig |
|
| 91 |
+} |
|
| 92 |
+ |
|
| 93 |
+type GCPolicy struct {
|
|
| 94 |
+ All bool `toml:"all"` |
|
| 95 |
+ KeepBytes int64 `toml:"keepBytes"` |
|
| 96 |
+ KeepDuration int64 `toml:"keepDuration"` |
|
| 97 |
+ Filters []string `toml:"filters"` |
|
| 98 |
+} |
|
| 99 |
+ |
|
| 100 |
+type DNSConfig struct {
|
|
| 101 |
+ Nameservers []string `toml:"nameservers"` |
|
| 102 |
+ Options []string `toml:"options"` |
|
| 103 |
+ SearchDomains []string `toml:"searchDomains"` |
|
| 104 |
+} |
| 0 | 105 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,31 @@ |
| 0 |
+package config |
|
| 1 |
+ |
|
| 2 |
+const defaultCap int64 = 2e9 // 2GB |
|
| 3 |
+ |
|
| 4 |
+func DefaultGCPolicy(p string, keep int64) []GCPolicy {
|
|
| 5 |
+ if keep == 0 {
|
|
| 6 |
+ keep = DetectDefaultGCCap(p) |
|
| 7 |
+ } |
|
| 8 |
+ return []GCPolicy{
|
|
| 9 |
+ // if build cache uses more than 512MB delete the most easily reproducible data after it has not been used for 2 days |
|
| 10 |
+ {
|
|
| 11 |
+ Filters: []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"},
|
|
| 12 |
+ KeepDuration: 48 * 3600, // 48h |
|
| 13 |
+ KeepBytes: 512 * 1e6, // 512MB |
|
| 14 |
+ }, |
|
| 15 |
+ // remove any data not used for 60 days |
|
| 16 |
+ {
|
|
| 17 |
+ KeepDuration: 60 * 24 * 3600, // 60d |
|
| 18 |
+ KeepBytes: keep, |
|
| 19 |
+ }, |
|
| 20 |
+ // keep the unshared build cache under cap |
|
| 21 |
+ {
|
|
| 22 |
+ KeepBytes: keep, |
|
| 23 |
+ }, |
|
| 24 |
+ // if previous policies were insufficient start deleting internal data to keep build cache under cap |
|
| 25 |
+ {
|
|
| 26 |
+ All: true, |
|
| 27 |
+ KeepBytes: keep, |
|
| 28 |
+ }, |
|
| 29 |
+ } |
|
| 30 |
+} |
| 0 | 31 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,17 @@ |
| 0 |
+// +build !windows |
|
| 1 |
+ |
|
| 2 |
+package config |
|
| 3 |
+ |
|
| 4 |
+import ( |
|
| 5 |
+ "syscall" |
|
| 6 |
+) |
|
| 7 |
+ |
|
| 8 |
+func DetectDefaultGCCap(root string) int64 {
|
|
| 9 |
+ var st syscall.Statfs_t |
|
| 10 |
+ if err := syscall.Statfs(root, &st); err != nil {
|
|
| 11 |
+ return defaultCap |
|
| 12 |
+ } |
|
| 13 |
+ diskSize := int64(st.Bsize) * int64(st.Blocks) |
|
| 14 |
+ avail := diskSize / 10 |
|
| 15 |
+ return (avail/(1<<30) + 1) * 1e9 // round up |
|
| 16 |
+} |
| ... | ... |
@@ -139,7 +139,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
|
| 139 | 139 |
buildContext = st |
| 140 | 140 |
} else if httpPrefix.MatchString(opts[localNameContext]) {
|
| 141 | 141 |
httpContext := llb.HTTP(opts[localNameContext], llb.Filename("context"), dockerfile2llb.WithInternalName("load remote build context"))
|
| 142 |
- def, err := httpContext.Marshal(marshalOpts...) |
|
| 142 |
+ def, err := httpContext.Marshal(ctx, marshalOpts...) |
|
| 143 | 143 |
if err != nil {
|
| 144 | 144 |
return nil, errors.Wrapf(err, "failed to marshal httpcontext") |
| 145 | 145 |
} |
| ... | ... |
@@ -221,7 +221,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
|
| 221 | 221 |
} |
| 222 | 222 |
} |
| 223 | 223 |
|
| 224 |
- def, err := src.Marshal(marshalOpts...) |
|
| 224 |
+ def, err := src.Marshal(ctx, marshalOpts...) |
|
| 225 | 225 |
if err != nil {
|
| 226 | 226 |
return nil, errors.Wrapf(err, "failed to marshal local source") |
| 227 | 227 |
} |
| ... | ... |
@@ -271,7 +271,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
|
| 271 | 271 |
) |
| 272 | 272 |
dockerignoreState = &st |
| 273 | 273 |
} |
| 274 |
- def, err := dockerignoreState.Marshal(marshalOpts...) |
|
| 274 |
+ def, err := dockerignoreState.Marshal(ctx, marshalOpts...) |
|
| 275 | 275 |
if err != nil {
|
| 276 | 276 |
return err |
| 277 | 277 |
} |
| ... | ... |
@@ -363,7 +363,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
|
| 363 | 363 |
return errors.Wrapf(err, "failed to create LLB definition") |
| 364 | 364 |
} |
| 365 | 365 |
|
| 366 |
- def, err := st.Marshal() |
|
| 366 |
+ def, err := st.Marshal(ctx) |
|
| 367 | 367 |
if err != nil {
|
| 368 | 368 |
return errors.Wrapf(err, "failed to marshal LLB definition") |
| 369 | 369 |
} |
| ... | ... |
@@ -456,9 +456,29 @@ func forwardGateway(ctx context.Context, c client.Client, ref string, cmdline st |
| 456 | 456 |
} |
| 457 | 457 |
opts["cmdline"] = cmdline |
| 458 | 458 |
opts["source"] = ref |
| 459 |
+ |
|
| 460 |
+ gwcaps := c.BuildOpts().Caps |
|
| 461 |
+ var frontendInputs map[string]*pb.Definition |
|
| 462 |
+ if (&gwcaps).Supports(gwpb.CapFrontendInputs) == nil {
|
|
| 463 |
+ inputs, err := c.Inputs(ctx) |
|
| 464 |
+ if err != nil {
|
|
| 465 |
+ return nil, errors.Wrapf(err, "failed to get frontend inputs") |
|
| 466 |
+ } |
|
| 467 |
+ |
|
| 468 |
+ frontendInputs = make(map[string]*pb.Definition) |
|
| 469 |
+ for name, state := range inputs {
|
|
| 470 |
+ def, err := state.Marshal(ctx) |
|
| 471 |
+ if err != nil {
|
|
| 472 |
+ return nil, err |
|
| 473 |
+ } |
|
| 474 |
+ frontendInputs[name] = def.ToPB() |
|
| 475 |
+ } |
|
| 476 |
+ } |
|
| 477 |
+ |
|
| 459 | 478 |
return c.Solve(ctx, client.SolveRequest{
|
| 460 |
- Frontend: "gateway.v0", |
|
| 461 |
- FrontendOpt: opts, |
|
| 479 |
+ Frontend: "gateway.v0", |
|
| 480 |
+ FrontendOpt: opts, |
|
| 481 |
+ FrontendInputs: frontendInputs, |
|
| 462 | 482 |
}) |
| 463 | 483 |
} |
| 464 | 484 |
|
| ... | ... |
@@ -462,7 +462,11 @@ type dispatchOpt struct {
|
| 462 | 462 |
func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
|
| 463 | 463 |
if ex, ok := cmd.Command.(instructions.SupportsSingleWordExpansion); ok {
|
| 464 | 464 |
err := ex.Expand(func(word string) (string, error) {
|
| 465 |
- return opt.shlex.ProcessWord(word, d.state.Env()) |
|
| 465 |
+ env, err := d.state.Env(context.TODO()) |
|
| 466 |
+ if err != nil {
|
|
| 467 |
+ return "", err |
|
| 468 |
+ } |
|
| 469 |
+ return opt.shlex.ProcessWord(word, env) |
|
| 466 | 470 |
}) |
| 467 | 471 |
if err != nil {
|
| 468 | 472 |
return err |
| ... | ... |
@@ -626,7 +630,10 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyE |
| 626 | 626 |
if c.PrependShell {
|
| 627 | 627 |
args = withShell(d.image, args) |
| 628 | 628 |
} |
| 629 |
- env := d.state.Env() |
|
| 629 |
+ env, err := d.state.Env(context.TODO()) |
|
| 630 |
+ if err != nil {
|
|
| 631 |
+ return err |
|
| 632 |
+ } |
|
| 630 | 633 |
opt := []llb.RunOption{llb.Args(args), dfCmd(c)}
|
| 631 | 634 |
if d.ignoreCache {
|
| 632 | 635 |
opt = append(opt, llb.IgnoreCache) |
| ... | ... |
@@ -661,7 +668,11 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyE |
| 661 | 661 |
shlex.RawQuotes = true |
| 662 | 662 |
shlex.SkipUnsetEnv = true |
| 663 | 663 |
|
| 664 |
- opt = append(opt, llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(&shlex, c.String(), env)), d.prefixPlatform, d.state.GetPlatform()))) |
|
| 664 |
+ pl, err := d.state.GetPlatform(context.TODO()) |
|
| 665 |
+ if err != nil {
|
|
| 666 |
+ return err |
|
| 667 |
+ } |
|
| 668 |
+ opt = append(opt, llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(&shlex, c.String(), env)), d.prefixPlatform, pl))) |
|
| 665 | 669 |
for _, h := range dopt.extraHosts {
|
| 666 | 670 |
opt = append(opt, llb.AddExtraHost(h.Host, h.IP)) |
| 667 | 671 |
} |
| ... | ... |
@@ -687,7 +698,11 @@ func dispatchWorkdir(d *dispatchState, c *instructions.WorkdirCommand, commit bo |
| 687 | 687 |
if d.platform != nil {
|
| 688 | 688 |
platform = *d.platform |
| 689 | 689 |
} |
| 690 |
- d.state = d.state.File(llb.Mkdir(wd, 0755, mkdirOpt...), llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(opt.shlex, c.String(), d.state.Env())), d.prefixPlatform, &platform))) |
|
| 690 |
+ env, err := d.state.Env(context.TODO()) |
|
| 691 |
+ if err != nil {
|
|
| 692 |
+ return err |
|
| 693 |
+ } |
|
| 694 |
+ d.state = d.state.File(llb.Mkdir(wd, 0755, mkdirOpt...), llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(opt.shlex, c.String(), env)), d.prefixPlatform, &platform))) |
|
| 691 | 695 |
withLayer = true |
| 692 | 696 |
} |
| 693 | 697 |
return commitToHistory(&d.image, "WORKDIR "+wd, withLayer, nil) |
| ... | ... |
@@ -696,7 +711,11 @@ func dispatchWorkdir(d *dispatchState, c *instructions.WorkdirCommand, commit bo |
| 696 | 696 |
} |
| 697 | 697 |
|
| 698 | 698 |
func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceState llb.State, isAddCommand bool, cmdToPrint fmt.Stringer, chown string, opt dispatchOpt) error {
|
| 699 |
- dest := path.Join("/", pathRelativeToWorkingDir(d.state, c.Dest()))
|
|
| 699 |
+ pp, err := pathRelativeToWorkingDir(d.state, c.Dest()) |
|
| 700 |
+ if err != nil {
|
|
| 701 |
+ return err |
|
| 702 |
+ } |
|
| 703 |
+ dest := path.Join("/", pp)
|
|
| 700 | 704 |
if c.Dest() == "." || c.Dest() == "" || c.Dest()[len(c.Dest())-1] == filepath.Separator {
|
| 701 | 705 |
dest += string(filepath.Separator) |
| 702 | 706 |
} |
| ... | ... |
@@ -772,7 +791,12 @@ func dispatchCopyFileOp(d *dispatchState, c instructions.SourcesAndDest, sourceS |
| 772 | 772 |
platform = *d.platform |
| 773 | 773 |
} |
| 774 | 774 |
|
| 775 |
- fileOpt := []llb.ConstraintsOpt{llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(opt.shlex, cmdToPrint.String(), d.state.Env())), d.prefixPlatform, &platform))}
|
|
| 775 |
+ env, err := d.state.Env(context.TODO()) |
|
| 776 |
+ if err != nil {
|
|
| 777 |
+ return err |
|
| 778 |
+ } |
|
| 779 |
+ |
|
| 780 |
+ fileOpt := []llb.ConstraintsOpt{llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(opt.shlex, cmdToPrint.String(), env)), d.prefixPlatform, &platform))}
|
|
| 776 | 781 |
if d.ignoreCache {
|
| 777 | 782 |
fileOpt = append(fileOpt, llb.IgnoreCache) |
| 778 | 783 |
} |
| ... | ... |
@@ -787,8 +811,11 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l |
| 787 | 787 |
} |
| 788 | 788 |
|
| 789 | 789 |
img := llb.Image(opt.copyImage, llb.MarkImageInternal, llb.Platform(opt.buildPlatforms[0]), WithInternalName("helper image for file operations"))
|
| 790 |
- |
|
| 791 |
- dest := path.Join(".", pathRelativeToWorkingDir(d.state, c.Dest()))
|
|
| 790 |
+ pp, err := pathRelativeToWorkingDir(d.state, c.Dest()) |
|
| 791 |
+ if err != nil {
|
|
| 792 |
+ return err |
|
| 793 |
+ } |
|
| 794 |
+ dest := path.Join(".", pp)
|
|
| 792 | 795 |
if c.Dest() == "." || c.Dest() == "" || c.Dest()[len(c.Dest())-1] == filepath.Separator {
|
| 793 | 796 |
dest += string(filepath.Separator) |
| 794 | 797 |
} |
| ... | ... |
@@ -861,7 +888,12 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l |
| 861 | 861 |
platform = *d.platform |
| 862 | 862 |
} |
| 863 | 863 |
|
| 864 |
- runOpt := []llb.RunOption{llb.Args(args), llb.Dir("/dest"), llb.ReadonlyRootFS(), dfCmd(cmdToPrint), llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(opt.shlex, cmdToPrint.String(), d.state.Env())), d.prefixPlatform, &platform))}
|
|
| 864 |
+ env, err := d.state.Env(context.TODO()) |
|
| 865 |
+ if err != nil {
|
|
| 866 |
+ return err |
|
| 867 |
+ } |
|
| 868 |
+ |
|
| 869 |
+ runOpt := []llb.RunOption{llb.Args(args), llb.Dir("/dest"), llb.ReadonlyRootFS(), dfCmd(cmdToPrint), llb.WithCustomName(prefixCommand(d, uppercaseCmd(processCmdEnv(opt.shlex, cmdToPrint.String(), env)), d.prefixPlatform, &platform))}
|
|
| 865 | 870 |
if d.ignoreCache {
|
| 866 | 871 |
runOpt = append(runOpt, llb.IgnoreCache) |
| 867 | 872 |
} |
| ... | ... |
@@ -936,8 +968,12 @@ func dispatchHealthcheck(d *dispatchState, c *instructions.HealthCheckCommand) e |
| 936 | 936 |
|
| 937 | 937 |
func dispatchExpose(d *dispatchState, c *instructions.ExposeCommand, shlex *shell.Lex) error {
|
| 938 | 938 |
ports := []string{}
|
| 939 |
+ env, err := d.state.Env(context.TODO()) |
|
| 940 |
+ if err != nil {
|
|
| 941 |
+ return err |
|
| 942 |
+ } |
|
| 939 | 943 |
for _, p := range c.Ports {
|
| 940 |
- ps, err := shlex.ProcessWords(p, d.state.Env()) |
|
| 944 |
+ ps, err := shlex.ProcessWords(p, env) |
|
| 941 | 945 |
if err != nil {
|
| 942 | 946 |
return err |
| 943 | 947 |
} |
| ... | ... |
@@ -1018,11 +1054,15 @@ func dispatchArg(d *dispatchState, c *instructions.ArgCommand, metaArgs []instru |
| 1018 | 1018 |
return commitToHistory(&d.image, commitStr, false, nil) |
| 1019 | 1019 |
} |
| 1020 | 1020 |
|
| 1021 |
-func pathRelativeToWorkingDir(s llb.State, p string) string {
|
|
| 1021 |
+func pathRelativeToWorkingDir(s llb.State, p string) (string, error) {
|
|
| 1022 | 1022 |
if path.IsAbs(p) {
|
| 1023 |
- return p |
|
| 1023 |
+ return p, nil |
|
| 1024 |
+ } |
|
| 1025 |
+ dir, err := s.GetDir(context.TODO()) |
|
| 1026 |
+ if err != nil {
|
|
| 1027 |
+ return "", err |
|
| 1024 | 1028 |
} |
| 1025 |
- return path.Join(s.GetDir(), p) |
|
| 1029 |
+ return path.Join(dir, p), nil |
|
| 1026 | 1030 |
} |
| 1027 | 1031 |
|
| 1028 | 1032 |
func splitWildcards(name string) (string, string) {
|
| ... | ... |
@@ -3,6 +3,7 @@ |
| 3 | 3 |
package dockerfile2llb |
| 4 | 4 |
|
| 5 | 5 |
import ( |
| 6 |
+ "context" |
|
| 6 | 7 |
"fmt" |
| 7 | 8 |
"os" |
| 8 | 9 |
"path" |
| ... | ... |
@@ -132,7 +133,11 @@ func dispatchRunMounts(d *dispatchState, c *instructions.RunCommand, sources []* |
| 132 | 132 |
} |
| 133 | 133 |
target := mount.Target |
| 134 | 134 |
if !filepath.IsAbs(filepath.Clean(mount.Target)) {
|
| 135 |
- target = filepath.Join("/", d.state.GetDir(), mount.Target)
|
|
| 135 |
+ dir, err := d.state.GetDir(context.TODO()) |
|
| 136 |
+ if err != nil {
|
|
| 137 |
+ return nil, err |
|
| 138 |
+ } |
|
| 139 |
+ target = filepath.Join("/", dir, mount.Target)
|
|
| 136 | 140 |
} |
| 137 | 141 |
if target == "/" {
|
| 138 | 142 |
return nil, errors.Errorf("invalid mount target %q", target)
|
| ... | ... |
@@ -135,7 +135,7 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten |
| 135 | 135 |
|
| 136 | 136 |
src := llb.Image(sourceRef.String(), &markTypeFrontend{})
|
| 137 | 137 |
|
| 138 |
- def, err := src.Marshal() |
|
| 138 |
+ def, err := src.Marshal(ctx) |
|
| 139 | 139 |
if err != nil {
|
| 140 | 140 |
return nil, err |
| 141 | 141 |
} |
| ... | ... |
@@ -3,6 +3,7 @@ module github.com/moby/buildkit |
| 3 | 3 |
go 1.13 |
| 4 | 4 |
|
| 5 | 5 |
require ( |
| 6 |
+ github.com/AkihiroSuda/containerd-fuse-overlayfs v0.0.0-20200220082720-bb896865146c |
|
| 6 | 7 |
github.com/BurntSushi/toml v0.3.1 |
| 7 | 8 |
github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 |
| 8 | 9 |
github.com/Microsoft/hcsshim v0.8.7 // indirect |
| ... | ... |
@@ -23,13 +24,12 @@ require ( |
| 23 | 23 |
github.com/docker/docker v0.0.0 |
| 24 | 24 |
github.com/docker/docker-credential-helpers v0.6.0 // indirect |
| 25 | 25 |
github.com/docker/go-connections v0.3.0 |
| 26 |
- github.com/docker/go-events v0.0.0-20170721190031-9461782956ad // indirect |
|
| 27 | 26 |
github.com/docker/libnetwork v0.8.0-dev.2.0.20200226230617-d8334ccdb9be |
| 28 | 27 |
github.com/gofrs/flock v0.7.0 |
| 29 | 28 |
github.com/gogo/googleapis v1.3.2 |
| 30 | 29 |
github.com/gogo/protobuf v1.3.1 |
| 31 | 30 |
github.com/golang/protobuf v1.3.3 |
| 32 |
- github.com/google/go-cmp v0.3.0 |
|
| 31 |
+ github.com/google/go-cmp v0.3.1 |
|
| 33 | 32 |
github.com/google/shlex v0.0.0-20150127133951-6f45313302b9 |
| 34 | 33 |
github.com/google/uuid v1.1.1 // indirect |
| 35 | 34 |
github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 |
| ... | ... |
@@ -54,7 +54,7 @@ require ( |
| 54 | 54 |
github.com/sirupsen/logrus v1.4.2 |
| 55 | 55 |
github.com/stretchr/testify v1.4.0 |
| 56 | 56 |
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect |
| 57 |
- github.com/tonistiigi/fsutil v0.0.0-20200225063759-013a9fe6aee2 |
|
| 57 |
+ github.com/tonistiigi/fsutil v0.0.0-20200326231323-c2c7d7b0e144 |
|
| 58 | 58 |
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea |
| 59 | 59 |
github.com/uber/jaeger-client-go v0.0.0-20180103221425-e02c85f9069e |
| 60 | 60 |
github.com/uber/jaeger-lib v1.2.1 // indirect |
| ... | ... |
@@ -150,8 +150,12 @@ type LoadedResult struct {
|
| 150 | 150 |
CacheKey *CacheKey |
| 151 | 151 |
} |
| 152 | 152 |
|
| 153 |
-func (c *cacheManager) filterResults(m map[string]Result, ck *CacheKey) (results []LoadedResult, err error) {
|
|
| 153 |
+func (c *cacheManager) filterResults(m map[string]Result, ck *CacheKey, visited map[string]struct{}) (results []LoadedResult, err error) {
|
|
| 154 | 154 |
id := c.getID(ck) |
| 155 |
+ if _, ok := visited[id]; ok {
|
|
| 156 |
+ return nil, nil |
|
| 157 |
+ } |
|
| 158 |
+ visited[id] = struct{}{}
|
|
| 155 | 159 |
if err := c.backend.WalkResults(id, func(cr CacheResult) error {
|
| 156 | 160 |
res, ok := m[id] |
| 157 | 161 |
if ok {
|
| ... | ... |
@@ -170,7 +174,7 @@ func (c *cacheManager) filterResults(m map[string]Result, ck *CacheKey) (results |
| 170 | 170 |
} |
| 171 | 171 |
for _, keys := range ck.Deps() {
|
| 172 | 172 |
for _, key := range keys {
|
| 173 |
- res, err := c.filterResults(m, key.CacheKey.CacheKey) |
|
| 173 |
+ res, err := c.filterResults(m, key.CacheKey.CacheKey, visited) |
|
| 174 | 174 |
if err != nil {
|
| 175 | 175 |
for _, r := range results {
|
| 176 | 176 |
r.Result.Release(context.TODO()) |
| ... | ... |
@@ -207,7 +211,7 @@ func (c *cacheManager) LoadWithParents(ctx context.Context, rec *CacheRecord) ([ |
| 207 | 207 |
return nil, err |
| 208 | 208 |
} |
| 209 | 209 |
|
| 210 |
- results, err := c.filterResults(m, rec.key) |
|
| 210 |
+ results, err := c.filterResults(m, rec.key, map[string]struct{}{})
|
|
| 211 | 211 |
if err != nil {
|
| 212 | 212 |
for _, r := range m {
|
| 213 | 213 |
r.Release(context.TODO()) |
| ... | ... |
@@ -20,11 +20,12 @@ func addBacklinks(t CacheExporterTarget, rec CacheExporterRecord, cm *cacheManag |
| 20 | 20 |
if rec == nil {
|
| 21 | 21 |
var ok bool |
| 22 | 22 |
rec, ok = bkm[id] |
| 23 |
- if ok {
|
|
| 23 |
+ if ok && rec != nil {
|
|
| 24 | 24 |
return rec, nil |
| 25 | 25 |
} |
| 26 | 26 |
_ = ok |
| 27 | 27 |
} |
| 28 |
+ bkm[id] = nil |
|
| 28 | 29 |
if err := cm.backend.WalkBacklinks(id, func(id string, link CacheInfoLink) error {
|
| 29 | 30 |
if rec == nil {
|
| 30 | 31 |
rec = t.Add(link.Digest) |
| ... | ... |
@@ -37,7 +38,9 @@ func addBacklinks(t CacheExporterTarget, rec CacheExporterRecord, cm *cacheManag |
| 37 | 37 |
return err |
| 38 | 38 |
} |
| 39 | 39 |
} |
| 40 |
- rec.LinkFrom(r, int(link.Input), link.Selector.String()) |
|
| 40 |
+ if r != nil {
|
|
| 41 |
+ rec.LinkFrom(r, int(link.Input), link.Selector.String()) |
|
| 42 |
+ } |
|
| 41 | 43 |
return nil |
| 42 | 44 |
}); err != nil {
|
| 43 | 45 |
return nil, err |
| ... | ... |
@@ -66,6 +69,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach |
| 66 | 66 |
if t.Visited(e) {
|
| 67 | 67 |
return e.res, nil |
| 68 | 68 |
} |
| 69 |
+ t.Visit(e) |
|
| 69 | 70 |
|
| 70 | 71 |
deps := e.k.Deps() |
| 71 | 72 |
|
| ... | ... |
@@ -177,7 +181,6 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach |
| 177 | 177 |
} |
| 178 | 178 |
|
| 179 | 179 |
e.res = allRec |
| 180 |
- t.Visit(e) |
|
| 181 | 180 |
|
| 182 | 181 |
return e.res, nil |
| 183 | 182 |
} |
| ... | ... |
@@ -272,8 +272,9 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe |
| 272 | 272 |
} |
| 273 | 273 |
args = append(args, "origin") |
| 274 | 274 |
if !isCommitSHA(ref) {
|
| 275 |
- args = append(args, ref+":tags/"+ref) |
|
| 276 |
- // local refs are needed so they would be advertised on next fetches |
|
| 275 |
+ args = append(args, "--force", ref+":tags/"+ref) |
|
| 276 |
+ // local refs are needed so they would be advertised on next fetches. Force is used |
|
| 277 |
+ // in case the ref is a branch and it now points to a different commit sha |
|
| 277 | 278 |
// TODO: is there a better way to do this? |
| 278 | 279 |
} |
| 279 | 280 |
if _, err := gitWithinDir(ctx, gitDir, "", args...); err != nil {
|
| ... | ... |
@@ -1,54 +1,219 @@ |
| 1 | 1 |
package resolver |
| 2 | 2 |
|
| 3 | 3 |
import ( |
| 4 |
- "math/rand" |
|
| 4 |
+ "context" |
|
| 5 |
+ "crypto/tls" |
|
| 6 |
+ "crypto/x509" |
|
| 7 |
+ "io/ioutil" |
|
| 8 |
+ "net" |
|
| 9 |
+ "net/http" |
|
| 10 |
+ "os" |
|
| 11 |
+ "path/filepath" |
|
| 12 |
+ "runtime" |
|
| 5 | 13 |
"strings" |
| 14 |
+ "time" |
|
| 6 | 15 |
|
| 16 |
+ "github.com/containerd/containerd/remotes" |
|
| 7 | 17 |
"github.com/containerd/containerd/remotes/docker" |
| 8 |
- "github.com/docker/distribution/reference" |
|
| 18 |
+ "github.com/moby/buildkit/cmd/buildkitd/config" |
|
| 19 |
+ "github.com/moby/buildkit/session" |
|
| 20 |
+ "github.com/moby/buildkit/session/auth" |
|
| 9 | 21 |
"github.com/moby/buildkit/util/tracing" |
| 22 |
+ "github.com/pkg/errors" |
|
| 10 | 23 |
) |
| 11 | 24 |
|
| 12 |
-type RegistryConf struct {
|
|
| 13 |
- Mirrors []string |
|
| 14 |
- PlainHTTP *bool |
|
| 25 |
+func fillInsecureOpts(host string, c config.RegistryConfig, h *docker.RegistryHost) error {
|
|
| 26 |
+ tc, err := loadTLSConfig(c) |
|
| 27 |
+ if err != nil {
|
|
| 28 |
+ return err |
|
| 29 |
+ } |
|
| 30 |
+ |
|
| 31 |
+ if c.PlainHTTP != nil && *c.PlainHTTP {
|
|
| 32 |
+ h.Scheme = "http" |
|
| 33 |
+ } else if c.Insecure != nil && *c.Insecure {
|
|
| 34 |
+ tc.InsecureSkipVerify = true |
|
| 35 |
+ } else if c.PlainHTTP == nil {
|
|
| 36 |
+ if ok, _ := docker.MatchLocalhost(host); ok {
|
|
| 37 |
+ h.Scheme = "http" |
|
| 38 |
+ } |
|
| 39 |
+ } |
|
| 40 |
+ |
|
| 41 |
+ transport := newDefaultTransport() |
|
| 42 |
+ transport.TLSClientConfig = tc |
|
| 43 |
+ |
|
| 44 |
+ h.Client = &http.Client{
|
|
| 45 |
+ Transport: tracing.NewTransport(transport), |
|
| 46 |
+ } |
|
| 47 |
+ return nil |
|
| 15 | 48 |
} |
| 16 | 49 |
|
| 17 |
-type ResolveOptionsFunc func(string) docker.ResolverOptions |
|
| 50 |
+func loadTLSConfig(c config.RegistryConfig) (*tls.Config, error) {
|
|
| 51 |
+ for _, d := range c.TLSConfigDir {
|
|
| 52 |
+ fs, err := ioutil.ReadDir(d) |
|
| 53 |
+ if err != nil && !os.IsNotExist(err) && !os.IsPermission(err) {
|
|
| 54 |
+ return nil, errors.WithStack(err) |
|
| 55 |
+ } |
|
| 56 |
+ for _, f := range fs {
|
|
| 57 |
+ if strings.HasSuffix(f.Name(), ".crt") {
|
|
| 58 |
+ c.RootCAs = append(c.RootCAs, filepath.Join(d, f.Name())) |
|
| 59 |
+ } |
|
| 60 |
+ if strings.HasSuffix(f.Name(), ".cert") {
|
|
| 61 |
+ c.KeyPairs = append(c.KeyPairs, config.TLSKeyPair{
|
|
| 62 |
+ Certificate: filepath.Join(d, f.Name()), |
|
| 63 |
+ Key: filepath.Join(d, strings.TrimSuffix(f.Name(), ".cert")+".key"), |
|
| 64 |
+ }) |
|
| 65 |
+ } |
|
| 66 |
+ } |
|
| 67 |
+ } |
|
| 18 | 68 |
|
| 19 |
-func NewResolveOptionsFunc(m map[string]RegistryConf) ResolveOptionsFunc {
|
|
| 20 |
- return func(ref string) docker.ResolverOptions {
|
|
| 21 |
- def := docker.ResolverOptions{
|
|
| 22 |
- Client: tracing.DefaultClient, |
|
| 69 |
+ tc := &tls.Config{}
|
|
| 70 |
+ if len(c.RootCAs) > 0 {
|
|
| 71 |
+ systemPool, err := x509.SystemCertPool() |
|
| 72 |
+ if err != nil {
|
|
| 73 |
+ if runtime.GOOS == "windows" {
|
|
| 74 |
+ systemPool = x509.NewCertPool() |
|
| 75 |
+ } else {
|
|
| 76 |
+ return nil, errors.Wrapf(err, "unable to get system cert pool") |
|
| 77 |
+ } |
|
| 23 | 78 |
} |
| 79 |
+ tc.RootCAs = systemPool |
|
| 80 |
+ } |
|
| 24 | 81 |
|
| 25 |
- parsed, err := reference.ParseNormalizedNamed(ref) |
|
| 82 |
+ for _, p := range c.RootCAs {
|
|
| 83 |
+ dt, err := ioutil.ReadFile(p) |
|
| 26 | 84 |
if err != nil {
|
| 27 |
- return def |
|
| 85 |
+ return nil, errors.Wrapf(err, "failed to read %s", p) |
|
| 28 | 86 |
} |
| 29 |
- host := reference.Domain(parsed) |
|
| 87 |
+ tc.RootCAs.AppendCertsFromPEM(dt) |
|
| 88 |
+ } |
|
| 30 | 89 |
|
| 31 |
- c, ok := m[host] |
|
| 32 |
- if !ok {
|
|
| 33 |
- return def |
|
| 90 |
+ for _, kp := range c.KeyPairs {
|
|
| 91 |
+ cert, err := tls.LoadX509KeyPair(kp.Certificate, kp.Key) |
|
| 92 |
+ if err != nil {
|
|
| 93 |
+ return nil, errors.Wrapf(err, "failed to load keypair for %s", kp.Certificate) |
|
| 34 | 94 |
} |
| 95 |
+ tc.Certificates = append(tc.Certificates, cert) |
|
| 96 |
+ } |
|
| 97 |
+ return tc, nil |
|
| 98 |
+} |
|
| 35 | 99 |
|
| 36 |
- var mirrorHost string |
|
| 37 |
- if len(c.Mirrors) > 0 {
|
|
| 38 |
- mirrorHost = c.Mirrors[rand.Intn(len(c.Mirrors))] |
|
| 39 |
- def.Host = func(string) (string, error) {
|
|
| 40 |
- return mirrorHost, nil |
|
| 100 |
+func NewRegistryConfig(m map[string]config.RegistryConfig) docker.RegistryHosts {
|
|
| 101 |
+ return docker.Registries( |
|
| 102 |
+ func(host string) ([]docker.RegistryHost, error) {
|
|
| 103 |
+ c, ok := m[host] |
|
| 104 |
+ if !ok {
|
|
| 105 |
+ return nil, nil |
|
| 106 |
+ } |
|
| 107 |
+ |
|
| 108 |
+ var out []docker.RegistryHost |
|
| 109 |
+ |
|
| 110 |
+ for _, mirror := range c.Mirrors {
|
|
| 111 |
+ h := docker.RegistryHost{
|
|
| 112 |
+ Scheme: "https", |
|
| 113 |
+ Client: newDefaultClient(), |
|
| 114 |
+ Host: mirror, |
|
| 115 |
+ Path: "/v2", |
|
| 116 |
+ Capabilities: docker.HostCapabilityPull | docker.HostCapabilityResolve, |
|
| 117 |
+ } |
|
| 118 |
+ |
|
| 119 |
+ if err := fillInsecureOpts(mirror, m[mirror], &h); err != nil {
|
|
| 120 |
+ return nil, err |
|
| 121 |
+ } |
|
| 122 |
+ |
|
| 123 |
+ out = append(out, h) |
|
| 124 |
+ } |
|
| 125 |
+ |
|
| 126 |
+ if host == "docker.io" {
|
|
| 127 |
+ host = "registry-1.docker.io" |
|
| 41 | 128 |
} |
| 42 |
- } |
|
| 43 | 129 |
|
| 44 |
- if c.PlainHTTP != nil {
|
|
| 45 |
- def.PlainHTTP = *c.PlainHTTP |
|
| 46 |
- } else {
|
|
| 47 |
- if mirrorHost == "localhost" || strings.HasPrefix(mirrorHost, "localhost:") {
|
|
| 48 |
- def.PlainHTTP = true |
|
| 130 |
+ h := docker.RegistryHost{
|
|
| 131 |
+ Scheme: "https", |
|
| 132 |
+ Client: newDefaultClient(), |
|
| 133 |
+ Host: host, |
|
| 134 |
+ Path: "/v2", |
|
| 135 |
+ Capabilities: docker.HostCapabilityPush | docker.HostCapabilityPull | docker.HostCapabilityResolve, |
|
| 49 | 136 |
} |
| 137 |
+ |
|
| 138 |
+ if err := fillInsecureOpts(host, c, &h); err != nil {
|
|
| 139 |
+ return nil, err |
|
| 140 |
+ } |
|
| 141 |
+ |
|
| 142 |
+ out = append(out, h) |
|
| 143 |
+ return out, nil |
|
| 144 |
+ }, |
|
| 145 |
+ docker.ConfigureDefaultRegistries( |
|
| 146 |
+ docker.WithClient(newDefaultClient()), |
|
| 147 |
+ docker.WithPlainHTTP(docker.MatchLocalhost), |
|
| 148 |
+ ), |
|
| 149 |
+ ) |
|
| 150 |
+} |
|
| 151 |
+ |
|
| 152 |
+func New(ctx context.Context, hosts docker.RegistryHosts, sm *session.Manager) remotes.Resolver {
|
|
| 153 |
+ return docker.NewResolver(docker.ResolverOptions{
|
|
| 154 |
+ Hosts: hostsWithCredentials(ctx, hosts, sm), |
|
| 155 |
+ }) |
|
| 156 |
+} |
|
| 157 |
+ |
|
| 158 |
+func hostsWithCredentials(ctx context.Context, hosts docker.RegistryHosts, sm *session.Manager) docker.RegistryHosts {
|
|
| 159 |
+ id := session.FromContext(ctx) |
|
| 160 |
+ if id == "" {
|
|
| 161 |
+ return hosts |
|
| 162 |
+ } |
|
| 163 |
+ return func(domain string) ([]docker.RegistryHost, error) {
|
|
| 164 |
+ res, err := hosts(domain) |
|
| 165 |
+ if err != nil {
|
|
| 166 |
+ return nil, err |
|
| 50 | 167 |
} |
| 168 |
+ if len(res) == 0 {
|
|
| 169 |
+ return nil, nil |
|
| 170 |
+ } |
|
| 171 |
+ |
|
| 172 |
+ timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) |
|
| 173 |
+ defer cancel() |
|
| 174 |
+ |
|
| 175 |
+ caller, err := sm.Get(timeoutCtx, id) |
|
| 176 |
+ if err != nil {
|
|
| 177 |
+ return nil, err |
|
| 178 |
+ } |
|
| 179 |
+ |
|
| 180 |
+ a := docker.NewDockerAuthorizer( |
|
| 181 |
+ docker.WithAuthClient(res[0].Client), |
|
| 182 |
+ docker.WithAuthCreds(auth.CredentialsFunc(context.TODO(), caller)), |
|
| 183 |
+ ) |
|
| 184 |
+ for i := range res {
|
|
| 185 |
+ res[i].Authorizer = a |
|
| 186 |
+ } |
|
| 187 |
+ return res, nil |
|
| 188 |
+ } |
|
| 189 |
+} |
|
| 190 |
+ |
|
| 191 |
+func newDefaultClient() *http.Client {
|
|
| 192 |
+ return &http.Client{
|
|
| 193 |
+ Transport: newDefaultTransport(), |
|
| 194 |
+ } |
|
| 195 |
+} |
|
| 51 | 196 |
|
| 52 |
- return def |
|
| 197 |
+// newDefaultTransport is for pull or push client |
|
| 198 |
+// |
|
| 199 |
+// NOTE: For push, there must disable http2 for https because the flow control |
|
| 200 |
+// will limit data transfer. The net/http package doesn't provide http2 tunable |
|
| 201 |
+// settings which limits push performance. |
|
| 202 |
+// |
|
| 203 |
+// REF: https://github.com/golang/go/issues/14077 |
|
| 204 |
+func newDefaultTransport() *http.Transport {
|
|
| 205 |
+ return &http.Transport{
|
|
| 206 |
+ Proxy: http.ProxyFromEnvironment, |
|
| 207 |
+ DialContext: (&net.Dialer{
|
|
| 208 |
+ Timeout: 30 * time.Second, |
|
| 209 |
+ KeepAlive: 30 * time.Second, |
|
| 210 |
+ DualStack: true, |
|
| 211 |
+ }).DialContext, |
|
| 212 |
+ MaxIdleConns: 10, |
|
| 213 |
+ IdleConnTimeout: 30 * time.Second, |
|
| 214 |
+ TLSHandshakeTimeout: 10 * time.Second, |
|
| 215 |
+ ExpectContinueTimeout: 5 * time.Second, |
|
| 216 |
+ DisableKeepAlives: true, |
|
| 217 |
+ TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper), |
|
| 53 | 218 |
} |
| 54 | 219 |
} |
| ... | ... |
@@ -74,6 +74,12 @@ type Transport struct {
|
| 74 | 74 |
http.RoundTripper |
| 75 | 75 |
} |
| 76 | 76 |
|
| 77 |
+func NewTransport(rt http.RoundTripper) http.RoundTripper {
|
|
| 78 |
+ return &Transport{
|
|
| 79 |
+ RoundTripper: &nethttp.Transport{RoundTripper: rt},
|
|
| 80 |
+ } |
|
| 81 |
+} |
|
| 82 |
+ |
|
| 77 | 83 |
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
|
| 78 | 84 |
span := opentracing.SpanFromContext(req.Context()) |
| 79 | 85 |
if span == nil { // no tracer connected with either request or transport
|
| ... | ... |
@@ -37,7 +37,11 @@ func WriteTar(ctx context.Context, fs FS, w io.Writer) error {
|
| 37 | 37 |
hdr.Linkname = stat.Linkname |
| 38 | 38 |
if hdr.Linkname != "" {
|
| 39 | 39 |
hdr.Size = 0 |
| 40 |
- hdr.Typeflag = tar.TypeLink |
|
| 40 |
+ if fi.Mode() & os.ModeSymlink != 0 {
|
|
| 41 |
+ hdr.Typeflag = tar.TypeSymlink |
|
| 42 |
+ } else {
|
|
| 43 |
+ hdr.Typeflag = tar.TypeLink |
|
| 44 |
+ } |
|
| 41 | 45 |
} |
| 42 | 46 |
|
| 43 | 47 |
if len(stat.Xattrs) > 0 {
|