Signed-off-by: Michael Crosby <crosbymichael@gmail.com>
Michael Crosby authored on 2017/12/05 23:44:42... | ... |
@@ -4,7 +4,7 @@ TOMLV_COMMIT=9baf8a8a9f2ed20a8e54160840c492f937eeaf9a |
4 | 4 |
|
5 | 5 |
# When updating RUNC_COMMIT, also update runc in vendor.conf accordingly |
6 | 6 |
RUNC_COMMIT=b2567b37d7b75eb4cf325b77297b140ea686ce8f |
7 |
-CONTAINERD_COMMIT=cc969fb42f427a68a8cc6870ef47f17304b83962 |
|
7 |
+CONTAINERD_COMMIT=v1.0.0 |
|
8 | 8 |
TINI_COMMIT=949e6facb77383876aeff8a6944dde66b3089574 |
9 | 9 |
LIBNETWORK_COMMIT=7b2b1feb1de4817d522cc372af149ff48d25028e |
10 | 10 |
VNDR_COMMIT=a6e196d8b4b0cbbdc29aebdb20c59ac6926bb384 |
... | ... |
@@ -103,7 +103,7 @@ github.com/googleapis/gax-go da06d194a00e19ce00d9011a13931c3f6f6887c7 |
103 | 103 |
google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 |
104 | 104 |
|
105 | 105 |
# containerd |
106 |
-github.com/containerd/containerd cc969fb42f427a68a8cc6870ef47f17304b83962 |
|
106 |
+github.com/containerd/containerd v1.0.0 |
|
107 | 107 |
github.com/containerd/fifo fbfb6a11ec671efbe94ad1c12c2e98773f19e1e6 |
108 | 108 |
github.com/containerd/continuity 35d55c5e8dd23b32037d56cf97174aff3efdfa83 |
109 | 109 |
github.com/containerd/cgroups 29da22c6171a4316169f9205ab6c49f59b5b852f |
... | ... |
@@ -13,13 +13,23 @@ containerd is designed to be embedded into a larger system, rather than being us |
13 | 13 |
|
14 | 14 |
## Getting Started |
15 | 15 |
|
16 |
-If you are interested in trying out containerd please see our [Getting Started Guide](docs/getting-started.md). |
|
16 |
+See our documentation on [containerd.io](containerd.io): |
|
17 |
+* [for ops and admins](docs/ops.md) |
|
18 |
+* [namespaces](docs/namespaces.md) |
|
19 |
+* [client options](docs/client-opts.md) |
|
20 |
+ |
|
21 |
+See how to build containerd from source at [BUILDING](BUILDING.md). |
|
22 |
+ |
|
23 |
+If you are interested in trying out containerd see our example at [Getting Started](docs/getting-started.md). |
|
24 |
+ |
|
17 | 25 |
|
18 | 26 |
## Runtime Requirements |
19 | 27 |
|
20 | 28 |
Runtime requirements for containerd are very minimal. Most interactions with |
21 | 29 |
the Linux and Windows container feature sets are handled via [runc](https://github.com/opencontainers/runc) and/or |
22 |
-OS-specific libraries (e.g. [hcsshim](https://github.com/Microsoft/hcsshim) for Microsoft). There are specific features |
|
30 |
+OS-specific libraries (e.g. [hcsshim](https://github.com/Microsoft/hcsshim) for Microsoft). The current required version of `runc` is always listed in [RUNC.md](/RUNC.md). |
|
31 |
+ |
|
32 |
+There are specific features |
|
23 | 33 |
used by containerd core code and snapshotters that will require a minimum kernel |
24 | 34 |
version on Linux. With the understood caveat of distro kernel versioning, a |
25 | 35 |
reasonable starting point for Linux is a minimum 4.x kernel version. |
... | ... |
@@ -33,9 +43,7 @@ distribution. |
33 | 33 |
To use Linux checkpoint and restore features, you will need `criu` installed on |
34 | 34 |
your system. See more details in [Checkpoint and Restore](#checkpoint-and-restore). |
35 | 35 |
|
36 |
-The current required version of runc is always listed in [RUNC.md](/RUNC.md). |
|
37 |
- |
|
38 |
-Build requirements for developers are listed in the [Developer Quick-Start](#developer-quick-start) section. |
|
36 |
+Build requirements for developers are listed in [BUILDING](BUILDING.md). |
|
39 | 37 |
|
40 | 38 |
## Features |
41 | 39 |
|
... | ... |
@@ -45,7 +53,11 @@ containerd offers a full client package to help you integrate containerd into yo |
45 | 45 |
|
46 | 46 |
```go |
47 | 47 |
|
48 |
-import "github.com/containerd/containerd" |
|
48 |
+import ( |
|
49 |
+ "github.com/containerd/containerd" |
|
50 |
+ "github.com/containerd/containerd/cio" |
|
51 |
+) |
|
52 |
+ |
|
49 | 53 |
|
50 | 54 |
func main() { |
51 | 55 |
client, err := containerd.New("/run/containerd/containerd.sock") |
... | ... |
@@ -61,7 +73,7 @@ Namespaces allow multiple consumers to use the same containerd without conflicti |
61 | 61 |
To set a namespace for requests to the API: |
62 | 62 |
|
63 | 63 |
```go |
64 |
-context = context.Background() |
|
64 |
+context = context.Background() |
|
65 | 65 |
// create a context for docker |
66 | 66 |
docker = namespaces.WithNamespace(context, "docker") |
67 | 67 |
|
... | ... |
@@ -133,7 +145,7 @@ Taking a container object and turning it into a runnable process on a system is |
133 | 133 |
|
134 | 134 |
```go |
135 | 135 |
// create a new task |
136 |
-task, err := redis.NewTask(context, containerd.Stdio) |
|
136 |
+task, err := redis.NewTask(context, cio.Stdio) |
|
137 | 137 |
defer task.Delete(context) |
138 | 138 |
|
139 | 139 |
// the task is now running and has a pid that can be use to setup networking |
... | ... |
@@ -165,37 +177,12 @@ checkpoint := image.Target() |
165 | 165 |
redis, err = client.NewContainer(context, "redis-master", containerd.WithCheckpoint(checkpoint, "redis-rootfs")) |
166 | 166 |
defer container.Delete(context) |
167 | 167 |
|
168 |
-task, err = redis.NewTask(context, containerd.Stdio, containerd.WithTaskCheckpoint(checkpoint)) |
|
168 |
+task, err = redis.NewTask(context, cio.Stdio, containerd.WithTaskCheckpoint(checkpoint)) |
|
169 | 169 |
defer task.Delete(context) |
170 | 170 |
|
171 | 171 |
err := task.Start(context) |
172 | 172 |
``` |
173 | 173 |
|
174 |
-## Developer Quick Start |
|
175 |
- |
|
176 |
-To build the daemon and `ctr` simple test client, the following build system dependencies are required: |
|
177 |
- |
|
178 |
-* Go 1.9.x or above |
|
179 |
-* Protoc 3.x compiler and headers (download at the [Google protobuf releases page](https://github.com/google/protobuf/releases)) |
|
180 |
-* Btrfs headers and libraries for your distribution. Note that building the btrfs driver can be disabled via build tag removing this dependency. |
|
181 |
- |
|
182 |
-For proper results, install the `protoc` release into `/usr/local` on your build system. For example, the following commands will download and install the 3.5.0 release for a 64-bit Linux host: |
|
183 |
- |
|
184 |
-``` |
|
185 |
-$ wget -c https://github.com/google/protobuf/releases/download/v3.5.0/protoc-3.5.0-linux-x86_64.zip |
|
186 |
-$ sudo unzip protoc-3.5.0-linux-x86_64.zip -d /usr/local |
|
187 |
-``` |
|
188 |
- |
|
189 |
-With the required dependencies installed, the `Makefile` target named **binaries** will compile the `ctr` and `containerd` binaries and place them in the `bin/` directory. Using `sudo make install` will place the binaries in `/usr/local/bin`. When making any changes to the gRPC API, `make generate` will use the installed `protoc` compiler to regenerate the API generated code packages. |
|
190 |
- |
|
191 |
-> *Note*: A build tag is currently available to disable building the btrfs snapshot driver. |
|
192 |
-> Adding `BUILDTAGS=no_btrfs` to your environment before calling the **binaries** |
|
193 |
-> Makefile target will disable the btrfs driver within the containerd Go build. |
|
194 |
- |
|
195 |
-Vendoring of external imports uses the [`vndr` tool](https://github.com/LK4D4/vndr) which uses a simple config file, `vendor.conf`, to provide the URL and version or hash details for each vendored import. After modifying `vendor.conf` run the `vndr` tool to update the `vendor/` directory contents. Combining the `vendor.conf` update with the changeset in `vendor/` after running `vndr` should become a single commit for a PR which relies on vendored updates. |
|
196 |
- |
|
197 |
-Please refer to [RUNC.md](/RUNC.md) for the currently supported version of `runc` that is used by containerd. |
|
198 |
- |
|
199 | 174 |
### Releases and API Stability |
200 | 175 |
|
201 | 176 |
Please see [RELEASES.md](RELEASES.md) for details on versioning and stability |
... | ... |
@@ -7,6 +7,7 @@ import ( |
7 | 7 |
"net/http" |
8 | 8 |
"runtime" |
9 | 9 |
"strconv" |
10 |
+ "strings" |
|
10 | 11 |
"sync" |
11 | 12 |
"time" |
12 | 13 |
|
... | ... |
@@ -29,7 +30,6 @@ import ( |
29 | 29 |
"github.com/containerd/containerd/namespaces" |
30 | 30 |
"github.com/containerd/containerd/platforms" |
31 | 31 |
"github.com/containerd/containerd/plugin" |
32 |
- "github.com/containerd/containerd/reference" |
|
33 | 32 |
"github.com/containerd/containerd/remotes" |
34 | 33 |
"github.com/containerd/containerd/remotes/docker" |
35 | 34 |
"github.com/containerd/containerd/remotes/docker/schema1" |
... | ... |
@@ -334,6 +334,14 @@ func (c *Client) Push(ctx context.Context, ref string, desc ocispec.Descriptor, |
334 | 334 |
for i := len(manifestStack) - 1; i >= 0; i-- { |
335 | 335 |
_, err := pushHandler(ctx, manifestStack[i]) |
336 | 336 |
if err != nil { |
337 |
+ // TODO(estesp): until we have a more complete method for index push, we need to report |
|
338 |
+ // missing dependencies in an index/manifest list by sensing the "400 Bad Request" |
|
339 |
+ // as a marker for this problem |
|
340 |
+ if (manifestStack[i].MediaType == ocispec.MediaTypeImageIndex || |
|
341 |
+ manifestStack[i].MediaType == images.MediaTypeDockerSchema2ManifestList) && |
|
342 |
+ errors.Cause(err) != nil && strings.Contains(errors.Cause(err).Error(), "400 Bad Request") { |
|
343 |
+ return errors.Wrap(err, "manifest list/index references to blobs and/or manifests are missing in your target registry") |
|
344 |
+ } |
|
337 | 345 |
return err |
338 | 346 |
} |
339 | 347 |
} |
... | ... |
@@ -494,95 +502,27 @@ func (c *Client) Version(ctx context.Context) (Version, error) { |
494 | 494 |
}, nil |
495 | 495 |
} |
496 | 496 |
|
497 |
-type imageFormat string |
|
498 |
- |
|
499 |
-const ( |
|
500 |
- ociImageFormat imageFormat = "oci" |
|
501 |
-) |
|
502 |
- |
|
503 | 497 |
type importOpts struct { |
504 |
- format imageFormat |
|
505 |
- refObject string |
|
506 |
- labels map[string]string |
|
507 | 498 |
} |
508 | 499 |
|
509 | 500 |
// ImportOpt allows the caller to specify import specific options |
510 | 501 |
type ImportOpt func(c *importOpts) error |
511 | 502 |
|
512 |
-// WithImportLabel sets a label to be associated with an imported image |
|
513 |
-func WithImportLabel(key, value string) ImportOpt { |
|
514 |
- return func(opts *importOpts) error { |
|
515 |
- if opts.labels == nil { |
|
516 |
- opts.labels = make(map[string]string) |
|
517 |
- } |
|
518 |
- |
|
519 |
- opts.labels[key] = value |
|
520 |
- return nil |
|
521 |
- } |
|
522 |
-} |
|
523 |
- |
|
524 |
-// WithImportLabels associates a set of labels to an imported image |
|
525 |
-func WithImportLabels(labels map[string]string) ImportOpt { |
|
526 |
- return func(opts *importOpts) error { |
|
527 |
- if opts.labels == nil { |
|
528 |
- opts.labels = make(map[string]string) |
|
529 |
- } |
|
530 |
- |
|
531 |
- for k, v := range labels { |
|
532 |
- opts.labels[k] = v |
|
533 |
- } |
|
534 |
- return nil |
|
535 |
- } |
|
536 |
-} |
|
537 |
- |
|
538 |
-// WithOCIImportFormat sets the import format for an OCI image format |
|
539 |
-func WithOCIImportFormat() ImportOpt { |
|
540 |
- return func(c *importOpts) error { |
|
541 |
- if c.format != "" { |
|
542 |
- return errors.New("format already set") |
|
543 |
- } |
|
544 |
- c.format = ociImageFormat |
|
545 |
- return nil |
|
546 |
- } |
|
547 |
-} |
|
548 |
- |
|
549 |
-// WithRefObject specifies the ref object to import. |
|
550 |
-// If refObject is empty, it is copied from the ref argument of Import(). |
|
551 |
-func WithRefObject(refObject string) ImportOpt { |
|
552 |
- return func(c *importOpts) error { |
|
553 |
- c.refObject = refObject |
|
554 |
- return nil |
|
555 |
- } |
|
556 |
-} |
|
557 |
- |
|
558 |
-func resolveImportOpt(ref string, opts ...ImportOpt) (importOpts, error) { |
|
503 |
+func resolveImportOpt(opts ...ImportOpt) (importOpts, error) { |
|
559 | 504 |
var iopts importOpts |
560 | 505 |
for _, o := range opts { |
561 | 506 |
if err := o(&iopts); err != nil { |
562 | 507 |
return iopts, err |
563 | 508 |
} |
564 | 509 |
} |
565 |
- // use OCI as the default format |
|
566 |
- if iopts.format == "" { |
|
567 |
- iopts.format = ociImageFormat |
|
568 |
- } |
|
569 |
- // if refObject is not explicitly specified, use the one specified in ref |
|
570 |
- if iopts.refObject == "" { |
|
571 |
- refSpec, err := reference.Parse(ref) |
|
572 |
- if err != nil { |
|
573 |
- return iopts, err |
|
574 |
- } |
|
575 |
- iopts.refObject = refSpec.Object |
|
576 |
- } |
|
577 | 510 |
return iopts, nil |
578 | 511 |
} |
579 | 512 |
|
580 | 513 |
// Import imports an image from a Tar stream using reader. |
581 |
-// OCI format is assumed by default. |
|
582 |
-// |
|
583 |
-// Note that unreferenced blobs are imported to the content store as well. |
|
584 |
-func (c *Client) Import(ctx context.Context, ref string, reader io.Reader, opts ...ImportOpt) (Image, error) { |
|
585 |
- iopts, err := resolveImportOpt(ref, opts...) |
|
514 |
+// Caller needs to specify importer. Future version may use oci.v1 as the default. |
|
515 |
+// Note that unreferrenced blobs may be imported to the content store as well. |
|
516 |
+func (c *Client) Import(ctx context.Context, importer images.Importer, reader io.Reader, opts ...ImportOpt) ([]Image, error) { |
|
517 |
+ _, err := resolveImportOpt(opts...) // unused now |
|
586 | 518 |
if err != nil { |
587 | 519 |
return nil, err |
588 | 520 |
} |
... | ... |
@@ -593,58 +533,66 @@ func (c *Client) Import(ctx context.Context, ref string, reader io.Reader, opts |
593 | 593 |
} |
594 | 594 |
defer done() |
595 | 595 |
|
596 |
- switch iopts.format { |
|
597 |
- case ociImageFormat: |
|
598 |
- return c.importFromOCITar(ctx, ref, reader, iopts) |
|
599 |
- default: |
|
600 |
- return nil, errors.Errorf("unsupported format: %s", iopts.format) |
|
596 |
+ imgrecs, err := importer.Import(ctx, c.ContentStore(), reader) |
|
597 |
+ if err != nil { |
|
598 |
+ // is.Update() is not called on error |
|
599 |
+ return nil, err |
|
600 |
+ } |
|
601 |
+ |
|
602 |
+ is := c.ImageService() |
|
603 |
+ var images []Image |
|
604 |
+ for _, imgrec := range imgrecs { |
|
605 |
+ if updated, err := is.Update(ctx, imgrec, "target"); err != nil { |
|
606 |
+ if !errdefs.IsNotFound(err) { |
|
607 |
+ return nil, err |
|
608 |
+ } |
|
609 |
+ |
|
610 |
+ created, err := is.Create(ctx, imgrec) |
|
611 |
+ if err != nil { |
|
612 |
+ return nil, err |
|
613 |
+ } |
|
614 |
+ |
|
615 |
+ imgrec = created |
|
616 |
+ } else { |
|
617 |
+ imgrec = updated |
|
618 |
+ } |
|
619 |
+ |
|
620 |
+ images = append(images, &image{ |
|
621 |
+ client: c, |
|
622 |
+ i: imgrec, |
|
623 |
+ }) |
|
601 | 624 |
} |
625 |
+ return images, nil |
|
602 | 626 |
} |
603 | 627 |
|
604 | 628 |
type exportOpts struct { |
605 |
- format imageFormat |
|
606 | 629 |
} |
607 | 630 |
|
608 |
-// ExportOpt allows callers to set export options |
|
631 |
+// ExportOpt allows the caller to specify export-specific options |
|
609 | 632 |
type ExportOpt func(c *exportOpts) error |
610 | 633 |
|
611 |
-// WithOCIExportFormat sets the OCI image format as the export target |
|
612 |
-func WithOCIExportFormat() ExportOpt { |
|
613 |
- return func(c *exportOpts) error { |
|
614 |
- if c.format != "" { |
|
615 |
- return errors.New("format already set") |
|
634 |
+func resolveExportOpt(opts ...ExportOpt) (exportOpts, error) { |
|
635 |
+ var eopts exportOpts |
|
636 |
+ for _, o := range opts { |
|
637 |
+ if err := o(&eopts); err != nil { |
|
638 |
+ return eopts, err |
|
616 | 639 |
} |
617 |
- c.format = ociImageFormat |
|
618 |
- return nil |
|
619 | 640 |
} |
641 |
+ return eopts, nil |
|
620 | 642 |
} |
621 | 643 |
|
622 |
-// TODO: add WithMediaTypeTranslation that transforms media types according to the format. |
|
623 |
-// e.g. application/vnd.docker.image.rootfs.diff.tar.gzip |
|
624 |
-// -> application/vnd.oci.image.layer.v1.tar+gzip |
|
625 |
- |
|
626 | 644 |
// Export exports an image to a Tar stream. |
627 | 645 |
// OCI format is used by default. |
628 | 646 |
// It is up to caller to put "org.opencontainers.image.ref.name" annotation to desc. |
629 |
-func (c *Client) Export(ctx context.Context, desc ocispec.Descriptor, opts ...ExportOpt) (io.ReadCloser, error) { |
|
630 |
- var eopts exportOpts |
|
631 |
- for _, o := range opts { |
|
632 |
- if err := o(&eopts); err != nil { |
|
633 |
- return nil, err |
|
634 |
- } |
|
635 |
- } |
|
636 |
- // use OCI as the default format |
|
637 |
- if eopts.format == "" { |
|
638 |
- eopts.format = ociImageFormat |
|
647 |
+// TODO(AkihiroSuda): support exporting multiple descriptors at once to a single archive stream. |
|
648 |
+func (c *Client) Export(ctx context.Context, exporter images.Exporter, desc ocispec.Descriptor, opts ...ExportOpt) (io.ReadCloser, error) { |
|
649 |
+ _, err := resolveExportOpt(opts...) // unused now |
|
650 |
+ if err != nil { |
|
651 |
+ return nil, err |
|
639 | 652 |
} |
640 | 653 |
pr, pw := io.Pipe() |
641 |
- switch eopts.format { |
|
642 |
- case ociImageFormat: |
|
643 |
- go func() { |
|
644 |
- pw.CloseWithError(c.exportToOCITar(ctx, desc, pw, eopts)) |
|
645 |
- }() |
|
646 |
- default: |
|
647 |
- return nil, errors.Errorf("unsupported format: %s", eopts.format) |
|
648 |
- } |
|
654 |
+ go func() { |
|
655 |
+ pw.CloseWithError(exporter.Export(ctx, c.ContentStore(), desc, pw)) |
|
656 |
+ }() |
|
649 | 657 |
return pr, nil |
650 | 658 |
} |
651 | 659 |
deleted file mode 100644 |
... | ... |
@@ -1,189 +0,0 @@ |
1 |
-package containerd |
|
2 |
- |
|
3 |
-import ( |
|
4 |
- "archive/tar" |
|
5 |
- "context" |
|
6 |
- "encoding/json" |
|
7 |
- "io" |
|
8 |
- "sort" |
|
9 |
- |
|
10 |
- "github.com/containerd/containerd/content" |
|
11 |
- "github.com/containerd/containerd/images" |
|
12 |
- "github.com/containerd/containerd/platforms" |
|
13 |
- ocispecs "github.com/opencontainers/image-spec/specs-go" |
|
14 |
- ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
|
15 |
- "github.com/pkg/errors" |
|
16 |
-) |
|
17 |
- |
|
18 |
-func (c *Client) exportToOCITar(ctx context.Context, desc ocispec.Descriptor, writer io.Writer, eopts exportOpts) error { |
|
19 |
- tw := tar.NewWriter(writer) |
|
20 |
- defer tw.Close() |
|
21 |
- |
|
22 |
- records := []tarRecord{ |
|
23 |
- ociLayoutFile(""), |
|
24 |
- ociIndexRecord(desc), |
|
25 |
- } |
|
26 |
- |
|
27 |
- cs := c.ContentStore() |
|
28 |
- algorithms := map[string]struct{}{} |
|
29 |
- exportHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { |
|
30 |
- records = append(records, blobRecord(cs, desc)) |
|
31 |
- algorithms[desc.Digest.Algorithm().String()] = struct{}{} |
|
32 |
- return nil, nil |
|
33 |
- } |
|
34 |
- |
|
35 |
- handlers := images.Handlers( |
|
36 |
- images.ChildrenHandler(cs, platforms.Default()), |
|
37 |
- images.HandlerFunc(exportHandler), |
|
38 |
- ) |
|
39 |
- |
|
40 |
- // Walk sequentially since the number of fetchs is likely one and doing in |
|
41 |
- // parallel requires locking the export handler |
|
42 |
- if err := images.Walk(ctx, handlers, desc); err != nil { |
|
43 |
- return err |
|
44 |
- } |
|
45 |
- |
|
46 |
- if len(algorithms) > 0 { |
|
47 |
- records = append(records, directoryRecord("blobs/", 0755)) |
|
48 |
- for alg := range algorithms { |
|
49 |
- records = append(records, directoryRecord("blobs/"+alg+"/", 0755)) |
|
50 |
- } |
|
51 |
- } |
|
52 |
- |
|
53 |
- return writeTar(ctx, tw, records) |
|
54 |
-} |
|
55 |
- |
|
56 |
-type tarRecord struct { |
|
57 |
- Header *tar.Header |
|
58 |
- CopyTo func(context.Context, io.Writer) (int64, error) |
|
59 |
-} |
|
60 |
- |
|
61 |
-func blobRecord(cs content.Store, desc ocispec.Descriptor) tarRecord { |
|
62 |
- path := "blobs/" + desc.Digest.Algorithm().String() + "/" + desc.Digest.Hex() |
|
63 |
- return tarRecord{ |
|
64 |
- Header: &tar.Header{ |
|
65 |
- Name: path, |
|
66 |
- Mode: 0444, |
|
67 |
- Size: desc.Size, |
|
68 |
- Typeflag: tar.TypeReg, |
|
69 |
- }, |
|
70 |
- CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { |
|
71 |
- r, err := cs.ReaderAt(ctx, desc.Digest) |
|
72 |
- if err != nil { |
|
73 |
- return 0, err |
|
74 |
- } |
|
75 |
- defer r.Close() |
|
76 |
- |
|
77 |
- // Verify digest |
|
78 |
- dgstr := desc.Digest.Algorithm().Digester() |
|
79 |
- |
|
80 |
- n, err := io.Copy(io.MultiWriter(w, dgstr.Hash()), content.NewReader(r)) |
|
81 |
- if err != nil { |
|
82 |
- return 0, err |
|
83 |
- } |
|
84 |
- if dgstr.Digest() != desc.Digest { |
|
85 |
- return 0, errors.Errorf("unexpected digest %s copied", dgstr.Digest()) |
|
86 |
- } |
|
87 |
- return n, nil |
|
88 |
- }, |
|
89 |
- } |
|
90 |
-} |
|
91 |
- |
|
92 |
-func directoryRecord(name string, mode int64) tarRecord { |
|
93 |
- return tarRecord{ |
|
94 |
- Header: &tar.Header{ |
|
95 |
- Name: name, |
|
96 |
- Mode: mode, |
|
97 |
- Typeflag: tar.TypeDir, |
|
98 |
- }, |
|
99 |
- } |
|
100 |
-} |
|
101 |
- |
|
102 |
-func ociLayoutFile(version string) tarRecord { |
|
103 |
- if version == "" { |
|
104 |
- version = ocispec.ImageLayoutVersion |
|
105 |
- } |
|
106 |
- layout := ocispec.ImageLayout{ |
|
107 |
- Version: version, |
|
108 |
- } |
|
109 |
- |
|
110 |
- b, err := json.Marshal(layout) |
|
111 |
- if err != nil { |
|
112 |
- panic(err) |
|
113 |
- } |
|
114 |
- |
|
115 |
- return tarRecord{ |
|
116 |
- Header: &tar.Header{ |
|
117 |
- Name: ocispec.ImageLayoutFile, |
|
118 |
- Mode: 0444, |
|
119 |
- Size: int64(len(b)), |
|
120 |
- Typeflag: tar.TypeReg, |
|
121 |
- }, |
|
122 |
- CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { |
|
123 |
- n, err := w.Write(b) |
|
124 |
- return int64(n), err |
|
125 |
- }, |
|
126 |
- } |
|
127 |
- |
|
128 |
-} |
|
129 |
- |
|
130 |
-func ociIndexRecord(manifests ...ocispec.Descriptor) tarRecord { |
|
131 |
- index := ocispec.Index{ |
|
132 |
- Versioned: ocispecs.Versioned{ |
|
133 |
- SchemaVersion: 2, |
|
134 |
- }, |
|
135 |
- Manifests: manifests, |
|
136 |
- } |
|
137 |
- |
|
138 |
- b, err := json.Marshal(index) |
|
139 |
- if err != nil { |
|
140 |
- panic(err) |
|
141 |
- } |
|
142 |
- |
|
143 |
- return tarRecord{ |
|
144 |
- Header: &tar.Header{ |
|
145 |
- Name: "index.json", |
|
146 |
- Mode: 0644, |
|
147 |
- Size: int64(len(b)), |
|
148 |
- Typeflag: tar.TypeReg, |
|
149 |
- }, |
|
150 |
- CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { |
|
151 |
- n, err := w.Write(b) |
|
152 |
- return int64(n), err |
|
153 |
- }, |
|
154 |
- } |
|
155 |
-} |
|
156 |
- |
|
157 |
-func writeTar(ctx context.Context, tw *tar.Writer, records []tarRecord) error { |
|
158 |
- sort.Sort(tarRecordsByName(records)) |
|
159 |
- |
|
160 |
- for _, record := range records { |
|
161 |
- if err := tw.WriteHeader(record.Header); err != nil { |
|
162 |
- return err |
|
163 |
- } |
|
164 |
- if record.CopyTo != nil { |
|
165 |
- n, err := record.CopyTo(ctx, tw) |
|
166 |
- if err != nil { |
|
167 |
- return err |
|
168 |
- } |
|
169 |
- if n != record.Header.Size { |
|
170 |
- return errors.Errorf("unexpected copy size for %s", record.Header.Name) |
|
171 |
- } |
|
172 |
- } else if record.Header.Size > 0 { |
|
173 |
- return errors.Errorf("no content to write to record with non-zero size for %s", record.Header.Name) |
|
174 |
- } |
|
175 |
- } |
|
176 |
- return nil |
|
177 |
-} |
|
178 |
- |
|
179 |
-type tarRecordsByName []tarRecord |
|
180 |
- |
|
181 |
-func (t tarRecordsByName) Len() int { |
|
182 |
- return len(t) |
|
183 |
-} |
|
184 |
-func (t tarRecordsByName) Swap(i, j int) { |
|
185 |
- t[i], t[j] = t[j], t[i] |
|
186 |
-} |
|
187 |
-func (t tarRecordsByName) Less(i, j int) bool { |
|
188 |
- return t[i].Header.Name < t[j].Header.Name |
|
189 |
-} |
... | ... |
@@ -3,7 +3,6 @@ package filters |
3 | 3 |
import ( |
4 | 4 |
"fmt" |
5 | 5 |
"io" |
6 |
- "strconv" |
|
7 | 6 |
|
8 | 7 |
"github.com/containerd/containerd/errdefs" |
9 | 8 |
"github.com/pkg/errors" |
... | ... |
@@ -134,7 +133,12 @@ func (p *parser) selector() (selector, error) { |
134 | 134 |
return selector{}, err |
135 | 135 |
} |
136 | 136 |
|
137 |
- value, err := p.value() |
|
137 |
+ var allowAltQuotes bool |
|
138 |
+ if op == operatorMatches { |
|
139 |
+ allowAltQuotes = true |
|
140 |
+ } |
|
141 |
+ |
|
142 |
+ value, err := p.value(allowAltQuotes) |
|
138 | 143 |
if err != nil { |
139 | 144 |
if err == io.EOF { |
140 | 145 |
return selector{}, io.ErrUnexpectedEOF |
... | ... |
@@ -188,7 +192,7 @@ func (p *parser) field() (string, error) { |
188 | 188 |
case tokenField: |
189 | 189 |
return s, nil |
190 | 190 |
case tokenQuoted: |
191 |
- return p.unquote(pos, s) |
|
191 |
+ return p.unquote(pos, s, false) |
|
192 | 192 |
} |
193 | 193 |
|
194 | 194 |
return "", p.mkerr(pos, "expected field or quoted") |
... | ... |
@@ -213,21 +217,25 @@ func (p *parser) operator() (operator, error) { |
213 | 213 |
return 0, p.mkerr(pos, `expected an operator ("=="|"!="|"~=")`) |
214 | 214 |
} |
215 | 215 |
|
216 |
-func (p *parser) value() (string, error) { |
|
216 |
+func (p *parser) value(allowAltQuotes bool) (string, error) { |
|
217 | 217 |
pos, tok, s := p.scanner.scan() |
218 | 218 |
|
219 | 219 |
switch tok { |
220 | 220 |
case tokenValue, tokenField: |
221 | 221 |
return s, nil |
222 | 222 |
case tokenQuoted: |
223 |
- return p.unquote(pos, s) |
|
223 |
+ return p.unquote(pos, s, allowAltQuotes) |
|
224 | 224 |
} |
225 | 225 |
|
226 | 226 |
return "", p.mkerr(pos, "expected value or quoted") |
227 | 227 |
} |
228 | 228 |
|
229 |
-func (p *parser) unquote(pos int, s string) (string, error) { |
|
230 |
- uq, err := strconv.Unquote(s) |
|
229 |
+func (p *parser) unquote(pos int, s string, allowAlts bool) (string, error) { |
|
230 |
+ if !allowAlts && s[0] != '\'' && s[0] != '"' { |
|
231 |
+ return "", p.mkerr(pos, "invalid quote encountered") |
|
232 |
+ } |
|
233 |
+ |
|
234 |
+ uq, err := unquote(s) |
|
231 | 235 |
if err != nil { |
232 | 236 |
return "", p.mkerr(pos, "unquoting failed: %v", err) |
233 | 237 |
} |
234 | 238 |
new file mode 100644 |
... | ... |
@@ -0,0 +1,237 @@ |
0 |
+package filters |
|
1 |
+ |
|
2 |
+import ( |
|
3 |
+ "unicode/utf8" |
|
4 |
+ |
|
5 |
+ "github.com/pkg/errors" |
|
6 |
+) |
|
7 |
+ |
|
8 |
+// NOTE(stevvooe): Most of this code in this file is copied from the stdlib |
|
9 |
+// strconv package and modified to be able to handle quoting with `/` and `|` |
|
10 |
+// as delimiters. The copyright is held by the Go authors. |
|
11 |
+ |
|
12 |
+var errQuoteSyntax = errors.New("quote syntax error") |
|
13 |
+ |
|
14 |
+// UnquoteChar decodes the first character or byte in the escaped string |
|
15 |
+// or character literal represented by the string s. |
|
16 |
+// It returns four values: |
|
17 |
+// |
|
18 |
+// 1) value, the decoded Unicode code point or byte value; |
|
19 |
+// 2) multibyte, a boolean indicating whether the decoded character requires a multibyte UTF-8 representation; |
|
20 |
+// 3) tail, the remainder of the string after the character; and |
|
21 |
+// 4) an error that will be nil if the character is syntactically valid. |
|
22 |
+// |
|
23 |
+// The second argument, quote, specifies the type of literal being parsed |
|
24 |
+// and therefore which escaped quote character is permitted. |
|
25 |
+// If set to a single quote, it permits the sequence \' and disallows unescaped '. |
|
26 |
+// If set to a double quote, it permits \" and disallows unescaped ". |
|
27 |
+// If set to zero, it does not permit either escape and allows both quote characters to appear unescaped. |
|
28 |
+// |
|
29 |
+// This is from Go strconv package, modified to support `|` and `/` as double |
|
30 |
+// quotes for use with regular expressions. |
|
31 |
+func unquoteChar(s string, quote byte) (value rune, multibyte bool, tail string, err error) { |
|
32 |
+ // easy cases |
|
33 |
+ switch c := s[0]; { |
|
34 |
+ case c == quote && (quote == '\'' || quote == '"' || quote == '/' || quote == '|'): |
|
35 |
+ err = errQuoteSyntax |
|
36 |
+ return |
|
37 |
+ case c >= utf8.RuneSelf: |
|
38 |
+ r, size := utf8.DecodeRuneInString(s) |
|
39 |
+ return r, true, s[size:], nil |
|
40 |
+ case c != '\\': |
|
41 |
+ return rune(s[0]), false, s[1:], nil |
|
42 |
+ } |
|
43 |
+ |
|
44 |
+ // hard case: c is backslash |
|
45 |
+ if len(s) <= 1 { |
|
46 |
+ err = errQuoteSyntax |
|
47 |
+ return |
|
48 |
+ } |
|
49 |
+ c := s[1] |
|
50 |
+ s = s[2:] |
|
51 |
+ |
|
52 |
+ switch c { |
|
53 |
+ case 'a': |
|
54 |
+ value = '\a' |
|
55 |
+ case 'b': |
|
56 |
+ value = '\b' |
|
57 |
+ case 'f': |
|
58 |
+ value = '\f' |
|
59 |
+ case 'n': |
|
60 |
+ value = '\n' |
|
61 |
+ case 'r': |
|
62 |
+ value = '\r' |
|
63 |
+ case 't': |
|
64 |
+ value = '\t' |
|
65 |
+ case 'v': |
|
66 |
+ value = '\v' |
|
67 |
+ case 'x', 'u', 'U': |
|
68 |
+ n := 0 |
|
69 |
+ switch c { |
|
70 |
+ case 'x': |
|
71 |
+ n = 2 |
|
72 |
+ case 'u': |
|
73 |
+ n = 4 |
|
74 |
+ case 'U': |
|
75 |
+ n = 8 |
|
76 |
+ } |
|
77 |
+ var v rune |
|
78 |
+ if len(s) < n { |
|
79 |
+ err = errQuoteSyntax |
|
80 |
+ return |
|
81 |
+ } |
|
82 |
+ for j := 0; j < n; j++ { |
|
83 |
+ x, ok := unhex(s[j]) |
|
84 |
+ if !ok { |
|
85 |
+ err = errQuoteSyntax |
|
86 |
+ return |
|
87 |
+ } |
|
88 |
+ v = v<<4 | x |
|
89 |
+ } |
|
90 |
+ s = s[n:] |
|
91 |
+ if c == 'x' { |
|
92 |
+ // single-byte string, possibly not UTF-8 |
|
93 |
+ value = v |
|
94 |
+ break |
|
95 |
+ } |
|
96 |
+ if v > utf8.MaxRune { |
|
97 |
+ err = errQuoteSyntax |
|
98 |
+ return |
|
99 |
+ } |
|
100 |
+ value = v |
|
101 |
+ multibyte = true |
|
102 |
+ case '0', '1', '2', '3', '4', '5', '6', '7': |
|
103 |
+ v := rune(c) - '0' |
|
104 |
+ if len(s) < 2 { |
|
105 |
+ err = errQuoteSyntax |
|
106 |
+ return |
|
107 |
+ } |
|
108 |
+ for j := 0; j < 2; j++ { // one digit already; two more |
|
109 |
+ x := rune(s[j]) - '0' |
|
110 |
+ if x < 0 || x > 7 { |
|
111 |
+ err = errQuoteSyntax |
|
112 |
+ return |
|
113 |
+ } |
|
114 |
+ v = (v << 3) | x |
|
115 |
+ } |
|
116 |
+ s = s[2:] |
|
117 |
+ if v > 255 { |
|
118 |
+ err = errQuoteSyntax |
|
119 |
+ return |
|
120 |
+ } |
|
121 |
+ value = v |
|
122 |
+ case '\\': |
|
123 |
+ value = '\\' |
|
124 |
+ case '\'', '"', '|', '/': |
|
125 |
+ if c != quote { |
|
126 |
+ err = errQuoteSyntax |
|
127 |
+ return |
|
128 |
+ } |
|
129 |
+ value = rune(c) |
|
130 |
+ default: |
|
131 |
+ err = errQuoteSyntax |
|
132 |
+ return |
|
133 |
+ } |
|
134 |
+ tail = s |
|
135 |
+ return |
|
136 |
+} |
|
137 |
+ |
|
138 |
+// unquote interprets s as a single-quoted, double-quoted, |
|
139 |
+// or backquoted Go string literal, returning the string value |
|
140 |
+// that s quotes. (If s is single-quoted, it would be a Go |
|
141 |
+// character literal; Unquote returns the corresponding |
|
142 |
+// one-character string.) |
|
143 |
+// |
|
144 |
+// This is modified from the standard library to support `|` and `/` as quote |
|
145 |
+// characters for use with regular expressions. |
|
146 |
+func unquote(s string) (string, error) { |
|
147 |
+ n := len(s) |
|
148 |
+ if n < 2 { |
|
149 |
+ return "", errQuoteSyntax |
|
150 |
+ } |
|
151 |
+ quote := s[0] |
|
152 |
+ if quote != s[n-1] { |
|
153 |
+ return "", errQuoteSyntax |
|
154 |
+ } |
|
155 |
+ s = s[1 : n-1] |
|
156 |
+ |
|
157 |
+ if quote == '`' { |
|
158 |
+ if contains(s, '`') { |
|
159 |
+ return "", errQuoteSyntax |
|
160 |
+ } |
|
161 |
+ if contains(s, '\r') { |
|
162 |
+ // -1 because we know there is at least one \r to remove. |
|
163 |
+ buf := make([]byte, 0, len(s)-1) |
|
164 |
+ for i := 0; i < len(s); i++ { |
|
165 |
+ if s[i] != '\r' { |
|
166 |
+ buf = append(buf, s[i]) |
|
167 |
+ } |
|
168 |
+ } |
|
169 |
+ return string(buf), nil |
|
170 |
+ } |
|
171 |
+ return s, nil |
|
172 |
+ } |
|
173 |
+ if quote != '"' && quote != '\'' && quote != '|' && quote != '/' { |
|
174 |
+ return "", errQuoteSyntax |
|
175 |
+ } |
|
176 |
+ if contains(s, '\n') { |
|
177 |
+ return "", errQuoteSyntax |
|
178 |
+ } |
|
179 |
+ |
|
180 |
+ // Is it trivial? Avoid allocation. |
|
181 |
+ if !contains(s, '\\') && !contains(s, quote) { |
|
182 |
+ switch quote { |
|
183 |
+ case '"', '/', '|': // pipe and slash are treated like double quote |
|
184 |
+ return s, nil |
|
185 |
+ case '\'': |
|
186 |
+ r, size := utf8.DecodeRuneInString(s) |
|
187 |
+ if size == len(s) && (r != utf8.RuneError || size != 1) { |
|
188 |
+ return s, nil |
|
189 |
+ } |
|
190 |
+ } |
|
191 |
+ } |
|
192 |
+ |
|
193 |
+ var runeTmp [utf8.UTFMax]byte |
|
194 |
+ buf := make([]byte, 0, 3*len(s)/2) // Try to avoid more allocations. |
|
195 |
+ for len(s) > 0 { |
|
196 |
+ c, multibyte, ss, err := unquoteChar(s, quote) |
|
197 |
+ if err != nil { |
|
198 |
+ return "", err |
|
199 |
+ } |
|
200 |
+ s = ss |
|
201 |
+ if c < utf8.RuneSelf || !multibyte { |
|
202 |
+ buf = append(buf, byte(c)) |
|
203 |
+ } else { |
|
204 |
+ n := utf8.EncodeRune(runeTmp[:], c) |
|
205 |
+ buf = append(buf, runeTmp[:n]...) |
|
206 |
+ } |
|
207 |
+ if quote == '\'' && len(s) != 0 { |
|
208 |
+ // single-quoted must be single character |
|
209 |
+ return "", errQuoteSyntax |
|
210 |
+ } |
|
211 |
+ } |
|
212 |
+ return string(buf), nil |
|
213 |
+} |
|
214 |
+ |
|
215 |
+// contains reports whether the string contains the byte c. |
|
216 |
+func contains(s string, c byte) bool { |
|
217 |
+ for i := 0; i < len(s); i++ { |
|
218 |
+ if s[i] == c { |
|
219 |
+ return true |
|
220 |
+ } |
|
221 |
+ } |
|
222 |
+ return false |
|
223 |
+} |
|
224 |
+ |
|
225 |
+func unhex(b byte) (v rune, ok bool) { |
|
226 |
+ c := rune(b) |
|
227 |
+ switch { |
|
228 |
+ case '0' <= c && c <= '9': |
|
229 |
+ return c - '0', true |
|
230 |
+ case 'a' <= c && c <= 'f': |
|
231 |
+ return c - 'a' + 10, true |
|
232 |
+ case 'A' <= c && c <= 'F': |
|
233 |
+ return c - 'A' + 10, true |
|
234 |
+ } |
|
235 |
+ return |
|
236 |
+} |
... | ... |
@@ -87,7 +87,7 @@ func (s *scanner) peek() rune { |
87 | 87 |
return ch |
88 | 88 |
} |
89 | 89 |
|
90 |
-func (s *scanner) scan() (int, token, string) { |
|
90 |
+func (s *scanner) scan() (nextp int, tk token, text string) { |
|
91 | 91 |
var ( |
92 | 92 |
ch = s.next() |
93 | 93 |
pos = s.pos |
... | ... |
@@ -101,6 +101,7 @@ chomp: |
101 | 101 |
s.scanQuoted(ch) |
102 | 102 |
return pos, tokenQuoted, s.input[pos:s.ppos] |
103 | 103 |
case isSeparatorRune(ch): |
104 |
+ s.value = false |
|
104 | 105 |
return pos, tokenSeparator, s.input[pos:s.ppos] |
105 | 106 |
case isOperatorRune(ch): |
106 | 107 |
s.scanOperator() |
... | ... |
@@ -241,7 +242,7 @@ func isOperatorRune(r rune) bool { |
241 | 241 |
|
242 | 242 |
func isQuoteRune(r rune) bool { |
243 | 243 |
switch r { |
244 |
- case '"': // maybe add single quoting? |
|
244 |
+ case '/', '|', '"': // maybe add single quoting? |
|
245 | 245 |
return true |
246 | 246 |
} |
247 | 247 |
|
... | ... |
@@ -222,8 +222,10 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err |
222 | 222 |
c1 = make(chan *currentPath) |
223 | 223 |
c2 = make(chan *currentPath) |
224 | 224 |
|
225 |
- f1, f2 *currentPath |
|
226 |
- rmdir string |
|
225 |
+ f1, f2 *currentPath |
|
226 |
+ rmdir string |
|
227 |
+ lastEmittedDir = string(filepath.Separator) |
|
228 |
+ parents []os.FileInfo |
|
227 | 229 |
) |
228 | 230 |
g.Go(func() error { |
229 | 231 |
defer close(c1) |
... | ... |
@@ -258,7 +260,10 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err |
258 | 258 |
continue |
259 | 259 |
} |
260 | 260 |
|
261 |
- var f os.FileInfo |
|
261 |
+ var ( |
|
262 |
+ f os.FileInfo |
|
263 |
+ emit = true |
|
264 |
+ ) |
|
262 | 265 |
k, p := pathChange(f1, f2) |
263 | 266 |
switch k { |
264 | 267 |
case ChangeKindAdd: |
... | ... |
@@ -294,13 +299,35 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err |
294 | 294 |
f2 = nil |
295 | 295 |
if same { |
296 | 296 |
if !isLinked(f) { |
297 |
- continue |
|
297 |
+ emit = false |
|
298 | 298 |
} |
299 | 299 |
k = ChangeKindUnmodified |
300 | 300 |
} |
301 | 301 |
} |
302 |
- if err := changeFn(k, p, f, nil); err != nil { |
|
303 |
- return err |
|
302 |
+ if emit { |
|
303 |
+ emittedDir, emitParents := commonParents(lastEmittedDir, p, parents) |
|
304 |
+ for _, pf := range emitParents { |
|
305 |
+ p := filepath.Join(emittedDir, pf.Name()) |
|
306 |
+ if err := changeFn(ChangeKindUnmodified, p, pf, nil); err != nil { |
|
307 |
+ return err |
|
308 |
+ } |
|
309 |
+ emittedDir = p |
|
310 |
+ } |
|
311 |
+ |
|
312 |
+ if err := changeFn(k, p, f, nil); err != nil { |
|
313 |
+ return err |
|
314 |
+ } |
|
315 |
+ |
|
316 |
+ if f != nil && f.IsDir() { |
|
317 |
+ lastEmittedDir = p |
|
318 |
+ } else { |
|
319 |
+ lastEmittedDir = emittedDir |
|
320 |
+ } |
|
321 |
+ |
|
322 |
+ parents = parents[:0] |
|
323 |
+ } else if f.IsDir() { |
|
324 |
+ lastEmittedDir, parents = commonParents(lastEmittedDir, p, parents) |
|
325 |
+ parents = append(parents, f) |
|
304 | 326 |
} |
305 | 327 |
} |
306 | 328 |
return nil |
... | ... |
@@ -308,3 +335,47 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err |
308 | 308 |
|
309 | 309 |
return g.Wait() |
310 | 310 |
} |
311 |
+ |
|
312 |
+func commonParents(base, updated string, dirs []os.FileInfo) (string, []os.FileInfo) { |
|
313 |
+ if basePrefix := makePrefix(base); strings.HasPrefix(updated, basePrefix) { |
|
314 |
+ var ( |
|
315 |
+ parents []os.FileInfo |
|
316 |
+ last = base |
|
317 |
+ ) |
|
318 |
+ for _, d := range dirs { |
|
319 |
+ next := filepath.Join(last, d.Name()) |
|
320 |
+ if strings.HasPrefix(updated, makePrefix(last)) { |
|
321 |
+ parents = append(parents, d) |
|
322 |
+ last = next |
|
323 |
+ } else { |
|
324 |
+ break |
|
325 |
+ } |
|
326 |
+ } |
|
327 |
+ return base, parents |
|
328 |
+ } |
|
329 |
+ |
|
330 |
+ baseS := strings.Split(base, string(filepath.Separator)) |
|
331 |
+ updatedS := strings.Split(updated, string(filepath.Separator)) |
|
332 |
+ commonS := []string{string(filepath.Separator)} |
|
333 |
+ |
|
334 |
+ min := len(baseS) |
|
335 |
+ if len(updatedS) < min { |
|
336 |
+ min = len(updatedS) |
|
337 |
+ } |
|
338 |
+ for i := 0; i < min; i++ { |
|
339 |
+ if baseS[i] == updatedS[i] { |
|
340 |
+ commonS = append(commonS, baseS[i]) |
|
341 |
+ } else { |
|
342 |
+ break |
|
343 |
+ } |
|
344 |
+ } |
|
345 |
+ |
|
346 |
+ return filepath.Join(commonS...), []os.FileInfo{} |
|
347 |
+} |
|
348 |
+ |
|
349 |
+func makePrefix(d string) string { |
|
350 |
+ if d == "" || d[len(d)-1] != filepath.Separator { |
|
351 |
+ return d + string(filepath.Separator) |
|
352 |
+ } |
|
353 |
+ return d |
|
354 |
+} |
... | ... |
@@ -1,5 +1,7 @@ |
1 | 1 |
package fs |
2 | 2 |
|
3 |
+import "context" |
|
4 |
+ |
|
3 | 5 |
// Usage of disk information |
4 | 6 |
type Usage struct { |
5 | 7 |
Inodes int64 |
... | ... |
@@ -11,3 +13,10 @@ type Usage struct { |
11 | 11 |
func DiskUsage(roots ...string) (Usage, error) { |
12 | 12 |
return diskUsage(roots...) |
13 | 13 |
} |
14 |
+ |
|
15 |
+// DiffUsage counts the numbers of inodes and disk usage in the |
|
16 |
+// diff between the 2 directories. The first path is intended |
|
17 |
+// as the base directory and the second as the changed directory. |
|
18 |
+func DiffUsage(ctx context.Context, a, b string) (Usage, error) { |
|
19 |
+ return diffUsage(ctx, a, b) |
|
20 |
+} |
... | ... |
@@ -3,17 +3,19 @@ |
3 | 3 |
package fs |
4 | 4 |
|
5 | 5 |
import ( |
6 |
+ "context" |
|
6 | 7 |
"os" |
7 | 8 |
"path/filepath" |
8 | 9 |
"syscall" |
9 | 10 |
) |
10 | 11 |
|
12 |
+type inode struct { |
|
13 |
+ // TODO(stevvooe): Can probably reduce memory usage by not tracking |
|
14 |
+ // device, but we can leave this right for now. |
|
15 |
+ dev, ino uint64 |
|
16 |
+} |
|
17 |
+ |
|
11 | 18 |
func diskUsage(roots ...string) (Usage, error) { |
12 |
- type inode struct { |
|
13 |
- // TODO(stevvooe): Can probably reduce memory usage by not tracking |
|
14 |
- // device, but we can leave this right for now. |
|
15 |
- dev, ino uint64 |
|
16 |
- } |
|
17 | 19 |
|
18 | 20 |
var ( |
19 | 21 |
size int64 |
... | ... |
@@ -45,3 +47,37 @@ func diskUsage(roots ...string) (Usage, error) { |
45 | 45 |
Size: size, |
46 | 46 |
}, nil |
47 | 47 |
} |
48 |
+ |
|
49 |
+func diffUsage(ctx context.Context, a, b string) (Usage, error) { |
|
50 |
+ var ( |
|
51 |
+ size int64 |
|
52 |
+ inodes = map[inode]struct{}{} // expensive! |
|
53 |
+ ) |
|
54 |
+ |
|
55 |
+ if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error { |
|
56 |
+ if err != nil { |
|
57 |
+ return err |
|
58 |
+ } |
|
59 |
+ |
|
60 |
+ if kind == ChangeKindAdd || kind == ChangeKindModify { |
|
61 |
+ stat := fi.Sys().(*syscall.Stat_t) |
|
62 |
+ |
|
63 |
+ inoKey := inode{dev: uint64(stat.Dev), ino: uint64(stat.Ino)} |
|
64 |
+ if _, ok := inodes[inoKey]; !ok { |
|
65 |
+ inodes[inoKey] = struct{}{} |
|
66 |
+ size += fi.Size() |
|
67 |
+ } |
|
68 |
+ |
|
69 |
+ return nil |
|
70 |
+ |
|
71 |
+ } |
|
72 |
+ return nil |
|
73 |
+ }); err != nil { |
|
74 |
+ return Usage{}, err |
|
75 |
+ } |
|
76 |
+ |
|
77 |
+ return Usage{ |
|
78 |
+ Inodes: int64(len(inodes)), |
|
79 |
+ Size: size, |
|
80 |
+ }, nil |
|
81 |
+} |
... | ... |
@@ -3,6 +3,7 @@ |
3 | 3 |
package fs |
4 | 4 |
|
5 | 5 |
import ( |
6 |
+ "context" |
|
6 | 7 |
"os" |
7 | 8 |
"path/filepath" |
8 | 9 |
) |
... | ... |
@@ -31,3 +32,29 @@ func diskUsage(roots ...string) (Usage, error) { |
31 | 31 |
Size: size, |
32 | 32 |
}, nil |
33 | 33 |
} |
34 |
+ |
|
35 |
+func diffUsage(ctx context.Context, a, b string) (Usage, error) { |
|
36 |
+ var ( |
|
37 |
+ size int64 |
|
38 |
+ ) |
|
39 |
+ |
|
40 |
+ if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error { |
|
41 |
+ if err != nil { |
|
42 |
+ return err |
|
43 |
+ } |
|
44 |
+ |
|
45 |
+ if kind == ChangeKindAdd || kind == ChangeKindModify { |
|
46 |
+ size += fi.Size() |
|
47 |
+ |
|
48 |
+ return nil |
|
49 |
+ |
|
50 |
+ } |
|
51 |
+ return nil |
|
52 |
+ }); err != nil { |
|
53 |
+ return Usage{}, err |
|
54 |
+ } |
|
55 |
+ |
|
56 |
+ return Usage{ |
|
57 |
+ Size: size, |
|
58 |
+ }, nil |
|
59 |
+} |
34 | 60 |
new file mode 100644 |
... | ... |
@@ -0,0 +1,21 @@ |
0 |
+package images |
|
1 |
+ |
|
2 |
+import ( |
|
3 |
+ "context" |
|
4 |
+ "io" |
|
5 |
+ |
|
6 |
+ "github.com/containerd/containerd/content" |
|
7 |
+ ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
|
8 |
+) |
|
9 |
+ |
|
10 |
+// Importer is the interface for image importer. |
|
11 |
+type Importer interface { |
|
12 |
+ // Import imports an image from a tar stream. |
|
13 |
+ Import(ctx context.Context, store content.Store, reader io.Reader) ([]Image, error) |
|
14 |
+} |
|
15 |
+ |
|
16 |
+// Exporter is the interface for image exporter. |
|
17 |
+type Exporter interface { |
|
18 |
+ // Export exports an image to a tar stream. |
|
19 |
+ Export(ctx context.Context, store content.Store, desc ocispec.Descriptor, writer io.Writer) error |
|
20 |
+} |
0 | 21 |
deleted file mode 100644 |
... | ... |
@@ -1,120 +0,0 @@ |
1 |
-package containerd |
|
2 |
- |
|
3 |
-import ( |
|
4 |
- "archive/tar" |
|
5 |
- "context" |
|
6 |
- "encoding/json" |
|
7 |
- "io" |
|
8 |
- "io/ioutil" |
|
9 |
- "strings" |
|
10 |
- |
|
11 |
- "github.com/containerd/containerd/content" |
|
12 |
- "github.com/containerd/containerd/errdefs" |
|
13 |
- "github.com/containerd/containerd/images" |
|
14 |
- "github.com/containerd/containerd/reference" |
|
15 |
- digest "github.com/opencontainers/go-digest" |
|
16 |
- ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
|
17 |
- "github.com/pkg/errors" |
|
18 |
-) |
|
19 |
- |
|
20 |
-func resolveOCIIndex(idx ocispec.Index, refObject string) (*ocispec.Descriptor, error) { |
|
21 |
- tag, dgst := reference.SplitObject(refObject) |
|
22 |
- if tag == "" && dgst == "" { |
|
23 |
- return nil, errors.Errorf("unexpected object: %q", refObject) |
|
24 |
- } |
|
25 |
- for _, m := range idx.Manifests { |
|
26 |
- if m.Digest == dgst { |
|
27 |
- return &m, nil |
|
28 |
- } |
|
29 |
- annot, ok := m.Annotations[ocispec.AnnotationRefName] |
|
30 |
- if ok && annot == tag && tag != "" { |
|
31 |
- return &m, nil |
|
32 |
- } |
|
33 |
- } |
|
34 |
- return nil, errors.Errorf("not found: %q", refObject) |
|
35 |
-} |
|
36 |
- |
|
37 |
-func (c *Client) importFromOCITar(ctx context.Context, ref string, reader io.Reader, iopts importOpts) (Image, error) { |
|
38 |
- tr := tar.NewReader(reader) |
|
39 |
- store := c.ContentStore() |
|
40 |
- var desc *ocispec.Descriptor |
|
41 |
- for { |
|
42 |
- hdr, err := tr.Next() |
|
43 |
- if err == io.EOF { |
|
44 |
- break |
|
45 |
- } |
|
46 |
- if err != nil { |
|
47 |
- return nil, err |
|
48 |
- } |
|
49 |
- if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { |
|
50 |
- continue |
|
51 |
- } |
|
52 |
- if hdr.Name == "index.json" { |
|
53 |
- desc, err = onUntarIndexJSON(tr, iopts.refObject) |
|
54 |
- if err != nil { |
|
55 |
- return nil, err |
|
56 |
- } |
|
57 |
- continue |
|
58 |
- } |
|
59 |
- if strings.HasPrefix(hdr.Name, "blobs/") { |
|
60 |
- if err := onUntarBlob(ctx, tr, store, hdr.Name, hdr.Size); err != nil { |
|
61 |
- return nil, err |
|
62 |
- } |
|
63 |
- } |
|
64 |
- } |
|
65 |
- if desc == nil { |
|
66 |
- return nil, errors.Errorf("no descriptor found for reference object %q", iopts.refObject) |
|
67 |
- } |
|
68 |
- imgrec := images.Image{ |
|
69 |
- Name: ref, |
|
70 |
- Target: *desc, |
|
71 |
- Labels: iopts.labels, |
|
72 |
- } |
|
73 |
- is := c.ImageService() |
|
74 |
- if updated, err := is.Update(ctx, imgrec, "target"); err != nil { |
|
75 |
- if !errdefs.IsNotFound(err) { |
|
76 |
- return nil, err |
|
77 |
- } |
|
78 |
- |
|
79 |
- created, err := is.Create(ctx, imgrec) |
|
80 |
- if err != nil { |
|
81 |
- return nil, err |
|
82 |
- } |
|
83 |
- |
|
84 |
- imgrec = created |
|
85 |
- } else { |
|
86 |
- imgrec = updated |
|
87 |
- } |
|
88 |
- |
|
89 |
- img := &image{ |
|
90 |
- client: c, |
|
91 |
- i: imgrec, |
|
92 |
- } |
|
93 |
- return img, nil |
|
94 |
-} |
|
95 |
- |
|
96 |
-func onUntarIndexJSON(r io.Reader, refObject string) (*ocispec.Descriptor, error) { |
|
97 |
- b, err := ioutil.ReadAll(r) |
|
98 |
- if err != nil { |
|
99 |
- return nil, err |
|
100 |
- } |
|
101 |
- var idx ocispec.Index |
|
102 |
- if err := json.Unmarshal(b, &idx); err != nil { |
|
103 |
- return nil, err |
|
104 |
- } |
|
105 |
- return resolveOCIIndex(idx, refObject) |
|
106 |
-} |
|
107 |
- |
|
108 |
-func onUntarBlob(ctx context.Context, r io.Reader, store content.Store, name string, size int64) error { |
|
109 |
- // name is like "blobs/sha256/deadbeef" |
|
110 |
- split := strings.Split(name, "/") |
|
111 |
- if len(split) != 3 { |
|
112 |
- return errors.Errorf("unexpected name: %q", name) |
|
113 |
- } |
|
114 |
- algo := digest.Algorithm(split[1]) |
|
115 |
- if !algo.Available() { |
|
116 |
- return errors.Errorf("unsupported algorithm: %s", algo) |
|
117 |
- } |
|
118 |
- dgst := digest.NewDigestFromHex(algo.String(), split[2]) |
|
119 |
- return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, size, dgst) |
|
120 |
-} |
... | ... |
@@ -114,6 +114,7 @@ func fetch(ctx context.Context, ingester content.Ingester, fetcher Fetcher, desc |
114 | 114 |
func commitOpts(desc ocispec.Descriptor, r io.Reader) (io.Reader, []content.Opt) { |
115 | 115 |
var childrenF func(r io.Reader) ([]ocispec.Descriptor, error) |
116 | 116 |
|
117 |
+ // TODO(AkihiroSuda): use images/oci.GetChildrenDescriptors? |
|
117 | 118 |
switch desc.MediaType { |
118 | 119 |
case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: |
119 | 120 |
childrenF = func(r io.Reader) ([]ocispec.Descriptor, error) { |