| ... | ... |
@@ -4,7 +4,7 @@ TOMLV_COMMIT=9baf8a8a9f2ed20a8e54160840c492f937eeaf9a |
| 4 | 4 |
|
| 5 | 5 |
# When updating RUNC_COMMIT, also update runc in vendor.conf accordingly |
| 6 | 6 |
RUNC_COMMIT=b2567b37d7b75eb4cf325b77297b140ea686ce8f |
| 7 |
-CONTAINERD_COMMIT=cc969fb42f427a68a8cc6870ef47f17304b83962 |
|
| 7 |
+CONTAINERD_COMMIT=v1.0.0 |
|
| 8 | 8 |
TINI_COMMIT=949e6facb77383876aeff8a6944dde66b3089574 |
| 9 | 9 |
LIBNETWORK_COMMIT=7b2b1feb1de4817d522cc372af149ff48d25028e |
| 10 | 10 |
VNDR_COMMIT=a6e196d8b4b0cbbdc29aebdb20c59ac6926bb384 |
| ... | ... |
@@ -681,7 +681,10 @@ func (c *client) processEventStream(ctx context.Context) {
|
| 681 | 681 |
}() |
| 682 | 682 |
|
| 683 | 683 |
eventStream, err = c.remote.EventService().Subscribe(ctx, &eventsapi.SubscribeRequest{
|
| 684 |
- Filters: []string{"namespace==" + c.namespace + ",topic~=/tasks/.+"},
|
|
| 684 |
+ Filters: []string{
|
|
| 685 |
+ "namespace==" + c.namespace, |
|
| 686 |
+ "topic~=/tasks/", |
|
| 687 |
+ }, |
|
| 685 | 688 |
}, grpc.FailFast(false)) |
| 686 | 689 |
if err != nil {
|
| 687 | 690 |
return |
| ... | ... |
@@ -103,7 +103,7 @@ github.com/googleapis/gax-go da06d194a00e19ce00d9011a13931c3f6f6887c7 |
| 103 | 103 |
google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 |
| 104 | 104 |
|
| 105 | 105 |
# containerd |
| 106 |
-github.com/containerd/containerd cc969fb42f427a68a8cc6870ef47f17304b83962 |
|
| 106 |
+github.com/containerd/containerd v1.0.0 |
|
| 107 | 107 |
github.com/containerd/fifo fbfb6a11ec671efbe94ad1c12c2e98773f19e1e6 |
| 108 | 108 |
github.com/containerd/continuity 35d55c5e8dd23b32037d56cf97174aff3efdfa83 |
| 109 | 109 |
github.com/containerd/cgroups 29da22c6171a4316169f9205ab6c49f59b5b852f |
| ... | ... |
@@ -13,13 +13,23 @@ containerd is designed to be embedded into a larger system, rather than being us |
| 13 | 13 |
|
| 14 | 14 |
## Getting Started |
| 15 | 15 |
|
| 16 |
-If you are interested in trying out containerd please see our [Getting Started Guide](docs/getting-started.md). |
|
| 16 |
+See our documentation on [containerd.io](containerd.io): |
|
| 17 |
+* [for ops and admins](docs/ops.md) |
|
| 18 |
+* [namespaces](docs/namespaces.md) |
|
| 19 |
+* [client options](docs/client-opts.md) |
|
| 20 |
+ |
|
| 21 |
+See how to build containerd from source at [BUILDING](BUILDING.md). |
|
| 22 |
+ |
|
| 23 |
+If you are interested in trying out containerd see our example at [Getting Started](docs/getting-started.md). |
|
| 24 |
+ |
|
| 17 | 25 |
|
| 18 | 26 |
## Runtime Requirements |
| 19 | 27 |
|
| 20 | 28 |
Runtime requirements for containerd are very minimal. Most interactions with |
| 21 | 29 |
the Linux and Windows container feature sets are handled via [runc](https://github.com/opencontainers/runc) and/or |
| 22 |
-OS-specific libraries (e.g. [hcsshim](https://github.com/Microsoft/hcsshim) for Microsoft). There are specific features |
|
| 30 |
+OS-specific libraries (e.g. [hcsshim](https://github.com/Microsoft/hcsshim) for Microsoft). The current required version of `runc` is always listed in [RUNC.md](/RUNC.md). |
|
| 31 |
+ |
|
| 32 |
+There are specific features |
|
| 23 | 33 |
used by containerd core code and snapshotters that will require a minimum kernel |
| 24 | 34 |
version on Linux. With the understood caveat of distro kernel versioning, a |
| 25 | 35 |
reasonable starting point for Linux is a minimum 4.x kernel version. |
| ... | ... |
@@ -33,9 +43,7 @@ distribution. |
| 33 | 33 |
To use Linux checkpoint and restore features, you will need `criu` installed on |
| 34 | 34 |
your system. See more details in [Checkpoint and Restore](#checkpoint-and-restore). |
| 35 | 35 |
|
| 36 |
-The current required version of runc is always listed in [RUNC.md](/RUNC.md). |
|
| 37 |
- |
|
| 38 |
-Build requirements for developers are listed in the [Developer Quick-Start](#developer-quick-start) section. |
|
| 36 |
+Build requirements for developers are listed in [BUILDING](BUILDING.md). |
|
| 39 | 37 |
|
| 40 | 38 |
## Features |
| 41 | 39 |
|
| ... | ... |
@@ -45,7 +53,11 @@ containerd offers a full client package to help you integrate containerd into yo |
| 45 | 45 |
|
| 46 | 46 |
```go |
| 47 | 47 |
|
| 48 |
-import "github.com/containerd/containerd" |
|
| 48 |
+import ( |
|
| 49 |
+ "github.com/containerd/containerd" |
|
| 50 |
+ "github.com/containerd/containerd/cio" |
|
| 51 |
+) |
|
| 52 |
+ |
|
| 49 | 53 |
|
| 50 | 54 |
func main() {
|
| 51 | 55 |
client, err := containerd.New("/run/containerd/containerd.sock")
|
| ... | ... |
@@ -61,7 +73,7 @@ Namespaces allow multiple consumers to use the same containerd without conflicti |
| 61 | 61 |
To set a namespace for requests to the API: |
| 62 | 62 |
|
| 63 | 63 |
```go |
| 64 |
-context = context.Background() |
|
| 64 |
+context = context.Background() |
|
| 65 | 65 |
// create a context for docker |
| 66 | 66 |
docker = namespaces.WithNamespace(context, "docker") |
| 67 | 67 |
|
| ... | ... |
@@ -133,7 +145,7 @@ Taking a container object and turning it into a runnable process on a system is |
| 133 | 133 |
|
| 134 | 134 |
```go |
| 135 | 135 |
// create a new task |
| 136 |
-task, err := redis.NewTask(context, containerd.Stdio) |
|
| 136 |
+task, err := redis.NewTask(context, cio.Stdio) |
|
| 137 | 137 |
defer task.Delete(context) |
| 138 | 138 |
|
| 139 | 139 |
// the task is now running and has a pid that can be use to setup networking |
| ... | ... |
@@ -165,37 +177,12 @@ checkpoint := image.Target() |
| 165 | 165 |
redis, err = client.NewContainer(context, "redis-master", containerd.WithCheckpoint(checkpoint, "redis-rootfs")) |
| 166 | 166 |
defer container.Delete(context) |
| 167 | 167 |
|
| 168 |
-task, err = redis.NewTask(context, containerd.Stdio, containerd.WithTaskCheckpoint(checkpoint)) |
|
| 168 |
+task, err = redis.NewTask(context, cio.Stdio, containerd.WithTaskCheckpoint(checkpoint)) |
|
| 169 | 169 |
defer task.Delete(context) |
| 170 | 170 |
|
| 171 | 171 |
err := task.Start(context) |
| 172 | 172 |
``` |
| 173 | 173 |
|
| 174 |
-## Developer Quick Start |
|
| 175 |
- |
|
| 176 |
-To build the daemon and `ctr` simple test client, the following build system dependencies are required: |
|
| 177 |
- |
|
| 178 |
-* Go 1.9.x or above |
|
| 179 |
-* Protoc 3.x compiler and headers (download at the [Google protobuf releases page](https://github.com/google/protobuf/releases)) |
|
| 180 |
-* Btrfs headers and libraries for your distribution. Note that building the btrfs driver can be disabled via build tag removing this dependency. |
|
| 181 |
- |
|
| 182 |
-For proper results, install the `protoc` release into `/usr/local` on your build system. For example, the following commands will download and install the 3.5.0 release for a 64-bit Linux host: |
|
| 183 |
- |
|
| 184 |
-``` |
|
| 185 |
-$ wget -c https://github.com/google/protobuf/releases/download/v3.5.0/protoc-3.5.0-linux-x86_64.zip |
|
| 186 |
-$ sudo unzip protoc-3.5.0-linux-x86_64.zip -d /usr/local |
|
| 187 |
-``` |
|
| 188 |
- |
|
| 189 |
-With the required dependencies installed, the `Makefile` target named **binaries** will compile the `ctr` and `containerd` binaries and place them in the `bin/` directory. Using `sudo make install` will place the binaries in `/usr/local/bin`. When making any changes to the gRPC API, `make generate` will use the installed `protoc` compiler to regenerate the API generated code packages. |
|
| 190 |
- |
|
| 191 |
-> *Note*: A build tag is currently available to disable building the btrfs snapshot driver. |
|
| 192 |
-> Adding `BUILDTAGS=no_btrfs` to your environment before calling the **binaries** |
|
| 193 |
-> Makefile target will disable the btrfs driver within the containerd Go build. |
|
| 194 |
- |
|
| 195 |
-Vendoring of external imports uses the [`vndr` tool](https://github.com/LK4D4/vndr) which uses a simple config file, `vendor.conf`, to provide the URL and version or hash details for each vendored import. After modifying `vendor.conf` run the `vndr` tool to update the `vendor/` directory contents. Combining the `vendor.conf` update with the changeset in `vendor/` after running `vndr` should become a single commit for a PR which relies on vendored updates. |
|
| 196 |
- |
|
| 197 |
-Please refer to [RUNC.md](/RUNC.md) for the currently supported version of `runc` that is used by containerd. |
|
| 198 |
- |
|
| 199 | 174 |
### Releases and API Stability |
| 200 | 175 |
|
| 201 | 176 |
Please see [RELEASES.md](RELEASES.md) for details on versioning and stability |
| ... | ... |
@@ -7,6 +7,7 @@ import ( |
| 7 | 7 |
"net/http" |
| 8 | 8 |
"runtime" |
| 9 | 9 |
"strconv" |
| 10 |
+ "strings" |
|
| 10 | 11 |
"sync" |
| 11 | 12 |
"time" |
| 12 | 13 |
|
| ... | ... |
@@ -29,7 +30,6 @@ import ( |
| 29 | 29 |
"github.com/containerd/containerd/namespaces" |
| 30 | 30 |
"github.com/containerd/containerd/platforms" |
| 31 | 31 |
"github.com/containerd/containerd/plugin" |
| 32 |
- "github.com/containerd/containerd/reference" |
|
| 33 | 32 |
"github.com/containerd/containerd/remotes" |
| 34 | 33 |
"github.com/containerd/containerd/remotes/docker" |
| 35 | 34 |
"github.com/containerd/containerd/remotes/docker/schema1" |
| ... | ... |
@@ -334,6 +334,14 @@ func (c *Client) Push(ctx context.Context, ref string, desc ocispec.Descriptor, |
| 334 | 334 |
for i := len(manifestStack) - 1; i >= 0; i-- {
|
| 335 | 335 |
_, err := pushHandler(ctx, manifestStack[i]) |
| 336 | 336 |
if err != nil {
|
| 337 |
+ // TODO(estesp): until we have a more complete method for index push, we need to report |
|
| 338 |
+ // missing dependencies in an index/manifest list by sensing the "400 Bad Request" |
|
| 339 |
+ // as a marker for this problem |
|
| 340 |
+ if (manifestStack[i].MediaType == ocispec.MediaTypeImageIndex || |
|
| 341 |
+ manifestStack[i].MediaType == images.MediaTypeDockerSchema2ManifestList) && |
|
| 342 |
+ errors.Cause(err) != nil && strings.Contains(errors.Cause(err).Error(), "400 Bad Request") {
|
|
| 343 |
+ return errors.Wrap(err, "manifest list/index references to blobs and/or manifests are missing in your target registry") |
|
| 344 |
+ } |
|
| 337 | 345 |
return err |
| 338 | 346 |
} |
| 339 | 347 |
} |
| ... | ... |
@@ -494,95 +502,27 @@ func (c *Client) Version(ctx context.Context) (Version, error) {
|
| 494 | 494 |
}, nil |
| 495 | 495 |
} |
| 496 | 496 |
|
| 497 |
-type imageFormat string |
|
| 498 |
- |
|
| 499 |
-const ( |
|
| 500 |
- ociImageFormat imageFormat = "oci" |
|
| 501 |
-) |
|
| 502 |
- |
|
| 503 | 497 |
type importOpts struct {
|
| 504 |
- format imageFormat |
|
| 505 |
- refObject string |
|
| 506 |
- labels map[string]string |
|
| 507 | 498 |
} |
| 508 | 499 |
|
| 509 | 500 |
// ImportOpt allows the caller to specify import specific options |
| 510 | 501 |
type ImportOpt func(c *importOpts) error |
| 511 | 502 |
|
| 512 |
-// WithImportLabel sets a label to be associated with an imported image |
|
| 513 |
-func WithImportLabel(key, value string) ImportOpt {
|
|
| 514 |
- return func(opts *importOpts) error {
|
|
| 515 |
- if opts.labels == nil {
|
|
| 516 |
- opts.labels = make(map[string]string) |
|
| 517 |
- } |
|
| 518 |
- |
|
| 519 |
- opts.labels[key] = value |
|
| 520 |
- return nil |
|
| 521 |
- } |
|
| 522 |
-} |
|
| 523 |
- |
|
| 524 |
-// WithImportLabels associates a set of labels to an imported image |
|
| 525 |
-func WithImportLabels(labels map[string]string) ImportOpt {
|
|
| 526 |
- return func(opts *importOpts) error {
|
|
| 527 |
- if opts.labels == nil {
|
|
| 528 |
- opts.labels = make(map[string]string) |
|
| 529 |
- } |
|
| 530 |
- |
|
| 531 |
- for k, v := range labels {
|
|
| 532 |
- opts.labels[k] = v |
|
| 533 |
- } |
|
| 534 |
- return nil |
|
| 535 |
- } |
|
| 536 |
-} |
|
| 537 |
- |
|
| 538 |
-// WithOCIImportFormat sets the import format for an OCI image format |
|
| 539 |
-func WithOCIImportFormat() ImportOpt {
|
|
| 540 |
- return func(c *importOpts) error {
|
|
| 541 |
- if c.format != "" {
|
|
| 542 |
- return errors.New("format already set")
|
|
| 543 |
- } |
|
| 544 |
- c.format = ociImageFormat |
|
| 545 |
- return nil |
|
| 546 |
- } |
|
| 547 |
-} |
|
| 548 |
- |
|
| 549 |
-// WithRefObject specifies the ref object to import. |
|
| 550 |
-// If refObject is empty, it is copied from the ref argument of Import(). |
|
| 551 |
-func WithRefObject(refObject string) ImportOpt {
|
|
| 552 |
- return func(c *importOpts) error {
|
|
| 553 |
- c.refObject = refObject |
|
| 554 |
- return nil |
|
| 555 |
- } |
|
| 556 |
-} |
|
| 557 |
- |
|
| 558 |
-func resolveImportOpt(ref string, opts ...ImportOpt) (importOpts, error) {
|
|
| 503 |
+func resolveImportOpt(opts ...ImportOpt) (importOpts, error) {
|
|
| 559 | 504 |
var iopts importOpts |
| 560 | 505 |
for _, o := range opts {
|
| 561 | 506 |
if err := o(&iopts); err != nil {
|
| 562 | 507 |
return iopts, err |
| 563 | 508 |
} |
| 564 | 509 |
} |
| 565 |
- // use OCI as the default format |
|
| 566 |
- if iopts.format == "" {
|
|
| 567 |
- iopts.format = ociImageFormat |
|
| 568 |
- } |
|
| 569 |
- // if refObject is not explicitly specified, use the one specified in ref |
|
| 570 |
- if iopts.refObject == "" {
|
|
| 571 |
- refSpec, err := reference.Parse(ref) |
|
| 572 |
- if err != nil {
|
|
| 573 |
- return iopts, err |
|
| 574 |
- } |
|
| 575 |
- iopts.refObject = refSpec.Object |
|
| 576 |
- } |
|
| 577 | 510 |
return iopts, nil |
| 578 | 511 |
} |
| 579 | 512 |
|
| 580 | 513 |
// Import imports an image from a Tar stream using reader. |
| 581 |
-// OCI format is assumed by default. |
|
| 582 |
-// |
|
| 583 |
-// Note that unreferenced blobs are imported to the content store as well. |
|
| 584 |
-func (c *Client) Import(ctx context.Context, ref string, reader io.Reader, opts ...ImportOpt) (Image, error) {
|
|
| 585 |
- iopts, err := resolveImportOpt(ref, opts...) |
|
| 514 |
+// Caller needs to specify importer. Future version may use oci.v1 as the default. |
|
| 515 |
+// Note that unreferrenced blobs may be imported to the content store as well. |
|
| 516 |
+func (c *Client) Import(ctx context.Context, importer images.Importer, reader io.Reader, opts ...ImportOpt) ([]Image, error) {
|
|
| 517 |
+ _, err := resolveImportOpt(opts...) // unused now |
|
| 586 | 518 |
if err != nil {
|
| 587 | 519 |
return nil, err |
| 588 | 520 |
} |
| ... | ... |
@@ -593,58 +533,66 @@ func (c *Client) Import(ctx context.Context, ref string, reader io.Reader, opts |
| 593 | 593 |
} |
| 594 | 594 |
defer done() |
| 595 | 595 |
|
| 596 |
- switch iopts.format {
|
|
| 597 |
- case ociImageFormat: |
|
| 598 |
- return c.importFromOCITar(ctx, ref, reader, iopts) |
|
| 599 |
- default: |
|
| 600 |
- return nil, errors.Errorf("unsupported format: %s", iopts.format)
|
|
| 596 |
+ imgrecs, err := importer.Import(ctx, c.ContentStore(), reader) |
|
| 597 |
+ if err != nil {
|
|
| 598 |
+ // is.Update() is not called on error |
|
| 599 |
+ return nil, err |
|
| 600 |
+ } |
|
| 601 |
+ |
|
| 602 |
+ is := c.ImageService() |
|
| 603 |
+ var images []Image |
|
| 604 |
+ for _, imgrec := range imgrecs {
|
|
| 605 |
+ if updated, err := is.Update(ctx, imgrec, "target"); err != nil {
|
|
| 606 |
+ if !errdefs.IsNotFound(err) {
|
|
| 607 |
+ return nil, err |
|
| 608 |
+ } |
|
| 609 |
+ |
|
| 610 |
+ created, err := is.Create(ctx, imgrec) |
|
| 611 |
+ if err != nil {
|
|
| 612 |
+ return nil, err |
|
| 613 |
+ } |
|
| 614 |
+ |
|
| 615 |
+ imgrec = created |
|
| 616 |
+ } else {
|
|
| 617 |
+ imgrec = updated |
|
| 618 |
+ } |
|
| 619 |
+ |
|
| 620 |
+ images = append(images, &image{
|
|
| 621 |
+ client: c, |
|
| 622 |
+ i: imgrec, |
|
| 623 |
+ }) |
|
| 601 | 624 |
} |
| 625 |
+ return images, nil |
|
| 602 | 626 |
} |
| 603 | 627 |
|
| 604 | 628 |
type exportOpts struct {
|
| 605 |
- format imageFormat |
|
| 606 | 629 |
} |
| 607 | 630 |
|
| 608 |
-// ExportOpt allows callers to set export options |
|
| 631 |
+// ExportOpt allows the caller to specify export-specific options |
|
| 609 | 632 |
type ExportOpt func(c *exportOpts) error |
| 610 | 633 |
|
| 611 |
-// WithOCIExportFormat sets the OCI image format as the export target |
|
| 612 |
-func WithOCIExportFormat() ExportOpt {
|
|
| 613 |
- return func(c *exportOpts) error {
|
|
| 614 |
- if c.format != "" {
|
|
| 615 |
- return errors.New("format already set")
|
|
| 634 |
+func resolveExportOpt(opts ...ExportOpt) (exportOpts, error) {
|
|
| 635 |
+ var eopts exportOpts |
|
| 636 |
+ for _, o := range opts {
|
|
| 637 |
+ if err := o(&eopts); err != nil {
|
|
| 638 |
+ return eopts, err |
|
| 616 | 639 |
} |
| 617 |
- c.format = ociImageFormat |
|
| 618 |
- return nil |
|
| 619 | 640 |
} |
| 641 |
+ return eopts, nil |
|
| 620 | 642 |
} |
| 621 | 643 |
|
| 622 |
-// TODO: add WithMediaTypeTranslation that transforms media types according to the format. |
|
| 623 |
-// e.g. application/vnd.docker.image.rootfs.diff.tar.gzip |
|
| 624 |
-// -> application/vnd.oci.image.layer.v1.tar+gzip |
|
| 625 |
- |
|
| 626 | 644 |
// Export exports an image to a Tar stream. |
| 627 | 645 |
// OCI format is used by default. |
| 628 | 646 |
// It is up to caller to put "org.opencontainers.image.ref.name" annotation to desc. |
| 629 |
-func (c *Client) Export(ctx context.Context, desc ocispec.Descriptor, opts ...ExportOpt) (io.ReadCloser, error) {
|
|
| 630 |
- var eopts exportOpts |
|
| 631 |
- for _, o := range opts {
|
|
| 632 |
- if err := o(&eopts); err != nil {
|
|
| 633 |
- return nil, err |
|
| 634 |
- } |
|
| 635 |
- } |
|
| 636 |
- // use OCI as the default format |
|
| 637 |
- if eopts.format == "" {
|
|
| 638 |
- eopts.format = ociImageFormat |
|
| 647 |
+// TODO(AkihiroSuda): support exporting multiple descriptors at once to a single archive stream. |
|
| 648 |
+func (c *Client) Export(ctx context.Context, exporter images.Exporter, desc ocispec.Descriptor, opts ...ExportOpt) (io.ReadCloser, error) {
|
|
| 649 |
+ _, err := resolveExportOpt(opts...) // unused now |
|
| 650 |
+ if err != nil {
|
|
| 651 |
+ return nil, err |
|
| 639 | 652 |
} |
| 640 | 653 |
pr, pw := io.Pipe() |
| 641 |
- switch eopts.format {
|
|
| 642 |
- case ociImageFormat: |
|
| 643 |
- go func() {
|
|
| 644 |
- pw.CloseWithError(c.exportToOCITar(ctx, desc, pw, eopts)) |
|
| 645 |
- }() |
|
| 646 |
- default: |
|
| 647 |
- return nil, errors.Errorf("unsupported format: %s", eopts.format)
|
|
| 648 |
- } |
|
| 654 |
+ go func() {
|
|
| 655 |
+ pw.CloseWithError(exporter.Export(ctx, c.ContentStore(), desc, pw)) |
|
| 656 |
+ }() |
|
| 649 | 657 |
return pr, nil |
| 650 | 658 |
} |
| 651 | 659 |
deleted file mode 100644 |
| ... | ... |
@@ -1,189 +0,0 @@ |
| 1 |
-package containerd |
|
| 2 |
- |
|
| 3 |
-import ( |
|
| 4 |
- "archive/tar" |
|
| 5 |
- "context" |
|
| 6 |
- "encoding/json" |
|
| 7 |
- "io" |
|
| 8 |
- "sort" |
|
| 9 |
- |
|
| 10 |
- "github.com/containerd/containerd/content" |
|
| 11 |
- "github.com/containerd/containerd/images" |
|
| 12 |
- "github.com/containerd/containerd/platforms" |
|
| 13 |
- ocispecs "github.com/opencontainers/image-spec/specs-go" |
|
| 14 |
- ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
|
| 15 |
- "github.com/pkg/errors" |
|
| 16 |
-) |
|
| 17 |
- |
|
| 18 |
-func (c *Client) exportToOCITar(ctx context.Context, desc ocispec.Descriptor, writer io.Writer, eopts exportOpts) error {
|
|
| 19 |
- tw := tar.NewWriter(writer) |
|
| 20 |
- defer tw.Close() |
|
| 21 |
- |
|
| 22 |
- records := []tarRecord{
|
|
| 23 |
- ociLayoutFile(""),
|
|
| 24 |
- ociIndexRecord(desc), |
|
| 25 |
- } |
|
| 26 |
- |
|
| 27 |
- cs := c.ContentStore() |
|
| 28 |
- algorithms := map[string]struct{}{}
|
|
| 29 |
- exportHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
|
|
| 30 |
- records = append(records, blobRecord(cs, desc)) |
|
| 31 |
- algorithms[desc.Digest.Algorithm().String()] = struct{}{}
|
|
| 32 |
- return nil, nil |
|
| 33 |
- } |
|
| 34 |
- |
|
| 35 |
- handlers := images.Handlers( |
|
| 36 |
- images.ChildrenHandler(cs, platforms.Default()), |
|
| 37 |
- images.HandlerFunc(exportHandler), |
|
| 38 |
- ) |
|
| 39 |
- |
|
| 40 |
- // Walk sequentially since the number of fetchs is likely one and doing in |
|
| 41 |
- // parallel requires locking the export handler |
|
| 42 |
- if err := images.Walk(ctx, handlers, desc); err != nil {
|
|
| 43 |
- return err |
|
| 44 |
- } |
|
| 45 |
- |
|
| 46 |
- if len(algorithms) > 0 {
|
|
| 47 |
- records = append(records, directoryRecord("blobs/", 0755))
|
|
| 48 |
- for alg := range algorithms {
|
|
| 49 |
- records = append(records, directoryRecord("blobs/"+alg+"/", 0755))
|
|
| 50 |
- } |
|
| 51 |
- } |
|
| 52 |
- |
|
| 53 |
- return writeTar(ctx, tw, records) |
|
| 54 |
-} |
|
| 55 |
- |
|
| 56 |
-type tarRecord struct {
|
|
| 57 |
- Header *tar.Header |
|
| 58 |
- CopyTo func(context.Context, io.Writer) (int64, error) |
|
| 59 |
-} |
|
| 60 |
- |
|
| 61 |
-func blobRecord(cs content.Store, desc ocispec.Descriptor) tarRecord {
|
|
| 62 |
- path := "blobs/" + desc.Digest.Algorithm().String() + "/" + desc.Digest.Hex() |
|
| 63 |
- return tarRecord{
|
|
| 64 |
- Header: &tar.Header{
|
|
| 65 |
- Name: path, |
|
| 66 |
- Mode: 0444, |
|
| 67 |
- Size: desc.Size, |
|
| 68 |
- Typeflag: tar.TypeReg, |
|
| 69 |
- }, |
|
| 70 |
- CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
|
|
| 71 |
- r, err := cs.ReaderAt(ctx, desc.Digest) |
|
| 72 |
- if err != nil {
|
|
| 73 |
- return 0, err |
|
| 74 |
- } |
|
| 75 |
- defer r.Close() |
|
| 76 |
- |
|
| 77 |
- // Verify digest |
|
| 78 |
- dgstr := desc.Digest.Algorithm().Digester() |
|
| 79 |
- |
|
| 80 |
- n, err := io.Copy(io.MultiWriter(w, dgstr.Hash()), content.NewReader(r)) |
|
| 81 |
- if err != nil {
|
|
| 82 |
- return 0, err |
|
| 83 |
- } |
|
| 84 |
- if dgstr.Digest() != desc.Digest {
|
|
| 85 |
- return 0, errors.Errorf("unexpected digest %s copied", dgstr.Digest())
|
|
| 86 |
- } |
|
| 87 |
- return n, nil |
|
| 88 |
- }, |
|
| 89 |
- } |
|
| 90 |
-} |
|
| 91 |
- |
|
| 92 |
-func directoryRecord(name string, mode int64) tarRecord {
|
|
| 93 |
- return tarRecord{
|
|
| 94 |
- Header: &tar.Header{
|
|
| 95 |
- Name: name, |
|
| 96 |
- Mode: mode, |
|
| 97 |
- Typeflag: tar.TypeDir, |
|
| 98 |
- }, |
|
| 99 |
- } |
|
| 100 |
-} |
|
| 101 |
- |
|
| 102 |
-func ociLayoutFile(version string) tarRecord {
|
|
| 103 |
- if version == "" {
|
|
| 104 |
- version = ocispec.ImageLayoutVersion |
|
| 105 |
- } |
|
| 106 |
- layout := ocispec.ImageLayout{
|
|
| 107 |
- Version: version, |
|
| 108 |
- } |
|
| 109 |
- |
|
| 110 |
- b, err := json.Marshal(layout) |
|
| 111 |
- if err != nil {
|
|
| 112 |
- panic(err) |
|
| 113 |
- } |
|
| 114 |
- |
|
| 115 |
- return tarRecord{
|
|
| 116 |
- Header: &tar.Header{
|
|
| 117 |
- Name: ocispec.ImageLayoutFile, |
|
| 118 |
- Mode: 0444, |
|
| 119 |
- Size: int64(len(b)), |
|
| 120 |
- Typeflag: tar.TypeReg, |
|
| 121 |
- }, |
|
| 122 |
- CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
|
|
| 123 |
- n, err := w.Write(b) |
|
| 124 |
- return int64(n), err |
|
| 125 |
- }, |
|
| 126 |
- } |
|
| 127 |
- |
|
| 128 |
-} |
|
| 129 |
- |
|
| 130 |
-func ociIndexRecord(manifests ...ocispec.Descriptor) tarRecord {
|
|
| 131 |
- index := ocispec.Index{
|
|
| 132 |
- Versioned: ocispecs.Versioned{
|
|
| 133 |
- SchemaVersion: 2, |
|
| 134 |
- }, |
|
| 135 |
- Manifests: manifests, |
|
| 136 |
- } |
|
| 137 |
- |
|
| 138 |
- b, err := json.Marshal(index) |
|
| 139 |
- if err != nil {
|
|
| 140 |
- panic(err) |
|
| 141 |
- } |
|
| 142 |
- |
|
| 143 |
- return tarRecord{
|
|
| 144 |
- Header: &tar.Header{
|
|
| 145 |
- Name: "index.json", |
|
| 146 |
- Mode: 0644, |
|
| 147 |
- Size: int64(len(b)), |
|
| 148 |
- Typeflag: tar.TypeReg, |
|
| 149 |
- }, |
|
| 150 |
- CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
|
|
| 151 |
- n, err := w.Write(b) |
|
| 152 |
- return int64(n), err |
|
| 153 |
- }, |
|
| 154 |
- } |
|
| 155 |
-} |
|
| 156 |
- |
|
| 157 |
-func writeTar(ctx context.Context, tw *tar.Writer, records []tarRecord) error {
|
|
| 158 |
- sort.Sort(tarRecordsByName(records)) |
|
| 159 |
- |
|
| 160 |
- for _, record := range records {
|
|
| 161 |
- if err := tw.WriteHeader(record.Header); err != nil {
|
|
| 162 |
- return err |
|
| 163 |
- } |
|
| 164 |
- if record.CopyTo != nil {
|
|
| 165 |
- n, err := record.CopyTo(ctx, tw) |
|
| 166 |
- if err != nil {
|
|
| 167 |
- return err |
|
| 168 |
- } |
|
| 169 |
- if n != record.Header.Size {
|
|
| 170 |
- return errors.Errorf("unexpected copy size for %s", record.Header.Name)
|
|
| 171 |
- } |
|
| 172 |
- } else if record.Header.Size > 0 {
|
|
| 173 |
- return errors.Errorf("no content to write to record with non-zero size for %s", record.Header.Name)
|
|
| 174 |
- } |
|
| 175 |
- } |
|
| 176 |
- return nil |
|
| 177 |
-} |
|
| 178 |
- |
|
| 179 |
-type tarRecordsByName []tarRecord |
|
| 180 |
- |
|
| 181 |
-func (t tarRecordsByName) Len() int {
|
|
| 182 |
- return len(t) |
|
| 183 |
-} |
|
| 184 |
-func (t tarRecordsByName) Swap(i, j int) {
|
|
| 185 |
- t[i], t[j] = t[j], t[i] |
|
| 186 |
-} |
|
| 187 |
-func (t tarRecordsByName) Less(i, j int) bool {
|
|
| 188 |
- return t[i].Header.Name < t[j].Header.Name |
|
| 189 |
-} |
| ... | ... |
@@ -3,7 +3,6 @@ package filters |
| 3 | 3 |
import ( |
| 4 | 4 |
"fmt" |
| 5 | 5 |
"io" |
| 6 |
- "strconv" |
|
| 7 | 6 |
|
| 8 | 7 |
"github.com/containerd/containerd/errdefs" |
| 9 | 8 |
"github.com/pkg/errors" |
| ... | ... |
@@ -134,7 +133,12 @@ func (p *parser) selector() (selector, error) {
|
| 134 | 134 |
return selector{}, err
|
| 135 | 135 |
} |
| 136 | 136 |
|
| 137 |
- value, err := p.value() |
|
| 137 |
+ var allowAltQuotes bool |
|
| 138 |
+ if op == operatorMatches {
|
|
| 139 |
+ allowAltQuotes = true |
|
| 140 |
+ } |
|
| 141 |
+ |
|
| 142 |
+ value, err := p.value(allowAltQuotes) |
|
| 138 | 143 |
if err != nil {
|
| 139 | 144 |
if err == io.EOF {
|
| 140 | 145 |
return selector{}, io.ErrUnexpectedEOF
|
| ... | ... |
@@ -188,7 +192,7 @@ func (p *parser) field() (string, error) {
|
| 188 | 188 |
case tokenField: |
| 189 | 189 |
return s, nil |
| 190 | 190 |
case tokenQuoted: |
| 191 |
- return p.unquote(pos, s) |
|
| 191 |
+ return p.unquote(pos, s, false) |
|
| 192 | 192 |
} |
| 193 | 193 |
|
| 194 | 194 |
return "", p.mkerr(pos, "expected field or quoted") |
| ... | ... |
@@ -213,21 +217,25 @@ func (p *parser) operator() (operator, error) {
|
| 213 | 213 |
return 0, p.mkerr(pos, `expected an operator ("=="|"!="|"~=")`)
|
| 214 | 214 |
} |
| 215 | 215 |
|
| 216 |
-func (p *parser) value() (string, error) {
|
|
| 216 |
+func (p *parser) value(allowAltQuotes bool) (string, error) {
|
|
| 217 | 217 |
pos, tok, s := p.scanner.scan() |
| 218 | 218 |
|
| 219 | 219 |
switch tok {
|
| 220 | 220 |
case tokenValue, tokenField: |
| 221 | 221 |
return s, nil |
| 222 | 222 |
case tokenQuoted: |
| 223 |
- return p.unquote(pos, s) |
|
| 223 |
+ return p.unquote(pos, s, allowAltQuotes) |
|
| 224 | 224 |
} |
| 225 | 225 |
|
| 226 | 226 |
return "", p.mkerr(pos, "expected value or quoted") |
| 227 | 227 |
} |
| 228 | 228 |
|
| 229 |
-func (p *parser) unquote(pos int, s string) (string, error) {
|
|
| 230 |
- uq, err := strconv.Unquote(s) |
|
| 229 |
+func (p *parser) unquote(pos int, s string, allowAlts bool) (string, error) {
|
|
| 230 |
+ if !allowAlts && s[0] != '\'' && s[0] != '"' {
|
|
| 231 |
+ return "", p.mkerr(pos, "invalid quote encountered") |
|
| 232 |
+ } |
|
| 233 |
+ |
|
| 234 |
+ uq, err := unquote(s) |
|
| 231 | 235 |
if err != nil {
|
| 232 | 236 |
return "", p.mkerr(pos, "unquoting failed: %v", err) |
| 233 | 237 |
} |
| 234 | 238 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,237 @@ |
| 0 |
+package filters |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "unicode/utf8" |
|
| 4 |
+ |
|
| 5 |
+ "github.com/pkg/errors" |
|
| 6 |
+) |
|
| 7 |
+ |
|
| 8 |
+// NOTE(stevvooe): Most of this code in this file is copied from the stdlib |
|
| 9 |
+// strconv package and modified to be able to handle quoting with `/` and `|` |
|
| 10 |
+// as delimiters. The copyright is held by the Go authors. |
|
| 11 |
+ |
|
| 12 |
+var errQuoteSyntax = errors.New("quote syntax error")
|
|
| 13 |
+ |
|
| 14 |
+// UnquoteChar decodes the first character or byte in the escaped string |
|
| 15 |
+// or character literal represented by the string s. |
|
| 16 |
+// It returns four values: |
|
| 17 |
+// |
|
| 18 |
+// 1) value, the decoded Unicode code point or byte value; |
|
| 19 |
+// 2) multibyte, a boolean indicating whether the decoded character requires a multibyte UTF-8 representation; |
|
| 20 |
+// 3) tail, the remainder of the string after the character; and |
|
| 21 |
+// 4) an error that will be nil if the character is syntactically valid. |
|
| 22 |
+// |
|
| 23 |
+// The second argument, quote, specifies the type of literal being parsed |
|
| 24 |
+// and therefore which escaped quote character is permitted. |
|
| 25 |
+// If set to a single quote, it permits the sequence \' and disallows unescaped '. |
|
| 26 |
+// If set to a double quote, it permits \" and disallows unescaped ". |
|
| 27 |
+// If set to zero, it does not permit either escape and allows both quote characters to appear unescaped. |
|
| 28 |
+// |
|
| 29 |
+// This is from Go strconv package, modified to support `|` and `/` as double |
|
| 30 |
+// quotes for use with regular expressions. |
|
| 31 |
+func unquoteChar(s string, quote byte) (value rune, multibyte bool, tail string, err error) {
|
|
| 32 |
+ // easy cases |
|
| 33 |
+ switch c := s[0]; {
|
|
| 34 |
+ case c == quote && (quote == '\'' || quote == '"' || quote == '/' || quote == '|'): |
|
| 35 |
+ err = errQuoteSyntax |
|
| 36 |
+ return |
|
| 37 |
+ case c >= utf8.RuneSelf: |
|
| 38 |
+ r, size := utf8.DecodeRuneInString(s) |
|
| 39 |
+ return r, true, s[size:], nil |
|
| 40 |
+ case c != '\\': |
|
| 41 |
+ return rune(s[0]), false, s[1:], nil |
|
| 42 |
+ } |
|
| 43 |
+ |
|
| 44 |
+ // hard case: c is backslash |
|
| 45 |
+ if len(s) <= 1 {
|
|
| 46 |
+ err = errQuoteSyntax |
|
| 47 |
+ return |
|
| 48 |
+ } |
|
| 49 |
+ c := s[1] |
|
| 50 |
+ s = s[2:] |
|
| 51 |
+ |
|
| 52 |
+ switch c {
|
|
| 53 |
+ case 'a': |
|
| 54 |
+ value = '\a' |
|
| 55 |
+ case 'b': |
|
| 56 |
+ value = '\b' |
|
| 57 |
+ case 'f': |
|
| 58 |
+ value = '\f' |
|
| 59 |
+ case 'n': |
|
| 60 |
+ value = '\n' |
|
| 61 |
+ case 'r': |
|
| 62 |
+ value = '\r' |
|
| 63 |
+ case 't': |
|
| 64 |
+ value = '\t' |
|
| 65 |
+ case 'v': |
|
| 66 |
+ value = '\v' |
|
| 67 |
+ case 'x', 'u', 'U': |
|
| 68 |
+ n := 0 |
|
| 69 |
+ switch c {
|
|
| 70 |
+ case 'x': |
|
| 71 |
+ n = 2 |
|
| 72 |
+ case 'u': |
|
| 73 |
+ n = 4 |
|
| 74 |
+ case 'U': |
|
| 75 |
+ n = 8 |
|
| 76 |
+ } |
|
| 77 |
+ var v rune |
|
| 78 |
+ if len(s) < n {
|
|
| 79 |
+ err = errQuoteSyntax |
|
| 80 |
+ return |
|
| 81 |
+ } |
|
| 82 |
+ for j := 0; j < n; j++ {
|
|
| 83 |
+ x, ok := unhex(s[j]) |
|
| 84 |
+ if !ok {
|
|
| 85 |
+ err = errQuoteSyntax |
|
| 86 |
+ return |
|
| 87 |
+ } |
|
| 88 |
+ v = v<<4 | x |
|
| 89 |
+ } |
|
| 90 |
+ s = s[n:] |
|
| 91 |
+ if c == 'x' {
|
|
| 92 |
+ // single-byte string, possibly not UTF-8 |
|
| 93 |
+ value = v |
|
| 94 |
+ break |
|
| 95 |
+ } |
|
| 96 |
+ if v > utf8.MaxRune {
|
|
| 97 |
+ err = errQuoteSyntax |
|
| 98 |
+ return |
|
| 99 |
+ } |
|
| 100 |
+ value = v |
|
| 101 |
+ multibyte = true |
|
| 102 |
+ case '0', '1', '2', '3', '4', '5', '6', '7': |
|
| 103 |
+ v := rune(c) - '0' |
|
| 104 |
+ if len(s) < 2 {
|
|
| 105 |
+ err = errQuoteSyntax |
|
| 106 |
+ return |
|
| 107 |
+ } |
|
| 108 |
+ for j := 0; j < 2; j++ { // one digit already; two more
|
|
| 109 |
+ x := rune(s[j]) - '0' |
|
| 110 |
+ if x < 0 || x > 7 {
|
|
| 111 |
+ err = errQuoteSyntax |
|
| 112 |
+ return |
|
| 113 |
+ } |
|
| 114 |
+ v = (v << 3) | x |
|
| 115 |
+ } |
|
| 116 |
+ s = s[2:] |
|
| 117 |
+ if v > 255 {
|
|
| 118 |
+ err = errQuoteSyntax |
|
| 119 |
+ return |
|
| 120 |
+ } |
|
| 121 |
+ value = v |
|
| 122 |
+ case '\\': |
|
| 123 |
+ value = '\\' |
|
| 124 |
+ case '\'', '"', '|', '/': |
|
| 125 |
+ if c != quote {
|
|
| 126 |
+ err = errQuoteSyntax |
|
| 127 |
+ return |
|
| 128 |
+ } |
|
| 129 |
+ value = rune(c) |
|
| 130 |
+ default: |
|
| 131 |
+ err = errQuoteSyntax |
|
| 132 |
+ return |
|
| 133 |
+ } |
|
| 134 |
+ tail = s |
|
| 135 |
+ return |
|
| 136 |
+} |
|
| 137 |
+ |
|
| 138 |
+// unquote interprets s as a single-quoted, double-quoted, |
|
| 139 |
+// or backquoted Go string literal, returning the string value |
|
| 140 |
+// that s quotes. (If s is single-quoted, it would be a Go |
|
| 141 |
+// character literal; Unquote returns the corresponding |
|
| 142 |
+// one-character string.) |
|
| 143 |
+// |
|
| 144 |
+// This is modified from the standard library to support `|` and `/` as quote |
|
| 145 |
+// characters for use with regular expressions. |
|
| 146 |
+func unquote(s string) (string, error) {
|
|
| 147 |
+ n := len(s) |
|
| 148 |
+ if n < 2 {
|
|
| 149 |
+ return "", errQuoteSyntax |
|
| 150 |
+ } |
|
| 151 |
+ quote := s[0] |
|
| 152 |
+ if quote != s[n-1] {
|
|
| 153 |
+ return "", errQuoteSyntax |
|
| 154 |
+ } |
|
| 155 |
+ s = s[1 : n-1] |
|
| 156 |
+ |
|
| 157 |
+ if quote == '`' {
|
|
| 158 |
+ if contains(s, '`') {
|
|
| 159 |
+ return "", errQuoteSyntax |
|
| 160 |
+ } |
|
| 161 |
+ if contains(s, '\r') {
|
|
| 162 |
+ // -1 because we know there is at least one \r to remove. |
|
| 163 |
+ buf := make([]byte, 0, len(s)-1) |
|
| 164 |
+ for i := 0; i < len(s); i++ {
|
|
| 165 |
+ if s[i] != '\r' {
|
|
| 166 |
+ buf = append(buf, s[i]) |
|
| 167 |
+ } |
|
| 168 |
+ } |
|
| 169 |
+ return string(buf), nil |
|
| 170 |
+ } |
|
| 171 |
+ return s, nil |
|
| 172 |
+ } |
|
| 173 |
+ if quote != '"' && quote != '\'' && quote != '|' && quote != '/' {
|
|
| 174 |
+ return "", errQuoteSyntax |
|
| 175 |
+ } |
|
| 176 |
+ if contains(s, '\n') {
|
|
| 177 |
+ return "", errQuoteSyntax |
|
| 178 |
+ } |
|
| 179 |
+ |
|
| 180 |
+ // Is it trivial? Avoid allocation. |
|
| 181 |
+ if !contains(s, '\\') && !contains(s, quote) {
|
|
| 182 |
+ switch quote {
|
|
| 183 |
+ case '"', '/', '|': // pipe and slash are treated like double quote |
|
| 184 |
+ return s, nil |
|
| 185 |
+ case '\'': |
|
| 186 |
+ r, size := utf8.DecodeRuneInString(s) |
|
| 187 |
+ if size == len(s) && (r != utf8.RuneError || size != 1) {
|
|
| 188 |
+ return s, nil |
|
| 189 |
+ } |
|
| 190 |
+ } |
|
| 191 |
+ } |
|
| 192 |
+ |
|
| 193 |
+ var runeTmp [utf8.UTFMax]byte |
|
| 194 |
+ buf := make([]byte, 0, 3*len(s)/2) // Try to avoid more allocations. |
|
| 195 |
+ for len(s) > 0 {
|
|
| 196 |
+ c, multibyte, ss, err := unquoteChar(s, quote) |
|
| 197 |
+ if err != nil {
|
|
| 198 |
+ return "", err |
|
| 199 |
+ } |
|
| 200 |
+ s = ss |
|
| 201 |
+ if c < utf8.RuneSelf || !multibyte {
|
|
| 202 |
+ buf = append(buf, byte(c)) |
|
| 203 |
+ } else {
|
|
| 204 |
+ n := utf8.EncodeRune(runeTmp[:], c) |
|
| 205 |
+ buf = append(buf, runeTmp[:n]...) |
|
| 206 |
+ } |
|
| 207 |
+ if quote == '\'' && len(s) != 0 {
|
|
| 208 |
+ // single-quoted must be single character |
|
| 209 |
+ return "", errQuoteSyntax |
|
| 210 |
+ } |
|
| 211 |
+ } |
|
| 212 |
+ return string(buf), nil |
|
| 213 |
+} |
|
| 214 |
+ |
|
| 215 |
+// contains reports whether the string contains the byte c. |
|
| 216 |
+func contains(s string, c byte) bool {
|
|
| 217 |
+ for i := 0; i < len(s); i++ {
|
|
| 218 |
+ if s[i] == c {
|
|
| 219 |
+ return true |
|
| 220 |
+ } |
|
| 221 |
+ } |
|
| 222 |
+ return false |
|
| 223 |
+} |
|
| 224 |
+ |
|
| 225 |
+func unhex(b byte) (v rune, ok bool) {
|
|
| 226 |
+ c := rune(b) |
|
| 227 |
+ switch {
|
|
| 228 |
+ case '0' <= c && c <= '9': |
|
| 229 |
+ return c - '0', true |
|
| 230 |
+ case 'a' <= c && c <= 'f': |
|
| 231 |
+ return c - 'a' + 10, true |
|
| 232 |
+ case 'A' <= c && c <= 'F': |
|
| 233 |
+ return c - 'A' + 10, true |
|
| 234 |
+ } |
|
| 235 |
+ return |
|
| 236 |
+} |
| ... | ... |
@@ -87,7 +87,7 @@ func (s *scanner) peek() rune {
|
| 87 | 87 |
return ch |
| 88 | 88 |
} |
| 89 | 89 |
|
| 90 |
-func (s *scanner) scan() (int, token, string) {
|
|
| 90 |
+func (s *scanner) scan() (nextp int, tk token, text string) {
|
|
| 91 | 91 |
var ( |
| 92 | 92 |
ch = s.next() |
| 93 | 93 |
pos = s.pos |
| ... | ... |
@@ -101,6 +101,7 @@ chomp: |
| 101 | 101 |
s.scanQuoted(ch) |
| 102 | 102 |
return pos, tokenQuoted, s.input[pos:s.ppos] |
| 103 | 103 |
case isSeparatorRune(ch): |
| 104 |
+ s.value = false |
|
| 104 | 105 |
return pos, tokenSeparator, s.input[pos:s.ppos] |
| 105 | 106 |
case isOperatorRune(ch): |
| 106 | 107 |
s.scanOperator() |
| ... | ... |
@@ -241,7 +242,7 @@ func isOperatorRune(r rune) bool {
|
| 241 | 241 |
|
| 242 | 242 |
func isQuoteRune(r rune) bool {
|
| 243 | 243 |
switch r {
|
| 244 |
- case '"': // maybe add single quoting? |
|
| 244 |
+ case '/', '|', '"': // maybe add single quoting? |
|
| 245 | 245 |
return true |
| 246 | 246 |
} |
| 247 | 247 |
|
| ... | ... |
@@ -222,8 +222,10 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err |
| 222 | 222 |
c1 = make(chan *currentPath) |
| 223 | 223 |
c2 = make(chan *currentPath) |
| 224 | 224 |
|
| 225 |
- f1, f2 *currentPath |
|
| 226 |
- rmdir string |
|
| 225 |
+ f1, f2 *currentPath |
|
| 226 |
+ rmdir string |
|
| 227 |
+ lastEmittedDir = string(filepath.Separator) |
|
| 228 |
+ parents []os.FileInfo |
|
| 227 | 229 |
) |
| 228 | 230 |
g.Go(func() error {
|
| 229 | 231 |
defer close(c1) |
| ... | ... |
@@ -258,7 +260,10 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err |
| 258 | 258 |
continue |
| 259 | 259 |
} |
| 260 | 260 |
|
| 261 |
- var f os.FileInfo |
|
| 261 |
+ var ( |
|
| 262 |
+ f os.FileInfo |
|
| 263 |
+ emit = true |
|
| 264 |
+ ) |
|
| 262 | 265 |
k, p := pathChange(f1, f2) |
| 263 | 266 |
switch k {
|
| 264 | 267 |
case ChangeKindAdd: |
| ... | ... |
@@ -294,13 +299,35 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err |
| 294 | 294 |
f2 = nil |
| 295 | 295 |
if same {
|
| 296 | 296 |
if !isLinked(f) {
|
| 297 |
- continue |
|
| 297 |
+ emit = false |
|
| 298 | 298 |
} |
| 299 | 299 |
k = ChangeKindUnmodified |
| 300 | 300 |
} |
| 301 | 301 |
} |
| 302 |
- if err := changeFn(k, p, f, nil); err != nil {
|
|
| 303 |
- return err |
|
| 302 |
+ if emit {
|
|
| 303 |
+ emittedDir, emitParents := commonParents(lastEmittedDir, p, parents) |
|
| 304 |
+ for _, pf := range emitParents {
|
|
| 305 |
+ p := filepath.Join(emittedDir, pf.Name()) |
|
| 306 |
+ if err := changeFn(ChangeKindUnmodified, p, pf, nil); err != nil {
|
|
| 307 |
+ return err |
|
| 308 |
+ } |
|
| 309 |
+ emittedDir = p |
|
| 310 |
+ } |
|
| 311 |
+ |
|
| 312 |
+ if err := changeFn(k, p, f, nil); err != nil {
|
|
| 313 |
+ return err |
|
| 314 |
+ } |
|
| 315 |
+ |
|
| 316 |
+ if f != nil && f.IsDir() {
|
|
| 317 |
+ lastEmittedDir = p |
|
| 318 |
+ } else {
|
|
| 319 |
+ lastEmittedDir = emittedDir |
|
| 320 |
+ } |
|
| 321 |
+ |
|
| 322 |
+ parents = parents[:0] |
|
| 323 |
+ } else if f.IsDir() {
|
|
| 324 |
+ lastEmittedDir, parents = commonParents(lastEmittedDir, p, parents) |
|
| 325 |
+ parents = append(parents, f) |
|
| 304 | 326 |
} |
| 305 | 327 |
} |
| 306 | 328 |
return nil |
| ... | ... |
@@ -308,3 +335,47 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err |
| 308 | 308 |
|
| 309 | 309 |
return g.Wait() |
| 310 | 310 |
} |
| 311 |
+ |
|
| 312 |
+func commonParents(base, updated string, dirs []os.FileInfo) (string, []os.FileInfo) {
|
|
| 313 |
+ if basePrefix := makePrefix(base); strings.HasPrefix(updated, basePrefix) {
|
|
| 314 |
+ var ( |
|
| 315 |
+ parents []os.FileInfo |
|
| 316 |
+ last = base |
|
| 317 |
+ ) |
|
| 318 |
+ for _, d := range dirs {
|
|
| 319 |
+ next := filepath.Join(last, d.Name()) |
|
| 320 |
+ if strings.HasPrefix(updated, makePrefix(last)) {
|
|
| 321 |
+ parents = append(parents, d) |
|
| 322 |
+ last = next |
|
| 323 |
+ } else {
|
|
| 324 |
+ break |
|
| 325 |
+ } |
|
| 326 |
+ } |
|
| 327 |
+ return base, parents |
|
| 328 |
+ } |
|
| 329 |
+ |
|
| 330 |
+ baseS := strings.Split(base, string(filepath.Separator)) |
|
| 331 |
+ updatedS := strings.Split(updated, string(filepath.Separator)) |
|
| 332 |
+ commonS := []string{string(filepath.Separator)}
|
|
| 333 |
+ |
|
| 334 |
+ min := len(baseS) |
|
| 335 |
+ if len(updatedS) < min {
|
|
| 336 |
+ min = len(updatedS) |
|
| 337 |
+ } |
|
| 338 |
+ for i := 0; i < min; i++ {
|
|
| 339 |
+ if baseS[i] == updatedS[i] {
|
|
| 340 |
+ commonS = append(commonS, baseS[i]) |
|
| 341 |
+ } else {
|
|
| 342 |
+ break |
|
| 343 |
+ } |
|
| 344 |
+ } |
|
| 345 |
+ |
|
| 346 |
+ return filepath.Join(commonS...), []os.FileInfo{}
|
|
| 347 |
+} |
|
| 348 |
+ |
|
| 349 |
+func makePrefix(d string) string {
|
|
| 350 |
+ if d == "" || d[len(d)-1] != filepath.Separator {
|
|
| 351 |
+ return d + string(filepath.Separator) |
|
| 352 |
+ } |
|
| 353 |
+ return d |
|
| 354 |
+} |
| ... | ... |
@@ -1,5 +1,7 @@ |
| 1 | 1 |
package fs |
| 2 | 2 |
|
| 3 |
+import "context" |
|
| 4 |
+ |
|
| 3 | 5 |
// Usage of disk information |
| 4 | 6 |
type Usage struct {
|
| 5 | 7 |
Inodes int64 |
| ... | ... |
@@ -11,3 +13,10 @@ type Usage struct {
|
| 11 | 11 |
func DiskUsage(roots ...string) (Usage, error) {
|
| 12 | 12 |
return diskUsage(roots...) |
| 13 | 13 |
} |
| 14 |
+ |
|
| 15 |
+// DiffUsage counts the numbers of inodes and disk usage in the |
|
| 16 |
+// diff between the 2 directories. The first path is intended |
|
| 17 |
+// as the base directory and the second as the changed directory. |
|
| 18 |
+func DiffUsage(ctx context.Context, a, b string) (Usage, error) {
|
|
| 19 |
+ return diffUsage(ctx, a, b) |
|
| 20 |
+} |
| ... | ... |
@@ -3,17 +3,19 @@ |
| 3 | 3 |
package fs |
| 4 | 4 |
|
| 5 | 5 |
import ( |
| 6 |
+ "context" |
|
| 6 | 7 |
"os" |
| 7 | 8 |
"path/filepath" |
| 8 | 9 |
"syscall" |
| 9 | 10 |
) |
| 10 | 11 |
|
| 12 |
+type inode struct {
|
|
| 13 |
+ // TODO(stevvooe): Can probably reduce memory usage by not tracking |
|
| 14 |
+ // device, but we can leave this right for now. |
|
| 15 |
+ dev, ino uint64 |
|
| 16 |
+} |
|
| 17 |
+ |
|
| 11 | 18 |
func diskUsage(roots ...string) (Usage, error) {
|
| 12 |
- type inode struct {
|
|
| 13 |
- // TODO(stevvooe): Can probably reduce memory usage by not tracking |
|
| 14 |
- // device, but we can leave this right for now. |
|
| 15 |
- dev, ino uint64 |
|
| 16 |
- } |
|
| 17 | 19 |
|
| 18 | 20 |
var ( |
| 19 | 21 |
size int64 |
| ... | ... |
@@ -45,3 +47,37 @@ func diskUsage(roots ...string) (Usage, error) {
|
| 45 | 45 |
Size: size, |
| 46 | 46 |
}, nil |
| 47 | 47 |
} |
| 48 |
+ |
|
| 49 |
+func diffUsage(ctx context.Context, a, b string) (Usage, error) {
|
|
| 50 |
+ var ( |
|
| 51 |
+ size int64 |
|
| 52 |
+ inodes = map[inode]struct{}{} // expensive!
|
|
| 53 |
+ ) |
|
| 54 |
+ |
|
| 55 |
+ if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error {
|
|
| 56 |
+ if err != nil {
|
|
| 57 |
+ return err |
|
| 58 |
+ } |
|
| 59 |
+ |
|
| 60 |
+ if kind == ChangeKindAdd || kind == ChangeKindModify {
|
|
| 61 |
+ stat := fi.Sys().(*syscall.Stat_t) |
|
| 62 |
+ |
|
| 63 |
+ inoKey := inode{dev: uint64(stat.Dev), ino: uint64(stat.Ino)}
|
|
| 64 |
+ if _, ok := inodes[inoKey]; !ok {
|
|
| 65 |
+ inodes[inoKey] = struct{}{}
|
|
| 66 |
+ size += fi.Size() |
|
| 67 |
+ } |
|
| 68 |
+ |
|
| 69 |
+ return nil |
|
| 70 |
+ |
|
| 71 |
+ } |
|
| 72 |
+ return nil |
|
| 73 |
+ }); err != nil {
|
|
| 74 |
+ return Usage{}, err
|
|
| 75 |
+ } |
|
| 76 |
+ |
|
| 77 |
+ return Usage{
|
|
| 78 |
+ Inodes: int64(len(inodes)), |
|
| 79 |
+ Size: size, |
|
| 80 |
+ }, nil |
|
| 81 |
+} |
| ... | ... |
@@ -3,6 +3,7 @@ |
| 3 | 3 |
package fs |
| 4 | 4 |
|
| 5 | 5 |
import ( |
| 6 |
+ "context" |
|
| 6 | 7 |
"os" |
| 7 | 8 |
"path/filepath" |
| 8 | 9 |
) |
| ... | ... |
@@ -31,3 +32,29 @@ func diskUsage(roots ...string) (Usage, error) {
|
| 31 | 31 |
Size: size, |
| 32 | 32 |
}, nil |
| 33 | 33 |
} |
| 34 |
+ |
|
| 35 |
+func diffUsage(ctx context.Context, a, b string) (Usage, error) {
|
|
| 36 |
+ var ( |
|
| 37 |
+ size int64 |
|
| 38 |
+ ) |
|
| 39 |
+ |
|
| 40 |
+ if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error {
|
|
| 41 |
+ if err != nil {
|
|
| 42 |
+ return err |
|
| 43 |
+ } |
|
| 44 |
+ |
|
| 45 |
+ if kind == ChangeKindAdd || kind == ChangeKindModify {
|
|
| 46 |
+ size += fi.Size() |
|
| 47 |
+ |
|
| 48 |
+ return nil |
|
| 49 |
+ |
|
| 50 |
+ } |
|
| 51 |
+ return nil |
|
| 52 |
+ }); err != nil {
|
|
| 53 |
+ return Usage{}, err
|
|
| 54 |
+ } |
|
| 55 |
+ |
|
| 56 |
+ return Usage{
|
|
| 57 |
+ Size: size, |
|
| 58 |
+ }, nil |
|
| 59 |
+} |
| 34 | 60 |
new file mode 100644 |
| ... | ... |
@@ -0,0 +1,21 @@ |
| 0 |
+package images |
|
| 1 |
+ |
|
| 2 |
+import ( |
|
| 3 |
+ "context" |
|
| 4 |
+ "io" |
|
| 5 |
+ |
|
| 6 |
+ "github.com/containerd/containerd/content" |
|
| 7 |
+ ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
|
| 8 |
+) |
|
| 9 |
+ |
|
| 10 |
+// Importer is the interface for image importer. |
|
| 11 |
+type Importer interface {
|
|
| 12 |
+ // Import imports an image from a tar stream. |
|
| 13 |
+ Import(ctx context.Context, store content.Store, reader io.Reader) ([]Image, error) |
|
| 14 |
+} |
|
| 15 |
+ |
|
| 16 |
+// Exporter is the interface for image exporter. |
|
| 17 |
+type Exporter interface {
|
|
| 18 |
+ // Export exports an image to a tar stream. |
|
| 19 |
+ Export(ctx context.Context, store content.Store, desc ocispec.Descriptor, writer io.Writer) error |
|
| 20 |
+} |
| 0 | 21 |
deleted file mode 100644 |
| ... | ... |
@@ -1,120 +0,0 @@ |
| 1 |
-package containerd |
|
| 2 |
- |
|
| 3 |
-import ( |
|
| 4 |
- "archive/tar" |
|
| 5 |
- "context" |
|
| 6 |
- "encoding/json" |
|
| 7 |
- "io" |
|
| 8 |
- "io/ioutil" |
|
| 9 |
- "strings" |
|
| 10 |
- |
|
| 11 |
- "github.com/containerd/containerd/content" |
|
| 12 |
- "github.com/containerd/containerd/errdefs" |
|
| 13 |
- "github.com/containerd/containerd/images" |
|
| 14 |
- "github.com/containerd/containerd/reference" |
|
| 15 |
- digest "github.com/opencontainers/go-digest" |
|
| 16 |
- ocispec "github.com/opencontainers/image-spec/specs-go/v1" |
|
| 17 |
- "github.com/pkg/errors" |
|
| 18 |
-) |
|
| 19 |
- |
|
| 20 |
-func resolveOCIIndex(idx ocispec.Index, refObject string) (*ocispec.Descriptor, error) {
|
|
| 21 |
- tag, dgst := reference.SplitObject(refObject) |
|
| 22 |
- if tag == "" && dgst == "" {
|
|
| 23 |
- return nil, errors.Errorf("unexpected object: %q", refObject)
|
|
| 24 |
- } |
|
| 25 |
- for _, m := range idx.Manifests {
|
|
| 26 |
- if m.Digest == dgst {
|
|
| 27 |
- return &m, nil |
|
| 28 |
- } |
|
| 29 |
- annot, ok := m.Annotations[ocispec.AnnotationRefName] |
|
| 30 |
- if ok && annot == tag && tag != "" {
|
|
| 31 |
- return &m, nil |
|
| 32 |
- } |
|
| 33 |
- } |
|
| 34 |
- return nil, errors.Errorf("not found: %q", refObject)
|
|
| 35 |
-} |
|
| 36 |
- |
|
| 37 |
-func (c *Client) importFromOCITar(ctx context.Context, ref string, reader io.Reader, iopts importOpts) (Image, error) {
|
|
| 38 |
- tr := tar.NewReader(reader) |
|
| 39 |
- store := c.ContentStore() |
|
| 40 |
- var desc *ocispec.Descriptor |
|
| 41 |
- for {
|
|
| 42 |
- hdr, err := tr.Next() |
|
| 43 |
- if err == io.EOF {
|
|
| 44 |
- break |
|
| 45 |
- } |
|
| 46 |
- if err != nil {
|
|
| 47 |
- return nil, err |
|
| 48 |
- } |
|
| 49 |
- if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
|
|
| 50 |
- continue |
|
| 51 |
- } |
|
| 52 |
- if hdr.Name == "index.json" {
|
|
| 53 |
- desc, err = onUntarIndexJSON(tr, iopts.refObject) |
|
| 54 |
- if err != nil {
|
|
| 55 |
- return nil, err |
|
| 56 |
- } |
|
| 57 |
- continue |
|
| 58 |
- } |
|
| 59 |
- if strings.HasPrefix(hdr.Name, "blobs/") {
|
|
| 60 |
- if err := onUntarBlob(ctx, tr, store, hdr.Name, hdr.Size); err != nil {
|
|
| 61 |
- return nil, err |
|
| 62 |
- } |
|
| 63 |
- } |
|
| 64 |
- } |
|
| 65 |
- if desc == nil {
|
|
| 66 |
- return nil, errors.Errorf("no descriptor found for reference object %q", iopts.refObject)
|
|
| 67 |
- } |
|
| 68 |
- imgrec := images.Image{
|
|
| 69 |
- Name: ref, |
|
| 70 |
- Target: *desc, |
|
| 71 |
- Labels: iopts.labels, |
|
| 72 |
- } |
|
| 73 |
- is := c.ImageService() |
|
| 74 |
- if updated, err := is.Update(ctx, imgrec, "target"); err != nil {
|
|
| 75 |
- if !errdefs.IsNotFound(err) {
|
|
| 76 |
- return nil, err |
|
| 77 |
- } |
|
| 78 |
- |
|
| 79 |
- created, err := is.Create(ctx, imgrec) |
|
| 80 |
- if err != nil {
|
|
| 81 |
- return nil, err |
|
| 82 |
- } |
|
| 83 |
- |
|
| 84 |
- imgrec = created |
|
| 85 |
- } else {
|
|
| 86 |
- imgrec = updated |
|
| 87 |
- } |
|
| 88 |
- |
|
| 89 |
- img := &image{
|
|
| 90 |
- client: c, |
|
| 91 |
- i: imgrec, |
|
| 92 |
- } |
|
| 93 |
- return img, nil |
|
| 94 |
-} |
|
| 95 |
- |
|
| 96 |
-func onUntarIndexJSON(r io.Reader, refObject string) (*ocispec.Descriptor, error) {
|
|
| 97 |
- b, err := ioutil.ReadAll(r) |
|
| 98 |
- if err != nil {
|
|
| 99 |
- return nil, err |
|
| 100 |
- } |
|
| 101 |
- var idx ocispec.Index |
|
| 102 |
- if err := json.Unmarshal(b, &idx); err != nil {
|
|
| 103 |
- return nil, err |
|
| 104 |
- } |
|
| 105 |
- return resolveOCIIndex(idx, refObject) |
|
| 106 |
-} |
|
| 107 |
- |
|
| 108 |
-func onUntarBlob(ctx context.Context, r io.Reader, store content.Store, name string, size int64) error {
|
|
| 109 |
- // name is like "blobs/sha256/deadbeef" |
|
| 110 |
- split := strings.Split(name, "/") |
|
| 111 |
- if len(split) != 3 {
|
|
| 112 |
- return errors.Errorf("unexpected name: %q", name)
|
|
| 113 |
- } |
|
| 114 |
- algo := digest.Algorithm(split[1]) |
|
| 115 |
- if !algo.Available() {
|
|
| 116 |
- return errors.Errorf("unsupported algorithm: %s", algo)
|
|
| 117 |
- } |
|
| 118 |
- dgst := digest.NewDigestFromHex(algo.String(), split[2]) |
|
| 119 |
- return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, size, dgst) |
|
| 120 |
-} |
| ... | ... |
@@ -114,6 +114,7 @@ func fetch(ctx context.Context, ingester content.Ingester, fetcher Fetcher, desc |
| 114 | 114 |
func commitOpts(desc ocispec.Descriptor, r io.Reader) (io.Reader, []content.Opt) {
|
| 115 | 115 |
var childrenF func(r io.Reader) ([]ocispec.Descriptor, error) |
| 116 | 116 |
|
| 117 |
+ // TODO(AkihiroSuda): use images/oci.GetChildrenDescriptors? |
|
| 117 | 118 |
switch desc.MediaType {
|
| 118 | 119 |
case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: |
| 119 | 120 |
childrenF = func(r io.Reader) ([]ocispec.Descriptor, error) {
|