Browse code

Update containerd to v1.0.0

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Michael Crosby authored on 2017/12/05 23:44:42
Showing 15 changed files
... ...
@@ -4,7 +4,7 @@ TOMLV_COMMIT=9baf8a8a9f2ed20a8e54160840c492f937eeaf9a
4 4
 
5 5
 # When updating RUNC_COMMIT, also update runc in vendor.conf accordingly
6 6
 RUNC_COMMIT=b2567b37d7b75eb4cf325b77297b140ea686ce8f
7
-CONTAINERD_COMMIT=cc969fb42f427a68a8cc6870ef47f17304b83962
7
+CONTAINERD_COMMIT=v1.0.0
8 8
 TINI_COMMIT=949e6facb77383876aeff8a6944dde66b3089574
9 9
 LIBNETWORK_COMMIT=7b2b1feb1de4817d522cc372af149ff48d25028e
10 10
 VNDR_COMMIT=a6e196d8b4b0cbbdc29aebdb20c59ac6926bb384
... ...
@@ -103,7 +103,7 @@ github.com/googleapis/gax-go da06d194a00e19ce00d9011a13931c3f6f6887c7
103 103
 google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944
104 104
 
105 105
 # containerd
106
-github.com/containerd/containerd cc969fb42f427a68a8cc6870ef47f17304b83962
106
+github.com/containerd/containerd v1.0.0
107 107
 github.com/containerd/fifo fbfb6a11ec671efbe94ad1c12c2e98773f19e1e6
108 108
 github.com/containerd/continuity 35d55c5e8dd23b32037d56cf97174aff3efdfa83
109 109
 github.com/containerd/cgroups 29da22c6171a4316169f9205ab6c49f59b5b852f
... ...
@@ -13,13 +13,23 @@ containerd is designed to be embedded into a larger system, rather than being us
13 13
 
14 14
 ## Getting Started
15 15
 
16
-If you are interested in trying out containerd please see our [Getting Started Guide](docs/getting-started.md).
16
+See our documentation on [containerd.io](containerd.io):
17
+* [for ops and admins](docs/ops.md)
18
+* [namespaces](docs/namespaces.md)
19
+* [client options](docs/client-opts.md)
20
+
21
+See how to build containerd from source at [BUILDING](BUILDING.md).
22
+
23
+If you are interested in trying out containerd see our example at [Getting Started](docs/getting-started.md).
24
+
17 25
 
18 26
 ## Runtime Requirements
19 27
 
20 28
 Runtime requirements for containerd are very minimal. Most interactions with
21 29
 the Linux and Windows container feature sets are handled via [runc](https://github.com/opencontainers/runc) and/or
22
-OS-specific libraries (e.g. [hcsshim](https://github.com/Microsoft/hcsshim) for Microsoft). There are specific features
30
+OS-specific libraries (e.g. [hcsshim](https://github.com/Microsoft/hcsshim) for Microsoft). The current required version of `runc` is always listed in [RUNC.md](/RUNC.md).
31
+
32
+There are specific features
23 33
 used by containerd core code and snapshotters that will require a minimum kernel
24 34
 version on Linux. With the understood caveat of distro kernel versioning, a
25 35
 reasonable starting point for Linux is a minimum 4.x kernel version.
... ...
@@ -33,9 +43,7 @@ distribution.
33 33
 To use Linux checkpoint and restore features, you will need `criu` installed on
34 34
 your system. See more details in [Checkpoint and Restore](#checkpoint-and-restore).
35 35
 
36
-The current required version of runc is always listed in [RUNC.md](/RUNC.md).
37
-
38
-Build requirements for developers are listed in the [Developer Quick-Start](#developer-quick-start) section.
36
+Build requirements for developers are listed in [BUILDING](BUILDING.md).
39 37
 
40 38
 ## Features
41 39
 
... ...
@@ -45,7 +53,11 @@ containerd offers a full client package to help you integrate containerd into yo
45 45
 
46 46
 ```go
47 47
 
48
-import "github.com/containerd/containerd"
48
+import (
49
+  "github.com/containerd/containerd"
50
+  "github.com/containerd/containerd/cio"
51
+)
52
+
49 53
 
50 54
 func main() {
51 55
 	client, err := containerd.New("/run/containerd/containerd.sock")
... ...
@@ -61,7 +73,7 @@ Namespaces allow multiple consumers to use the same containerd without conflicti
61 61
 To set a namespace for requests to the API:
62 62
 
63 63
 ```go
64
-context    = context.Background()
64
+context = context.Background()
65 65
 // create a context for docker
66 66
 docker = namespaces.WithNamespace(context, "docker")
67 67
 
... ...
@@ -133,7 +145,7 @@ Taking a container object and turning it into a runnable process on a system is
133 133
 
134 134
 ```go
135 135
 // create a new task
136
-task, err := redis.NewTask(context, containerd.Stdio)
136
+task, err := redis.NewTask(context, cio.Stdio)
137 137
 defer task.Delete(context)
138 138
 
139 139
 // the task is now running and has a pid that can be use to setup networking
... ...
@@ -165,37 +177,12 @@ checkpoint := image.Target()
165 165
 redis, err = client.NewContainer(context, "redis-master", containerd.WithCheckpoint(checkpoint, "redis-rootfs"))
166 166
 defer container.Delete(context)
167 167
 
168
-task, err = redis.NewTask(context, containerd.Stdio, containerd.WithTaskCheckpoint(checkpoint))
168
+task, err = redis.NewTask(context, cio.Stdio, containerd.WithTaskCheckpoint(checkpoint))
169 169
 defer task.Delete(context)
170 170
 
171 171
 err := task.Start(context)
172 172
 ```
173 173
 
174
-## Developer Quick Start
175
-
176
-To build the daemon and `ctr` simple test client, the following build system dependencies are required:
177
-
178
-* Go 1.9.x or above
179
-* Protoc 3.x compiler and headers (download at the [Google protobuf releases page](https://github.com/google/protobuf/releases))
180
-* Btrfs headers and libraries for your distribution. Note that building the btrfs driver can be disabled via build tag removing this dependency.
181
-
182
-For proper results, install the `protoc` release into `/usr/local` on your build system. For example, the following commands will download and install the 3.5.0 release for a 64-bit Linux host:
183
-
184
-```
185
-$ wget -c https://github.com/google/protobuf/releases/download/v3.5.0/protoc-3.5.0-linux-x86_64.zip
186
-$ sudo unzip protoc-3.5.0-linux-x86_64.zip -d /usr/local
187
-```
188
-
189
-With the required dependencies installed, the `Makefile` target named **binaries** will compile the `ctr` and `containerd` binaries and place them in the `bin/` directory. Using `sudo make install` will place the binaries in `/usr/local/bin`. When making any changes to the gRPC API, `make generate` will use the installed `protoc` compiler to regenerate the API generated code packages.
190
-
191
-> *Note*: A build tag is currently available to disable building the btrfs snapshot driver.
192
-> Adding `BUILDTAGS=no_btrfs` to your environment before calling the **binaries**
193
-> Makefile target will disable the btrfs driver within the containerd Go build.
194
-
195
-Vendoring of external imports uses the [`vndr` tool](https://github.com/LK4D4/vndr) which uses a simple config file, `vendor.conf`, to provide the URL and version or hash details for each vendored import. After modifying `vendor.conf` run the `vndr` tool to update the `vendor/` directory contents. Combining the `vendor.conf` update with the changeset in `vendor/` after running `vndr` should become a single commit for a PR which relies on vendored updates.
196
-
197
-Please refer to [RUNC.md](/RUNC.md) for the currently supported version of `runc` that is used by containerd.
198
-
199 174
 ### Releases and API Stability
200 175
 
201 176
 Please see [RELEASES.md](RELEASES.md) for details on versioning and stability
... ...
@@ -7,6 +7,7 @@ import (
7 7
 	"net/http"
8 8
 	"runtime"
9 9
 	"strconv"
10
+	"strings"
10 11
 	"sync"
11 12
 	"time"
12 13
 
... ...
@@ -29,7 +30,6 @@ import (
29 29
 	"github.com/containerd/containerd/namespaces"
30 30
 	"github.com/containerd/containerd/platforms"
31 31
 	"github.com/containerd/containerd/plugin"
32
-	"github.com/containerd/containerd/reference"
33 32
 	"github.com/containerd/containerd/remotes"
34 33
 	"github.com/containerd/containerd/remotes/docker"
35 34
 	"github.com/containerd/containerd/remotes/docker/schema1"
... ...
@@ -334,6 +334,14 @@ func (c *Client) Push(ctx context.Context, ref string, desc ocispec.Descriptor,
334 334
 	for i := len(manifestStack) - 1; i >= 0; i-- {
335 335
 		_, err := pushHandler(ctx, manifestStack[i])
336 336
 		if err != nil {
337
+			// TODO(estesp): until we have a more complete method for index push, we need to report
338
+			// missing dependencies in an index/manifest list by sensing the "400 Bad Request"
339
+			// as a marker for this problem
340
+			if (manifestStack[i].MediaType == ocispec.MediaTypeImageIndex ||
341
+				manifestStack[i].MediaType == images.MediaTypeDockerSchema2ManifestList) &&
342
+				errors.Cause(err) != nil && strings.Contains(errors.Cause(err).Error(), "400 Bad Request") {
343
+				return errors.Wrap(err, "manifest list/index references to blobs and/or manifests are missing in your target registry")
344
+			}
337 345
 			return err
338 346
 		}
339 347
 	}
... ...
@@ -494,95 +502,27 @@ func (c *Client) Version(ctx context.Context) (Version, error) {
494 494
 	}, nil
495 495
 }
496 496
 
497
-type imageFormat string
498
-
499
-const (
500
-	ociImageFormat imageFormat = "oci"
501
-)
502
-
503 497
 type importOpts struct {
504
-	format    imageFormat
505
-	refObject string
506
-	labels    map[string]string
507 498
 }
508 499
 
509 500
 // ImportOpt allows the caller to specify import specific options
510 501
 type ImportOpt func(c *importOpts) error
511 502
 
512
-// WithImportLabel sets a label to be associated with an imported image
513
-func WithImportLabel(key, value string) ImportOpt {
514
-	return func(opts *importOpts) error {
515
-		if opts.labels == nil {
516
-			opts.labels = make(map[string]string)
517
-		}
518
-
519
-		opts.labels[key] = value
520
-		return nil
521
-	}
522
-}
523
-
524
-// WithImportLabels associates a set of labels to an imported image
525
-func WithImportLabels(labels map[string]string) ImportOpt {
526
-	return func(opts *importOpts) error {
527
-		if opts.labels == nil {
528
-			opts.labels = make(map[string]string)
529
-		}
530
-
531
-		for k, v := range labels {
532
-			opts.labels[k] = v
533
-		}
534
-		return nil
535
-	}
536
-}
537
-
538
-// WithOCIImportFormat sets the import format for an OCI image format
539
-func WithOCIImportFormat() ImportOpt {
540
-	return func(c *importOpts) error {
541
-		if c.format != "" {
542
-			return errors.New("format already set")
543
-		}
544
-		c.format = ociImageFormat
545
-		return nil
546
-	}
547
-}
548
-
549
-// WithRefObject specifies the ref object to import.
550
-// If refObject is empty, it is copied from the ref argument of Import().
551
-func WithRefObject(refObject string) ImportOpt {
552
-	return func(c *importOpts) error {
553
-		c.refObject = refObject
554
-		return nil
555
-	}
556
-}
557
-
558
-func resolveImportOpt(ref string, opts ...ImportOpt) (importOpts, error) {
503
+func resolveImportOpt(opts ...ImportOpt) (importOpts, error) {
559 504
 	var iopts importOpts
560 505
 	for _, o := range opts {
561 506
 		if err := o(&iopts); err != nil {
562 507
 			return iopts, err
563 508
 		}
564 509
 	}
565
-	// use OCI as the default format
566
-	if iopts.format == "" {
567
-		iopts.format = ociImageFormat
568
-	}
569
-	// if refObject is not explicitly specified, use the one specified in ref
570
-	if iopts.refObject == "" {
571
-		refSpec, err := reference.Parse(ref)
572
-		if err != nil {
573
-			return iopts, err
574
-		}
575
-		iopts.refObject = refSpec.Object
576
-	}
577 510
 	return iopts, nil
578 511
 }
579 512
 
580 513
 // Import imports an image from a Tar stream using reader.
581
-// OCI format is assumed by default.
582
-//
583
-// Note that unreferenced blobs are imported to the content store as well.
584
-func (c *Client) Import(ctx context.Context, ref string, reader io.Reader, opts ...ImportOpt) (Image, error) {
585
-	iopts, err := resolveImportOpt(ref, opts...)
514
+// Caller needs to specify importer. Future version may use oci.v1 as the default.
515
+// Note that unreferrenced blobs may be imported to the content store as well.
516
+func (c *Client) Import(ctx context.Context, importer images.Importer, reader io.Reader, opts ...ImportOpt) ([]Image, error) {
517
+	_, err := resolveImportOpt(opts...) // unused now
586 518
 	if err != nil {
587 519
 		return nil, err
588 520
 	}
... ...
@@ -593,58 +533,66 @@ func (c *Client) Import(ctx context.Context, ref string, reader io.Reader, opts
593 593
 	}
594 594
 	defer done()
595 595
 
596
-	switch iopts.format {
597
-	case ociImageFormat:
598
-		return c.importFromOCITar(ctx, ref, reader, iopts)
599
-	default:
600
-		return nil, errors.Errorf("unsupported format: %s", iopts.format)
596
+	imgrecs, err := importer.Import(ctx, c.ContentStore(), reader)
597
+	if err != nil {
598
+		// is.Update() is not called on error
599
+		return nil, err
600
+	}
601
+
602
+	is := c.ImageService()
603
+	var images []Image
604
+	for _, imgrec := range imgrecs {
605
+		if updated, err := is.Update(ctx, imgrec, "target"); err != nil {
606
+			if !errdefs.IsNotFound(err) {
607
+				return nil, err
608
+			}
609
+
610
+			created, err := is.Create(ctx, imgrec)
611
+			if err != nil {
612
+				return nil, err
613
+			}
614
+
615
+			imgrec = created
616
+		} else {
617
+			imgrec = updated
618
+		}
619
+
620
+		images = append(images, &image{
621
+			client: c,
622
+			i:      imgrec,
623
+		})
601 624
 	}
625
+	return images, nil
602 626
 }
603 627
 
604 628
 type exportOpts struct {
605
-	format imageFormat
606 629
 }
607 630
 
608
-// ExportOpt allows callers to set export options
631
+// ExportOpt allows the caller to specify export-specific options
609 632
 type ExportOpt func(c *exportOpts) error
610 633
 
611
-// WithOCIExportFormat sets the OCI image format as the export target
612
-func WithOCIExportFormat() ExportOpt {
613
-	return func(c *exportOpts) error {
614
-		if c.format != "" {
615
-			return errors.New("format already set")
634
+func resolveExportOpt(opts ...ExportOpt) (exportOpts, error) {
635
+	var eopts exportOpts
636
+	for _, o := range opts {
637
+		if err := o(&eopts); err != nil {
638
+			return eopts, err
616 639
 		}
617
-		c.format = ociImageFormat
618
-		return nil
619 640
 	}
641
+	return eopts, nil
620 642
 }
621 643
 
622
-// TODO: add WithMediaTypeTranslation that transforms media types according to the format.
623
-// e.g. application/vnd.docker.image.rootfs.diff.tar.gzip
624
-//      -> application/vnd.oci.image.layer.v1.tar+gzip
625
-
626 644
 // Export exports an image to a Tar stream.
627 645
 // OCI format is used by default.
628 646
 // It is up to caller to put "org.opencontainers.image.ref.name" annotation to desc.
629
-func (c *Client) Export(ctx context.Context, desc ocispec.Descriptor, opts ...ExportOpt) (io.ReadCloser, error) {
630
-	var eopts exportOpts
631
-	for _, o := range opts {
632
-		if err := o(&eopts); err != nil {
633
-			return nil, err
634
-		}
635
-	}
636
-	// use OCI as the default format
637
-	if eopts.format == "" {
638
-		eopts.format = ociImageFormat
647
+// TODO(AkihiroSuda): support exporting multiple descriptors at once to a single archive stream.
648
+func (c *Client) Export(ctx context.Context, exporter images.Exporter, desc ocispec.Descriptor, opts ...ExportOpt) (io.ReadCloser, error) {
649
+	_, err := resolveExportOpt(opts...) // unused now
650
+	if err != nil {
651
+		return nil, err
639 652
 	}
640 653
 	pr, pw := io.Pipe()
641
-	switch eopts.format {
642
-	case ociImageFormat:
643
-		go func() {
644
-			pw.CloseWithError(c.exportToOCITar(ctx, desc, pw, eopts))
645
-		}()
646
-	default:
647
-		return nil, errors.Errorf("unsupported format: %s", eopts.format)
648
-	}
654
+	go func() {
655
+		pw.CloseWithError(exporter.Export(ctx, c.ContentStore(), desc, pw))
656
+	}()
649 657
 	return pr, nil
650 658
 }
651 659
deleted file mode 100644
... ...
@@ -1,189 +0,0 @@
1
-package containerd
2
-
3
-import (
4
-	"archive/tar"
5
-	"context"
6
-	"encoding/json"
7
-	"io"
8
-	"sort"
9
-
10
-	"github.com/containerd/containerd/content"
11
-	"github.com/containerd/containerd/images"
12
-	"github.com/containerd/containerd/platforms"
13
-	ocispecs "github.com/opencontainers/image-spec/specs-go"
14
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
15
-	"github.com/pkg/errors"
16
-)
17
-
18
-func (c *Client) exportToOCITar(ctx context.Context, desc ocispec.Descriptor, writer io.Writer, eopts exportOpts) error {
19
-	tw := tar.NewWriter(writer)
20
-	defer tw.Close()
21
-
22
-	records := []tarRecord{
23
-		ociLayoutFile(""),
24
-		ociIndexRecord(desc),
25
-	}
26
-
27
-	cs := c.ContentStore()
28
-	algorithms := map[string]struct{}{}
29
-	exportHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
30
-		records = append(records, blobRecord(cs, desc))
31
-		algorithms[desc.Digest.Algorithm().String()] = struct{}{}
32
-		return nil, nil
33
-	}
34
-
35
-	handlers := images.Handlers(
36
-		images.ChildrenHandler(cs, platforms.Default()),
37
-		images.HandlerFunc(exportHandler),
38
-	)
39
-
40
-	// Walk sequentially since the number of fetchs is likely one and doing in
41
-	// parallel requires locking the export handler
42
-	if err := images.Walk(ctx, handlers, desc); err != nil {
43
-		return err
44
-	}
45
-
46
-	if len(algorithms) > 0 {
47
-		records = append(records, directoryRecord("blobs/", 0755))
48
-		for alg := range algorithms {
49
-			records = append(records, directoryRecord("blobs/"+alg+"/", 0755))
50
-		}
51
-	}
52
-
53
-	return writeTar(ctx, tw, records)
54
-}
55
-
56
-type tarRecord struct {
57
-	Header *tar.Header
58
-	CopyTo func(context.Context, io.Writer) (int64, error)
59
-}
60
-
61
-func blobRecord(cs content.Store, desc ocispec.Descriptor) tarRecord {
62
-	path := "blobs/" + desc.Digest.Algorithm().String() + "/" + desc.Digest.Hex()
63
-	return tarRecord{
64
-		Header: &tar.Header{
65
-			Name:     path,
66
-			Mode:     0444,
67
-			Size:     desc.Size,
68
-			Typeflag: tar.TypeReg,
69
-		},
70
-		CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
71
-			r, err := cs.ReaderAt(ctx, desc.Digest)
72
-			if err != nil {
73
-				return 0, err
74
-			}
75
-			defer r.Close()
76
-
77
-			// Verify digest
78
-			dgstr := desc.Digest.Algorithm().Digester()
79
-
80
-			n, err := io.Copy(io.MultiWriter(w, dgstr.Hash()), content.NewReader(r))
81
-			if err != nil {
82
-				return 0, err
83
-			}
84
-			if dgstr.Digest() != desc.Digest {
85
-				return 0, errors.Errorf("unexpected digest %s copied", dgstr.Digest())
86
-			}
87
-			return n, nil
88
-		},
89
-	}
90
-}
91
-
92
-func directoryRecord(name string, mode int64) tarRecord {
93
-	return tarRecord{
94
-		Header: &tar.Header{
95
-			Name:     name,
96
-			Mode:     mode,
97
-			Typeflag: tar.TypeDir,
98
-		},
99
-	}
100
-}
101
-
102
-func ociLayoutFile(version string) tarRecord {
103
-	if version == "" {
104
-		version = ocispec.ImageLayoutVersion
105
-	}
106
-	layout := ocispec.ImageLayout{
107
-		Version: version,
108
-	}
109
-
110
-	b, err := json.Marshal(layout)
111
-	if err != nil {
112
-		panic(err)
113
-	}
114
-
115
-	return tarRecord{
116
-		Header: &tar.Header{
117
-			Name:     ocispec.ImageLayoutFile,
118
-			Mode:     0444,
119
-			Size:     int64(len(b)),
120
-			Typeflag: tar.TypeReg,
121
-		},
122
-		CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
123
-			n, err := w.Write(b)
124
-			return int64(n), err
125
-		},
126
-	}
127
-
128
-}
129
-
130
-func ociIndexRecord(manifests ...ocispec.Descriptor) tarRecord {
131
-	index := ocispec.Index{
132
-		Versioned: ocispecs.Versioned{
133
-			SchemaVersion: 2,
134
-		},
135
-		Manifests: manifests,
136
-	}
137
-
138
-	b, err := json.Marshal(index)
139
-	if err != nil {
140
-		panic(err)
141
-	}
142
-
143
-	return tarRecord{
144
-		Header: &tar.Header{
145
-			Name:     "index.json",
146
-			Mode:     0644,
147
-			Size:     int64(len(b)),
148
-			Typeflag: tar.TypeReg,
149
-		},
150
-		CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
151
-			n, err := w.Write(b)
152
-			return int64(n), err
153
-		},
154
-	}
155
-}
156
-
157
-func writeTar(ctx context.Context, tw *tar.Writer, records []tarRecord) error {
158
-	sort.Sort(tarRecordsByName(records))
159
-
160
-	for _, record := range records {
161
-		if err := tw.WriteHeader(record.Header); err != nil {
162
-			return err
163
-		}
164
-		if record.CopyTo != nil {
165
-			n, err := record.CopyTo(ctx, tw)
166
-			if err != nil {
167
-				return err
168
-			}
169
-			if n != record.Header.Size {
170
-				return errors.Errorf("unexpected copy size for %s", record.Header.Name)
171
-			}
172
-		} else if record.Header.Size > 0 {
173
-			return errors.Errorf("no content to write to record with non-zero size for %s", record.Header.Name)
174
-		}
175
-	}
176
-	return nil
177
-}
178
-
179
-type tarRecordsByName []tarRecord
180
-
181
-func (t tarRecordsByName) Len() int {
182
-	return len(t)
183
-}
184
-func (t tarRecordsByName) Swap(i, j int) {
185
-	t[i], t[j] = t[j], t[i]
186
-}
187
-func (t tarRecordsByName) Less(i, j int) bool {
188
-	return t[i].Header.Name < t[j].Header.Name
189
-}
... ...
@@ -3,7 +3,6 @@ package filters
3 3
 import (
4 4
 	"fmt"
5 5
 	"io"
6
-	"strconv"
7 6
 
8 7
 	"github.com/containerd/containerd/errdefs"
9 8
 	"github.com/pkg/errors"
... ...
@@ -134,7 +133,12 @@ func (p *parser) selector() (selector, error) {
134 134
 		return selector{}, err
135 135
 	}
136 136
 
137
-	value, err := p.value()
137
+	var allowAltQuotes bool
138
+	if op == operatorMatches {
139
+		allowAltQuotes = true
140
+	}
141
+
142
+	value, err := p.value(allowAltQuotes)
138 143
 	if err != nil {
139 144
 		if err == io.EOF {
140 145
 			return selector{}, io.ErrUnexpectedEOF
... ...
@@ -188,7 +192,7 @@ func (p *parser) field() (string, error) {
188 188
 	case tokenField:
189 189
 		return s, nil
190 190
 	case tokenQuoted:
191
-		return p.unquote(pos, s)
191
+		return p.unquote(pos, s, false)
192 192
 	}
193 193
 
194 194
 	return "", p.mkerr(pos, "expected field or quoted")
... ...
@@ -213,21 +217,25 @@ func (p *parser) operator() (operator, error) {
213 213
 	return 0, p.mkerr(pos, `expected an operator ("=="|"!="|"~=")`)
214 214
 }
215 215
 
216
-func (p *parser) value() (string, error) {
216
+func (p *parser) value(allowAltQuotes bool) (string, error) {
217 217
 	pos, tok, s := p.scanner.scan()
218 218
 
219 219
 	switch tok {
220 220
 	case tokenValue, tokenField:
221 221
 		return s, nil
222 222
 	case tokenQuoted:
223
-		return p.unquote(pos, s)
223
+		return p.unquote(pos, s, allowAltQuotes)
224 224
 	}
225 225
 
226 226
 	return "", p.mkerr(pos, "expected value or quoted")
227 227
 }
228 228
 
229
-func (p *parser) unquote(pos int, s string) (string, error) {
230
-	uq, err := strconv.Unquote(s)
229
+func (p *parser) unquote(pos int, s string, allowAlts bool) (string, error) {
230
+	if !allowAlts && s[0] != '\'' && s[0] != '"' {
231
+		return "", p.mkerr(pos, "invalid quote encountered")
232
+	}
233
+
234
+	uq, err := unquote(s)
231 235
 	if err != nil {
232 236
 		return "", p.mkerr(pos, "unquoting failed: %v", err)
233 237
 	}
234 238
new file mode 100644
... ...
@@ -0,0 +1,237 @@
0
+package filters
1
+
2
+import (
3
+	"unicode/utf8"
4
+
5
+	"github.com/pkg/errors"
6
+)
7
+
8
+// NOTE(stevvooe): Most of this code in this file is copied from the stdlib
9
+// strconv package and modified to be able to handle quoting with `/` and `|`
10
+// as delimiters.  The copyright is held by the Go authors.
11
+
12
+var errQuoteSyntax = errors.New("quote syntax error")
13
+
14
+// UnquoteChar decodes the first character or byte in the escaped string
15
+// or character literal represented by the string s.
16
+// It returns four values:
17
+//
18
+//	1) value, the decoded Unicode code point or byte value;
19
+//	2) multibyte, a boolean indicating whether the decoded character requires a multibyte UTF-8 representation;
20
+//	3) tail, the remainder of the string after the character; and
21
+//	4) an error that will be nil if the character is syntactically valid.
22
+//
23
+// The second argument, quote, specifies the type of literal being parsed
24
+// and therefore which escaped quote character is permitted.
25
+// If set to a single quote, it permits the sequence \' and disallows unescaped '.
26
+// If set to a double quote, it permits \" and disallows unescaped ".
27
+// If set to zero, it does not permit either escape and allows both quote characters to appear unescaped.
28
+//
29
+// This is from Go strconv package, modified to support `|` and `/` as double
30
+// quotes for use with regular expressions.
31
+func unquoteChar(s string, quote byte) (value rune, multibyte bool, tail string, err error) {
32
+	// easy cases
33
+	switch c := s[0]; {
34
+	case c == quote && (quote == '\'' || quote == '"' || quote == '/' || quote == '|'):
35
+		err = errQuoteSyntax
36
+		return
37
+	case c >= utf8.RuneSelf:
38
+		r, size := utf8.DecodeRuneInString(s)
39
+		return r, true, s[size:], nil
40
+	case c != '\\':
41
+		return rune(s[0]), false, s[1:], nil
42
+	}
43
+
44
+	// hard case: c is backslash
45
+	if len(s) <= 1 {
46
+		err = errQuoteSyntax
47
+		return
48
+	}
49
+	c := s[1]
50
+	s = s[2:]
51
+
52
+	switch c {
53
+	case 'a':
54
+		value = '\a'
55
+	case 'b':
56
+		value = '\b'
57
+	case 'f':
58
+		value = '\f'
59
+	case 'n':
60
+		value = '\n'
61
+	case 'r':
62
+		value = '\r'
63
+	case 't':
64
+		value = '\t'
65
+	case 'v':
66
+		value = '\v'
67
+	case 'x', 'u', 'U':
68
+		n := 0
69
+		switch c {
70
+		case 'x':
71
+			n = 2
72
+		case 'u':
73
+			n = 4
74
+		case 'U':
75
+			n = 8
76
+		}
77
+		var v rune
78
+		if len(s) < n {
79
+			err = errQuoteSyntax
80
+			return
81
+		}
82
+		for j := 0; j < n; j++ {
83
+			x, ok := unhex(s[j])
84
+			if !ok {
85
+				err = errQuoteSyntax
86
+				return
87
+			}
88
+			v = v<<4 | x
89
+		}
90
+		s = s[n:]
91
+		if c == 'x' {
92
+			// single-byte string, possibly not UTF-8
93
+			value = v
94
+			break
95
+		}
96
+		if v > utf8.MaxRune {
97
+			err = errQuoteSyntax
98
+			return
99
+		}
100
+		value = v
101
+		multibyte = true
102
+	case '0', '1', '2', '3', '4', '5', '6', '7':
103
+		v := rune(c) - '0'
104
+		if len(s) < 2 {
105
+			err = errQuoteSyntax
106
+			return
107
+		}
108
+		for j := 0; j < 2; j++ { // one digit already; two more
109
+			x := rune(s[j]) - '0'
110
+			if x < 0 || x > 7 {
111
+				err = errQuoteSyntax
112
+				return
113
+			}
114
+			v = (v << 3) | x
115
+		}
116
+		s = s[2:]
117
+		if v > 255 {
118
+			err = errQuoteSyntax
119
+			return
120
+		}
121
+		value = v
122
+	case '\\':
123
+		value = '\\'
124
+	case '\'', '"', '|', '/':
125
+		if c != quote {
126
+			err = errQuoteSyntax
127
+			return
128
+		}
129
+		value = rune(c)
130
+	default:
131
+		err = errQuoteSyntax
132
+		return
133
+	}
134
+	tail = s
135
+	return
136
+}
137
+
138
+// unquote interprets s as a single-quoted, double-quoted,
139
+// or backquoted Go string literal, returning the string value
140
+// that s quotes.  (If s is single-quoted, it would be a Go
141
+// character literal; Unquote returns the corresponding
142
+// one-character string.)
143
+//
144
+// This is modified from the standard library to support `|` and `/` as quote
145
+// characters for use with regular expressions.
146
+func unquote(s string) (string, error) {
147
+	n := len(s)
148
+	if n < 2 {
149
+		return "", errQuoteSyntax
150
+	}
151
+	quote := s[0]
152
+	if quote != s[n-1] {
153
+		return "", errQuoteSyntax
154
+	}
155
+	s = s[1 : n-1]
156
+
157
+	if quote == '`' {
158
+		if contains(s, '`') {
159
+			return "", errQuoteSyntax
160
+		}
161
+		if contains(s, '\r') {
162
+			// -1 because we know there is at least one \r to remove.
163
+			buf := make([]byte, 0, len(s)-1)
164
+			for i := 0; i < len(s); i++ {
165
+				if s[i] != '\r' {
166
+					buf = append(buf, s[i])
167
+				}
168
+			}
169
+			return string(buf), nil
170
+		}
171
+		return s, nil
172
+	}
173
+	if quote != '"' && quote != '\'' && quote != '|' && quote != '/' {
174
+		return "", errQuoteSyntax
175
+	}
176
+	if contains(s, '\n') {
177
+		return "", errQuoteSyntax
178
+	}
179
+
180
+	// Is it trivial?  Avoid allocation.
181
+	if !contains(s, '\\') && !contains(s, quote) {
182
+		switch quote {
183
+		case '"', '/', '|': // pipe and slash are treated like double quote
184
+			return s, nil
185
+		case '\'':
186
+			r, size := utf8.DecodeRuneInString(s)
187
+			if size == len(s) && (r != utf8.RuneError || size != 1) {
188
+				return s, nil
189
+			}
190
+		}
191
+	}
192
+
193
+	var runeTmp [utf8.UTFMax]byte
194
+	buf := make([]byte, 0, 3*len(s)/2) // Try to avoid more allocations.
195
+	for len(s) > 0 {
196
+		c, multibyte, ss, err := unquoteChar(s, quote)
197
+		if err != nil {
198
+			return "", err
199
+		}
200
+		s = ss
201
+		if c < utf8.RuneSelf || !multibyte {
202
+			buf = append(buf, byte(c))
203
+		} else {
204
+			n := utf8.EncodeRune(runeTmp[:], c)
205
+			buf = append(buf, runeTmp[:n]...)
206
+		}
207
+		if quote == '\'' && len(s) != 0 {
208
+			// single-quoted must be single character
209
+			return "", errQuoteSyntax
210
+		}
211
+	}
212
+	return string(buf), nil
213
+}
214
+
215
+// contains reports whether the string contains the byte c.
216
+func contains(s string, c byte) bool {
217
+	for i := 0; i < len(s); i++ {
218
+		if s[i] == c {
219
+			return true
220
+		}
221
+	}
222
+	return false
223
+}
224
+
225
+func unhex(b byte) (v rune, ok bool) {
226
+	c := rune(b)
227
+	switch {
228
+	case '0' <= c && c <= '9':
229
+		return c - '0', true
230
+	case 'a' <= c && c <= 'f':
231
+		return c - 'a' + 10, true
232
+	case 'A' <= c && c <= 'F':
233
+		return c - 'A' + 10, true
234
+	}
235
+	return
236
+}
... ...
@@ -87,7 +87,7 @@ func (s *scanner) peek() rune {
87 87
 	return ch
88 88
 }
89 89
 
90
-func (s *scanner) scan() (int, token, string) {
90
+func (s *scanner) scan() (nextp int, tk token, text string) {
91 91
 	var (
92 92
 		ch  = s.next()
93 93
 		pos = s.pos
... ...
@@ -101,6 +101,7 @@ chomp:
101 101
 		s.scanQuoted(ch)
102 102
 		return pos, tokenQuoted, s.input[pos:s.ppos]
103 103
 	case isSeparatorRune(ch):
104
+		s.value = false
104 105
 		return pos, tokenSeparator, s.input[pos:s.ppos]
105 106
 	case isOperatorRune(ch):
106 107
 		s.scanOperator()
... ...
@@ -241,7 +242,7 @@ func isOperatorRune(r rune) bool {
241 241
 
242 242
 func isQuoteRune(r rune) bool {
243 243
 	switch r {
244
-	case '"': // maybe add single quoting?
244
+	case '/', '|', '"': // maybe add single quoting?
245 245
 		return true
246 246
 	}
247 247
 
... ...
@@ -222,8 +222,10 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err
222 222
 		c1 = make(chan *currentPath)
223 223
 		c2 = make(chan *currentPath)
224 224
 
225
-		f1, f2 *currentPath
226
-		rmdir  string
225
+		f1, f2         *currentPath
226
+		rmdir          string
227
+		lastEmittedDir = string(filepath.Separator)
228
+		parents        []os.FileInfo
227 229
 	)
228 230
 	g.Go(func() error {
229 231
 		defer close(c1)
... ...
@@ -258,7 +260,10 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err
258 258
 				continue
259 259
 			}
260 260
 
261
-			var f os.FileInfo
261
+			var (
262
+				f    os.FileInfo
263
+				emit = true
264
+			)
262 265
 			k, p := pathChange(f1, f2)
263 266
 			switch k {
264 267
 			case ChangeKindAdd:
... ...
@@ -294,13 +299,35 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err
294 294
 				f2 = nil
295 295
 				if same {
296 296
 					if !isLinked(f) {
297
-						continue
297
+						emit = false
298 298
 					}
299 299
 					k = ChangeKindUnmodified
300 300
 				}
301 301
 			}
302
-			if err := changeFn(k, p, f, nil); err != nil {
303
-				return err
302
+			if emit {
303
+				emittedDir, emitParents := commonParents(lastEmittedDir, p, parents)
304
+				for _, pf := range emitParents {
305
+					p := filepath.Join(emittedDir, pf.Name())
306
+					if err := changeFn(ChangeKindUnmodified, p, pf, nil); err != nil {
307
+						return err
308
+					}
309
+					emittedDir = p
310
+				}
311
+
312
+				if err := changeFn(k, p, f, nil); err != nil {
313
+					return err
314
+				}
315
+
316
+				if f != nil && f.IsDir() {
317
+					lastEmittedDir = p
318
+				} else {
319
+					lastEmittedDir = emittedDir
320
+				}
321
+
322
+				parents = parents[:0]
323
+			} else if f.IsDir() {
324
+				lastEmittedDir, parents = commonParents(lastEmittedDir, p, parents)
325
+				parents = append(parents, f)
304 326
 			}
305 327
 		}
306 328
 		return nil
... ...
@@ -308,3 +335,47 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err
308 308
 
309 309
 	return g.Wait()
310 310
 }
311
+
312
+func commonParents(base, updated string, dirs []os.FileInfo) (string, []os.FileInfo) {
313
+	if basePrefix := makePrefix(base); strings.HasPrefix(updated, basePrefix) {
314
+		var (
315
+			parents []os.FileInfo
316
+			last    = base
317
+		)
318
+		for _, d := range dirs {
319
+			next := filepath.Join(last, d.Name())
320
+			if strings.HasPrefix(updated, makePrefix(last)) {
321
+				parents = append(parents, d)
322
+				last = next
323
+			} else {
324
+				break
325
+			}
326
+		}
327
+		return base, parents
328
+	}
329
+
330
+	baseS := strings.Split(base, string(filepath.Separator))
331
+	updatedS := strings.Split(updated, string(filepath.Separator))
332
+	commonS := []string{string(filepath.Separator)}
333
+
334
+	min := len(baseS)
335
+	if len(updatedS) < min {
336
+		min = len(updatedS)
337
+	}
338
+	for i := 0; i < min; i++ {
339
+		if baseS[i] == updatedS[i] {
340
+			commonS = append(commonS, baseS[i])
341
+		} else {
342
+			break
343
+		}
344
+	}
345
+
346
+	return filepath.Join(commonS...), []os.FileInfo{}
347
+}
348
+
349
+func makePrefix(d string) string {
350
+	if d == "" || d[len(d)-1] != filepath.Separator {
351
+		return d + string(filepath.Separator)
352
+	}
353
+	return d
354
+}
... ...
@@ -1,5 +1,7 @@
1 1
 package fs
2 2
 
3
+import "context"
4
+
3 5
 // Usage of disk information
4 6
 type Usage struct {
5 7
 	Inodes int64
... ...
@@ -11,3 +13,10 @@ type Usage struct {
11 11
 func DiskUsage(roots ...string) (Usage, error) {
12 12
 	return diskUsage(roots...)
13 13
 }
14
+
15
+// DiffUsage counts the numbers of inodes and disk usage in the
16
+// diff between the 2 directories. The first path is intended
17
+// as the base directory and the second as the changed directory.
18
+func DiffUsage(ctx context.Context, a, b string) (Usage, error) {
19
+	return diffUsage(ctx, a, b)
20
+}
... ...
@@ -3,17 +3,19 @@
3 3
 package fs
4 4
 
5 5
 import (
6
+	"context"
6 7
 	"os"
7 8
 	"path/filepath"
8 9
 	"syscall"
9 10
 )
10 11
 
12
+type inode struct {
13
+	// TODO(stevvooe): Can probably reduce memory usage by not tracking
14
+	// device, but we can leave this right for now.
15
+	dev, ino uint64
16
+}
17
+
11 18
 func diskUsage(roots ...string) (Usage, error) {
12
-	type inode struct {
13
-		// TODO(stevvooe): Can probably reduce memory usage by not tracking
14
-		// device, but we can leave this right for now.
15
-		dev, ino uint64
16
-	}
17 19
 
18 20
 	var (
19 21
 		size   int64
... ...
@@ -45,3 +47,37 @@ func diskUsage(roots ...string) (Usage, error) {
45 45
 		Size:   size,
46 46
 	}, nil
47 47
 }
48
+
49
+func diffUsage(ctx context.Context, a, b string) (Usage, error) {
50
+	var (
51
+		size   int64
52
+		inodes = map[inode]struct{}{} // expensive!
53
+	)
54
+
55
+	if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error {
56
+		if err != nil {
57
+			return err
58
+		}
59
+
60
+		if kind == ChangeKindAdd || kind == ChangeKindModify {
61
+			stat := fi.Sys().(*syscall.Stat_t)
62
+
63
+			inoKey := inode{dev: uint64(stat.Dev), ino: uint64(stat.Ino)}
64
+			if _, ok := inodes[inoKey]; !ok {
65
+				inodes[inoKey] = struct{}{}
66
+				size += fi.Size()
67
+			}
68
+
69
+			return nil
70
+
71
+		}
72
+		return nil
73
+	}); err != nil {
74
+		return Usage{}, err
75
+	}
76
+
77
+	return Usage{
78
+		Inodes: int64(len(inodes)),
79
+		Size:   size,
80
+	}, nil
81
+}
... ...
@@ -3,6 +3,7 @@
3 3
 package fs
4 4
 
5 5
 import (
6
+	"context"
6 7
 	"os"
7 8
 	"path/filepath"
8 9
 )
... ...
@@ -31,3 +32,29 @@ func diskUsage(roots ...string) (Usage, error) {
31 31
 		Size: size,
32 32
 	}, nil
33 33
 }
34
+
35
+func diffUsage(ctx context.Context, a, b string) (Usage, error) {
36
+	var (
37
+		size int64
38
+	)
39
+
40
+	if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error {
41
+		if err != nil {
42
+			return err
43
+		}
44
+
45
+		if kind == ChangeKindAdd || kind == ChangeKindModify {
46
+			size += fi.Size()
47
+
48
+			return nil
49
+
50
+		}
51
+		return nil
52
+	}); err != nil {
53
+		return Usage{}, err
54
+	}
55
+
56
+	return Usage{
57
+		Size: size,
58
+	}, nil
59
+}
34 60
new file mode 100644
... ...
@@ -0,0 +1,21 @@
0
+package images
1
+
2
+import (
3
+	"context"
4
+	"io"
5
+
6
+	"github.com/containerd/containerd/content"
7
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
8
+)
9
+
10
+// Importer is the interface for image importer.
11
+type Importer interface {
12
+	// Import imports an image from a tar stream.
13
+	Import(ctx context.Context, store content.Store, reader io.Reader) ([]Image, error)
14
+}
15
+
16
+// Exporter is the interface for image exporter.
17
+type Exporter interface {
18
+	// Export exports an image to a tar stream.
19
+	Export(ctx context.Context, store content.Store, desc ocispec.Descriptor, writer io.Writer) error
20
+}
0 21
deleted file mode 100644
... ...
@@ -1,120 +0,0 @@
1
-package containerd
2
-
3
-import (
4
-	"archive/tar"
5
-	"context"
6
-	"encoding/json"
7
-	"io"
8
-	"io/ioutil"
9
-	"strings"
10
-
11
-	"github.com/containerd/containerd/content"
12
-	"github.com/containerd/containerd/errdefs"
13
-	"github.com/containerd/containerd/images"
14
-	"github.com/containerd/containerd/reference"
15
-	digest "github.com/opencontainers/go-digest"
16
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
17
-	"github.com/pkg/errors"
18
-)
19
-
20
-func resolveOCIIndex(idx ocispec.Index, refObject string) (*ocispec.Descriptor, error) {
21
-	tag, dgst := reference.SplitObject(refObject)
22
-	if tag == "" && dgst == "" {
23
-		return nil, errors.Errorf("unexpected object: %q", refObject)
24
-	}
25
-	for _, m := range idx.Manifests {
26
-		if m.Digest == dgst {
27
-			return &m, nil
28
-		}
29
-		annot, ok := m.Annotations[ocispec.AnnotationRefName]
30
-		if ok && annot == tag && tag != "" {
31
-			return &m, nil
32
-		}
33
-	}
34
-	return nil, errors.Errorf("not found: %q", refObject)
35
-}
36
-
37
-func (c *Client) importFromOCITar(ctx context.Context, ref string, reader io.Reader, iopts importOpts) (Image, error) {
38
-	tr := tar.NewReader(reader)
39
-	store := c.ContentStore()
40
-	var desc *ocispec.Descriptor
41
-	for {
42
-		hdr, err := tr.Next()
43
-		if err == io.EOF {
44
-			break
45
-		}
46
-		if err != nil {
47
-			return nil, err
48
-		}
49
-		if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
50
-			continue
51
-		}
52
-		if hdr.Name == "index.json" {
53
-			desc, err = onUntarIndexJSON(tr, iopts.refObject)
54
-			if err != nil {
55
-				return nil, err
56
-			}
57
-			continue
58
-		}
59
-		if strings.HasPrefix(hdr.Name, "blobs/") {
60
-			if err := onUntarBlob(ctx, tr, store, hdr.Name, hdr.Size); err != nil {
61
-				return nil, err
62
-			}
63
-		}
64
-	}
65
-	if desc == nil {
66
-		return nil, errors.Errorf("no descriptor found for reference object %q", iopts.refObject)
67
-	}
68
-	imgrec := images.Image{
69
-		Name:   ref,
70
-		Target: *desc,
71
-		Labels: iopts.labels,
72
-	}
73
-	is := c.ImageService()
74
-	if updated, err := is.Update(ctx, imgrec, "target"); err != nil {
75
-		if !errdefs.IsNotFound(err) {
76
-			return nil, err
77
-		}
78
-
79
-		created, err := is.Create(ctx, imgrec)
80
-		if err != nil {
81
-			return nil, err
82
-		}
83
-
84
-		imgrec = created
85
-	} else {
86
-		imgrec = updated
87
-	}
88
-
89
-	img := &image{
90
-		client: c,
91
-		i:      imgrec,
92
-	}
93
-	return img, nil
94
-}
95
-
96
-func onUntarIndexJSON(r io.Reader, refObject string) (*ocispec.Descriptor, error) {
97
-	b, err := ioutil.ReadAll(r)
98
-	if err != nil {
99
-		return nil, err
100
-	}
101
-	var idx ocispec.Index
102
-	if err := json.Unmarshal(b, &idx); err != nil {
103
-		return nil, err
104
-	}
105
-	return resolveOCIIndex(idx, refObject)
106
-}
107
-
108
-func onUntarBlob(ctx context.Context, r io.Reader, store content.Store, name string, size int64) error {
109
-	// name is like "blobs/sha256/deadbeef"
110
-	split := strings.Split(name, "/")
111
-	if len(split) != 3 {
112
-		return errors.Errorf("unexpected name: %q", name)
113
-	}
114
-	algo := digest.Algorithm(split[1])
115
-	if !algo.Available() {
116
-		return errors.Errorf("unsupported algorithm: %s", algo)
117
-	}
118
-	dgst := digest.NewDigestFromHex(algo.String(), split[2])
119
-	return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, size, dgst)
120
-}
... ...
@@ -114,6 +114,7 @@ func fetch(ctx context.Context, ingester content.Ingester, fetcher Fetcher, desc
114 114
 func commitOpts(desc ocispec.Descriptor, r io.Reader) (io.Reader, []content.Opt) {
115 115
 	var childrenF func(r io.Reader) ([]ocispec.Descriptor, error)
116 116
 
117
+	// TODO(AkihiroSuda): use images/oci.GetChildrenDescriptors?
117 118
 	switch desc.MediaType {
118 119
 	case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
119 120
 		childrenF = func(r io.Reader) ([]ocispec.Descriptor, error) {