Browse code

Merge pull request #34568 from Microsoft/jjh/singletagstore

Move to a single tag-store

John Stephens authored on 2017/08/23 09:50:36
Showing 14 changed files
... ...
@@ -75,7 +75,6 @@ type daemonStore struct {
75 75
 	imageStore                image.Store
76 76
 	layerStore                layer.Store
77 77
 	distributionMetadataStore dmetadata.Store
78
-	referenceStore            refstore.Store
79 78
 }
80 79
 
81 80
 // Daemon holds information about the Docker daemon.
... ...
@@ -103,7 +102,8 @@ type Daemon struct {
103 103
 	shutdown              bool
104 104
 	idMappings            *idtools.IDMappings
105 105
 	stores                map[string]daemonStore // By container target platform
106
-	PluginStore           *plugin.Store          // todo: remove
106
+	referenceStore        refstore.Store
107
+	PluginStore           *plugin.Store // todo: remove
107 108
 	pluginManager         *plugin.Manager
108 109
 	linkIndex             *linkIndex
109 110
 	containerd            libcontainerd.Client
... ...
@@ -691,7 +691,6 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe
691 691
 	d.downloadManager = xfer.NewLayerDownloadManager(lsMap, *config.MaxConcurrentDownloads)
692 692
 	logrus.Debugf("Max Concurrent Uploads: %d", *config.MaxConcurrentUploads)
693 693
 	d.uploadManager = xfer.NewLayerUploadManager(*config.MaxConcurrentUploads)
694
-
695 694
 	for platform, ds := range d.stores {
696 695
 		imageRoot := filepath.Join(config.Root, "image", ds.graphDriver)
697 696
 		ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb"))
... ...
@@ -728,18 +727,30 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe
728 728
 
729 729
 	eventsService := events.New()
730 730
 
731
+	// We have a single tag/reference store for the daemon globally. However, it's
732
+	// stored under the graphdriver. On host platforms which only support a single
733
+	// container OS, but multiple selectable graphdrivers, this means depending on which
734
+	// graphdriver is chosen, the global reference store is under there. For
735
+	// platforms which support multiple container operating systems, this is slightly
736
+	// more problematic as where does the global ref store get located? Fortunately,
737
+	// for Windows, which is currently the only daemon supporting multiple container
738
+	// operating systems, the list of graphdrivers available isn't user configurable.
739
+	// For backwards compatibility, we just put it under the windowsfilter
740
+	// directory regardless.
741
+	refStoreLocation := filepath.Join(d.stores[runtime.GOOS].imageRoot, `repositories.json`)
742
+	rs, err := refstore.NewReferenceStore(refStoreLocation)
743
+	if err != nil {
744
+		return nil, fmt.Errorf("Couldn't create reference store repository: %s", err)
745
+	}
746
+	d.referenceStore = rs
747
+
731 748
 	for platform, ds := range d.stores {
732 749
 		dms, err := dmetadata.NewFSMetadataStore(filepath.Join(ds.imageRoot, "distribution"), platform)
733 750
 		if err != nil {
734 751
 			return nil, err
735 752
 		}
736 753
 
737
-		rs, err := refstore.NewReferenceStore(filepath.Join(ds.imageRoot, "repositories.json"), platform)
738
-		if err != nil {
739
-			return nil, fmt.Errorf("Couldn't create Tag store repositories: %s", err)
740
-		}
741 754
 		ds.distributionMetadataStore = dms
742
-		ds.referenceStore = rs
743 755
 		d.stores[platform] = ds
744 756
 
745 757
 		// No content-addressability migration on Windows as it never supported pre-CA
... ...
@@ -20,7 +20,7 @@ func (daemon *Daemon) getLayerRefs(platform string) map[layer.ChainID]int {
20 20
 	layerRefs := map[layer.ChainID]int{}
21 21
 	for id, img := range tmpImages {
22 22
 		dgst := digest.Digest(id)
23
-		if len(daemon.stores[platform].referenceStore.References(dgst)) == 0 && len(daemon.stores[platform].imageStore.Children(id)) != 0 {
23
+		if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.stores[platform].imageStore.Children(id)) != 0 {
24 24
 			continue
25 25
 		}
26 26
 
... ...
@@ -2,6 +2,7 @@ package daemon
2 2
 
3 3
 import (
4 4
 	"fmt"
5
+	"runtime"
5 6
 
6 7
 	"github.com/docker/distribution/reference"
7 8
 	"github.com/docker/docker/image"
... ...
@@ -45,10 +46,17 @@ func (daemon *Daemon) GetImageIDAndPlatform(refOrID string) (image.ID, string, e
45 45
 		return "", "", errImageDoesNotExist{ref}
46 46
 	}
47 47
 
48
-	for platform := range daemon.stores {
49
-		if id, err := daemon.stores[platform].referenceStore.Get(namedRef); err == nil {
50
-			return image.IDFromDigest(id), platform, nil
48
+	if digest, err := daemon.referenceStore.Get(namedRef); err == nil {
49
+		// Search the image stores to get the platform, defaulting to host OS.
50
+		imagePlatform := runtime.GOOS
51
+		id := image.IDFromDigest(digest)
52
+		for platform := range daemon.stores {
53
+			if img, err := daemon.stores[platform].imageStore.Get(id); err == nil {
54
+				imagePlatform = img.Platform()
55
+				break
56
+			}
51 57
 		}
58
+		return id, imagePlatform, nil
52 59
 	}
53 60
 
54 61
 	// deprecated: repo:shortid https://github.com/docker/docker/pull/799
... ...
@@ -56,7 +64,7 @@ func (daemon *Daemon) GetImageIDAndPlatform(refOrID string) (image.ID, string, e
56 56
 		if tag := tagged.Tag(); stringid.IsShortID(stringid.TruncateID(tag)) {
57 57
 			for platform := range daemon.stores {
58 58
 				if id, err := daemon.stores[platform].imageStore.Search(tag); err == nil {
59
-					for _, storeRef := range daemon.stores[platform].referenceStore.References(id.Digest()) {
59
+					for _, storeRef := range daemon.referenceStore.References(id.Digest()) {
60 60
 						if storeRef.Name() == namedRef.Name() {
61 61
 							return id, platform, nil
62 62
 						}
... ...
@@ -70,7 +70,7 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I
70 70
 		return nil, err
71 71
 	}
72 72
 
73
-	repoRefs := daemon.stores[platform].referenceStore.References(imgID.Digest())
73
+	repoRefs := daemon.referenceStore.References(imgID.Digest())
74 74
 
75 75
 	var removedRepositoryRef bool
76 76
 	if !isImageIDPrefix(imgID.String(), imageRef) {
... ...
@@ -104,7 +104,7 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I
104 104
 		daemon.LogImageEvent(imgID.String(), imgID.String(), "untag")
105 105
 		records = append(records, untaggedRecord)
106 106
 
107
-		repoRefs = daemon.stores[platform].referenceStore.References(imgID.Digest())
107
+		repoRefs = daemon.referenceStore.References(imgID.Digest())
108 108
 
109 109
 		// If a tag reference was removed and the only remaining
110 110
 		// references to the same repository are digest references,
... ...
@@ -237,7 +237,7 @@ func (daemon *Daemon) removeImageRef(platform string, ref reference.Named) (refe
237 237
 	// Ignore the boolean value returned, as far as we're concerned, this
238 238
 	// is an idempotent operation and it's okay if the reference didn't
239 239
 	// exist in the first place.
240
-	_, err := daemon.stores[platform].referenceStore.Delete(ref)
240
+	_, err := daemon.referenceStore.Delete(ref)
241 241
 
242 242
 	return ref, err
243 243
 }
... ...
@@ -248,7 +248,7 @@ func (daemon *Daemon) removeImageRef(platform string, ref reference.Named) (refe
248 248
 // daemon's event service. An "Untagged" types.ImageDeleteResponseItem is added to the
249 249
 // given list of records.
250 250
 func (daemon *Daemon) removeAllReferencesToImageID(imgID image.ID, platform string, records *[]types.ImageDeleteResponseItem) error {
251
-	imageRefs := daemon.stores[platform].referenceStore.References(imgID.Digest())
251
+	imageRefs := daemon.referenceStore.References(imgID.Digest())
252 252
 
253 253
 	for _, imageRef := range imageRefs {
254 254
 		parsedRef, err := daemon.removeImageRef(platform, imageRef)
... ...
@@ -383,7 +383,7 @@ func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, platform string,
383 383
 	}
384 384
 
385 385
 	// Check if any repository tags/digest reference this image.
386
-	if mask&conflictActiveReference != 0 && len(daemon.stores[platform].referenceStore.References(imgID.Digest())) > 0 {
386
+	if mask&conflictActiveReference != 0 && len(daemon.referenceStore.References(imgID.Digest())) > 0 {
387 387
 		return &imageDeleteConflict{
388 388
 			imgID:   imgID,
389 389
 			message: "image is referenced in multiple repositories",
... ...
@@ -411,5 +411,5 @@ func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, platform string,
411 411
 // that there are no repository references to the given image and it has no
412 412
 // child images.
413 413
 func (daemon *Daemon) imageIsDangling(imgID image.ID, platform string) bool {
414
-	return !(len(daemon.stores[platform].referenceStore.References(imgID.Digest())) > 0 || len(daemon.stores[platform].imageStore.Children(imgID)) > 0)
414
+	return !(len(daemon.referenceStore.References(imgID.Digest())) > 0 || len(daemon.stores[platform].imageStore.Children(imgID)) > 0)
415 415
 }
... ...
@@ -19,7 +19,7 @@ func (daemon *Daemon) ExportImage(names []string, outStream io.Writer) error {
19 19
 	if system.LCOWSupported() {
20 20
 		platform = "linux"
21 21
 	}
22
-	imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.stores[platform].referenceStore, daemon)
22
+	imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.referenceStore, daemon)
23 23
 	return imageExporter.Save(names, outStream)
24 24
 }
25 25
 
... ...
@@ -32,6 +32,6 @@ func (daemon *Daemon) LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet
32 32
 	if system.LCOWSupported() {
33 33
 		platform = "linux"
34 34
 	}
35
-	imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.stores[platform].referenceStore, daemon)
35
+	imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.referenceStore, daemon)
36 36
 	return imageExporter.Load(inTar, outStream, quiet)
37 37
 }
... ...
@@ -69,7 +69,7 @@ func (daemon *Daemon) ImageHistory(name string) ([]*image.HistoryResponseItem, e
69 69
 		h.ID = id.String()
70 70
 
71 71
 		var tags []string
72
-		for _, r := range daemon.stores[platform].referenceStore.References(id.Digest()) {
72
+		for _, r := range daemon.referenceStore.References(id.Digest()) {
73 73
 			if _, ok := r.(reference.NamedTagged); ok {
74 74
 				tags = append(tags, reference.FamiliarString(r))
75 75
 			}
... ...
@@ -24,7 +24,7 @@ func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) {
24 24
 		platform = runtime.GOOS
25 25
 	}
26 26
 
27
-	refs := daemon.stores[platform].referenceStore.References(img.ID().Digest())
27
+	refs := daemon.referenceStore.References(img.ID().Digest())
28 28
 	repoTags := []string{}
29 29
 	repoDigests := []string{}
30 30
 	for _, ref := range refs {
... ...
@@ -74,7 +74,7 @@ func (daemon *Daemon) pullImageWithReference(ctx context.Context, ref reference.
74 74
 			ImageEventLogger: daemon.LogImageEvent,
75 75
 			MetadataStore:    daemon.stores[platform].distributionMetadataStore,
76 76
 			ImageStore:       distribution.NewImageConfigStoreFromStore(daemon.stores[platform].imageStore),
77
-			ReferenceStore:   daemon.stores[platform].referenceStore,
77
+			ReferenceStore:   daemon.referenceStore,
78 78
 		},
79 79
 		DownloadManager: daemon.downloadManager,
80 80
 		Schema2Types:    distribution.ImageTypes,
... ...
@@ -56,7 +56,7 @@ func (daemon *Daemon) PushImage(ctx context.Context, image, tag string, metaHead
56 56
 			ImageEventLogger: daemon.LogImageEvent,
57 57
 			MetadataStore:    daemon.stores[platform].distributionMetadataStore,
58 58
 			ImageStore:       distribution.NewImageConfigStoreFromStore(daemon.stores[platform].imageStore),
59
-			ReferenceStore:   daemon.stores[platform].referenceStore,
59
+			ReferenceStore:   daemon.referenceStore,
60 60
 		},
61 61
 		ConfigMediaType: schema2.MediaTypeImageConfig,
62 62
 		LayerStore:      distribution.NewLayerProviderFromStore(daemon.stores[platform].layerStore),
... ...
@@ -28,7 +28,7 @@ func (daemon *Daemon) TagImage(imageName, repository, tag string) error {
28 28
 
29 29
 // TagImageWithReference adds the given reference to the image ID provided.
30 30
 func (daemon *Daemon) TagImageWithReference(imageID image.ID, platform string, newTag reference.Named) error {
31
-	if err := daemon.stores[platform].referenceStore.AddTag(newTag, imageID.Digest(), true); err != nil {
31
+	if err := daemon.referenceStore.AddTag(newTag, imageID.Digest(), true); err != nil {
32 32
 		return err
33 33
 	}
34 34
 
... ...
@@ -149,7 +149,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs
149 149
 
150 150
 		newImage := newImage(img, size)
151 151
 
152
-		for _, ref := range daemon.stores[platform].referenceStore.References(id.Digest()) {
152
+		for _, ref := range daemon.referenceStore.References(id.Digest()) {
153 153
 			if imageFilters.Include("reference") {
154 154
 				var found bool
155 155
 				var matchErr error
... ...
@@ -221,7 +221,7 @@ func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args
221 221
 			return nil, ctx.Err()
222 222
 		default:
223 223
 			dgst := digest.Digest(id)
224
-			if len(daemon.stores[platform].referenceStore.References(dgst)) == 0 && len(daemon.stores[platform].imageStore.Children(id)) != 0 {
224
+			if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.stores[platform].imageStore.Children(id)) != 0 {
225 225
 				continue
226 226
 			}
227 227
 			if !until.IsZero() && img.Created.After(until) {
... ...
@@ -252,7 +252,7 @@ deleteImagesLoop:
252 252
 		}
253 253
 
254 254
 		deletedImages := []types.ImageDeleteResponseItem{}
255
-		refs := daemon.stores[platform].referenceStore.References(dgst)
255
+		refs := daemon.referenceStore.References(dgst)
256 256
 		if len(refs) > 0 {
257 257
 			shouldDelete := !danglingOnly
258 258
 			if !shouldDelete {
... ...
@@ -26,7 +26,7 @@ type Association struct {
26 26
 	ID  digest.Digest
27 27
 }
28 28
 
29
-// Store provides the set of methods which can operate on a tag store.
29
+// Store provides the set of methods which can operate on a reference store.
30 30
 type Store interface {
31 31
 	References(id digest.Digest) []reference.Named
32 32
 	ReferencesByName(ref reference.Named) []Association
... ...
@@ -46,9 +46,6 @@ type store struct {
46 46
 	// referencesByIDCache is a cache of references indexed by ID, to speed
47 47
 	// up References.
48 48
 	referencesByIDCache map[digest.Digest]map[string]reference.Named
49
-	// platform is the container target platform for this store (which may be
50
-	// different to the host operating system
51
-	platform string
52 49
 }
53 50
 
54 51
 // Repository maps tags to digests. The key is a stringified Reference,
... ...
@@ -73,7 +70,7 @@ func (a lexicalAssociations) Less(i, j int) bool {
73 73
 
74 74
 // NewReferenceStore creates a new reference store, tied to a file path where
75 75
 // the set of references are serialized in JSON format.
76
-func NewReferenceStore(jsonPath, platform string) (Store, error) {
76
+func NewReferenceStore(jsonPath string) (Store, error) {
77 77
 	abspath, err := filepath.Abs(jsonPath)
78 78
 	if err != nil {
79 79
 		return nil, err
... ...
@@ -83,7 +80,6 @@ func NewReferenceStore(jsonPath, platform string) (Store, error) {
83 83
 		jsonPath:            abspath,
84 84
 		Repositories:        make(map[string]repository),
85 85
 		referencesByIDCache: make(map[digest.Digest]map[string]reference.Named),
86
-		platform:            platform,
87 86
 	}
88 87
 	// Load the json file if it exists, otherwise create it.
89 88
 	if err := store.reload(); os.IsNotExist(err) {
... ...
@@ -5,7 +5,6 @@ import (
5 5
 	"io/ioutil"
6 6
 	"os"
7 7
 	"path/filepath"
8
-	"runtime"
9 8
 	"strings"
10 9
 	"testing"
11 10
 
... ...
@@ -41,7 +40,7 @@ func TestLoad(t *testing.T) {
41 41
 	}
42 42
 	jsonFile.Close()
43 43
 
44
-	store, err := NewReferenceStore(jsonFile.Name(), runtime.GOOS)
44
+	store, err := NewReferenceStore(jsonFile.Name())
45 45
 	if err != nil {
46 46
 		t.Fatalf("error creating tag store: %v", err)
47 47
 	}
... ...
@@ -70,7 +69,7 @@ func TestSave(t *testing.T) {
70 70
 	jsonFile.Close()
71 71
 	defer os.RemoveAll(jsonFile.Name())
72 72
 
73
-	store, err := NewReferenceStore(jsonFile.Name(), runtime.GOOS)
73
+	store, err := NewReferenceStore(jsonFile.Name())
74 74
 	if err != nil {
75 75
 		t.Fatalf("error creating tag store: %v", err)
76 76
 	}
... ...
@@ -112,7 +111,7 @@ func TestAddDeleteGet(t *testing.T) {
112 112
 	jsonFile.Close()
113 113
 	defer os.RemoveAll(jsonFile.Name())
114 114
 
115
-	store, err := NewReferenceStore(jsonFile.Name(), runtime.GOOS)
115
+	store, err := NewReferenceStore(jsonFile.Name())
116 116
 	if err != nil {
117 117
 		t.Fatalf("error creating tag store: %v", err)
118 118
 	}
... ...
@@ -329,7 +328,7 @@ func TestInvalidTags(t *testing.T) {
329 329
 	tmpDir, err := ioutil.TempDir("", "tag-store-test")
330 330
 	defer os.RemoveAll(tmpDir)
331 331
 
332
-	store, err := NewReferenceStore(filepath.Join(tmpDir, "repositories.json"), runtime.GOOS)
332
+	store, err := NewReferenceStore(filepath.Join(tmpDir, "repositories.json"))
333 333
 	if err != nil {
334 334
 		t.Fatalf("error creating tag store: %v", err)
335 335
 	}