Move to a single tag-store
John Stephens authored on 2017/08/23 09:50:36... | ... |
@@ -75,7 +75,6 @@ type daemonStore struct { |
75 | 75 |
imageStore image.Store |
76 | 76 |
layerStore layer.Store |
77 | 77 |
distributionMetadataStore dmetadata.Store |
78 |
- referenceStore refstore.Store |
|
79 | 78 |
} |
80 | 79 |
|
81 | 80 |
// Daemon holds information about the Docker daemon. |
... | ... |
@@ -103,7 +102,8 @@ type Daemon struct { |
103 | 103 |
shutdown bool |
104 | 104 |
idMappings *idtools.IDMappings |
105 | 105 |
stores map[string]daemonStore // By container target platform |
106 |
- PluginStore *plugin.Store // todo: remove |
|
106 |
+ referenceStore refstore.Store |
|
107 |
+ PluginStore *plugin.Store // todo: remove |
|
107 | 108 |
pluginManager *plugin.Manager |
108 | 109 |
linkIndex *linkIndex |
109 | 110 |
containerd libcontainerd.Client |
... | ... |
@@ -691,7 +691,6 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe |
691 | 691 |
d.downloadManager = xfer.NewLayerDownloadManager(lsMap, *config.MaxConcurrentDownloads) |
692 | 692 |
logrus.Debugf("Max Concurrent Uploads: %d", *config.MaxConcurrentUploads) |
693 | 693 |
d.uploadManager = xfer.NewLayerUploadManager(*config.MaxConcurrentUploads) |
694 |
- |
|
695 | 694 |
for platform, ds := range d.stores { |
696 | 695 |
imageRoot := filepath.Join(config.Root, "image", ds.graphDriver) |
697 | 696 |
ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb")) |
... | ... |
@@ -728,18 +727,30 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe |
728 | 728 |
|
729 | 729 |
eventsService := events.New() |
730 | 730 |
|
731 |
+ // We have a single tag/reference store for the daemon globally. However, it's |
|
732 |
+ // stored under the graphdriver. On host platforms which only support a single |
|
733 |
+ // container OS, but multiple selectable graphdrivers, this means depending on which |
|
734 |
+ // graphdriver is chosen, the global reference store is under there. For |
|
735 |
+ // platforms which support multiple container operating systems, this is slightly |
|
736 |
+ // more problematic as where does the global ref store get located? Fortunately, |
|
737 |
+ // for Windows, which is currently the only daemon supporting multiple container |
|
738 |
+ // operating systems, the list of graphdrivers available isn't user configurable. |
|
739 |
+ // For backwards compatibility, we just put it under the windowsfilter |
|
740 |
+ // directory regardless. |
|
741 |
+ refStoreLocation := filepath.Join(d.stores[runtime.GOOS].imageRoot, `repositories.json`) |
|
742 |
+ rs, err := refstore.NewReferenceStore(refStoreLocation) |
|
743 |
+ if err != nil { |
|
744 |
+ return nil, fmt.Errorf("Couldn't create reference store repository: %s", err) |
|
745 |
+ } |
|
746 |
+ d.referenceStore = rs |
|
747 |
+ |
|
731 | 748 |
for platform, ds := range d.stores { |
732 | 749 |
dms, err := dmetadata.NewFSMetadataStore(filepath.Join(ds.imageRoot, "distribution"), platform) |
733 | 750 |
if err != nil { |
734 | 751 |
return nil, err |
735 | 752 |
} |
736 | 753 |
|
737 |
- rs, err := refstore.NewReferenceStore(filepath.Join(ds.imageRoot, "repositories.json"), platform) |
|
738 |
- if err != nil { |
|
739 |
- return nil, fmt.Errorf("Couldn't create Tag store repositories: %s", err) |
|
740 |
- } |
|
741 | 754 |
ds.distributionMetadataStore = dms |
742 |
- ds.referenceStore = rs |
|
743 | 755 |
d.stores[platform] = ds |
744 | 756 |
|
745 | 757 |
// No content-addressability migration on Windows as it never supported pre-CA |
... | ... |
@@ -20,7 +20,7 @@ func (daemon *Daemon) getLayerRefs(platform string) map[layer.ChainID]int { |
20 | 20 |
layerRefs := map[layer.ChainID]int{} |
21 | 21 |
for id, img := range tmpImages { |
22 | 22 |
dgst := digest.Digest(id) |
23 |
- if len(daemon.stores[platform].referenceStore.References(dgst)) == 0 && len(daemon.stores[platform].imageStore.Children(id)) != 0 { |
|
23 |
+ if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.stores[platform].imageStore.Children(id)) != 0 { |
|
24 | 24 |
continue |
25 | 25 |
} |
26 | 26 |
|
... | ... |
@@ -2,6 +2,7 @@ package daemon |
2 | 2 |
|
3 | 3 |
import ( |
4 | 4 |
"fmt" |
5 |
+ "runtime" |
|
5 | 6 |
|
6 | 7 |
"github.com/docker/distribution/reference" |
7 | 8 |
"github.com/docker/docker/image" |
... | ... |
@@ -45,10 +46,17 @@ func (daemon *Daemon) GetImageIDAndPlatform(refOrID string) (image.ID, string, e |
45 | 45 |
return "", "", errImageDoesNotExist{ref} |
46 | 46 |
} |
47 | 47 |
|
48 |
- for platform := range daemon.stores { |
|
49 |
- if id, err := daemon.stores[platform].referenceStore.Get(namedRef); err == nil { |
|
50 |
- return image.IDFromDigest(id), platform, nil |
|
48 |
+ if digest, err := daemon.referenceStore.Get(namedRef); err == nil { |
|
49 |
+ // Search the image stores to get the platform, defaulting to host OS. |
|
50 |
+ imagePlatform := runtime.GOOS |
|
51 |
+ id := image.IDFromDigest(digest) |
|
52 |
+ for platform := range daemon.stores { |
|
53 |
+ if img, err := daemon.stores[platform].imageStore.Get(id); err == nil { |
|
54 |
+ imagePlatform = img.Platform() |
|
55 |
+ break |
|
56 |
+ } |
|
51 | 57 |
} |
58 |
+ return id, imagePlatform, nil |
|
52 | 59 |
} |
53 | 60 |
|
54 | 61 |
// deprecated: repo:shortid https://github.com/docker/docker/pull/799 |
... | ... |
@@ -56,7 +64,7 @@ func (daemon *Daemon) GetImageIDAndPlatform(refOrID string) (image.ID, string, e |
56 | 56 |
if tag := tagged.Tag(); stringid.IsShortID(stringid.TruncateID(tag)) { |
57 | 57 |
for platform := range daemon.stores { |
58 | 58 |
if id, err := daemon.stores[platform].imageStore.Search(tag); err == nil { |
59 |
- for _, storeRef := range daemon.stores[platform].referenceStore.References(id.Digest()) { |
|
59 |
+ for _, storeRef := range daemon.referenceStore.References(id.Digest()) { |
|
60 | 60 |
if storeRef.Name() == namedRef.Name() { |
61 | 61 |
return id, platform, nil |
62 | 62 |
} |
... | ... |
@@ -70,7 +70,7 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I |
70 | 70 |
return nil, err |
71 | 71 |
} |
72 | 72 |
|
73 |
- repoRefs := daemon.stores[platform].referenceStore.References(imgID.Digest()) |
|
73 |
+ repoRefs := daemon.referenceStore.References(imgID.Digest()) |
|
74 | 74 |
|
75 | 75 |
var removedRepositoryRef bool |
76 | 76 |
if !isImageIDPrefix(imgID.String(), imageRef) { |
... | ... |
@@ -104,7 +104,7 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I |
104 | 104 |
daemon.LogImageEvent(imgID.String(), imgID.String(), "untag") |
105 | 105 |
records = append(records, untaggedRecord) |
106 | 106 |
|
107 |
- repoRefs = daemon.stores[platform].referenceStore.References(imgID.Digest()) |
|
107 |
+ repoRefs = daemon.referenceStore.References(imgID.Digest()) |
|
108 | 108 |
|
109 | 109 |
// If a tag reference was removed and the only remaining |
110 | 110 |
// references to the same repository are digest references, |
... | ... |
@@ -237,7 +237,7 @@ func (daemon *Daemon) removeImageRef(platform string, ref reference.Named) (refe |
237 | 237 |
// Ignore the boolean value returned, as far as we're concerned, this |
238 | 238 |
// is an idempotent operation and it's okay if the reference didn't |
239 | 239 |
// exist in the first place. |
240 |
- _, err := daemon.stores[platform].referenceStore.Delete(ref) |
|
240 |
+ _, err := daemon.referenceStore.Delete(ref) |
|
241 | 241 |
|
242 | 242 |
return ref, err |
243 | 243 |
} |
... | ... |
@@ -248,7 +248,7 @@ func (daemon *Daemon) removeImageRef(platform string, ref reference.Named) (refe |
248 | 248 |
// daemon's event service. An "Untagged" types.ImageDeleteResponseItem is added to the |
249 | 249 |
// given list of records. |
250 | 250 |
func (daemon *Daemon) removeAllReferencesToImageID(imgID image.ID, platform string, records *[]types.ImageDeleteResponseItem) error { |
251 |
- imageRefs := daemon.stores[platform].referenceStore.References(imgID.Digest()) |
|
251 |
+ imageRefs := daemon.referenceStore.References(imgID.Digest()) |
|
252 | 252 |
|
253 | 253 |
for _, imageRef := range imageRefs { |
254 | 254 |
parsedRef, err := daemon.removeImageRef(platform, imageRef) |
... | ... |
@@ -383,7 +383,7 @@ func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, platform string, |
383 | 383 |
} |
384 | 384 |
|
385 | 385 |
// Check if any repository tags/digest reference this image. |
386 |
- if mask&conflictActiveReference != 0 && len(daemon.stores[platform].referenceStore.References(imgID.Digest())) > 0 { |
|
386 |
+ if mask&conflictActiveReference != 0 && len(daemon.referenceStore.References(imgID.Digest())) > 0 { |
|
387 | 387 |
return &imageDeleteConflict{ |
388 | 388 |
imgID: imgID, |
389 | 389 |
message: "image is referenced in multiple repositories", |
... | ... |
@@ -411,5 +411,5 @@ func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, platform string, |
411 | 411 |
// that there are no repository references to the given image and it has no |
412 | 412 |
// child images. |
413 | 413 |
func (daemon *Daemon) imageIsDangling(imgID image.ID, platform string) bool { |
414 |
- return !(len(daemon.stores[platform].referenceStore.References(imgID.Digest())) > 0 || len(daemon.stores[platform].imageStore.Children(imgID)) > 0) |
|
414 |
+ return !(len(daemon.referenceStore.References(imgID.Digest())) > 0 || len(daemon.stores[platform].imageStore.Children(imgID)) > 0) |
|
415 | 415 |
} |
... | ... |
@@ -19,7 +19,7 @@ func (daemon *Daemon) ExportImage(names []string, outStream io.Writer) error { |
19 | 19 |
if system.LCOWSupported() { |
20 | 20 |
platform = "linux" |
21 | 21 |
} |
22 |
- imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.stores[platform].referenceStore, daemon) |
|
22 |
+ imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.referenceStore, daemon) |
|
23 | 23 |
return imageExporter.Save(names, outStream) |
24 | 24 |
} |
25 | 25 |
|
... | ... |
@@ -32,6 +32,6 @@ func (daemon *Daemon) LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet |
32 | 32 |
if system.LCOWSupported() { |
33 | 33 |
platform = "linux" |
34 | 34 |
} |
35 |
- imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.stores[platform].referenceStore, daemon) |
|
35 |
+ imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.referenceStore, daemon) |
|
36 | 36 |
return imageExporter.Load(inTar, outStream, quiet) |
37 | 37 |
} |
... | ... |
@@ -69,7 +69,7 @@ func (daemon *Daemon) ImageHistory(name string) ([]*image.HistoryResponseItem, e |
69 | 69 |
h.ID = id.String() |
70 | 70 |
|
71 | 71 |
var tags []string |
72 |
- for _, r := range daemon.stores[platform].referenceStore.References(id.Digest()) { |
|
72 |
+ for _, r := range daemon.referenceStore.References(id.Digest()) { |
|
73 | 73 |
if _, ok := r.(reference.NamedTagged); ok { |
74 | 74 |
tags = append(tags, reference.FamiliarString(r)) |
75 | 75 |
} |
... | ... |
@@ -24,7 +24,7 @@ func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) { |
24 | 24 |
platform = runtime.GOOS |
25 | 25 |
} |
26 | 26 |
|
27 |
- refs := daemon.stores[platform].referenceStore.References(img.ID().Digest()) |
|
27 |
+ refs := daemon.referenceStore.References(img.ID().Digest()) |
|
28 | 28 |
repoTags := []string{} |
29 | 29 |
repoDigests := []string{} |
30 | 30 |
for _, ref := range refs { |
... | ... |
@@ -74,7 +74,7 @@ func (daemon *Daemon) pullImageWithReference(ctx context.Context, ref reference. |
74 | 74 |
ImageEventLogger: daemon.LogImageEvent, |
75 | 75 |
MetadataStore: daemon.stores[platform].distributionMetadataStore, |
76 | 76 |
ImageStore: distribution.NewImageConfigStoreFromStore(daemon.stores[platform].imageStore), |
77 |
- ReferenceStore: daemon.stores[platform].referenceStore, |
|
77 |
+ ReferenceStore: daemon.referenceStore, |
|
78 | 78 |
}, |
79 | 79 |
DownloadManager: daemon.downloadManager, |
80 | 80 |
Schema2Types: distribution.ImageTypes, |
... | ... |
@@ -56,7 +56,7 @@ func (daemon *Daemon) PushImage(ctx context.Context, image, tag string, metaHead |
56 | 56 |
ImageEventLogger: daemon.LogImageEvent, |
57 | 57 |
MetadataStore: daemon.stores[platform].distributionMetadataStore, |
58 | 58 |
ImageStore: distribution.NewImageConfigStoreFromStore(daemon.stores[platform].imageStore), |
59 |
- ReferenceStore: daemon.stores[platform].referenceStore, |
|
59 |
+ ReferenceStore: daemon.referenceStore, |
|
60 | 60 |
}, |
61 | 61 |
ConfigMediaType: schema2.MediaTypeImageConfig, |
62 | 62 |
LayerStore: distribution.NewLayerProviderFromStore(daemon.stores[platform].layerStore), |
... | ... |
@@ -28,7 +28,7 @@ func (daemon *Daemon) TagImage(imageName, repository, tag string) error { |
28 | 28 |
|
29 | 29 |
// TagImageWithReference adds the given reference to the image ID provided. |
30 | 30 |
func (daemon *Daemon) TagImageWithReference(imageID image.ID, platform string, newTag reference.Named) error { |
31 |
- if err := daemon.stores[platform].referenceStore.AddTag(newTag, imageID.Digest(), true); err != nil { |
|
31 |
+ if err := daemon.referenceStore.AddTag(newTag, imageID.Digest(), true); err != nil { |
|
32 | 32 |
return err |
33 | 33 |
} |
34 | 34 |
|
... | ... |
@@ -149,7 +149,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs |
149 | 149 |
|
150 | 150 |
newImage := newImage(img, size) |
151 | 151 |
|
152 |
- for _, ref := range daemon.stores[platform].referenceStore.References(id.Digest()) { |
|
152 |
+ for _, ref := range daemon.referenceStore.References(id.Digest()) { |
|
153 | 153 |
if imageFilters.Include("reference") { |
154 | 154 |
var found bool |
155 | 155 |
var matchErr error |
... | ... |
@@ -221,7 +221,7 @@ func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args |
221 | 221 |
return nil, ctx.Err() |
222 | 222 |
default: |
223 | 223 |
dgst := digest.Digest(id) |
224 |
- if len(daemon.stores[platform].referenceStore.References(dgst)) == 0 && len(daemon.stores[platform].imageStore.Children(id)) != 0 { |
|
224 |
+ if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.stores[platform].imageStore.Children(id)) != 0 { |
|
225 | 225 |
continue |
226 | 226 |
} |
227 | 227 |
if !until.IsZero() && img.Created.After(until) { |
... | ... |
@@ -252,7 +252,7 @@ deleteImagesLoop: |
252 | 252 |
} |
253 | 253 |
|
254 | 254 |
deletedImages := []types.ImageDeleteResponseItem{} |
255 |
- refs := daemon.stores[platform].referenceStore.References(dgst) |
|
255 |
+ refs := daemon.referenceStore.References(dgst) |
|
256 | 256 |
if len(refs) > 0 { |
257 | 257 |
shouldDelete := !danglingOnly |
258 | 258 |
if !shouldDelete { |
... | ... |
@@ -26,7 +26,7 @@ type Association struct { |
26 | 26 |
ID digest.Digest |
27 | 27 |
} |
28 | 28 |
|
29 |
-// Store provides the set of methods which can operate on a tag store. |
|
29 |
+// Store provides the set of methods which can operate on a reference store. |
|
30 | 30 |
type Store interface { |
31 | 31 |
References(id digest.Digest) []reference.Named |
32 | 32 |
ReferencesByName(ref reference.Named) []Association |
... | ... |
@@ -46,9 +46,6 @@ type store struct { |
46 | 46 |
// referencesByIDCache is a cache of references indexed by ID, to speed |
47 | 47 |
// up References. |
48 | 48 |
referencesByIDCache map[digest.Digest]map[string]reference.Named |
49 |
- // platform is the container target platform for this store (which may be |
|
50 |
- // different to the host operating system |
|
51 |
- platform string |
|
52 | 49 |
} |
53 | 50 |
|
54 | 51 |
// Repository maps tags to digests. The key is a stringified Reference, |
... | ... |
@@ -73,7 +70,7 @@ func (a lexicalAssociations) Less(i, j int) bool { |
73 | 73 |
|
74 | 74 |
// NewReferenceStore creates a new reference store, tied to a file path where |
75 | 75 |
// the set of references are serialized in JSON format. |
76 |
-func NewReferenceStore(jsonPath, platform string) (Store, error) { |
|
76 |
+func NewReferenceStore(jsonPath string) (Store, error) { |
|
77 | 77 |
abspath, err := filepath.Abs(jsonPath) |
78 | 78 |
if err != nil { |
79 | 79 |
return nil, err |
... | ... |
@@ -83,7 +80,6 @@ func NewReferenceStore(jsonPath, platform string) (Store, error) { |
83 | 83 |
jsonPath: abspath, |
84 | 84 |
Repositories: make(map[string]repository), |
85 | 85 |
referencesByIDCache: make(map[digest.Digest]map[string]reference.Named), |
86 |
- platform: platform, |
|
87 | 86 |
} |
88 | 87 |
// Load the json file if it exists, otherwise create it. |
89 | 88 |
if err := store.reload(); os.IsNotExist(err) { |
... | ... |
@@ -5,7 +5,6 @@ import ( |
5 | 5 |
"io/ioutil" |
6 | 6 |
"os" |
7 | 7 |
"path/filepath" |
8 |
- "runtime" |
|
9 | 8 |
"strings" |
10 | 9 |
"testing" |
11 | 10 |
|
... | ... |
@@ -41,7 +40,7 @@ func TestLoad(t *testing.T) { |
41 | 41 |
} |
42 | 42 |
jsonFile.Close() |
43 | 43 |
|
44 |
- store, err := NewReferenceStore(jsonFile.Name(), runtime.GOOS) |
|
44 |
+ store, err := NewReferenceStore(jsonFile.Name()) |
|
45 | 45 |
if err != nil { |
46 | 46 |
t.Fatalf("error creating tag store: %v", err) |
47 | 47 |
} |
... | ... |
@@ -70,7 +69,7 @@ func TestSave(t *testing.T) { |
70 | 70 |
jsonFile.Close() |
71 | 71 |
defer os.RemoveAll(jsonFile.Name()) |
72 | 72 |
|
73 |
- store, err := NewReferenceStore(jsonFile.Name(), runtime.GOOS) |
|
73 |
+ store, err := NewReferenceStore(jsonFile.Name()) |
|
74 | 74 |
if err != nil { |
75 | 75 |
t.Fatalf("error creating tag store: %v", err) |
76 | 76 |
} |
... | ... |
@@ -112,7 +111,7 @@ func TestAddDeleteGet(t *testing.T) { |
112 | 112 |
jsonFile.Close() |
113 | 113 |
defer os.RemoveAll(jsonFile.Name()) |
114 | 114 |
|
115 |
- store, err := NewReferenceStore(jsonFile.Name(), runtime.GOOS) |
|
115 |
+ store, err := NewReferenceStore(jsonFile.Name()) |
|
116 | 116 |
if err != nil { |
117 | 117 |
t.Fatalf("error creating tag store: %v", err) |
118 | 118 |
} |
... | ... |
@@ -329,7 +328,7 @@ func TestInvalidTags(t *testing.T) { |
329 | 329 |
tmpDir, err := ioutil.TempDir("", "tag-store-test") |
330 | 330 |
defer os.RemoveAll(tmpDir) |
331 | 331 |
|
332 |
- store, err := NewReferenceStore(filepath.Join(tmpDir, "repositories.json"), runtime.GOOS) |
|
332 |
+ store, err := NewReferenceStore(filepath.Join(tmpDir, "repositories.json")) |
|
333 | 333 |
if err != nil { |
334 | 334 |
t.Fatalf("error creating tag store: %v", err) |
335 | 335 |
} |