layer: protect from same-name races
As pointed out by Tonis, there's a race between ReleaseRWLayer()
and GetRWLayer():
```
----- goroutine 1 ----- ----- goroutine 2 -----
ReleaseRWLayer()
m := ls.mounts[l.Name()]
...
m.deleteReference(l)
m.hasReferences()
... GetRWLayer()
... mount := ls.mounts[id]
ls.driver.Remove(m.mountID)
ls.store.RemoveMount(m.name) return mount.getReference()
delete(ls.mounts, m.Name())
----------------------- -----------------------
```
When something like this happens, GetRWLayer will return
an RWLayer without a storage. Oops.
There might be more races like this, and it seems the best
solution is to lock by layer id/name by using pkg/locker.
With this in place, name collision could not happen, so remove
the part of previous commit that protected against it in
CreateRWLayer (temporary nil assigmment and associated rollback).
So, now we have
* layerStore.mountL sync.Mutex to protect layerStore.mount map[]
(against concurrent access);
* mountedLayer's embedded `sync.Mutex` to protect its references map[];
* layerStore.layerL (which I haven't touched);
* per-id locker, to avoid name conflicts and concurrent operations
on the same rw layer.
The whole rig seems to look more readable now (mutexes use is
straightforward, no nested locks).
Reported-by: Tonis Tiigi <tonistiigi@gmail.com>
Signed-off-by: Kir Kolyshkin <kolyshkin@gmail.com>
Signed-off-by: Kir Kolyshkin <kolyshkin@gmail.com>
(cherry picked from commit af433dd200f8287305b1531d5058780be36b7e2e)
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
Kir Kolyshkin authored on 2019/05/21 06:46:54Showing 1 changed files
| ... | ... |
@@ -10,6 +10,7 @@ import ( |
| 10 | 10 |
"github.com/docker/distribution" |
| 11 | 11 |
"github.com/docker/docker/daemon/graphdriver" |
| 12 | 12 |
"github.com/docker/docker/pkg/idtools" |
| 13 |
+ "github.com/docker/docker/pkg/locker" |
|
| 13 | 14 |
"github.com/docker/docker/pkg/plugingetter" |
| 14 | 15 |
"github.com/docker/docker/pkg/stringid" |
| 15 | 16 |
"github.com/docker/docker/pkg/system" |
| ... | ... |
@@ -36,7 +37,11 @@ type layerStore struct {
|
| 36 | 36 |
|
| 37 | 37 |
mounts map[string]*mountedLayer |
| 38 | 38 |
mountL sync.Mutex |
| 39 |
- os string |
|
| 39 |
+ |
|
| 40 |
+ // protect *RWLayer() methods from operating on the same name/id |
|
| 41 |
+ locker *locker.Locker |
|
| 42 |
+ |
|
| 43 |
+ os string |
|
| 40 | 44 |
} |
| 41 | 45 |
|
| 42 | 46 |
// StoreOptions are the options used to create a new Store instance |
| ... | ... |
@@ -92,6 +97,7 @@ func newStoreFromGraphDriver(root string, driver graphdriver.Driver, os string) |
| 92 | 92 |
driver: driver, |
| 93 | 93 |
layerMap: map[ChainID]*roLayer{},
|
| 94 | 94 |
mounts: map[string]*mountedLayer{},
|
| 95 |
+ locker: locker.New(), |
|
| 95 | 96 |
useTarSplit: !caps.ReproducesExactDiffs, |
| 96 | 97 |
os: os, |
| 97 | 98 |
} |
| ... | ... |
@@ -191,7 +197,7 @@ func (ls *layerStore) loadLayer(layer ChainID) (*roLayer, error) {
|
| 191 | 191 |
func (ls *layerStore) loadMount(mount string) error {
|
| 192 | 192 |
ls.mountL.Lock() |
| 193 | 193 |
defer ls.mountL.Unlock() |
| 194 |
- if m := ls.mounts[mount]; m != nil {
|
|
| 194 |
+ if _, ok := ls.mounts[mount]; ok {
|
|
| 195 | 195 |
return nil |
| 196 | 196 |
} |
| 197 | 197 |
|
| ... | ... |
@@ -492,21 +498,15 @@ func (ls *layerStore) CreateRWLayer(name string, parent ChainID, opts *CreateRWL |
| 492 | 492 |
initFunc = opts.InitFunc |
| 493 | 493 |
} |
| 494 | 494 |
|
| 495 |
+ ls.locker.Lock(name) |
|
| 496 |
+ defer ls.locker.Unlock(name) |
|
| 497 |
+ |
|
| 495 | 498 |
ls.mountL.Lock() |
| 496 |
- if _, ok := ls.mounts[name]; ok {
|
|
| 497 |
- ls.mountL.Unlock() |
|
| 499 |
+ _, ok := ls.mounts[name] |
|
| 500 |
+ ls.mountL.Unlock() |
|
| 501 |
+ if ok {
|
|
| 498 | 502 |
return nil, ErrMountNameConflict |
| 499 | 503 |
} |
| 500 |
- // Avoid name collision by temporary assigning nil |
|
| 501 |
- ls.mounts[name] = nil |
|
| 502 |
- ls.mountL.Unlock() |
|
| 503 |
- defer func() {
|
|
| 504 |
- if err != nil {
|
|
| 505 |
- ls.mountL.Lock() |
|
| 506 |
- delete(ls.mounts, name) |
|
| 507 |
- ls.mountL.Unlock() |
|
| 508 |
- } |
|
| 509 |
- }() |
|
| 510 | 504 |
|
| 511 | 505 |
var pid string |
| 512 | 506 |
var p *roLayer |
| ... | ... |
@@ -558,6 +558,9 @@ func (ls *layerStore) CreateRWLayer(name string, parent ChainID, opts *CreateRWL |
| 558 | 558 |
} |
| 559 | 559 |
|
| 560 | 560 |
func (ls *layerStore) GetRWLayer(id string) (RWLayer, error) {
|
| 561 |
+ ls.locker.Lock(id) |
|
| 562 |
+ defer ls.locker.Unlock(id) |
|
| 563 |
+ |
|
| 561 | 564 |
ls.mountL.Lock() |
| 562 | 565 |
mount := ls.mounts[id] |
| 563 | 566 |
ls.mountL.Unlock() |
| ... | ... |
@@ -570,8 +573,9 @@ func (ls *layerStore) GetRWLayer(id string) (RWLayer, error) {
|
| 570 | 570 |
|
| 571 | 571 |
func (ls *layerStore) GetMountID(id string) (string, error) {
|
| 572 | 572 |
ls.mountL.Lock() |
| 573 |
- defer ls.mountL.Unlock() |
|
| 574 | 573 |
mount := ls.mounts[id] |
| 574 |
+ ls.mountL.Unlock() |
|
| 575 |
+ |
|
| 575 | 576 |
if mount == nil {
|
| 576 | 577 |
return "", ErrMountDoesNotExist |
| 577 | 578 |
} |
| ... | ... |
@@ -581,8 +585,12 @@ func (ls *layerStore) GetMountID(id string) (string, error) {
|
| 581 | 581 |
} |
| 582 | 582 |
|
| 583 | 583 |
func (ls *layerStore) ReleaseRWLayer(l RWLayer) ([]Metadata, error) {
|
| 584 |
+ name := l.Name() |
|
| 585 |
+ ls.locker.Lock(name) |
|
| 586 |
+ defer ls.locker.Unlock(name) |
|
| 587 |
+ |
|
| 584 | 588 |
ls.mountL.Lock() |
| 585 |
- m := ls.mounts[l.Name()] |
|
| 589 |
+ m := ls.mounts[name] |
|
| 586 | 590 |
ls.mountL.Unlock() |
| 587 | 591 |
if m == nil {
|
| 588 | 592 |
return []Metadata{}, nil
|
| ... | ... |
@@ -617,7 +625,7 @@ func (ls *layerStore) ReleaseRWLayer(l RWLayer) ([]Metadata, error) {
|
| 617 | 617 |
} |
| 618 | 618 |
|
| 619 | 619 |
ls.mountL.Lock() |
| 620 |
- delete(ls.mounts, m.Name()) |
|
| 620 |
+ delete(ls.mounts, name) |
|
| 621 | 621 |
ls.mountL.Unlock() |
| 622 | 622 |
|
| 623 | 623 |
ls.layerL.Lock() |