devmapper: Save and restore NextDeviceId in a file
The way thin-pool right now is designed, user space is supposed to keep
track of what device ids have already been used. If user space tries to
create a new thin/snap device and device id has already been used, thin
pool retuns -EEXIST.
Upon receiving -EEXIST, current docker implementation simply tries the
NextDeviceId++ and keeps on doing this till it finds a free device id.
This approach has two issues.
- It is little suboptimal.
- If device id already exists, current kenrel implementation spits out
a messsage on console.
[17991.140135] device-mapper: thin: Creation of new snapshot 33 of device 3 failed.
Here kenrel is trying to tell user that device id 33 has already been used.
And this shows up for every device id docker tries till it reaches a point
where device ids are not used. So if there are thousands of container and
one is trying to create a new container after fresh docker start, expect
thousands of such warnings to flood console.
This patch saves the NextDeviceId in a file in
/var/lib/docker/devmapper/metadata/deviceset-metadata and reads it back
when docker starts. This way we don't retry lots of device ids which
have already been used.
There might be some device ids which are free but we will get back to them
once device numbers wrap around (24bit limit on device ids).
This patch should cut down on number of kernel warnings.
Notice that I am creating a deviceset metadata file which is a global file
for this pool. So down the line if we need to save more data we should be
able to do that.
Signed-off-by: Vivek Goyal <vgoyal@redhat.com>
Vivek Goyal authored on 2014/11/05 23:25:02Showing 1 changed files
| ... | ... |
@@ -62,25 +62,25 @@ type MetaData struct {
|
| 62 | 62 |
} |
| 63 | 63 |
|
| 64 | 64 |
type DeviceSet struct {
|
| 65 |
- MetaData |
|
| 66 |
- sync.Mutex // Protects Devices map and serializes calls into libdevmapper |
|
| 67 |
- root string |
|
| 68 |
- devicePrefix string |
|
| 69 |
- TransactionId uint64 |
|
| 70 |
- NewTransactionId uint64 |
|
| 71 |
- NextDeviceId int |
|
| 65 |
+ MetaData `json:"-"` |
|
| 66 |
+ sync.Mutex `json:"-"` // Protects Devices map and serializes calls into libdevmapper |
|
| 67 |
+ root string `json:"-"` |
|
| 68 |
+ devicePrefix string `json:"-"` |
|
| 69 |
+ TransactionId uint64 `json:"-"` |
|
| 70 |
+ NewTransactionId uint64 `json:"-"` |
|
| 71 |
+ NextDeviceId int `json:"next_device_id"` |
|
| 72 | 72 |
|
| 73 | 73 |
// Options |
| 74 |
- dataLoopbackSize int64 |
|
| 75 |
- metaDataLoopbackSize int64 |
|
| 76 |
- baseFsSize uint64 |
|
| 77 |
- filesystem string |
|
| 78 |
- mountOptions string |
|
| 79 |
- mkfsArgs []string |
|
| 80 |
- dataDevice string |
|
| 81 |
- metadataDevice string |
|
| 82 |
- doBlkDiscard bool |
|
| 83 |
- thinpBlockSize uint32 |
|
| 74 |
+ dataLoopbackSize int64 `json:"-"` |
|
| 75 |
+ metaDataLoopbackSize int64 `json:"-"` |
|
| 76 |
+ baseFsSize uint64 `json:"-"` |
|
| 77 |
+ filesystem string `json:"-"` |
|
| 78 |
+ mountOptions string `json:"-"` |
|
| 79 |
+ mkfsArgs []string `json:"-"` |
|
| 80 |
+ dataDevice string `json:"-"` |
|
| 81 |
+ metadataDevice string `json:"-"` |
|
| 82 |
+ doBlkDiscard bool `json:"-"` |
|
| 83 |
+ thinpBlockSize uint32 `json:"-"` |
|
| 84 | 84 |
} |
| 85 | 85 |
|
| 86 | 86 |
type DiskUsage struct {
|
| ... | ... |
@@ -138,6 +138,10 @@ func (devices *DeviceSet) metadataFile(info *DevInfo) string {
|
| 138 | 138 |
return path.Join(devices.metadataDir(), file) |
| 139 | 139 |
} |
| 140 | 140 |
|
| 141 |
+func (devices *DeviceSet) deviceSetMetaFile() string {
|
|
| 142 |
+ return path.Join(devices.metadataDir(), "deviceset-metadata") |
|
| 143 |
+} |
|
| 144 |
+ |
|
| 141 | 145 |
func (devices *DeviceSet) oldMetadataFile() string {
|
| 142 | 146 |
return path.Join(devices.loopbackDir(), "json") |
| 143 | 147 |
} |
| ... | ... |
@@ -545,6 +549,34 @@ func (devices *DeviceSet) ResizePool(size int64) error {
|
| 545 | 545 |
return nil |
| 546 | 546 |
} |
| 547 | 547 |
|
| 548 |
+func (devices *DeviceSet) loadDeviceSetMetaData() error {
|
|
| 549 |
+ jsonData, err := ioutil.ReadFile(devices.deviceSetMetaFile()) |
|
| 550 |
+ if err != nil {
|
|
| 551 |
+ return nil |
|
| 552 |
+ } |
|
| 553 |
+ |
|
| 554 |
+ if err := json.Unmarshal(jsonData, devices); err != nil {
|
|
| 555 |
+ return nil |
|
| 556 |
+ } |
|
| 557 |
+ |
|
| 558 |
+ return nil |
|
| 559 |
+} |
|
| 560 |
+ |
|
| 561 |
+func (devices *DeviceSet) saveDeviceSetMetaData() error {
|
|
| 562 |
+ jsonData, err := json.Marshal(devices) |
|
| 563 |
+ |
|
| 564 |
+ if err != nil {
|
|
| 565 |
+ return fmt.Errorf("Error encoding metadata to json: %s", err)
|
|
| 566 |
+ } |
|
| 567 |
+ |
|
| 568 |
+ err = devices.writeMetaFile(jsonData, devices.deviceSetMetaFile()) |
|
| 569 |
+ if err != nil {
|
|
| 570 |
+ return err |
|
| 571 |
+ } |
|
| 572 |
+ |
|
| 573 |
+ return nil |
|
| 574 |
+} |
|
| 575 |
+ |
|
| 548 | 576 |
func (devices *DeviceSet) initDevmapper(doInit bool) error {
|
| 549 | 577 |
logInit(devices) |
| 550 | 578 |
|
| ... | ... |
@@ -676,6 +708,10 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error {
|
| 676 | 676 |
} |
| 677 | 677 |
} |
| 678 | 678 |
|
| 679 |
+ // Right now this loads only NextDeviceId. If there is more metatadata |
|
| 680 |
+ // down the line, we might have to move it earlier. |
|
| 681 |
+ devices.loadDeviceSetMetaData() |
|
| 682 |
+ |
|
| 679 | 683 |
// Setup the base image |
| 680 | 684 |
if doInit {
|
| 681 | 685 |
if err := devices.setupBaseImage(); err != nil {
|
| ... | ... |
@@ -955,6 +991,8 @@ func (devices *DeviceSet) Shutdown() error {
|
| 955 | 955 |
if err := devices.deactivatePool(); err != nil {
|
| 956 | 956 |
log.Debugf("Shutdown deactivate pool , error: %s", err)
|
| 957 | 957 |
} |
| 958 |
+ |
|
| 959 |
+ devices.saveDeviceSetMetaData() |
|
| 958 | 960 |
devices.Unlock() |
| 959 | 961 |
|
| 960 | 962 |
return nil |