Browse code

Merge pull request #41330 from BtbN/vol-pquota

Add size option to volumes on linux/unix via xfs pquota

Sebastiaan van Stijn authored on 2020/10/08 00:50:39
Showing 20 changed files
... ...
@@ -13,8 +13,8 @@ import (
13 13
 	"unsafe"
14 14
 
15 15
 	"github.com/docker/docker/daemon/graphdriver"
16
-	"github.com/docker/docker/daemon/graphdriver/quota"
17 16
 	"github.com/docker/docker/pkg/stringid"
17
+	"github.com/docker/docker/quota"
18 18
 	units "github.com/docker/go-units"
19 19
 	"golang.org/x/sys/unix"
20 20
 	"gotest.tools/v3/assert"
... ...
@@ -18,7 +18,6 @@ import (
18 18
 	"github.com/containerd/containerd/sys"
19 19
 	"github.com/docker/docker/daemon/graphdriver"
20 20
 	"github.com/docker/docker/daemon/graphdriver/overlayutils"
21
-	"github.com/docker/docker/daemon/graphdriver/quota"
22 21
 	"github.com/docker/docker/pkg/archive"
23 22
 	"github.com/docker/docker/pkg/chrootarchive"
24 23
 	"github.com/docker/docker/pkg/containerfs"
... ...
@@ -27,6 +26,7 @@ import (
27 27
 	"github.com/docker/docker/pkg/idtools"
28 28
 	"github.com/docker/docker/pkg/parsers"
29 29
 	"github.com/docker/docker/pkg/system"
30
+	"github.com/docker/docker/quota"
30 31
 	units "github.com/docker/go-units"
31 32
 	"github.com/moby/locker"
32 33
 	"github.com/moby/sys/mount"
33 34
deleted file mode 100644
... ...
@@ -1,19 +0,0 @@
1
-package quota // import "github.com/docker/docker/daemon/graphdriver/quota"
2
-
3
-import "github.com/docker/docker/errdefs"
4
-
5
-var (
6
-	_ errdefs.ErrNotImplemented = (*errQuotaNotSupported)(nil)
7
-)
8
-
9
-// ErrQuotaNotSupported indicates if were found the FS didn't have projects quotas available
10
-var ErrQuotaNotSupported = errQuotaNotSupported{}
11
-
12
-type errQuotaNotSupported struct {
13
-}
14
-
15
-func (e errQuotaNotSupported) NotImplemented() {}
16
-
17
-func (e errQuotaNotSupported) Error() string {
18
-	return "Filesystem does not support, or has not enabled quotas"
19
-}
20 1
deleted file mode 100644
... ...
@@ -1,377 +0,0 @@
1
-// +build linux,!exclude_disk_quota,cgo
2
-
3
-//
4
-// projectquota.go - implements XFS project quota controls
5
-// for setting quota limits on a newly created directory.
6
-// It currently supports the legacy XFS specific ioctls.
7
-//
8
-// TODO: use generic quota control ioctl FS_IOC_FS{GET,SET}XATTR
9
-//       for both xfs/ext4 for kernel version >= v4.5
10
-//
11
-
12
-package quota // import "github.com/docker/docker/daemon/graphdriver/quota"
13
-
14
-/*
15
-#include <stdlib.h>
16
-#include <dirent.h>
17
-#include <linux/fs.h>
18
-#include <linux/quota.h>
19
-#include <linux/dqblk_xfs.h>
20
-
21
-#ifndef FS_XFLAG_PROJINHERIT
22
-struct fsxattr {
23
-	__u32		fsx_xflags;
24
-	__u32		fsx_extsize;
25
-	__u32		fsx_nextents;
26
-	__u32		fsx_projid;
27
-	unsigned char	fsx_pad[12];
28
-};
29
-#define FS_XFLAG_PROJINHERIT	0x00000200
30
-#endif
31
-#ifndef FS_IOC_FSGETXATTR
32
-#define FS_IOC_FSGETXATTR		_IOR ('X', 31, struct fsxattr)
33
-#endif
34
-#ifndef FS_IOC_FSSETXATTR
35
-#define FS_IOC_FSSETXATTR		_IOW ('X', 32, struct fsxattr)
36
-#endif
37
-
38
-#ifndef PRJQUOTA
39
-#define PRJQUOTA	2
40
-#endif
41
-#ifndef XFS_PROJ_QUOTA
42
-#define XFS_PROJ_QUOTA	2
43
-#endif
44
-#ifndef Q_XSETPQLIM
45
-#define Q_XSETPQLIM QCMD(Q_XSETQLIM, PRJQUOTA)
46
-#endif
47
-#ifndef Q_XGETPQUOTA
48
-#define Q_XGETPQUOTA QCMD(Q_XGETQUOTA, PRJQUOTA)
49
-#endif
50
-
51
-const int Q_XGETQSTAT_PRJQUOTA = QCMD(Q_XGETQSTAT, PRJQUOTA);
52
-*/
53
-import "C"
54
-import (
55
-	"io/ioutil"
56
-	"path"
57
-	"path/filepath"
58
-	"unsafe"
59
-
60
-	"github.com/containerd/containerd/sys"
61
-	"github.com/pkg/errors"
62
-	"github.com/sirupsen/logrus"
63
-	"golang.org/x/sys/unix"
64
-)
65
-
66
-// NewControl - initialize project quota support.
67
-// Test to make sure that quota can be set on a test dir and find
68
-// the first project id to be used for the next container create.
69
-//
70
-// Returns nil (and error) if project quota is not supported.
71
-//
72
-// First get the project id of the home directory.
73
-// This test will fail if the backing fs is not xfs.
74
-//
75
-// xfs_quota tool can be used to assign a project id to the driver home directory, e.g.:
76
-//    echo 999:/var/lib/docker/overlay2 >> /etc/projects
77
-//    echo docker:999 >> /etc/projid
78
-//    xfs_quota -x -c 'project -s docker' /<xfs mount point>
79
-//
80
-// In that case, the home directory project id will be used as a "start offset"
81
-// and all containers will be assigned larger project ids (e.g. >= 1000).
82
-// This is a way to prevent xfs_quota management from conflicting with docker.
83
-//
84
-// Then try to create a test directory with the next project id and set a quota
85
-// on it. If that works, continue to scan existing containers to map allocated
86
-// project ids.
87
-//
88
-func NewControl(basePath string) (*Control, error) {
89
-	//
90
-	// If we are running in a user namespace quota won't be supported for
91
-	// now since makeBackingFsDev() will try to mknod().
92
-	//
93
-	if sys.RunningInUserNS() {
94
-		return nil, ErrQuotaNotSupported
95
-	}
96
-
97
-	//
98
-	// create backing filesystem device node
99
-	//
100
-	backingFsBlockDev, err := makeBackingFsDev(basePath)
101
-	if err != nil {
102
-		return nil, err
103
-	}
104
-
105
-	// check if we can call quotactl with project quotas
106
-	// as a mechanism to determine (early) if we have support
107
-	hasQuotaSupport, err := hasQuotaSupport(backingFsBlockDev)
108
-	if err != nil {
109
-		return nil, err
110
-	}
111
-	if !hasQuotaSupport {
112
-		return nil, ErrQuotaNotSupported
113
-	}
114
-
115
-	//
116
-	// Get project id of parent dir as minimal id to be used by driver
117
-	//
118
-	minProjectID, err := getProjectID(basePath)
119
-	if err != nil {
120
-		return nil, err
121
-	}
122
-	minProjectID++
123
-
124
-	//
125
-	// Test if filesystem supports project quotas by trying to set
126
-	// a quota on the first available project id
127
-	//
128
-	quota := Quota{
129
-		Size: 0,
130
-	}
131
-	if err := setProjectQuota(backingFsBlockDev, minProjectID, quota); err != nil {
132
-		return nil, err
133
-	}
134
-
135
-	q := Control{
136
-		backingFsBlockDev: backingFsBlockDev,
137
-		nextProjectID:     minProjectID + 1,
138
-		quotas:            make(map[string]uint32),
139
-	}
140
-
141
-	//
142
-	// get first project id to be used for next container
143
-	//
144
-	err = q.findNextProjectID(basePath)
145
-	if err != nil {
146
-		return nil, err
147
-	}
148
-
149
-	logrus.Debugf("NewControl(%s): nextProjectID = %d", basePath, q.nextProjectID)
150
-	return &q, nil
151
-}
152
-
153
-// SetQuota - assign a unique project id to directory and set the quota limits
154
-// for that project id
155
-func (q *Control) SetQuota(targetPath string, quota Quota) error {
156
-	q.RLock()
157
-	projectID, ok := q.quotas[targetPath]
158
-	q.RUnlock()
159
-	if !ok {
160
-		q.Lock()
161
-		projectID = q.nextProjectID
162
-
163
-		//
164
-		// assign project id to new container directory
165
-		//
166
-		err := setProjectID(targetPath, projectID)
167
-		if err != nil {
168
-			q.Unlock()
169
-			return err
170
-		}
171
-		q.quotas[targetPath] = projectID
172
-		q.nextProjectID++
173
-		q.Unlock()
174
-	}
175
-
176
-	//
177
-	// set the quota limit for the container's project id
178
-	//
179
-	logrus.Debugf("SetQuota(%s, %d): projectID=%d", targetPath, quota.Size, projectID)
180
-	return setProjectQuota(q.backingFsBlockDev, projectID, quota)
181
-}
182
-
183
-// setProjectQuota - set the quota for project id on xfs block device
184
-func setProjectQuota(backingFsBlockDev string, projectID uint32, quota Quota) error {
185
-	var d C.fs_disk_quota_t
186
-	d.d_version = C.FS_DQUOT_VERSION
187
-	d.d_id = C.__u32(projectID)
188
-	d.d_flags = C.XFS_PROJ_QUOTA
189
-
190
-	d.d_fieldmask = C.FS_DQ_BHARD | C.FS_DQ_BSOFT
191
-	d.d_blk_hardlimit = C.__u64(quota.Size / 512)
192
-	d.d_blk_softlimit = d.d_blk_hardlimit
193
-
194
-	var cs = C.CString(backingFsBlockDev)
195
-	defer C.free(unsafe.Pointer(cs))
196
-
197
-	_, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XSETPQLIM,
198
-		uintptr(unsafe.Pointer(cs)), uintptr(d.d_id),
199
-		uintptr(unsafe.Pointer(&d)), 0, 0)
200
-	if errno != 0 {
201
-		return errors.Wrapf(errno, "failed to set quota limit for projid %d on %s",
202
-			projectID, backingFsBlockDev)
203
-	}
204
-
205
-	return nil
206
-}
207
-
208
-// GetQuota - get the quota limits of a directory that was configured with SetQuota
209
-func (q *Control) GetQuota(targetPath string, quota *Quota) error {
210
-	q.RLock()
211
-	projectID, ok := q.quotas[targetPath]
212
-	q.RUnlock()
213
-	if !ok {
214
-		return errors.Errorf("quota not found for path: %s", targetPath)
215
-	}
216
-
217
-	//
218
-	// get the quota limit for the container's project id
219
-	//
220
-	var d C.fs_disk_quota_t
221
-
222
-	var cs = C.CString(q.backingFsBlockDev)
223
-	defer C.free(unsafe.Pointer(cs))
224
-
225
-	_, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XGETPQUOTA,
226
-		uintptr(unsafe.Pointer(cs)), uintptr(C.__u32(projectID)),
227
-		uintptr(unsafe.Pointer(&d)), 0, 0)
228
-	if errno != 0 {
229
-		return errors.Wrapf(errno, "Failed to get quota limit for projid %d on %s",
230
-			projectID, q.backingFsBlockDev)
231
-	}
232
-	quota.Size = uint64(d.d_blk_hardlimit) * 512
233
-
234
-	return nil
235
-}
236
-
237
-// getProjectID - get the project id of path on xfs
238
-func getProjectID(targetPath string) (uint32, error) {
239
-	dir, err := openDir(targetPath)
240
-	if err != nil {
241
-		return 0, err
242
-	}
243
-	defer closeDir(dir)
244
-
245
-	var fsx C.struct_fsxattr
246
-	_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR,
247
-		uintptr(unsafe.Pointer(&fsx)))
248
-	if errno != 0 {
249
-		return 0, errors.Wrapf(errno, "failed to get projid for %s", targetPath)
250
-	}
251
-
252
-	return uint32(fsx.fsx_projid), nil
253
-}
254
-
255
-// setProjectID - set the project id of path on xfs
256
-func setProjectID(targetPath string, projectID uint32) error {
257
-	dir, err := openDir(targetPath)
258
-	if err != nil {
259
-		return err
260
-	}
261
-	defer closeDir(dir)
262
-
263
-	var fsx C.struct_fsxattr
264
-	_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR,
265
-		uintptr(unsafe.Pointer(&fsx)))
266
-	if errno != 0 {
267
-		return errors.Wrapf(errno, "failed to get projid for %s", targetPath)
268
-	}
269
-	fsx.fsx_projid = C.__u32(projectID)
270
-	fsx.fsx_xflags |= C.FS_XFLAG_PROJINHERIT
271
-	_, _, errno = unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSSETXATTR,
272
-		uintptr(unsafe.Pointer(&fsx)))
273
-	if errno != 0 {
274
-		return errors.Wrapf(errno, "failed to set projid for %s", targetPath)
275
-	}
276
-
277
-	return nil
278
-}
279
-
280
-// findNextProjectID - find the next project id to be used for containers
281
-// by scanning driver home directory to find used project ids
282
-func (q *Control) findNextProjectID(home string) error {
283
-	q.Lock()
284
-	defer q.Unlock()
285
-	files, err := ioutil.ReadDir(home)
286
-	if err != nil {
287
-		return errors.Errorf("read directory failed: %s", home)
288
-	}
289
-	for _, file := range files {
290
-		if !file.IsDir() {
291
-			continue
292
-		}
293
-		path := filepath.Join(home, file.Name())
294
-		projid, err := getProjectID(path)
295
-		if err != nil {
296
-			return err
297
-		}
298
-		if projid > 0 {
299
-			q.quotas[path] = projid
300
-		}
301
-		if q.nextProjectID <= projid {
302
-			q.nextProjectID = projid + 1
303
-		}
304
-	}
305
-
306
-	return nil
307
-}
308
-
309
-func free(p *C.char) {
310
-	C.free(unsafe.Pointer(p))
311
-}
312
-
313
-func openDir(path string) (*C.DIR, error) {
314
-	Cpath := C.CString(path)
315
-	defer free(Cpath)
316
-
317
-	dir := C.opendir(Cpath)
318
-	if dir == nil {
319
-		return nil, errors.Errorf("failed to open dir: %s", path)
320
-	}
321
-	return dir, nil
322
-}
323
-
324
-func closeDir(dir *C.DIR) {
325
-	if dir != nil {
326
-		C.closedir(dir)
327
-	}
328
-}
329
-
330
-func getDirFd(dir *C.DIR) uintptr {
331
-	return uintptr(C.dirfd(dir))
332
-}
333
-
334
-// Get the backing block device of the driver home directory
335
-// and create a block device node under the home directory
336
-// to be used by quotactl commands
337
-func makeBackingFsDev(home string) (string, error) {
338
-	var stat unix.Stat_t
339
-	if err := unix.Stat(home, &stat); err != nil {
340
-		return "", err
341
-	}
342
-
343
-	backingFsBlockDev := path.Join(home, "backingFsBlockDev")
344
-	// Re-create just in case someone copied the home directory over to a new device
345
-	unix.Unlink(backingFsBlockDev)
346
-	err := unix.Mknod(backingFsBlockDev, unix.S_IFBLK|0600, int(stat.Dev))
347
-	switch err {
348
-	case nil:
349
-		return backingFsBlockDev, nil
350
-
351
-	case unix.ENOSYS, unix.EPERM:
352
-		return "", ErrQuotaNotSupported
353
-
354
-	default:
355
-		return "", errors.Wrapf(err, "failed to mknod %s", backingFsBlockDev)
356
-	}
357
-}
358
-
359
-func hasQuotaSupport(backingFsBlockDev string) (bool, error) {
360
-	var cs = C.CString(backingFsBlockDev)
361
-	defer free(cs)
362
-	var qstat C.fs_quota_stat_t
363
-
364
-	_, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, uintptr(C.Q_XGETQSTAT_PRJQUOTA), uintptr(unsafe.Pointer(cs)), 0, uintptr(unsafe.Pointer(&qstat)), 0, 0)
365
-	if errno == 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ENFD > 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ACCT > 0 {
366
-		return true, nil
367
-	}
368
-
369
-	switch errno {
370
-	// These are the known fatal errors, consider all other errors (ENOTTY, etc.. not supporting quota)
371
-	case unix.EFAULT, unix.ENOENT, unix.ENOTBLK, unix.EPERM:
372
-	default:
373
-		return false, nil
374
-	}
375
-
376
-	return false, errno
377
-}
378 1
deleted file mode 100644
... ...
@@ -1,152 +0,0 @@
1
-// +build linux
2
-
3
-package quota // import "github.com/docker/docker/daemon/graphdriver/quota"
4
-
5
-import (
6
-	"io"
7
-	"io/ioutil"
8
-	"os"
9
-	"os/exec"
10
-	"path/filepath"
11
-	"testing"
12
-
13
-	"golang.org/x/sys/unix"
14
-	"gotest.tools/v3/assert"
15
-	is "gotest.tools/v3/assert/cmp"
16
-	"gotest.tools/v3/fs"
17
-)
18
-
19
-// 10MB
20
-const testQuotaSize = 10 * 1024 * 1024
21
-const imageSize = 64 * 1024 * 1024
22
-
23
-func TestBlockDev(t *testing.T) {
24
-	mkfs, err := exec.LookPath("mkfs.xfs")
25
-	if err != nil {
26
-		t.Skip("mkfs.xfs not found in PATH")
27
-	}
28
-
29
-	// create a sparse image
30
-	imageFile, err := ioutil.TempFile("", "xfs-image")
31
-	if err != nil {
32
-		t.Fatal(err)
33
-	}
34
-	imageFileName := imageFile.Name()
35
-	defer os.Remove(imageFileName)
36
-	if _, err = imageFile.Seek(imageSize-1, 0); err != nil {
37
-		t.Fatal(err)
38
-	}
39
-	if _, err = imageFile.Write([]byte{0}); err != nil {
40
-		t.Fatal(err)
41
-	}
42
-	if err = imageFile.Close(); err != nil {
43
-		t.Fatal(err)
44
-	}
45
-
46
-	// The reason for disabling these options is sometimes people run with a newer userspace
47
-	// than kernelspace
48
-	out, err := exec.Command(mkfs, "-m", "crc=0,finobt=0", imageFileName).CombinedOutput()
49
-	if len(out) > 0 {
50
-		t.Log(string(out))
51
-	}
52
-	if err != nil {
53
-		t.Fatal(err)
54
-	}
55
-
56
-	t.Run("testBlockDevQuotaDisabled", wrapMountTest(imageFileName, false, testBlockDevQuotaDisabled))
57
-	t.Run("testBlockDevQuotaEnabled", wrapMountTest(imageFileName, true, testBlockDevQuotaEnabled))
58
-	t.Run("testSmallerThanQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testSmallerThanQuota)))
59
-	t.Run("testBiggerThanQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testBiggerThanQuota)))
60
-	t.Run("testRetrieveQuota", wrapMountTest(imageFileName, true, wrapQuotaTest(testRetrieveQuota)))
61
-}
62
-
63
-func wrapMountTest(imageFileName string, enableQuota bool, testFunc func(t *testing.T, mountPoint, backingFsDev string)) func(*testing.T) {
64
-	return func(t *testing.T) {
65
-		mountOptions := "loop"
66
-
67
-		if enableQuota {
68
-			mountOptions = mountOptions + ",prjquota"
69
-		}
70
-
71
-		mountPointDir := fs.NewDir(t, "xfs-mountPoint")
72
-		defer mountPointDir.Remove()
73
-		mountPoint := mountPointDir.Path()
74
-
75
-		out, err := exec.Command("mount", "-o", mountOptions, imageFileName, mountPoint).CombinedOutput()
76
-		if err != nil {
77
-			_, err := os.Stat("/proc/fs/xfs")
78
-			if os.IsNotExist(err) {
79
-				t.Skip("no /proc/fs/xfs")
80
-			}
81
-		}
82
-
83
-		assert.NilError(t, err, "mount failed: %s", out)
84
-
85
-		defer func() {
86
-			assert.NilError(t, unix.Unmount(mountPoint, 0))
87
-		}()
88
-
89
-		backingFsDev, err := makeBackingFsDev(mountPoint)
90
-		assert.NilError(t, err)
91
-
92
-		testFunc(t, mountPoint, backingFsDev)
93
-	}
94
-}
95
-
96
-func testBlockDevQuotaDisabled(t *testing.T, mountPoint, backingFsDev string) {
97
-	hasSupport, err := hasQuotaSupport(backingFsDev)
98
-	assert.NilError(t, err)
99
-	assert.Check(t, !hasSupport)
100
-}
101
-
102
-func testBlockDevQuotaEnabled(t *testing.T, mountPoint, backingFsDev string) {
103
-	hasSupport, err := hasQuotaSupport(backingFsDev)
104
-	assert.NilError(t, err)
105
-	assert.Check(t, hasSupport)
106
-}
107
-
108
-func wrapQuotaTest(testFunc func(t *testing.T, ctrl *Control, mountPoint, testDir, testSubDir string)) func(t *testing.T, mountPoint, backingFsDev string) {
109
-	return func(t *testing.T, mountPoint, backingFsDev string) {
110
-		testDir, err := ioutil.TempDir(mountPoint, "per-test")
111
-		assert.NilError(t, err)
112
-		defer os.RemoveAll(testDir)
113
-
114
-		ctrl, err := NewControl(testDir)
115
-		assert.NilError(t, err)
116
-
117
-		testSubDir, err := ioutil.TempDir(testDir, "quota-test")
118
-		assert.NilError(t, err)
119
-		testFunc(t, ctrl, mountPoint, testDir, testSubDir)
120
-	}
121
-
122
-}
123
-
124
-func testSmallerThanQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) {
125
-	assert.NilError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize}))
126
-	smallerThanQuotaFile := filepath.Join(testSubDir, "smaller-than-quota")
127
-	assert.NilError(t, ioutil.WriteFile(smallerThanQuotaFile, make([]byte, testQuotaSize/2), 0644))
128
-	assert.NilError(t, os.Remove(smallerThanQuotaFile))
129
-}
130
-
131
-func testBiggerThanQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) {
132
-	// Make sure the quota is being enforced
133
-	// TODO: When we implement this under EXT4, we need to shed CAP_SYS_RESOURCE, otherwise
134
-	// we're able to violate quota without issue
135
-	assert.NilError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize}))
136
-
137
-	biggerThanQuotaFile := filepath.Join(testSubDir, "bigger-than-quota")
138
-	err := ioutil.WriteFile(biggerThanQuotaFile, make([]byte, testQuotaSize+1), 0644)
139
-	assert.Assert(t, is.ErrorContains(err, ""))
140
-	if err == io.ErrShortWrite {
141
-		assert.NilError(t, os.Remove(biggerThanQuotaFile))
142
-	}
143
-}
144
-
145
-func testRetrieveQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) {
146
-	// Validate that we can retrieve quota
147
-	assert.NilError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize}))
148
-
149
-	var q Quota
150
-	assert.NilError(t, ctrl.GetQuota(testSubDir, &q))
151
-	assert.Check(t, is.Equal(uint64(testQuotaSize), q.Size))
152
-}
153 1
deleted file mode 100644
... ...
@@ -1,18 +0,0 @@
1
-// +build linux,exclude_disk_quota linux,!cgo
2
-
3
-package quota // import "github.com/docker/docker/daemon/graphdriver/quota"
4
-
5
-func NewControl(basePath string) (*Control, error) {
6
-	return nil, ErrQuotaNotSupported
7
-}
8
-
9
-// SetQuota - assign a unique project id to directory and set the quota limits
10
-// for that project id
11
-func (q *Control) SetQuota(targetPath string, quota Quota) error {
12
-	return ErrQuotaNotSupported
13
-}
14
-
15
-// GetQuota - get the quota limits of a directory that was configured with SetQuota
16
-func (q *Control) GetQuota(targetPath string, quota *Quota) error {
17
-	return ErrQuotaNotSupported
18
-}
19 1
deleted file mode 100644
... ...
@@ -1,19 +0,0 @@
1
-// +build linux
2
-
3
-package quota // import "github.com/docker/docker/daemon/graphdriver/quota"
4
-
5
-import "sync"
6
-
7
-// Quota limit params - currently we only control blocks hard limit
8
-type Quota struct {
9
-	Size uint64
10
-}
11
-
12
-// Control - Context to be used by storage driver (e.g. overlay)
13
-// who wants to apply project quotas to container dirs
14
-type Control struct {
15
-	backingFsBlockDev string
16
-	sync.RWMutex      // protect nextProjectID and quotas map
17
-	nextProjectID     uint32
18
-	quotas            map[string]uint32
19
-}
... ...
@@ -6,12 +6,12 @@ import (
6 6
 	"path/filepath"
7 7
 
8 8
 	"github.com/docker/docker/daemon/graphdriver"
9
-	"github.com/docker/docker/daemon/graphdriver/quota"
10 9
 	"github.com/docker/docker/errdefs"
11 10
 	"github.com/docker/docker/pkg/containerfs"
12 11
 	"github.com/docker/docker/pkg/idtools"
13 12
 	"github.com/docker/docker/pkg/parsers"
14 13
 	"github.com/docker/docker/pkg/system"
14
+	"github.com/docker/docker/quota"
15 15
 	units "github.com/docker/go-units"
16 16
 	"github.com/opencontainers/selinux/go-selinux/label"
17 17
 	"github.com/pkg/errors"
... ...
@@ -1,7 +1,7 @@
1 1
 package vfs // import "github.com/docker/docker/daemon/graphdriver/vfs"
2 2
 
3 3
 import (
4
-	"github.com/docker/docker/daemon/graphdriver/quota"
4
+	"github.com/docker/docker/quota"
5 5
 	"github.com/sirupsen/logrus"
6 6
 )
7 7
 
... ...
@@ -2,7 +2,7 @@
2 2
 
3 3
 package vfs // import "github.com/docker/docker/daemon/graphdriver/vfs"
4 4
 
5
-import "github.com/docker/docker/daemon/graphdriver/quota"
5
+import "github.com/docker/docker/quota"
6 6
 
7 7
 type driverQuota struct {
8 8
 }
9 9
new file mode 100644
... ...
@@ -0,0 +1,19 @@
0
+package quota // import "github.com/docker/docker/quota"
1
+
2
+import "github.com/docker/docker/errdefs"
3
+
4
+var (
5
+	_ errdefs.ErrNotImplemented = (*errQuotaNotSupported)(nil)
6
+)
7
+
8
+// ErrQuotaNotSupported indicates if were found the FS didn't have projects quotas available
9
+var ErrQuotaNotSupported = errQuotaNotSupported{}
10
+
11
+type errQuotaNotSupported struct {
12
+}
13
+
14
+func (e errQuotaNotSupported) NotImplemented() {}
15
+
16
+func (e errQuotaNotSupported) Error() string {
17
+	return "Filesystem does not support, or has not enabled quotas"
18
+}
0 19
new file mode 100644
... ...
@@ -0,0 +1,442 @@
0
+// +build linux,!exclude_disk_quota,cgo
1
+
2
+//
3
+// projectquota.go - implements XFS project quota controls
4
+// for setting quota limits on a newly created directory.
5
+// It currently supports the legacy XFS specific ioctls.
6
+//
7
+// TODO: use generic quota control ioctl FS_IOC_FS{GET,SET}XATTR
8
+//       for both xfs/ext4 for kernel version >= v4.5
9
+//
10
+
11
+package quota // import "github.com/docker/docker/quota"
12
+
13
+/*
14
+#include <stdlib.h>
15
+#include <dirent.h>
16
+#include <linux/fs.h>
17
+#include <linux/quota.h>
18
+#include <linux/dqblk_xfs.h>
19
+
20
+#ifndef FS_XFLAG_PROJINHERIT
21
+struct fsxattr {
22
+	__u32		fsx_xflags;
23
+	__u32		fsx_extsize;
24
+	__u32		fsx_nextents;
25
+	__u32		fsx_projid;
26
+	unsigned char	fsx_pad[12];
27
+};
28
+#define FS_XFLAG_PROJINHERIT	0x00000200
29
+#endif
30
+#ifndef FS_IOC_FSGETXATTR
31
+#define FS_IOC_FSGETXATTR		_IOR ('X', 31, struct fsxattr)
32
+#endif
33
+#ifndef FS_IOC_FSSETXATTR
34
+#define FS_IOC_FSSETXATTR		_IOW ('X', 32, struct fsxattr)
35
+#endif
36
+
37
+#ifndef PRJQUOTA
38
+#define PRJQUOTA	2
39
+#endif
40
+#ifndef XFS_PROJ_QUOTA
41
+#define XFS_PROJ_QUOTA	2
42
+#endif
43
+#ifndef Q_XSETPQLIM
44
+#define Q_XSETPQLIM QCMD(Q_XSETQLIM, PRJQUOTA)
45
+#endif
46
+#ifndef Q_XGETPQUOTA
47
+#define Q_XGETPQUOTA QCMD(Q_XGETQUOTA, PRJQUOTA)
48
+#endif
49
+
50
+const int Q_XGETQSTAT_PRJQUOTA = QCMD(Q_XGETQSTAT, PRJQUOTA);
51
+*/
52
+import "C"
53
+import (
54
+	"io/ioutil"
55
+	"path"
56
+	"path/filepath"
57
+	"sync"
58
+	"unsafe"
59
+
60
+	"github.com/containerd/containerd/sys"
61
+	"github.com/pkg/errors"
62
+	"github.com/sirupsen/logrus"
63
+	"golang.org/x/sys/unix"
64
+)
65
+
66
+type pquotaState struct {
67
+	sync.Mutex
68
+	nextProjectID uint32
69
+}
70
+
71
+var pquotaStateInst *pquotaState
72
+var pquotaStateOnce sync.Once
73
+
74
+// getPquotaState - get global pquota state tracker instance
75
+func getPquotaState() *pquotaState {
76
+	pquotaStateOnce.Do(func() {
77
+		pquotaStateInst = &pquotaState{
78
+			nextProjectID: 1,
79
+		}
80
+	})
81
+	return pquotaStateInst
82
+}
83
+
84
+// registerBasePath - register a new base path and update nextProjectID
85
+func (state *pquotaState) updateMinProjID(minProjectID uint32) {
86
+	state.Lock()
87
+	defer state.Unlock()
88
+	if state.nextProjectID <= minProjectID {
89
+		state.nextProjectID = minProjectID + 1
90
+	}
91
+}
92
+
93
+// NewControl - initialize project quota support.
94
+// Test to make sure that quota can be set on a test dir and find
95
+// the first project id to be used for the next container create.
96
+//
97
+// Returns nil (and error) if project quota is not supported.
98
+//
99
+// First get the project id of the home directory.
100
+// This test will fail if the backing fs is not xfs.
101
+//
102
+// xfs_quota tool can be used to assign a project id to the driver home directory, e.g.:
103
+//    echo 999:/var/lib/docker/overlay2 >> /etc/projects
104
+//    echo docker:999 >> /etc/projid
105
+//    xfs_quota -x -c 'project -s docker' /<xfs mount point>
106
+//
107
+// In that case, the home directory project id will be used as a "start offset"
108
+// and all containers will be assigned larger project ids (e.g. >= 1000).
109
+// This is a way to prevent xfs_quota management from conflicting with docker.
110
+//
111
+// Then try to create a test directory with the next project id and set a quota
112
+// on it. If that works, continue to scan existing containers to map allocated
113
+// project ids.
114
+//
115
+func NewControl(basePath string) (*Control, error) {
116
+	//
117
+	// If we are running in a user namespace quota won't be supported for
118
+	// now since makeBackingFsDev() will try to mknod().
119
+	//
120
+	if sys.RunningInUserNS() {
121
+		return nil, ErrQuotaNotSupported
122
+	}
123
+
124
+	//
125
+	// create backing filesystem device node
126
+	//
127
+	backingFsBlockDev, err := makeBackingFsDev(basePath)
128
+	if err != nil {
129
+		return nil, err
130
+	}
131
+
132
+	// check if we can call quotactl with project quotas
133
+	// as a mechanism to determine (early) if we have support
134
+	hasQuotaSupport, err := hasQuotaSupport(backingFsBlockDev)
135
+	if err != nil {
136
+		return nil, err
137
+	}
138
+	if !hasQuotaSupport {
139
+		return nil, ErrQuotaNotSupported
140
+	}
141
+
142
+	//
143
+	// Get project id of parent dir as minimal id to be used by driver
144
+	//
145
+	baseProjectID, err := getProjectID(basePath)
146
+	if err != nil {
147
+		return nil, err
148
+	}
149
+	minProjectID := baseProjectID + 1
150
+
151
+	//
152
+	// Test if filesystem supports project quotas by trying to set
153
+	// a quota on the first available project id
154
+	//
155
+	quota := Quota{
156
+		Size: 0,
157
+	}
158
+	if err := setProjectQuota(backingFsBlockDev, minProjectID, quota); err != nil {
159
+		return nil, err
160
+	}
161
+
162
+	q := Control{
163
+		backingFsBlockDev: backingFsBlockDev,
164
+		quotas:            make(map[string]uint32),
165
+	}
166
+
167
+	//
168
+	// update minimum project ID
169
+	//
170
+	state := getPquotaState()
171
+	state.updateMinProjID(minProjectID)
172
+
173
+	//
174
+	// get first project id to be used for next container
175
+	//
176
+	err = q.findNextProjectID(basePath, baseProjectID)
177
+	if err != nil {
178
+		return nil, err
179
+	}
180
+
181
+	logrus.Debugf("NewControl(%s): nextProjectID = %d", basePath, state.nextProjectID)
182
+	return &q, nil
183
+}
184
+
185
+// SetQuota - assign a unique project id to directory and set the quota limits
186
+// for that project id
187
+func (q *Control) SetQuota(targetPath string, quota Quota) error {
188
+	q.RLock()
189
+	projectID, ok := q.quotas[targetPath]
190
+	q.RUnlock()
191
+	if !ok {
192
+		state := getPquotaState()
193
+		state.Lock()
194
+		projectID = state.nextProjectID
195
+
196
+		//
197
+		// assign project id to new container directory
198
+		//
199
+		err := setProjectID(targetPath, projectID)
200
+		if err != nil {
201
+			state.Unlock()
202
+			return err
203
+		}
204
+
205
+		state.nextProjectID++
206
+		state.Unlock()
207
+
208
+		q.Lock()
209
+		q.quotas[targetPath] = projectID
210
+		q.Unlock()
211
+	}
212
+
213
+	//
214
+	// set the quota limit for the container's project id
215
+	//
216
+	logrus.Debugf("SetQuota(%s, %d): projectID=%d", targetPath, quota.Size, projectID)
217
+	return setProjectQuota(q.backingFsBlockDev, projectID, quota)
218
+}
219
+
220
+// setProjectQuota - set the quota for project id on xfs block device
221
+func setProjectQuota(backingFsBlockDev string, projectID uint32, quota Quota) error {
222
+	var d C.fs_disk_quota_t
223
+	d.d_version = C.FS_DQUOT_VERSION
224
+	d.d_id = C.__u32(projectID)
225
+	d.d_flags = C.XFS_PROJ_QUOTA
226
+
227
+	d.d_fieldmask = C.FS_DQ_BHARD | C.FS_DQ_BSOFT
228
+	d.d_blk_hardlimit = C.__u64(quota.Size / 512)
229
+	d.d_blk_softlimit = d.d_blk_hardlimit
230
+
231
+	var cs = C.CString(backingFsBlockDev)
232
+	defer C.free(unsafe.Pointer(cs))
233
+
234
+	_, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XSETPQLIM,
235
+		uintptr(unsafe.Pointer(cs)), uintptr(d.d_id),
236
+		uintptr(unsafe.Pointer(&d)), 0, 0)
237
+	if errno != 0 {
238
+		return errors.Wrapf(errno, "failed to set quota limit for projid %d on %s",
239
+			projectID, backingFsBlockDev)
240
+	}
241
+
242
+	return nil
243
+}
244
+
245
+// GetQuota - get the quota limits of a directory that was configured with SetQuota
246
+func (q *Control) GetQuota(targetPath string, quota *Quota) error {
247
+	q.RLock()
248
+	projectID, ok := q.quotas[targetPath]
249
+	q.RUnlock()
250
+	if !ok {
251
+		return errors.Errorf("quota not found for path: %s", targetPath)
252
+	}
253
+
254
+	//
255
+	// get the quota limit for the container's project id
256
+	//
257
+	var d C.fs_disk_quota_t
258
+
259
+	var cs = C.CString(q.backingFsBlockDev)
260
+	defer C.free(unsafe.Pointer(cs))
261
+
262
+	_, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XGETPQUOTA,
263
+		uintptr(unsafe.Pointer(cs)), uintptr(C.__u32(projectID)),
264
+		uintptr(unsafe.Pointer(&d)), 0, 0)
265
+	if errno != 0 {
266
+		return errors.Wrapf(errno, "Failed to get quota limit for projid %d on %s",
267
+			projectID, q.backingFsBlockDev)
268
+	}
269
+	quota.Size = uint64(d.d_blk_hardlimit) * 512
270
+
271
+	return nil
272
+}
273
+
274
+// getProjectID - get the project id of path on xfs
275
+func getProjectID(targetPath string) (uint32, error) {
276
+	dir, err := openDir(targetPath)
277
+	if err != nil {
278
+		return 0, err
279
+	}
280
+	defer closeDir(dir)
281
+
282
+	var fsx C.struct_fsxattr
283
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR,
284
+		uintptr(unsafe.Pointer(&fsx)))
285
+	if errno != 0 {
286
+		return 0, errors.Wrapf(errno, "failed to get projid for %s", targetPath)
287
+	}
288
+
289
+	return uint32(fsx.fsx_projid), nil
290
+}
291
+
292
+// setProjectID - set the project id of path on xfs
293
+func setProjectID(targetPath string, projectID uint32) error {
294
+	dir, err := openDir(targetPath)
295
+	if err != nil {
296
+		return err
297
+	}
298
+	defer closeDir(dir)
299
+
300
+	var fsx C.struct_fsxattr
301
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR,
302
+		uintptr(unsafe.Pointer(&fsx)))
303
+	if errno != 0 {
304
+		return errors.Wrapf(errno, "failed to get projid for %s", targetPath)
305
+	}
306
+	fsx.fsx_projid = C.__u32(projectID)
307
+	fsx.fsx_xflags |= C.FS_XFLAG_PROJINHERIT
308
+	_, _, errno = unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSSETXATTR,
309
+		uintptr(unsafe.Pointer(&fsx)))
310
+	if errno != 0 {
311
+		return errors.Wrapf(errno, "failed to set projid for %s", targetPath)
312
+	}
313
+
314
+	return nil
315
+}
316
+
317
+// findNextProjectID - find the next project id to be used for containers
318
+// by scanning driver home directory to find used project ids
319
+func (q *Control) findNextProjectID(home string, baseID uint32) error {
320
+	state := getPquotaState()
321
+	state.Lock()
322
+	defer state.Unlock()
323
+
324
+	checkProjID := func(path string) (uint32, error) {
325
+		projid, err := getProjectID(path)
326
+		if err != nil {
327
+			return projid, err
328
+		}
329
+		if projid > 0 {
330
+			q.quotas[path] = projid
331
+		}
332
+		if state.nextProjectID <= projid {
333
+			state.nextProjectID = projid + 1
334
+		}
335
+		return projid, nil
336
+	}
337
+
338
+	files, err := ioutil.ReadDir(home)
339
+	if err != nil {
340
+		return errors.Errorf("read directory failed: %s", home)
341
+	}
342
+	for _, file := range files {
343
+		if !file.IsDir() {
344
+			continue
345
+		}
346
+		path := filepath.Join(home, file.Name())
347
+		projid, err := checkProjID(path)
348
+		if err != nil {
349
+			return err
350
+		}
351
+		if projid > 0 && projid != baseID {
352
+			continue
353
+		}
354
+		subfiles, err := ioutil.ReadDir(path)
355
+		if err != nil {
356
+			return errors.Errorf("read directory failed: %s", path)
357
+		}
358
+		for _, subfile := range subfiles {
359
+			if !subfile.IsDir() {
360
+				continue
361
+			}
362
+			subpath := filepath.Join(path, subfile.Name())
363
+			_, err := checkProjID(subpath)
364
+			if err != nil {
365
+				return err
366
+			}
367
+		}
368
+	}
369
+
370
+	return nil
371
+}
372
+
373
+func free(p *C.char) {
374
+	C.free(unsafe.Pointer(p))
375
+}
376
+
377
+func openDir(path string) (*C.DIR, error) {
378
+	Cpath := C.CString(path)
379
+	defer free(Cpath)
380
+
381
+	dir := C.opendir(Cpath)
382
+	if dir == nil {
383
+		return nil, errors.Errorf("failed to open dir: %s", path)
384
+	}
385
+	return dir, nil
386
+}
387
+
388
+func closeDir(dir *C.DIR) {
389
+	if dir != nil {
390
+		C.closedir(dir)
391
+	}
392
+}
393
+
394
+func getDirFd(dir *C.DIR) uintptr {
395
+	return uintptr(C.dirfd(dir))
396
+}
397
+
398
+// Get the backing block device of the driver home directory
399
+// and create a block device node under the home directory
400
+// to be used by quotactl commands
401
+func makeBackingFsDev(home string) (string, error) {
402
+	var stat unix.Stat_t
403
+	if err := unix.Stat(home, &stat); err != nil {
404
+		return "", err
405
+	}
406
+
407
+	backingFsBlockDev := path.Join(home, "backingFsBlockDev")
408
+	// Re-create just in case someone copied the home directory over to a new device
409
+	unix.Unlink(backingFsBlockDev)
410
+	err := unix.Mknod(backingFsBlockDev, unix.S_IFBLK|0600, int(stat.Dev))
411
+	switch err {
412
+	case nil:
413
+		return backingFsBlockDev, nil
414
+
415
+	case unix.ENOSYS, unix.EPERM:
416
+		return "", ErrQuotaNotSupported
417
+
418
+	default:
419
+		return "", errors.Wrapf(err, "failed to mknod %s", backingFsBlockDev)
420
+	}
421
+}
422
+
423
+func hasQuotaSupport(backingFsBlockDev string) (bool, error) {
424
+	var cs = C.CString(backingFsBlockDev)
425
+	defer free(cs)
426
+	var qstat C.fs_quota_stat_t
427
+
428
+	_, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, uintptr(C.Q_XGETQSTAT_PRJQUOTA), uintptr(unsafe.Pointer(cs)), 0, uintptr(unsafe.Pointer(&qstat)), 0, 0)
429
+	if errno == 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ENFD > 0 && qstat.qs_flags&C.FS_QUOTA_PDQ_ACCT > 0 {
430
+		return true, nil
431
+	}
432
+
433
+	switch errno {
434
+	// These are the known fatal errors, consider all other errors (ENOTTY, etc.. not supporting quota)
435
+	case unix.EFAULT, unix.ENOENT, unix.ENOTBLK, unix.EPERM:
436
+	default:
437
+		return false, nil
438
+	}
439
+
440
+	return false, errno
441
+}
0 442
new file mode 100644
... ...
@@ -0,0 +1,78 @@
0
+// +build linux
1
+
2
+package quota // import "github.com/docker/docker/quota"
3
+
4
+import (
5
+	"io"
6
+	"io/ioutil"
7
+	"os"
8
+	"path/filepath"
9
+	"testing"
10
+
11
+	"gotest.tools/v3/assert"
12
+	is "gotest.tools/v3/assert/cmp"
13
+)
14
+
15
+// 10MB
16
+const testQuotaSize = 10 * 1024 * 1024
17
+
18
+func TestBlockDev(t *testing.T) {
19
+	if msg, ok := CanTestQuota(); !ok {
20
+		t.Skip(msg)
21
+	}
22
+
23
+	// get sparse xfs test image
24
+	imageFileName, err := PrepareQuotaTestImage(t)
25
+	if err != nil {
26
+		t.Fatal(err)
27
+	}
28
+	defer os.Remove(imageFileName)
29
+
30
+	t.Run("testBlockDevQuotaDisabled", WrapMountTest(imageFileName, false, testBlockDevQuotaDisabled))
31
+	t.Run("testBlockDevQuotaEnabled", WrapMountTest(imageFileName, true, testBlockDevQuotaEnabled))
32
+	t.Run("testSmallerThanQuota", WrapMountTest(imageFileName, true, WrapQuotaTest(testSmallerThanQuota)))
33
+	t.Run("testBiggerThanQuota", WrapMountTest(imageFileName, true, WrapQuotaTest(testBiggerThanQuota)))
34
+	t.Run("testRetrieveQuota", WrapMountTest(imageFileName, true, WrapQuotaTest(testRetrieveQuota)))
35
+}
36
+
37
+func testBlockDevQuotaDisabled(t *testing.T, mountPoint, backingFsDev, testDir string) {
38
+	hasSupport, err := hasQuotaSupport(backingFsDev)
39
+	assert.NilError(t, err)
40
+	assert.Check(t, !hasSupport)
41
+}
42
+
43
+func testBlockDevQuotaEnabled(t *testing.T, mountPoint, backingFsDev, testDir string) {
44
+	hasSupport, err := hasQuotaSupport(backingFsDev)
45
+	assert.NilError(t, err)
46
+	assert.Check(t, hasSupport)
47
+}
48
+
49
+func testSmallerThanQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) {
50
+	assert.NilError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize}))
51
+	smallerThanQuotaFile := filepath.Join(testSubDir, "smaller-than-quota")
52
+	assert.NilError(t, ioutil.WriteFile(smallerThanQuotaFile, make([]byte, testQuotaSize/2), 0644))
53
+	assert.NilError(t, os.Remove(smallerThanQuotaFile))
54
+}
55
+
56
+func testBiggerThanQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) {
57
+	// Make sure the quota is being enforced
58
+	// TODO: When we implement this under EXT4, we need to shed CAP_SYS_RESOURCE, otherwise
59
+	// we're able to violate quota without issue
60
+	assert.NilError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize}))
61
+
62
+	biggerThanQuotaFile := filepath.Join(testSubDir, "bigger-than-quota")
63
+	err := ioutil.WriteFile(biggerThanQuotaFile, make([]byte, testQuotaSize+1), 0644)
64
+	assert.Assert(t, is.ErrorContains(err, ""))
65
+	if err == io.ErrShortWrite {
66
+		assert.NilError(t, os.Remove(biggerThanQuotaFile))
67
+	}
68
+}
69
+
70
+func testRetrieveQuota(t *testing.T, ctrl *Control, homeDir, testDir, testSubDir string) {
71
+	// Validate that we can retrieve quota
72
+	assert.NilError(t, ctrl.SetQuota(testSubDir, Quota{testQuotaSize}))
73
+
74
+	var q Quota
75
+	assert.NilError(t, ctrl.GetQuota(testSubDir, &q))
76
+	assert.Check(t, is.Equal(uint64(testQuotaSize), q.Size))
77
+}
0 78
new file mode 100644
... ...
@@ -0,0 +1,18 @@
0
+// +build linux,exclude_disk_quota linux,!cgo !linux
1
+
2
+package quota // import "github.com/docker/docker/quota"
3
+
4
+func NewControl(basePath string) (*Control, error) {
5
+	return nil, ErrQuotaNotSupported
6
+}
7
+
8
+// SetQuota - assign a unique project id to directory and set the quota limits
9
+// for that project id
10
+func (q *Control) SetQuota(targetPath string, quota Quota) error {
11
+	return ErrQuotaNotSupported
12
+}
13
+
14
+// GetQuota - get the quota limits of a directory that was configured with SetQuota
15
+func (q *Control) GetQuota(targetPath string, quota *Quota) error {
16
+	return ErrQuotaNotSupported
17
+}
0 18
new file mode 100644
... ...
@@ -0,0 +1,122 @@
0
+// +build linux
1
+
2
+package quota // import "github.com/docker/docker/quota"
3
+
4
+import (
5
+	"io/ioutil"
6
+	"os"
7
+	"os/exec"
8
+	"testing"
9
+
10
+	"golang.org/x/sys/unix"
11
+	"gotest.tools/v3/assert"
12
+	"gotest.tools/v3/fs"
13
+)
14
+
15
+const imageSize = 64 * 1024 * 1024
16
+
17
+// CanTestQuota - checks if xfs prjquota can be tested
18
+// returns a reason if not
19
+func CanTestQuota() (string, bool) {
20
+	if os.Getuid() != 0 {
21
+		return "requires mounts", false
22
+	}
23
+	_, err := exec.LookPath("mkfs.xfs")
24
+	if err != nil {
25
+		return "mkfs.xfs not found in PATH", false
26
+	}
27
+	return "", true
28
+}
29
+
30
+// PrepareQuotaTestImage - prepares an xfs prjquota test image
31
+// returns the path the the image on success
32
+func PrepareQuotaTestImage(t *testing.T) (string, error) {
33
+	mkfs, err := exec.LookPath("mkfs.xfs")
34
+	if err != nil {
35
+		return "", err
36
+	}
37
+
38
+	// create a sparse image
39
+	imageFile, err := ioutil.TempFile("", "xfs-image")
40
+	if err != nil {
41
+		return "", err
42
+	}
43
+	imageFileName := imageFile.Name()
44
+	if _, err = imageFile.Seek(imageSize-1, 0); err != nil {
45
+		os.Remove(imageFileName)
46
+		return "", err
47
+	}
48
+	if _, err = imageFile.Write([]byte{0}); err != nil {
49
+		os.Remove(imageFileName)
50
+		return "", err
51
+	}
52
+	if err = imageFile.Close(); err != nil {
53
+		os.Remove(imageFileName)
54
+		return "", err
55
+	}
56
+
57
+	// The reason for disabling these options is sometimes people run with a newer userspace
58
+	// than kernelspace
59
+	out, err := exec.Command(mkfs, "-m", "crc=0,finobt=0", imageFileName).CombinedOutput()
60
+	if len(out) > 0 {
61
+		t.Log(string(out))
62
+	}
63
+	if err != nil {
64
+		os.Remove(imageFileName)
65
+		return "", err
66
+	}
67
+
68
+	return imageFileName, nil
69
+}
70
+
71
+// WrapMountTest - wraps a test function such that it has easy access to a mountPoint and testDir
72
+// with guaranteed prjquota or guaranteed no prjquota support.
73
+func WrapMountTest(imageFileName string, enableQuota bool, testFunc func(t *testing.T, mountPoint, backingFsDev, testDir string)) func(*testing.T) {
74
+	return func(t *testing.T) {
75
+		mountOptions := "loop"
76
+
77
+		if enableQuota {
78
+			mountOptions = mountOptions + ",prjquota"
79
+		}
80
+
81
+		mountPointDir := fs.NewDir(t, "xfs-mountPoint")
82
+		defer mountPointDir.Remove()
83
+		mountPoint := mountPointDir.Path()
84
+
85
+		out, err := exec.Command("mount", "-o", mountOptions, imageFileName, mountPoint).CombinedOutput()
86
+		if err != nil {
87
+			_, err := os.Stat("/proc/fs/xfs")
88
+			if os.IsNotExist(err) {
89
+				t.Skip("no /proc/fs/xfs")
90
+			}
91
+		}
92
+
93
+		assert.NilError(t, err, "mount failed: %s", out)
94
+
95
+		defer func() {
96
+			assert.NilError(t, unix.Unmount(mountPoint, 0))
97
+		}()
98
+
99
+		backingFsDev, err := makeBackingFsDev(mountPoint)
100
+		assert.NilError(t, err)
101
+
102
+		testDir, err := ioutil.TempDir(mountPoint, "per-test")
103
+		assert.NilError(t, err)
104
+		defer os.RemoveAll(testDir)
105
+
106
+		testFunc(t, mountPoint, backingFsDev, testDir)
107
+	}
108
+}
109
+
110
+// WrapQuotaTest - wraps a test function such that is has easy and guaranteed access to a quota Control
111
+// instance with a quota test dir under its control.
112
+func WrapQuotaTest(testFunc func(t *testing.T, ctrl *Control, mountPoint, testDir, testSubDir string)) func(t *testing.T, mountPoint, backingFsDev, testDir string) {
113
+	return func(t *testing.T, mountPoint, backingFsDev, testDir string) {
114
+		ctrl, err := NewControl(testDir)
115
+		assert.NilError(t, err)
116
+
117
+		testSubDir, err := ioutil.TempDir(testDir, "quota-test")
118
+		assert.NilError(t, err)
119
+		testFunc(t, ctrl, mountPoint, testDir, testSubDir)
120
+	}
121
+}
0 122
new file mode 100644
... ...
@@ -0,0 +1,16 @@
0
+package quota // import "github.com/docker/docker/quota"
1
+
2
+import "sync"
3
+
4
+// Quota limit params - currently we only control blocks hard limit
5
+type Quota struct {
6
+	Size uint64
7
+}
8
+
9
+// Control - Context to be used by storage driver (e.g. overlay)
10
+// who wants to apply project quotas to container dirs
11
+type Control struct {
12
+	backingFsBlockDev string
13
+	sync.RWMutex      // protect nextProjectID and quotas map
14
+	quotas            map[string]uint32
15
+}
... ...
@@ -16,10 +16,12 @@ import (
16 16
 	"github.com/docker/docker/daemon/names"
17 17
 	"github.com/docker/docker/errdefs"
18 18
 	"github.com/docker/docker/pkg/idtools"
19
+	"github.com/docker/docker/quota"
19 20
 	"github.com/docker/docker/volume"
20 21
 	"github.com/moby/sys/mount"
21 22
 	"github.com/moby/sys/mountinfo"
22 23
 	"github.com/pkg/errors"
24
+	"github.com/sirupsen/logrus"
23 25
 )
24 26
 
25 27
 // VolumeDataPathName is the name of the directory where the volume data is stored.
... ...
@@ -66,6 +68,10 @@ func New(scope string, rootIdentity idtools.Identity) (*Root, error) {
66 66
 		return nil, err
67 67
 	}
68 68
 
69
+	if r.quotaCtl, err = quota.NewControl(rootDirectory); err != nil {
70
+		logrus.Debugf("No quota support for local volumes in %s: %v", rootDirectory, err)
71
+	}
72
+
69 73
 	for _, d := range dirs {
70 74
 		if !d.IsDir() {
71 75
 			continue
... ...
@@ -76,6 +82,7 @@ func New(scope string, rootIdentity idtools.Identity) (*Root, error) {
76 76
 			driverName: r.Name(),
77 77
 			name:       name,
78 78
 			path:       r.DataPath(name),
79
+			quotaCtl:   r.quotaCtl,
79 80
 		}
80 81
 		r.volumes[name] = v
81 82
 		optsFilePath := filepath.Join(rootDirectory, name, "opts.json")
... ...
@@ -105,6 +112,7 @@ type Root struct {
105 105
 	m            sync.Mutex
106 106
 	scope        string
107 107
 	path         string
108
+	quotaCtl     *quota.Control
108 109
 	volumes      map[string]*localVolume
109 110
 	rootIdentity idtools.Identity
110 111
 }
... ...
@@ -162,6 +170,7 @@ func (r *Root) Create(name string, opts map[string]string) (volume.Volume, error
162 162
 		driverName: r.Name(),
163 163
 		name:       name,
164 164
 		path:       path,
165
+		quotaCtl:   r.quotaCtl,
165 166
 	}
166 167
 
167 168
 	if len(opts) != 0 {
... ...
@@ -273,6 +282,8 @@ type localVolume struct {
273 273
 	opts *optsConfig
274 274
 	// active refcounts the active mounts
275 275
 	active activeMount
276
+	// reference to Root instances quotaCtl
277
+	quotaCtl *quota.Control
276 278
 }
277 279
 
278 280
 // Name returns the name of the given Volume.
... ...
@@ -300,7 +311,7 @@ func (v *localVolume) CachedPath() string {
300 300
 func (v *localVolume) Mount(id string) (string, error) {
301 301
 	v.m.Lock()
302 302
 	defer v.m.Unlock()
303
-	if v.opts != nil {
303
+	if v.needsMount() {
304 304
 		if !v.active.mounted {
305 305
 			if err := v.mount(); err != nil {
306 306
 				return "", errdefs.System(err)
... ...
@@ -309,6 +320,9 @@ func (v *localVolume) Mount(id string) (string, error) {
309 309
 		}
310 310
 		v.active.count++
311 311
 	}
312
+	if err := v.postMount(); err != nil {
313
+		return "", err
314
+	}
312 315
 	return v.path, nil
313 316
 }
314 317
 
... ...
@@ -322,7 +336,7 @@ func (v *localVolume) Unmount(id string) error {
322 322
 	// Essentially docker doesn't care if this fails, it will send an error, but
323 323
 	// ultimately there's nothing that can be done. If we don't decrement the count
324 324
 	// this volume can never be removed until a daemon restart occurs.
325
-	if v.opts != nil {
325
+	if v.needsMount() {
326 326
 		v.active.count--
327 327
 	}
328 328
 
... ...
@@ -334,7 +348,7 @@ func (v *localVolume) Unmount(id string) error {
334 334
 }
335 335
 
336 336
 func (v *localVolume) unmount() error {
337
-	if v.opts != nil {
337
+	if v.needsMount() {
338 338
 		if err := mount.Unmount(v.path); err != nil {
339 339
 			if mounted, mErr := mountinfo.Mounted(v.path); mounted || mErr != nil {
340 340
 				return errdefs.System(err)
341 341
new file mode 100644
... ...
@@ -0,0 +1,97 @@
0
+// +build linux
1
+
2
+package local // import "github.com/docker/docker/volume/local"
3
+
4
+import (
5
+	"io/ioutil"
6
+	"os"
7
+	"path/filepath"
8
+	"testing"
9
+
10
+	"github.com/docker/docker/pkg/idtools"
11
+	"github.com/docker/docker/quota"
12
+	"gotest.tools/v3/assert"
13
+	is "gotest.tools/v3/assert/cmp"
14
+)
15
+
16
+const quotaSize = 1024 * 1024
17
+const quotaSizeLiteral = "1M"
18
+
19
+func TestQuota(t *testing.T) {
20
+	if msg, ok := quota.CanTestQuota(); !ok {
21
+		t.Skip(msg)
22
+	}
23
+
24
+	// get sparse xfs test image
25
+	imageFileName, err := quota.PrepareQuotaTestImage(t)
26
+	if err != nil {
27
+		t.Fatal(err)
28
+	}
29
+	defer os.Remove(imageFileName)
30
+
31
+	t.Run("testVolWithQuota", quota.WrapMountTest(imageFileName, true, testVolWithQuota))
32
+	t.Run("testVolQuotaUnsupported", quota.WrapMountTest(imageFileName, false, testVolQuotaUnsupported))
33
+}
34
+
35
+func testVolWithQuota(t *testing.T, mountPoint, backingFsDev, testDir string) {
36
+	r, err := New(testDir, idtools.Identity{UID: os.Geteuid(), GID: os.Getegid()})
37
+	if err != nil {
38
+		t.Fatal(err)
39
+	}
40
+	assert.Assert(t, r.quotaCtl != nil)
41
+
42
+	vol, err := r.Create("testing", map[string]string{"size": quotaSizeLiteral})
43
+	if err != nil {
44
+		t.Fatal(err)
45
+	}
46
+
47
+	dir, err := vol.Mount("1234")
48
+	if err != nil {
49
+		t.Fatal(err)
50
+	}
51
+	defer func() {
52
+		if err := vol.Unmount("1234"); err != nil {
53
+			t.Fatal(err)
54
+		}
55
+	}()
56
+
57
+	testfile := filepath.Join(dir, "testfile")
58
+
59
+	// test writing file smaller than quota
60
+	assert.NilError(t, ioutil.WriteFile(testfile, make([]byte, quotaSize/2), 0644))
61
+	assert.NilError(t, os.Remove(testfile))
62
+
63
+	// test writing fiel larger than quota
64
+	err = ioutil.WriteFile(testfile, make([]byte, quotaSize+1), 0644)
65
+	assert.ErrorContains(t, err, "")
66
+	if _, err := os.Stat(testfile); err == nil {
67
+		assert.NilError(t, os.Remove(testfile))
68
+	}
69
+}
70
+
71
+func testVolQuotaUnsupported(t *testing.T, mountPoint, backingFsDev, testDir string) {
72
+	r, err := New(testDir, idtools.Identity{UID: os.Geteuid(), GID: os.Getegid()})
73
+	if err != nil {
74
+		t.Fatal(err)
75
+	}
76
+	assert.Assert(t, is.Nil(r.quotaCtl))
77
+
78
+	_, err = r.Create("testing", map[string]string{"size": quotaSizeLiteral})
79
+	assert.ErrorContains(t, err, "no quota support")
80
+
81
+	vol, err := r.Create("testing", nil)
82
+	if err != nil {
83
+		t.Fatal(err)
84
+	}
85
+
86
+	// this could happen if someone moves volumes from storage with
87
+	// quota support to some place without
88
+	lv, ok := vol.(*localVolume)
89
+	assert.Assert(t, ok)
90
+	lv.opts = &optsConfig{
91
+		Quota: quota.Quota{Size: quotaSize},
92
+	}
93
+
94
+	_, err = vol.Mount("1234")
95
+	assert.ErrorContains(t, err, "no quota support")
96
+}
... ...
@@ -15,6 +15,8 @@ import (
15 15
 	"time"
16 16
 
17 17
 	"github.com/docker/docker/errdefs"
18
+	"github.com/docker/docker/quota"
19
+	units "github.com/docker/go-units"
18 20
 	"github.com/moby/sys/mount"
19 21
 	"github.com/pkg/errors"
20 22
 )
... ...
@@ -26,10 +28,12 @@ var (
26 26
 		"type":   {}, // specify the filesystem type for mount, e.g. nfs
27 27
 		"o":      {}, // generic mount options
28 28
 		"device": {}, // device to mount from
29
+		"size":   {}, // quota size limit
29 30
 	}
30
-	mandatoryOpts = map[string]struct{}{
31
-		"device": {},
32
-		"type":   {},
31
+	mandatoryOpts = map[string][]string{
32
+		"device": []string{"type"},
33
+		"type":   []string{"device"},
34
+		"o":      []string{"device", "type"},
33 35
 	}
34 36
 )
35 37
 
... ...
@@ -37,10 +41,11 @@ type optsConfig struct {
37 37
 	MountType   string
38 38
 	MountOpts   string
39 39
 	MountDevice string
40
+	Quota       quota.Quota
40 41
 }
41 42
 
42 43
 func (o *optsConfig) String() string {
43
-	return fmt.Sprintf("type='%s' device='%s' o='%s'", o.MountType, o.MountDevice, o.MountOpts)
44
+	return fmt.Sprintf("type='%s' device='%s' o='%s' size='%d'", o.MountType, o.MountDevice, o.MountOpts, o.Quota.Size)
44 45
 }
45 46
 
46 47
 // scopedPath verifies that the path where the volume is located
... ...
@@ -63,15 +68,25 @@ func setOpts(v *localVolume, opts map[string]string) error {
63 63
 	if len(opts) == 0 {
64 64
 		return nil
65 65
 	}
66
-	if err := validateOpts(opts); err != nil {
66
+	err := validateOpts(opts)
67
+	if err != nil {
67 68
 		return err
68 69
 	}
69
-
70 70
 	v.opts = &optsConfig{
71 71
 		MountType:   opts["type"],
72 72
 		MountOpts:   opts["o"],
73 73
 		MountDevice: opts["device"],
74 74
 	}
75
+	if val, ok := opts["size"]; ok {
76
+		size, err := units.RAMInBytes(val)
77
+		if err != nil {
78
+			return err
79
+		}
80
+		if size > 0 && v.quotaCtl == nil {
81
+			return errdefs.InvalidParameter(errors.Errorf("quota size requested but no quota support"))
82
+		}
83
+		v.opts.Quota.Size = uint64(size)
84
+	}
75 85
 	return nil
76 86
 }
77 87
 
... ...
@@ -84,14 +99,28 @@ func validateOpts(opts map[string]string) error {
84 84
 			return errdefs.InvalidParameter(errors.Errorf("invalid option: %q", opt))
85 85
 		}
86 86
 	}
87
-	for opt := range mandatoryOpts {
88
-		if _, ok := opts[opt]; !ok {
89
-			return errdefs.InvalidParameter(errors.Errorf("missing required option: %q", opt))
87
+	for opt, reqopts := range mandatoryOpts {
88
+		if _, ok := opts[opt]; ok {
89
+			for _, reqopt := range reqopts {
90
+				if _, ok := opts[reqopt]; !ok {
91
+					return errdefs.InvalidParameter(errors.Errorf("missing required option: %q", reqopt))
92
+				}
93
+			}
90 94
 		}
91 95
 	}
92 96
 	return nil
93 97
 }
94 98
 
99
+func (v *localVolume) needsMount() bool {
100
+	if v.opts == nil {
101
+		return false
102
+	}
103
+	if v.opts.MountDevice != "" || v.opts.MountType != "" {
104
+		return true
105
+	}
106
+	return false
107
+}
108
+
95 109
 func (v *localVolume) mount() error {
96 110
 	if v.opts.MountDevice == "" {
97 111
 		return fmt.Errorf("missing device in volume options")
... ...
@@ -111,6 +140,23 @@ func (v *localVolume) mount() error {
111 111
 	return errors.Wrap(err, "failed to mount local volume")
112 112
 }
113 113
 
114
+func (v *localVolume) postMount() error {
115
+	if v.opts == nil {
116
+		return nil
117
+	}
118
+	if v.opts.Quota.Size > 0 {
119
+		if v.quotaCtl != nil {
120
+			err := v.quotaCtl.SetQuota(v.path, v.opts.Quota)
121
+			if err != nil {
122
+				return err
123
+			}
124
+		} else {
125
+			return fmt.Errorf("size quota requested for volume but no quota support")
126
+		}
127
+	}
128
+	return nil
129
+}
130
+
114 131
 func (v *localVolume) CreatedAt() (time.Time, error) {
115 132
 	fileInfo, err := os.Stat(v.path)
116 133
 	if err != nil {
... ...
@@ -32,10 +32,18 @@ func setOpts(v *localVolume, opts map[string]string) error {
32 32
 	return nil
33 33
 }
34 34
 
35
+func (v *localVolume) needsMount() bool {
36
+	return false
37
+}
38
+
35 39
 func (v *localVolume) mount() error {
36 40
 	return nil
37 41
 }
38 42
 
43
+func (v *localVolume) postMount() error {
44
+	return nil
45
+}
46
+
39 47
 func (v *localVolume) CreatedAt() (time.Time, error) {
40 48
 	fileInfo, err := os.Stat(v.path)
41 49
 	if err != nil {