Browse code

Remove the graph driver from the daemon, move it into the layer store.

Support restoreCustomImage for windows with a new interface to extract
the graph driver from the LayerStore.

Signed-off-by: Daniel Nephin <dnephin@docker.com>

Daniel Nephin authored on 2015/12/17 05:32:16
Showing 14 changed files
... ...
@@ -289,7 +289,8 @@ func (daemon *Daemon) getSize(container *container.Container) (int64, int64) {
289 289
 
290 290
 	sizeRw, err = container.RWLayer.Size()
291 291
 	if err != nil {
292
-		logrus.Errorf("Driver %s couldn't return diff size of container %s: %s", daemon.driver, container.ID, err)
292
+		logrus.Errorf("Driver %s couldn't return diff size of container %s: %s",
293
+			daemon.GraphDriverName(), container.ID, err)
293 294
 		// FIXME: GetSize should return an error. Not changing it now in case
294 295
 		// there is a side-effect.
295 296
 		sizeRw = -1
... ...
@@ -30,7 +30,6 @@ import (
30 30
 	"github.com/docker/docker/daemon/exec"
31 31
 	"github.com/docker/docker/daemon/execdriver"
32 32
 	"github.com/docker/docker/daemon/execdriver/execdrivers"
33
-	"github.com/docker/docker/daemon/graphdriver"
34 33
 	_ "github.com/docker/docker/daemon/graphdriver/vfs" // register vfs
35 34
 	"github.com/docker/docker/daemon/logger"
36 35
 	"github.com/docker/docker/daemon/network"
... ...
@@ -146,7 +145,6 @@ type Daemon struct {
146 146
 	idIndex                   *truncindex.TruncIndex
147 147
 	configStore               *Config
148 148
 	containerGraphDB          *graphdb.Database
149
-	driver                    graphdriver.Driver
150 149
 	execDriver                execdriver.Driver
151 150
 	statsCollector            *statsCollector
152 151
 	defaultLogConfig          containertypes.LogConfig
... ...
@@ -297,7 +295,7 @@ func (daemon *Daemon) restore() error {
297 297
 
298 298
 	var (
299 299
 		debug         = os.Getenv("DEBUG") != ""
300
-		currentDriver = daemon.driver.String()
300
+		currentDriver = daemon.GraphDriverName()
301 301
 		containers    = make(map[string]*cr)
302 302
 	)
303 303
 
... ...
@@ -520,7 +518,7 @@ func (daemon *Daemon) newContainer(name string, config *containertypes.Config, i
520 520
 	base.ImageID = imgID
521 521
 	base.NetworkSettings = &network.Settings{IsAnonymousEndpoint: noExplicitName}
522 522
 	base.Name = name
523
-	base.Driver = daemon.driver.String()
523
+	base.Driver = daemon.GraphDriverName()
524 524
 
525 525
 	return base, err
526 526
 }
... ...
@@ -703,20 +701,9 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo
703 703
 	}
704 704
 	os.Setenv("TMPDIR", realTmp)
705 705
 
706
-	// Set the default driver
707
-	graphdriver.DefaultDriver = config.GraphDriver
708
-
709
-	// Load storage driver
710
-	driver, err := graphdriver.New(config.Root, config.GraphOptions, uidMaps, gidMaps)
711
-	if err != nil {
712
-		return nil, fmt.Errorf("error initializing graphdriver: %v", err)
713
-	}
714
-	logrus.Debugf("Using graph driver %s", driver)
715
-
716 706
 	d := &Daemon{}
717
-	d.driver = driver
718
-
719
-	// Ensure the graph driver is shutdown at a later point
707
+	// Ensure the daemon is properly shutdown if there is a failure during
708
+	// initialization
720 709
 	defer func() {
721 710
 		if err != nil {
722 711
 			if err := d.Shutdown(); err != nil {
... ...
@@ -733,25 +720,32 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo
733 733
 	}
734 734
 	logrus.Debugf("Using default logging driver %s", config.LogConfig.Type)
735 735
 
736
-	// Configure and validate the kernels security support
737
-	if err := configureKernelSecuritySupport(config, d.driver.String()); err != nil {
738
-		return nil, err
739
-	}
740
-
741 736
 	daemonRepo := filepath.Join(config.Root, "containers")
742
-
743 737
 	if err := idtools.MkdirAllAs(daemonRepo, 0700, rootUID, rootGID); err != nil && !os.IsExist(err) {
744 738
 		return nil, err
745 739
 	}
746 740
 
747
-	imageRoot := filepath.Join(config.Root, "image", d.driver.String())
748
-	fms, err := layer.NewFSMetadataStore(filepath.Join(imageRoot, "layerdb"))
741
+	driverName := os.Getenv("DOCKER_DRIVER")
742
+	if driverName == "" {
743
+		driverName = config.GraphDriver
744
+	}
745
+	d.layerStore, err = layer.NewStoreFromOptions(layer.StoreOptions{
746
+		StorePath:                 config.Root,
747
+		MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s"),
748
+		GraphDriver:               driverName,
749
+		GraphDriverOptions:        config.GraphOptions,
750
+		UIDMaps:                   uidMaps,
751
+		GIDMaps:                   gidMaps,
752
+	})
749 753
 	if err != nil {
750 754
 		return nil, err
751 755
 	}
752 756
 
753
-	d.layerStore, err = layer.NewStore(fms, d.driver)
754
-	if err != nil {
757
+	graphDriver := d.layerStore.DriverName()
758
+	imageRoot := filepath.Join(config.Root, "image", graphDriver)
759
+
760
+	// Configure and validate the kernels security support
761
+	if err := configureKernelSecuritySupport(config, graphDriver); err != nil {
755 762
 		return nil, err
756 763
 	}
757 764
 
... ...
@@ -797,11 +791,11 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo
797 797
 		return nil, fmt.Errorf("Couldn't create Tag store repositories: %s", err)
798 798
 	}
799 799
 
800
-	if err := restoreCustomImage(d.driver, d.imageStore, d.layerStore, referenceStore); err != nil {
800
+	if err := restoreCustomImage(d.imageStore, d.layerStore, referenceStore); err != nil {
801 801
 		return nil, fmt.Errorf("Couldn't restore custom images: %s", err)
802 802
 	}
803 803
 
804
-	if err := v1.Migrate(config.Root, d.driver.String(), d.layerStore, d.imageStore, referenceStore, distributionMetadataStore); err != nil {
804
+	if err := v1.Migrate(config.Root, graphDriver, d.layerStore, d.imageStore, referenceStore, distributionMetadataStore); err != nil {
805 805
 		return nil, err
806 806
 	}
807 807
 
... ...
@@ -953,9 +947,9 @@ func (daemon *Daemon) Shutdown() error {
953 953
 		}
954 954
 	}
955 955
 
956
-	if daemon.driver != nil {
957
-		if err := daemon.driver.Cleanup(); err != nil {
958
-			logrus.Errorf("Error during graph storage driver.Cleanup(): %v", err)
956
+	if daemon.layerStore != nil {
957
+		if err := daemon.layerStore.Cleanup(); err != nil {
958
+			logrus.Errorf("Error during layer Store.Cleanup(): %v", err)
959 959
 		}
960 960
 	}
961 961
 
... ...
@@ -982,7 +976,7 @@ func (daemon *Daemon) Mount(container *container.Container) error {
982 982
 		if container.BaseFS != "" && runtime.GOOS != "windows" {
983 983
 			daemon.Unmount(container)
984 984
 			return fmt.Errorf("Error: driver %s is returning inconsistent paths for container %s ('%s' then '%s')",
985
-				daemon.driver, container.ID, container.BaseFS, dir)
985
+				daemon.GraphDriverName(), container.ID, container.BaseFS, dir)
986 986
 		}
987 987
 	}
988 988
 	container.BaseFS = dir // TODO: combine these fields
... ...
@@ -1193,7 +1187,7 @@ func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) {
1193 1193
 		VirtualSize:     size, // TODO: field unused, deprecate
1194 1194
 	}
1195 1195
 
1196
-	imageInspect.GraphDriver.Name = daemon.driver.String()
1196
+	imageInspect.GraphDriver.Name = daemon.GraphDriverName()
1197 1197
 
1198 1198
 	imageInspect.GraphDriver.Data = layerMetadata
1199 1199
 
... ...
@@ -1322,10 +1316,9 @@ func (daemon *Daemon) GetImage(refOrID string) (*image.Image, error) {
1322 1322
 	return daemon.imageStore.Get(imgID)
1323 1323
 }
1324 1324
 
1325
-// GraphDriver returns the currently used driver for processing
1326
-// container layers.
1327
-func (daemon *Daemon) GraphDriver() graphdriver.Driver {
1328
-	return daemon.driver
1325
+// GraphDriverName returns the name of the graph driver used by the layer.Store
1326
+func (daemon *Daemon) GraphDriverName() string {
1327
+	return daemon.layerStore.DriverName()
1329 1328
 }
1330 1329
 
1331 1330
 // ExecutionDriver returns the currently used driver for creating and
... ...
@@ -15,7 +15,6 @@ import (
15 15
 	pblkiodev "github.com/docker/docker/api/types/blkiodev"
16 16
 	containertypes "github.com/docker/docker/api/types/container"
17 17
 	"github.com/docker/docker/container"
18
-	"github.com/docker/docker/daemon/graphdriver"
19 18
 	derr "github.com/docker/docker/errors"
20 19
 	"github.com/docker/docker/image"
21 20
 	"github.com/docker/docker/layer"
... ...
@@ -728,7 +727,7 @@ func (daemon *Daemon) conditionalUnmountOnCleanup(container *container.Container
728 728
 	daemon.Unmount(container)
729 729
 }
730 730
 
731
-func restoreCustomImage(driver graphdriver.Driver, is image.Store, ls layer.Store, rs reference.Store) error {
731
+func restoreCustomImage(is image.Store, ls layer.Store, rs reference.Store) error {
732 732
 	// Unix has no custom images to register
733 733
 	return nil
734 734
 }
... ...
@@ -156,67 +156,78 @@ func (daemon *Daemon) conditionalUnmountOnCleanup(container *container.Container
156 156
 	}
157 157
 }
158 158
 
159
-func restoreCustomImage(driver graphdriver.Driver, is image.Store, ls layer.Store, rs reference.Store) error {
160
-	if wd, ok := driver.(*windows.Driver); ok {
161
-		imageInfos, err := wd.GetCustomImageInfos()
159
+func restoreCustomImage(is image.Store, ls layer.Store, rs reference.Store) error {
160
+	type graphDriverStore interface {
161
+		GraphDriver() graphdriver.Driver
162
+	}
163
+
164
+	gds, ok := ls.(graphDriverStore)
165
+	if !ok {
166
+		return nil
167
+	}
168
+
169
+	driver := gds.GraphDriver()
170
+	wd, ok := driver.(*windows.Driver)
171
+	if !ok {
172
+		return nil
173
+	}
174
+
175
+	imageInfos, err := wd.GetCustomImageInfos()
176
+	if err != nil {
177
+		return err
178
+	}
179
+
180
+	// Convert imageData to valid image configuration
181
+	for i := range imageInfos {
182
+		name := strings.ToLower(imageInfos[i].Name)
183
+
184
+		type registrar interface {
185
+			RegisterDiffID(graphID string, size int64) (layer.Layer, error)
186
+		}
187
+		r, ok := ls.(registrar)
188
+		if !ok {
189
+			return errors.New("Layerstore doesn't support RegisterDiffID")
190
+		}
191
+		if _, err := r.RegisterDiffID(imageInfos[i].ID, imageInfos[i].Size); err != nil {
192
+			return err
193
+		}
194
+		// layer is intentionally not released
195
+
196
+		rootFS := image.NewRootFS()
197
+		rootFS.BaseLayer = filepath.Base(imageInfos[i].Path)
198
+
199
+		// Create history for base layer
200
+		config, err := json.Marshal(&image.Image{
201
+			V1Image: image.V1Image{
202
+				DockerVersion: dockerversion.Version,
203
+				Architecture:  runtime.GOARCH,
204
+				OS:            runtime.GOOS,
205
+				Created:       imageInfos[i].CreatedTime,
206
+			},
207
+			RootFS:  rootFS,
208
+			History: []image.History{},
209
+		})
210
+
211
+		named, err := reference.ParseNamed(name)
162 212
 		if err != nil {
163 213
 			return err
164 214
 		}
165 215
 
166
-		// Convert imageData to valid image configuration
167
-		for i := range imageInfos {
168
-			name := strings.ToLower(imageInfos[i].Name)
169
-
170
-			type registrar interface {
171
-				RegisterDiffID(graphID string, size int64) (layer.Layer, error)
172
-			}
173
-			r, ok := ls.(registrar)
174
-			if !ok {
175
-				return errors.New("Layerstore doesn't support RegisterDiffID")
176
-			}
177
-			if _, err := r.RegisterDiffID(imageInfos[i].ID, imageInfos[i].Size); err != nil {
178
-				return err
179
-			}
180
-			// layer is intentionally not released
181
-
182
-			rootFS := image.NewRootFS()
183
-			rootFS.BaseLayer = filepath.Base(imageInfos[i].Path)
184
-
185
-			// Create history for base layer
186
-			config, err := json.Marshal(&image.Image{
187
-				V1Image: image.V1Image{
188
-					DockerVersion: dockerversion.Version,
189
-					Architecture:  runtime.GOARCH,
190
-					OS:            runtime.GOOS,
191
-					Created:       imageInfos[i].CreatedTime,
192
-				},
193
-				RootFS:  rootFS,
194
-				History: []image.History{},
195
-			})
196
-
197
-			named, err := reference.ParseNamed(name)
198
-			if err != nil {
199
-				return err
200
-			}
201
-
202
-			ref, err := reference.WithTag(named, imageInfos[i].Version)
203
-			if err != nil {
204
-				return err
205
-			}
206
-
207
-			id, err := is.Create(config)
208
-			if err != nil {
209
-				return err
210
-			}
211
-
212
-			if err := rs.AddTag(ref, id, true); err != nil {
213
-				return err
214
-			}
215
-
216
-			logrus.Debugf("Registered base layer %s as %s", ref, id)
216
+		ref, err := reference.WithTag(named, imageInfos[i].Version)
217
+		if err != nil {
218
+			return err
217 219
 		}
218 220
 
219
-	}
221
+		id, err := is.Create(config)
222
+		if err != nil {
223
+			return err
224
+		}
225
+
226
+		if err := rs.AddTag(ref, id, true); err != nil {
227
+			return err
228
+		}
220 229
 
230
+		logrus.Debugf("Registered base layer %s as %s", ref, id)
231
+	}
221 232
 	return nil
222 233
 }
... ...
@@ -133,7 +133,7 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, forceRemo
133 133
 	metadata, err := daemon.layerStore.ReleaseRWLayer(container.RWLayer)
134 134
 	layer.LogReleaseMetadata(metadata)
135 135
 	if err != nil && err != layer.ErrMountDoesNotExist {
136
-		return derr.ErrorCodeRmDriverFS.WithArgs(daemon.driver, container.ID, err)
136
+		return derr.ErrorCodeRmDriverFS.WithArgs(daemon.GraphDriverName(), container.ID, err)
137 137
 	}
138 138
 
139 139
 	if err = daemon.execDriver.Clean(container.ID); err != nil {
... ...
@@ -22,8 +22,6 @@ const (
22 22
 )
23 23
 
24 24
 var (
25
-	// DefaultDriver if a storage driver is not specified.
26
-	DefaultDriver string
27 25
 	// All registered drivers
28 26
 	drivers map[string]InitFunc
29 27
 
... ...
@@ -130,12 +128,10 @@ func getBuiltinDriver(name, home string, options []string, uidMaps, gidMaps []id
130 130
 }
131 131
 
132 132
 // New creates the driver and initializes it at the specified root.
133
-func New(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (driver Driver, err error) {
134
-	for _, name := range []string{os.Getenv("DOCKER_DRIVER"), DefaultDriver} {
135
-		if name != "" {
136
-			logrus.Debugf("[graphdriver] trying provided driver %q", name) // so the logs show specified driver
137
-			return GetDriver(name, root, options, uidMaps, gidMaps)
138
-		}
133
+func New(root string, name string, options []string, uidMaps, gidMaps []idtools.IDMap) (driver Driver, err error) {
134
+	if name != "" {
135
+		logrus.Debugf("[graphdriver] trying provided driver %q", name) // so the logs show specified driver
136
+		return GetDriver(name, root, options, uidMaps, gidMaps)
139 137
 	}
140 138
 
141 139
 	// Guess for prior driver
... ...
@@ -58,8 +58,8 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) {
58 58
 		ID:                 daemon.ID,
59 59
 		Containers:         len(daemon.List()),
60 60
 		Images:             len(daemon.imageStore.Map()),
61
-		Driver:             daemon.GraphDriver().String(),
62
-		DriverStatus:       daemon.GraphDriver().Status(),
61
+		Driver:             daemon.GraphDriverName(),
62
+		DriverStatus:       daemon.layerStore.DriverStatus(),
63 63
 		Plugins:            daemon.showPluginsInfo(),
64 64
 		IPv4Forwarding:     !sysInfo.IPv4ForwardingDisabled,
65 65
 		BridgeNfIptables:   !sysInfo.BridgeNfCallIptablesDisabled,
... ...
@@ -128,6 +128,18 @@ func (ls *mockLayerStore) ReleaseRWLayer(layer.RWLayer) ([]layer.Metadata, error
128 128
 
129 129
 }
130 130
 
131
+func (ls *mockLayerStore) Cleanup() error {
132
+	return nil
133
+}
134
+
135
+func (ls *mockLayerStore) DriverStatus() [][2]string {
136
+	return [][2]string{}
137
+}
138
+
139
+func (ls *mockLayerStore) DriverName() string {
140
+	return "mock"
141
+}
142
+
131 143
 type mockDownloadDescriptor struct {
132 144
 	currentDownloads *int32
133 145
 	id               string
... ...
@@ -240,7 +240,7 @@ func (cli *DaemonCli) CmdDaemon(args ...string) error {
240 240
 		"version":     dockerversion.Version,
241 241
 		"commit":      dockerversion.GitCommit,
242 242
 		"execdriver":  d.ExecutionDriver().Name(),
243
-		"graphdriver": d.GraphDriver().String(),
243
+		"graphdriver": d.GraphDriverName(),
244 244
 	}).Info("Docker daemon")
245 245
 
246 246
 	api.InitRouters(d)
... ...
@@ -170,6 +170,10 @@ type Store interface {
170 170
 	CreateRWLayer(id string, parent ChainID, mountLabel string, initFunc MountInit) (RWLayer, error)
171 171
 	GetRWLayer(id string) (RWLayer, error)
172 172
 	ReleaseRWLayer(RWLayer) ([]Metadata, error)
173
+
174
+	Cleanup() error
175
+	DriverStatus() [][2]string
176
+	DriverName() string
173 177
 }
174 178
 
175 179
 // MetadataTransaction represents functions for setting layer metadata
... ...
@@ -11,6 +11,7 @@ import (
11 11
 	"github.com/docker/distribution/digest"
12 12
 	"github.com/docker/docker/daemon/graphdriver"
13 13
 	"github.com/docker/docker/pkg/archive"
14
+	"github.com/docker/docker/pkg/idtools"
14 15
 	"github.com/docker/docker/pkg/stringid"
15 16
 	"github.com/vbatts/tar-split/tar/asm"
16 17
 	"github.com/vbatts/tar-split/tar/storage"
... ...
@@ -34,11 +35,41 @@ type layerStore struct {
34 34
 	mountL sync.Mutex
35 35
 }
36 36
 
37
-// NewStore creates a new Store instance using
38
-// the provided metadata store and graph driver.
39
-// The metadata store will be used to restore
37
+// StoreOptions are the options used to create a new Store instance
38
+type StoreOptions struct {
39
+	StorePath                 string
40
+	MetadataStorePathTemplate string
41
+	GraphDriver               string
42
+	GraphDriverOptions        []string
43
+	UIDMaps                   []idtools.IDMap
44
+	GIDMaps                   []idtools.IDMap
45
+}
46
+
47
+// NewStoreFromOptions creates a new Store instance
48
+func NewStoreFromOptions(options StoreOptions) (Store, error) {
49
+	driver, err := graphdriver.New(
50
+		options.StorePath,
51
+		options.GraphDriver,
52
+		options.GraphDriverOptions,
53
+		options.UIDMaps,
54
+		options.GIDMaps)
55
+	if err != nil {
56
+		return nil, fmt.Errorf("error initializing graphdriver: %v", err)
57
+	}
58
+	logrus.Debugf("Using graph driver %s", driver)
59
+
60
+	fms, err := NewFSMetadataStore(fmt.Sprintf(options.MetadataStorePathTemplate, driver))
61
+	if err != nil {
62
+		return nil, err
63
+	}
64
+
65
+	return NewStoreFromGraphDriver(fms, driver)
66
+}
67
+
68
+// NewStoreFromGraphDriver creates a new Store instance using the provided
69
+// metadata store and graph driver. The metadata store will be used to restore
40 70
 // the Store.
41
-func NewStore(store MetadataStore, driver graphdriver.Driver) (Store, error) {
71
+func NewStoreFromGraphDriver(store MetadataStore, driver graphdriver.Driver) (Store, error) {
42 72
 	ls := &layerStore{
43 73
 		store:    store,
44 74
 		driver:   driver,
... ...
@@ -581,6 +612,18 @@ func (ls *layerStore) assembleTar(graphID string, metadata io.ReadCloser, size *
581 581
 	return pR, nil
582 582
 }
583 583
 
584
+func (ls *layerStore) Cleanup() error {
585
+	return ls.driver.Cleanup()
586
+}
587
+
588
+func (ls *layerStore) DriverStatus() [][2]string {
589
+	return ls.driver.Status()
590
+}
591
+
592
+func (ls *layerStore) DriverName() string {
593
+	return ls.driver.String()
594
+}
595
+
584 596
 type naiveDiffPathDriver struct {
585 597
 	graphdriver.Driver
586 598
 }
... ...
@@ -67,7 +67,7 @@ func newTestStore(t *testing.T) (Store, func()) {
67 67
 	if err != nil {
68 68
 		t.Fatal(err)
69 69
 	}
70
-	ls, err := NewStore(fms, graph)
70
+	ls, err := NewStoreFromGraphDriver(fms, graph)
71 71
 	if err != nil {
72 72
 		t.Fatal(err)
73 73
 	}
... ...
@@ -398,7 +398,7 @@ func TestStoreRestore(t *testing.T) {
398 398
 
399 399
 	assertActivityCount(t, m, 0)
400 400
 
401
-	ls2, err := NewStore(ls.(*layerStore).store, ls.(*layerStore).driver)
401
+	ls2, err := NewStoreFromGraphDriver(ls.(*layerStore).store, ls.(*layerStore).driver)
402 402
 	if err != nil {
403 403
 		t.Fatal(err)
404 404
 	}
... ...
@@ -6,6 +6,7 @@ import (
6 6
 
7 7
 	"github.com/Sirupsen/logrus"
8 8
 	"github.com/docker/distribution/digest"
9
+	"github.com/docker/docker/daemon/graphdriver"
9 10
 )
10 11
 
11 12
 // GetLayerPath returns the path to a layer
... ...
@@ -94,3 +95,7 @@ func (ls *layerStore) mountID(name string) string {
94 94
 	// windows has issues if container ID doesn't match mount ID
95 95
 	return name
96 96
 }
97
+
98
+func (ls *layerStore) GraphDriver() graphdriver.Driver {
99
+	return ls.driver
100
+}
... ...
@@ -89,7 +89,7 @@ func TestLayerMigration(t *testing.T) {
89 89
 	if err != nil {
90 90
 		t.Fatal(err)
91 91
 	}
92
-	ls, err := NewStore(fms, graph)
92
+	ls, err := NewStoreFromGraphDriver(fms, graph)
93 93
 	if err != nil {
94 94
 		t.Fatal(err)
95 95
 	}
... ...
@@ -205,7 +205,7 @@ func TestLayerMigrationNoTarsplit(t *testing.T) {
205 205
 	if err != nil {
206 206
 		t.Fatal(err)
207 207
 	}
208
-	ls, err := NewStore(fms, graph)
208
+	ls, err := NewStoreFromGraphDriver(fms, graph)
209 209
 	if err != nil {
210 210
 		t.Fatal(err)
211 211
 	}