Browse code

daemon: reload runtimes w/o breaking containers

The existing runtimes reload logic went to great lengths to replace the
directory containing runtime wrapper scripts as atomically as possible
within the limitations of the Linux filesystem ABI. Trouble is,
atomically swapping the wrapper scripts directory solves the wrong
problem! The runtime configuration is "locked in" when a container is
started, including the path to the runC binary. If a container is
started with a runtime which requires a daemon-managed wrapper script
and then the daemon is reloaded with a config which no longer requires
the wrapper script (i.e. some args -> no args, or the runtime is dropped
from the config), that container would become unmanageable. Any attempts
to stop, exec or otherwise perform lifecycle management operations on
the container are likely to fail due to the wrapper script no longer
existing at its original path.

Atomically swapping the wrapper scripts is also incompatible with the
read-copy-update paradigm for reloading configuration. A handler in the
daemon could retain a reference to the pre-reload configuration for an
indeterminate amount of time after the daemon configuration has been
reloaded and updated. It is possible for the daemon to attempt to start
a container using a deleted wrapper script if a request to run a
container races a reload.

Solve the problem of deleting referenced wrapper scripts by ensuring
that all wrapper scripts are *immutable* for the lifetime of the daemon
process. Any given runtime wrapper script must always exist with the
same contents, no matter how many times the daemon config is reloaded,
or what changes are made to the config. This is accomplished by using
everyone's favourite design pattern: content-addressable storage. Each
wrapper script file name is suffixed with the SHA-256 digest of its
contents to (probabilistically) guarantee immutability without needing
any concurrency control. Stale runtime wrapper scripts are only cleaned
up on the next daemon restart.

Split the derived runtimes configuration from the user-supplied
configuration to have a place to store derived state without mutating
the user-supplied configuration or exposing daemon internals in API
struct types. Hold the derived state and the user-supplied configuration
in a single struct value so that they can be updated as an atomic unit.

Signed-off-by: Cory Snider <csnider@mirantis.com>

Cory Snider authored on 2022/09/01 05:12:30
Showing 40 changed files
... ...
@@ -16,7 +16,6 @@ import (
16 16
 	"github.com/docker/docker/api/types/swarm"
17 17
 	"github.com/docker/docker/api/types/volume"
18 18
 	"github.com/docker/go-connections/nat"
19
-	"github.com/opencontainers/runtime-spec/specs-go/features"
20 19
 )
21 20
 
22 21
 const (
... ...
@@ -657,16 +656,6 @@ type Runtime struct {
657 657
 
658 658
 	Type    string                 `json:"runtimeType,omitempty"`
659 659
 	Options map[string]interface{} `json:"options,omitempty"`
660
-
661
-	// This is exposed here only for internal use
662
-	ShimConfig *ShimConfig        `json:"-"`
663
-	Features   *features.Features `json:"-"`
664
-}
665
-
666
-// ShimConfig is used by runtime to configure containerd shims
667
-type ShimConfig struct {
668
-	Binary string
669
-	Opts   interface{}
670 660
 }
671 661
 
672 662
 // DiskUsageObject represents an object type used for disk usage query filtering.
... ...
@@ -268,7 +268,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
268 268
 	// Restart all autostart containers which has a swarm endpoint
269 269
 	// and is not yet running now that we have successfully
270 270
 	// initialized the cluster.
271
-	d.RestartSwarmContainers(cli.Config)
271
+	d.RestartSwarmContainers()
272 272
 
273 273
 	logrus.Info("Daemon has completed initialization")
274 274
 
... ...
@@ -81,15 +81,6 @@ type Config struct {
81 81
 	Rootless   bool   `json:"rootless,omitempty"`
82 82
 }
83 83
 
84
-// GetRuntime returns the runtime path and arguments for a given
85
-// runtime name
86
-func (conf *Config) GetRuntime(name string) *types.Runtime {
87
-	if rt, ok := conf.Runtimes[name]; ok {
88
-		return &rt
89
-	}
90
-	return nil
91
-}
92
-
93 84
 // GetAllRuntimes returns a copy of the runtimes map
94 85
 func (conf *Config) GetAllRuntimes() map[string]types.Runtime {
95 86
 	return conf.Runtimes
... ...
@@ -30,12 +30,6 @@ type Config struct {
30 30
 	// for the Windows daemon.)
31 31
 }
32 32
 
33
-// GetRuntime returns the runtime path and arguments for a given
34
-// runtime name
35
-func (conf *Config) GetRuntime(name string) *types.Runtime {
36
-	return nil
37
-}
38
-
39 33
 // GetAllRuntimes returns a copy of the runtimes map
40 34
 func (conf *Config) GetAllRuntimes() map[string]types.Runtime {
41 35
 	return map[string]types.Runtime{}
... ...
@@ -235,7 +235,7 @@ func (daemon *Daemon) setHostConfig(container *container.Container, hostConfig *
235 235
 
236 236
 // verifyContainerSettings performs validation of the hostconfig and config
237 237
 // structures.
238
-func (daemon *Daemon) verifyContainerSettings(daemonCfg *config.Config, hostConfig *containertypes.HostConfig, config *containertypes.Config, update bool) (warnings []string, err error) {
238
+func (daemon *Daemon) verifyContainerSettings(daemonCfg *configStore, hostConfig *containertypes.HostConfig, config *containertypes.Config, update bool) (warnings []string, err error) {
239 239
 	// First perform verification of settings common across all platforms.
240 240
 	if err = validateContainerConfig(config); err != nil {
241 241
 		return warnings, err
... ...
@@ -1075,7 +1075,7 @@ func (daemon *Daemon) ConnectToNetwork(container *container.Container, idOrName
1075 1075
 			}
1076 1076
 		}
1077 1077
 	} else {
1078
-		if err := daemon.connectToNetwork(daemon.config(), container, idOrName, endpointConfig, true); err != nil {
1078
+		if err := daemon.connectToNetwork(&daemon.config().Config, container, idOrName, endpointConfig, true); err != nil {
1079 1079
 			return err
1080 1080
 		}
1081 1081
 	}
... ...
@@ -31,11 +31,14 @@ func TestContainerWarningHostAndPublishPorts(t *testing.T) {
31 31
 			NetworkMode:  "host",
32 32
 			PortBindings: tc.ports,
33 33
 		}
34
-		cs := &config.Config{}
35
-		configureRuntimes(cs)
36 34
 		d := &Daemon{}
37
-		d.configStore.Store(cs)
38
-		wrns, err := d.verifyContainerSettings(cs, hostConfig, &containertypes.Config{}, false)
35
+		cfg, err := config.New()
36
+		assert.NilError(t, err)
37
+		configureRuntimes(cfg)
38
+		runtimes, err := setupRuntimes(cfg)
39
+		assert.NilError(t, err)
40
+		daemonCfg := &configStore{Config: *cfg, Runtimes: runtimes}
41
+		wrns, err := d.verifyContainerSettings(daemonCfg, hostConfig, &containertypes.Config{}, false)
39 42
 		assert.NilError(t, err)
40 43
 		assert.DeepEqual(t, tc.warnings, wrns)
41 44
 	}
... ...
@@ -57,7 +57,7 @@ func (daemon *Daemon) ContainerCreateIgnoreImagesArgsEscaped(ctx context.Context
57 57
 	})
58 58
 }
59 59
 
60
-func (daemon *Daemon) containerCreate(ctx context.Context, daemonCfg *config.Config, opts createOpts) (containertypes.CreateResponse, error) {
60
+func (daemon *Daemon) containerCreate(ctx context.Context, daemonCfg *configStore, opts createOpts) (containertypes.CreateResponse, error) {
61 61
 	start := time.Now()
62 62
 	if opts.params.Config == nil {
63 63
 		return containertypes.CreateResponse{}, errdefs.InvalidParameter(errors.New("Config cannot be empty in order to create a container"))
... ...
@@ -95,12 +95,12 @@ func (daemon *Daemon) containerCreate(ctx context.Context, daemonCfg *config.Con
95 95
 	if opts.params.HostConfig == nil {
96 96
 		opts.params.HostConfig = &containertypes.HostConfig{}
97 97
 	}
98
-	err = daemon.adaptContainerSettings(daemonCfg, opts.params.HostConfig, opts.params.AdjustCPUShares)
98
+	err = daemon.adaptContainerSettings(&daemonCfg.Config, opts.params.HostConfig, opts.params.AdjustCPUShares)
99 99
 	if err != nil {
100 100
 		return containertypes.CreateResponse{Warnings: warnings}, errdefs.InvalidParameter(err)
101 101
 	}
102 102
 
103
-	ctr, err := daemon.create(ctx, daemonCfg, opts)
103
+	ctr, err := daemon.create(ctx, &daemonCfg.Config, opts)
104 104
 	if err != nil {
105 105
 		return containertypes.CreateResponse{Warnings: warnings}, err
106 106
 	}
... ...
@@ -77,6 +77,12 @@ import (
77 77
 	"resenje.org/singleflight"
78 78
 )
79 79
 
80
+type configStore struct {
81
+	config.Config
82
+
83
+	Runtimes runtimes
84
+}
85
+
80 86
 // Daemon holds information about the Docker daemon.
81 87
 type Daemon struct {
82 88
 	id                    string
... ...
@@ -85,7 +91,7 @@ type Daemon struct {
85 85
 	containersReplica     *container.ViewDB
86 86
 	execCommands          *container.ExecStore
87 87
 	imageService          ImageService
88
-	configStore           atomic.Pointer[config.Config]
88
+	configStore           atomic.Pointer[configStore]
89 89
 	configReload          sync.Mutex
90 90
 	statsCollector        *stats.Collector
91 91
 	defaultLogConfig      containertypes.LogConfig
... ...
@@ -159,10 +165,10 @@ func (daemon *Daemon) StoreHosts(hosts []string) {
159 159
 // lifetime of an operation, the configuration pointer should be passed down the
160 160
 // call stack, like one would a [context.Context] value. Only the entrypoints
161 161
 // for operations, the outermost functions, should call this function.
162
-func (daemon *Daemon) config() *config.Config {
162
+func (daemon *Daemon) config() *configStore {
163 163
 	cfg := daemon.configStore.Load()
164 164
 	if cfg == nil {
165
-		return &config.Config{}
165
+		return &configStore{}
166 166
 	}
167 167
 	return cfg
168 168
 }
... ...
@@ -247,7 +253,7 @@ type layerAccessor interface {
247 247
 	GetLayerByID(cid string) (layer.RWLayer, error)
248 248
 }
249 249
 
250
-func (daemon *Daemon) restore(cfg *config.Config) error {
250
+func (daemon *Daemon) restore(cfg *configStore) error {
251 251
 	var mapLock sync.Mutex
252 252
 	containers := make(map[string]*container.Container)
253 253
 
... ...
@@ -467,7 +473,7 @@ func (daemon *Daemon) restore(cfg *config.Config) error {
467 467
 
468 468
 				c.ResetRestartManager(false)
469 469
 				if !c.HostConfig.NetworkMode.IsContainer() && c.IsRunning() {
470
-					options, err := daemon.buildSandboxOptions(cfg, c)
470
+					options, err := daemon.buildSandboxOptions(&cfg.Config, c)
471 471
 					if err != nil {
472 472
 						logger(c).WithError(err).Warn("failed to build sandbox option to restore container")
473 473
 					}
... ...
@@ -523,7 +529,7 @@ func (daemon *Daemon) restore(cfg *config.Config) error {
523 523
 	//
524 524
 	// Note that we cannot initialize the network controller earlier, as it
525 525
 	// needs to know if there's active sandboxes (running containers).
526
-	if err = daemon.initNetworkController(cfg, activeSandboxes); err != nil {
526
+	if err = daemon.initNetworkController(&cfg.Config, activeSandboxes); err != nil {
527 527
 		return fmt.Errorf("Error initializing network controller: %v", err)
528 528
 	}
529 529
 
... ...
@@ -586,7 +592,7 @@ func (daemon *Daemon) restore(cfg *config.Config) error {
586 586
 		go func(cid string) {
587 587
 			_ = sem.Acquire(context.Background(), 1)
588 588
 
589
-			if err := daemon.containerRm(cfg, cid, &types.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil {
589
+			if err := daemon.containerRm(&cfg.Config, cid, &types.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil {
590 590
 				logrus.WithField("container", cid).WithError(err).Error("failed to remove container")
591 591
 			}
592 592
 
... ...
@@ -634,9 +640,11 @@ func (daemon *Daemon) restore(cfg *config.Config) error {
634 634
 
635 635
 // RestartSwarmContainers restarts any autostart container which has a
636 636
 // swarm endpoint.
637
-func (daemon *Daemon) RestartSwarmContainers(cfg *config.Config) {
638
-	ctx := context.Background()
637
+func (daemon *Daemon) RestartSwarmContainers() {
638
+	daemon.restartSwarmContainers(context.Background(), daemon.config())
639
+}
639 640
 
641
+func (daemon *Daemon) restartSwarmContainers(ctx context.Context, cfg *configStore) {
640 642
 	// parallelLimit is the maximum number of parallel startup jobs that we
641 643
 	// allow (this is the limited used for all startup semaphores). The multipler
642 644
 	// (128) was chosen after some fairly significant benchmarking -- don't change
... ...
@@ -806,11 +814,23 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
806 806
 		os.Setenv("TMPDIR", realTmp)
807 807
 	}
808 808
 
809
+	if err := initRuntimesDir(config); err != nil {
810
+		return nil, err
811
+	}
812
+	runtimes, err := setupRuntimes(config)
813
+	if err != nil {
814
+		return nil, err
815
+	}
816
+
809 817
 	d := &Daemon{
810 818
 		PluginStore: pluginStore,
811 819
 		startupDone: make(chan struct{}),
812 820
 	}
813
-	d.configStore.Store(config)
821
+	configStore := &configStore{
822
+		Config:   *config,
823
+		Runtimes: runtimes,
824
+	}
825
+	d.configStore.Store(configStore)
814 826
 
815 827
 	// TEST_INTEGRATION_USE_SNAPSHOTTER is used for integration tests only.
816 828
 	if os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" {
... ...
@@ -830,27 +850,27 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
830 830
 		}
831 831
 	}()
832 832
 
833
-	if err := d.setGenericResources(config); err != nil {
833
+	if err := d.setGenericResources(&configStore.Config); err != nil {
834 834
 		return nil, err
835 835
 	}
836 836
 	// set up SIGUSR1 handler on Unix-like systems, or a Win32 global event
837 837
 	// on Windows to dump Go routine stacks
838
-	stackDumpDir := config.Root
839
-	if execRoot := config.GetExecRoot(); execRoot != "" {
838
+	stackDumpDir := configStore.Root
839
+	if execRoot := configStore.GetExecRoot(); execRoot != "" {
840 840
 		stackDumpDir = execRoot
841 841
 	}
842 842
 	d.setupDumpStackTrap(stackDumpDir)
843 843
 
844
-	if err := d.setupSeccompProfile(config); err != nil {
844
+	if err := d.setupSeccompProfile(&configStore.Config); err != nil {
845 845
 		return nil, err
846 846
 	}
847 847
 
848 848
 	// Set the default isolation mode (only applicable on Windows)
849
-	if err := d.setDefaultIsolation(config); err != nil {
849
+	if err := d.setDefaultIsolation(&configStore.Config); err != nil {
850 850
 		return nil, fmt.Errorf("error setting default isolation mode: %v", err)
851 851
 	}
852 852
 
853
-	if err := configureMaxThreads(config); err != nil {
853
+	if err := configureMaxThreads(&configStore.Config); err != nil {
854 854
 		logrus.Warnf("Failed to configure golang's threads limit: %v", err)
855 855
 	}
856 856
 
... ...
@@ -859,7 +879,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
859 859
 		logrus.Errorf(err.Error())
860 860
 	}
861 861
 
862
-	daemonRepo := filepath.Join(config.Root, "containers")
862
+	daemonRepo := filepath.Join(configStore.Root, "containers")
863 863
 	if err := idtools.MkdirAllAndChown(daemonRepo, 0o710, idtools.Identity{
864 864
 		UID: idtools.CurrentIdentity().UID,
865 865
 		GID: rootIDs.GID,
... ...
@@ -867,20 +887,11 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
867 867
 		return nil, err
868 868
 	}
869 869
 
870
-	// Create the directory where we'll store the runtime scripts (i.e. in
871
-	// order to support runtimeArgs)
872
-	if err = os.Mkdir(filepath.Join(config.Root, "runtimes"), 0o700); err != nil && !errors.Is(err, os.ErrExist) {
873
-		return nil, err
874
-	}
875
-	if err := d.loadRuntimes(); err != nil {
876
-		return nil, err
877
-	}
878
-
879 870
 	if isWindows {
880 871
 		// Note that permissions (0o700) are ignored on Windows; passing them to
881 872
 		// show intent only. We could consider using idtools.MkdirAndChown here
882 873
 		// to apply an ACL.
883
-		if err = os.Mkdir(filepath.Join(config.Root, "credentialspecs"), 0o700); err != nil && !errors.Is(err, os.ErrExist) {
874
+		if err = os.Mkdir(filepath.Join(configStore.Root, "credentialspecs"), 0o700); err != nil && !errors.Is(err, os.ErrExist) {
884 875
 			return nil, err
885 876
 		}
886 877
 	}
... ...
@@ -888,7 +899,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
888 888
 	d.registryService = registryService
889 889
 	dlogger.RegisterPluginGetter(d.PluginStore)
890 890
 
891
-	metricsSockPath, err := d.listenMetricsSock(config)
891
+	metricsSockPath, err := d.listenMetricsSock(&configStore.Config)
892 892
 	if err != nil {
893 893
 		return nil, err
894 894
 	}
... ...
@@ -927,20 +938,20 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
927 927
 		grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
928 928
 	}
929 929
 
930
-	if config.ContainerdAddr != "" {
931
-		d.containerdCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(config.ContainerdNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
930
+	if configStore.ContainerdAddr != "" {
931
+		d.containerdCli, err = containerd.New(configStore.ContainerdAddr, containerd.WithDefaultNamespace(configStore.ContainerdNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
932 932
 		if err != nil {
933
-			return nil, errors.Wrapf(err, "failed to dial %q", config.ContainerdAddr)
933
+			return nil, errors.Wrapf(err, "failed to dial %q", configStore.ContainerdAddr)
934 934
 		}
935 935
 	}
936 936
 
937 937
 	createPluginExec := func(m *plugin.Manager) (plugin.Executor, error) {
938 938
 		var pluginCli *containerd.Client
939 939
 
940
-		if config.ContainerdAddr != "" {
941
-			pluginCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(config.ContainerdPluginNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
940
+		if configStore.ContainerdAddr != "" {
941
+			pluginCli, err = containerd.New(configStore.ContainerdAddr, containerd.WithDefaultNamespace(configStore.ContainerdPluginNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
942 942
 			if err != nil {
943
-				return nil, errors.Wrapf(err, "failed to dial %q", config.ContainerdAddr)
943
+				return nil, errors.Wrapf(err, "failed to dial %q", configStore.ContainerdAddr)
944 944
 			}
945 945
 		}
946 946
 
... ...
@@ -949,22 +960,22 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
949 949
 			shimOpts interface{}
950 950
 		)
951 951
 		if runtime.GOOS != "windows" {
952
-			shim, shimOpts, err = d.getRuntime(config, config.DefaultRuntime)
952
+			shim, shimOpts, err = runtimes.Get(configStore.DefaultRuntime)
953 953
 			if err != nil {
954 954
 				return nil, err
955 955
 			}
956 956
 		}
957
-		return pluginexec.New(ctx, getPluginExecRoot(config), pluginCli, config.ContainerdPluginNamespace, m, shim, shimOpts)
957
+		return pluginexec.New(ctx, getPluginExecRoot(&configStore.Config), pluginCli, configStore.ContainerdPluginNamespace, m, shim, shimOpts)
958 958
 	}
959 959
 
960 960
 	// Plugin system initialization should happen before restore. Do not change order.
961 961
 	d.pluginManager, err = plugin.NewManager(plugin.ManagerConfig{
962
-		Root:               filepath.Join(config.Root, "plugins"),
963
-		ExecRoot:           getPluginExecRoot(config),
962
+		Root:               filepath.Join(configStore.Root, "plugins"),
963
+		ExecRoot:           getPluginExecRoot(&configStore.Config),
964 964
 		Store:              d.PluginStore,
965 965
 		CreateExecutor:     createPluginExec,
966 966
 		RegistryService:    registryService,
967
-		LiveRestoreEnabled: config.LiveRestoreEnabled,
967
+		LiveRestoreEnabled: configStore.LiveRestoreEnabled,
968 968
 		LogPluginEvent:     d.LogPluginEvent, // todo: make private
969 969
 		AuthzMiddleware:    authzMiddleware,
970 970
 	})
... ...
@@ -972,13 +983,13 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
972 972
 		return nil, errors.Wrap(err, "couldn't create plugin manager")
973 973
 	}
974 974
 
975
-	d.defaultLogConfig, err = defaultLogConfig(config)
975
+	d.defaultLogConfig, err = defaultLogConfig(&configStore.Config)
976 976
 	if err != nil {
977 977
 		return nil, errors.Wrap(err, "failed to set log opts")
978 978
 	}
979 979
 	logrus.Debugf("Using default logging driver %s", d.defaultLogConfig.Type)
980 980
 
981
-	d.volumes, err = volumesservice.NewVolumeService(config.Root, d.PluginStore, rootIDs, d)
981
+	d.volumes, err = volumesservice.NewVolumeService(configStore.Root, d.PluginStore, rootIDs, d)
982 982
 	if err != nil {
983 983
 		return nil, err
984 984
 	}
... ...
@@ -991,11 +1002,11 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
991 991
 	// at this point.
992 992
 	//
993 993
 	// TODO(thaJeztah) add a utility to only collect the CgroupDevicesEnabled information
994
-	if runtime.GOOS == "linux" && !userns.RunningInUserNS() && !getSysInfo(config).CgroupDevicesEnabled {
994
+	if runtime.GOOS == "linux" && !userns.RunningInUserNS() && !getSysInfo(&configStore.Config).CgroupDevicesEnabled {
995 995
 		return nil, errors.New("Devices cgroup isn't mounted")
996 996
 	}
997 997
 
998
-	d.id, err = loadOrCreateID(filepath.Join(config.Root, "engine-id"))
998
+	d.id, err = loadOrCreateID(filepath.Join(configStore.Root, "engine-id"))
999 999
 	if err != nil {
1000 1000
 		return nil, err
1001 1001
 	}
... ...
@@ -1008,7 +1019,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
1008 1008
 	d.statsCollector = d.newStatsCollector(1 * time.Second)
1009 1009
 
1010 1010
 	d.EventsService = events.New()
1011
-	d.root = config.Root
1011
+	d.root = configStore.Root
1012 1012
 	d.idMapping = idMapping
1013 1013
 
1014 1014
 	d.linkIndex = newLinkIndex()
... ...
@@ -1023,7 +1034,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
1023 1023
 	} else if driverName != "" {
1024 1024
 		logrus.Infof("Setting the storage driver from the $DOCKER_DRIVER environment variable (%s)", driverName)
1025 1025
 	} else {
1026
-		driverName = config.GraphDriver
1026
+		driverName = configStore.GraphDriver
1027 1027
 	}
1028 1028
 
1029 1029
 	if d.UsesSnapshotter() {
... ...
@@ -1039,7 +1050,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
1039 1039
 
1040 1040
 		// Configure and validate the kernels security support. Note this is a Linux/FreeBSD
1041 1041
 		// operation only, so it is safe to pass *just* the runtime OS graphdriver.
1042
-		if err := configureKernelSecuritySupport(config, driverName); err != nil {
1042
+		if err := configureKernelSecuritySupport(&configStore.Config, driverName); err != nil {
1043 1043
 			return nil, err
1044 1044
 		}
1045 1045
 		d.imageService = ctrd.NewService(ctrd.ImageServiceConfig{
... ...
@@ -1052,13 +1063,13 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
1052 1052
 		})
1053 1053
 	} else {
1054 1054
 		layerStore, err := layer.NewStoreFromOptions(layer.StoreOptions{
1055
-			Root:                      config.Root,
1056
-			MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"),
1055
+			Root:                      configStore.Root,
1056
+			MetadataStorePathTemplate: filepath.Join(configStore.Root, "image", "%s", "layerdb"),
1057 1057
 			GraphDriver:               driverName,
1058
-			GraphDriverOptions:        config.GraphOptions,
1058
+			GraphDriverOptions:        configStore.GraphOptions,
1059 1059
 			IDMapping:                 idMapping,
1060 1060
 			PluginGetter:              d.PluginStore,
1061
-			ExperimentalEnabled:       config.Experimental,
1061
+			ExperimentalEnabled:       configStore.Experimental,
1062 1062
 		})
1063 1063
 		if err != nil {
1064 1064
 			return nil, err
... ...
@@ -1066,11 +1077,11 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
1066 1066
 
1067 1067
 		// Configure and validate the kernels security support. Note this is a Linux/FreeBSD
1068 1068
 		// operation only, so it is safe to pass *just* the runtime OS graphdriver.
1069
-		if err := configureKernelSecuritySupport(config, layerStore.DriverName()); err != nil {
1069
+		if err := configureKernelSecuritySupport(&configStore.Config, layerStore.DriverName()); err != nil {
1070 1070
 			return nil, err
1071 1071
 		}
1072 1072
 
1073
-		imageRoot := filepath.Join(config.Root, "image", layerStore.DriverName())
1073
+		imageRoot := filepath.Join(configStore.Root, "image", layerStore.DriverName())
1074 1074
 		ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb"))
1075 1075
 		if err != nil {
1076 1076
 			return nil, err
... ...
@@ -1144,11 +1155,11 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
1144 1144
 
1145 1145
 	go d.execCommandGC()
1146 1146
 
1147
-	if err := d.initLibcontainerd(ctx, config); err != nil {
1147
+	if err := d.initLibcontainerd(ctx, &configStore.Config); err != nil {
1148 1148
 		return nil, err
1149 1149
 	}
1150 1150
 
1151
-	if err := d.restore(config); err != nil {
1151
+	if err := d.restore(configStore); err != nil {
1152 1152
 		return nil, err
1153 1153
 	}
1154 1154
 	close(d.startupDone)
... ...
@@ -1210,7 +1221,7 @@ func (daemon *Daemon) shutdownContainer(c *container.Container) error {
1210 1210
 // A negative (-1) timeout means "indefinitely", which means that containers
1211 1211
 // are not forcibly killed, and the daemon shuts down after all containers exit.
1212 1212
 func (daemon *Daemon) ShutdownTimeout() int {
1213
-	return daemon.shutdownTimeout(daemon.config())
1213
+	return daemon.shutdownTimeout(&daemon.config().Config)
1214 1214
 }
1215 1215
 
1216 1216
 func (daemon *Daemon) shutdownTimeout(cfg *config.Config) int {
... ...
@@ -1241,7 +1252,7 @@ func (daemon *Daemon) Shutdown(ctx context.Context) error {
1241 1241
 	// Keep mounts and networking running on daemon shutdown if
1242 1242
 	// we are to keep containers running and restore them.
1243 1243
 
1244
-	cfg := daemon.config()
1244
+	cfg := &daemon.config().Config
1245 1245
 	if cfg.LiveRestoreEnabled && daemon.containers != nil {
1246 1246
 		// check if there are any running containers, if none we should do some cleanup
1247 1247
 		if ls, err := daemon.Containers(ctx, &types.ContainerListOptions{}); len(ls) != 0 || err != nil {
... ...
@@ -1523,7 +1534,7 @@ func (daemon *Daemon) RawSysInfo() *sysinfo.SysInfo {
1523 1523
 		// We check if sysInfo is not set here, to allow some test to
1524 1524
 		// override the actual sysInfo.
1525 1525
 		if daemon.sysInfo == nil {
1526
-			daemon.sysInfo = getSysInfo(daemon.config())
1526
+			daemon.sysInfo = getSysInfo(&daemon.config().Config)
1527 1527
 		}
1528 1528
 	})
1529 1529
 
... ...
@@ -239,18 +239,18 @@ func kernelSupportsRecursivelyReadOnly() error {
239 239
 	return kernelSupportsRROErr
240 240
 }
241 241
 
242
-func supportsRecursivelyReadOnly(cfg *config.Config, runtime string) error {
242
+func supportsRecursivelyReadOnly(cfg *configStore, runtime string) error {
243 243
 	if err := kernelSupportsRecursivelyReadOnly(); err != nil {
244 244
 		return fmt.Errorf("rro is not supported: %w (kernel is older than 5.12?)", err)
245 245
 	}
246 246
 	if runtime == "" {
247 247
 		runtime = cfg.DefaultRuntime
248 248
 	}
249
-	rt := cfg.GetRuntime(runtime)
250
-	if rt.Features == nil {
249
+	features := cfg.Runtimes.Features(runtime)
250
+	if features == nil {
251 251
 		return fmt.Errorf("rro is not supported by runtime %q: OCI features struct is not available", runtime)
252 252
 	}
253
-	for _, s := range rt.Features.MountOptions {
253
+	for _, s := range features.MountOptions {
254 254
 		if s == "rro" {
255 255
 			return nil
256 256
 		}
... ...
@@ -10,7 +10,6 @@ import (
10 10
 	"testing"
11 11
 
12 12
 	containertypes "github.com/docker/docker/api/types/container"
13
-	"github.com/docker/docker/daemon/config"
14 13
 	"github.com/docker/docker/libnetwork/testutils"
15 14
 	"github.com/docker/docker/libnetwork/types"
16 15
 	"github.com/google/go-cmp/cmp/cmpopts"
... ...
@@ -178,7 +177,7 @@ func TestNotCleanupMounts(t *testing.T) {
178 178
 func TestValidateContainerIsolationLinux(t *testing.T) {
179 179
 	d := Daemon{}
180 180
 
181
-	_, err := d.verifyContainerSettings(&config.Config{}, &containertypes.HostConfig{Isolation: containertypes.IsolationHyperV}, nil, false)
181
+	_, err := d.verifyContainerSettings(&configStore{}, &containertypes.HostConfig{Isolation: containertypes.IsolationHyperV}, nil, false)
182 182
 	assert.Check(t, is.Error(err, "invalid isolation 'hyperv' on linux"))
183 183
 }
184 184
 
... ...
@@ -250,7 +249,7 @@ func TestRootMountCleanup(t *testing.T) {
250 250
 	testRoot, err := os.MkdirTemp("", t.Name())
251 251
 	assert.NilError(t, err)
252 252
 	defer os.RemoveAll(testRoot)
253
-	cfg := &config.Config{}
253
+	cfg := &configStore{}
254 254
 
255 255
 	err = mount.MakePrivate(testRoot)
256 256
 	assert.NilError(t, err)
... ...
@@ -266,16 +265,16 @@ func TestRootMountCleanup(t *testing.T) {
266 266
 
267 267
 	d := &Daemon{root: cfg.Root}
268 268
 	d.configStore.Store(cfg)
269
-	unmountFile := getUnmountOnShutdownPath(cfg)
269
+	unmountFile := getUnmountOnShutdownPath(&cfg.Config)
270 270
 
271 271
 	t.Run("regular dir no mountpoint", func(t *testing.T) {
272
-		err = setupDaemonRootPropagation(cfg)
272
+		err = setupDaemonRootPropagation(&cfg.Config)
273 273
 		assert.NilError(t, err)
274 274
 		_, err = os.Stat(unmountFile)
275 275
 		assert.NilError(t, err)
276 276
 		checkMounted(t, cfg.Root, true)
277 277
 
278
-		assert.Assert(t, d.cleanupMounts(cfg))
278
+		assert.Assert(t, d.cleanupMounts(&cfg.Config))
279 279
 		checkMounted(t, cfg.Root, false)
280 280
 
281 281
 		_, err = os.Stat(unmountFile)
... ...
@@ -287,13 +286,13 @@ func TestRootMountCleanup(t *testing.T) {
287 287
 		assert.NilError(t, err)
288 288
 		defer mount.Unmount(cfg.Root)
289 289
 
290
-		err = setupDaemonRootPropagation(cfg)
290
+		err = setupDaemonRootPropagation(&cfg.Config)
291 291
 		assert.NilError(t, err)
292 292
 		assert.Check(t, ensureShared(cfg.Root))
293 293
 
294 294
 		_, err = os.Stat(unmountFile)
295 295
 		assert.Assert(t, os.IsNotExist(err))
296
-		assert.Assert(t, d.cleanupMounts(cfg))
296
+		assert.Assert(t, d.cleanupMounts(&cfg.Config))
297 297
 		checkMounted(t, cfg.Root, true)
298 298
 	})
299 299
 
... ...
@@ -303,14 +302,14 @@ func TestRootMountCleanup(t *testing.T) {
303 303
 		assert.NilError(t, err)
304 304
 		defer mount.Unmount(cfg.Root)
305 305
 
306
-		err = setupDaemonRootPropagation(cfg)
306
+		err = setupDaemonRootPropagation(&cfg.Config)
307 307
 		assert.NilError(t, err)
308 308
 
309 309
 		if _, err := os.Stat(unmountFile); err == nil {
310 310
 			t.Fatal("unmount file should not exist")
311 311
 		}
312 312
 
313
-		assert.Assert(t, d.cleanupMounts(cfg))
313
+		assert.Assert(t, d.cleanupMounts(&cfg.Config))
314 314
 		checkMounted(t, cfg.Root, true)
315 315
 		assert.Assert(t, mount.Unmount(cfg.Root))
316 316
 	})
... ...
@@ -323,13 +322,13 @@ func TestRootMountCleanup(t *testing.T) {
323 323
 		err = os.WriteFile(unmountFile, nil, 0644)
324 324
 		assert.NilError(t, err)
325 325
 
326
-		err = setupDaemonRootPropagation(cfg)
326
+		err = setupDaemonRootPropagation(&cfg.Config)
327 327
 		assert.NilError(t, err)
328 328
 
329 329
 		_, err = os.Stat(unmountFile)
330 330
 		assert.Check(t, os.IsNotExist(err), err)
331 331
 		checkMounted(t, cfg.Root, false)
332
-		assert.Assert(t, d.cleanupMounts(cfg))
332
+		assert.Assert(t, d.cleanupMounts(&cfg.Config))
333 333
 	})
334 334
 }
335 335
 
... ...
@@ -8,7 +8,6 @@ import (
8 8
 
9 9
 	containertypes "github.com/docker/docker/api/types/container"
10 10
 	"github.com/docker/docker/container"
11
-	"github.com/docker/docker/daemon/config"
12 11
 	"github.com/docker/docker/errdefs"
13 12
 	"github.com/docker/docker/libnetwork"
14 13
 	"github.com/docker/docker/pkg/idtools"
... ...
@@ -301,7 +300,7 @@ func TestMerge(t *testing.T) {
301 301
 func TestValidateContainerIsolation(t *testing.T) {
302 302
 	d := Daemon{}
303 303
 
304
-	_, err := d.verifyContainerSettings(&config.Config{}, &containertypes.HostConfig{Isolation: containertypes.Isolation("invalid")}, nil, false)
304
+	_, err := d.verifyContainerSettings(&configStore{}, &containertypes.HostConfig{Isolation: containertypes.Isolation("invalid")}, nil, false)
305 305
 	assert.Check(t, is.Error(err, "invalid isolation 'invalid' on "+runtime.GOOS))
306 306
 }
307 307
 
... ...
@@ -639,7 +639,7 @@ func isRunningSystemd() bool {
639 639
 
640 640
 // verifyPlatformContainerSettings performs platform-specific validation of the
641 641
 // hostconfig and config structures.
642
-func verifyPlatformContainerSettings(daemon *Daemon, daemonCfg *config.Config, hostConfig *containertypes.HostConfig, update bool) (warnings []string, err error) {
642
+func verifyPlatformContainerSettings(daemon *Daemon, daemonCfg *configStore, hostConfig *containertypes.HostConfig, update bool) (warnings []string, err error) {
643 643
 	if hostConfig == nil {
644 644
 		return nil, nil
645 645
 	}
... ...
@@ -691,7 +691,7 @@ func verifyPlatformContainerSettings(daemon *Daemon, daemonCfg *config.Config, h
691 691
 			return warnings, fmt.Errorf("cannot share the host PID namespace when user namespaces are enabled")
692 692
 		}
693 693
 	}
694
-	if hostConfig.CgroupParent != "" && UsingSystemd(daemonCfg) {
694
+	if hostConfig.CgroupParent != "" && UsingSystemd(&daemonCfg.Config) {
695 695
 		// CgroupParent for systemd cgroup should be named as "xxx.slice"
696 696
 		if len(hostConfig.CgroupParent) <= 6 || !strings.HasSuffix(hostConfig.CgroupParent, ".slice") {
697 697
 			return warnings, fmt.Errorf("cgroup-parent for systemd cgroup should be a valid slice named as \"xxx.slice\"")
... ...
@@ -701,7 +701,7 @@ func verifyPlatformContainerSettings(daemon *Daemon, daemonCfg *config.Config, h
701 701
 		hostConfig.Runtime = daemonCfg.DefaultRuntime
702 702
 	}
703 703
 
704
-	if _, _, err := daemon.getRuntime(daemonCfg, hostConfig.Runtime); err != nil {
704
+	if _, _, err := daemonCfg.Runtimes.Get(hostConfig.Runtime); err != nil {
705 705
 		return warnings, err
706 706
 	}
707 707
 
... ...
@@ -757,7 +757,7 @@ func verifyDaemonSettings(conf *config.Config) error {
757 757
 
758 758
 	configureRuntimes(conf)
759 759
 	if rtName := conf.DefaultRuntime; rtName != "" {
760
-		if conf.GetRuntime(rtName) == nil {
760
+		if _, ok := conf.Runtimes[rtName]; !ok {
761 761
 			if !config.IsPermissibleC8dRuntimeName(rtName) {
762 762
 				return fmt.Errorf("specified default runtime '%s' does not exist", rtName)
763 763
 			}
... ...
@@ -245,7 +245,7 @@ func TestParseSecurityOpt(t *testing.T) {
245 245
 }
246 246
 
247 247
 func TestParseNNPSecurityOptions(t *testing.T) {
248
-	daemonCfg := &config.Config{NoNewPrivileges: true}
248
+	daemonCfg := &configStore{Config: config.Config{NoNewPrivileges: true}}
249 249
 	daemon := &Daemon{}
250 250
 	daemon.configStore.Store(daemonCfg)
251 251
 	opts := &container.SecurityOptions{}
... ...
@@ -254,7 +254,7 @@ func TestParseNNPSecurityOptions(t *testing.T) {
254 254
 	// test NNP when "daemon:true" and "no-new-privileges=false""
255 255
 	cfg.SecurityOpt = []string{"no-new-privileges=false"}
256 256
 
257
-	if err := daemon.parseSecurityOpt(daemonCfg, opts, cfg); err != nil {
257
+	if err := daemon.parseSecurityOpt(&daemonCfg.Config, opts, cfg); err != nil {
258 258
 		t.Fatalf("Unexpected daemon.parseSecurityOpt error: %v", err)
259 259
 	}
260 260
 	if opts.NoNewPrivileges {
... ...
@@ -265,7 +265,7 @@ func TestParseNNPSecurityOptions(t *testing.T) {
265 265
 	daemonCfg.NoNewPrivileges = false
266 266
 	cfg.SecurityOpt = []string{"no-new-privileges=true"}
267 267
 
268
-	if err := daemon.parseSecurityOpt(daemonCfg, opts, cfg); err != nil {
268
+	if err := daemon.parseSecurityOpt(&daemonCfg.Config, opts, cfg); err != nil {
269 269
 		t.Fatalf("Unexpected daemon.parseSecurityOpt error: %v", err)
270 270
 	}
271 271
 	if !opts.NoNewPrivileges {
... ...
@@ -171,7 +171,7 @@ func verifyPlatformContainerResources(resources *containertypes.Resources, isHyp
171 171
 
172 172
 // verifyPlatformContainerSettings performs platform-specific validation of the
173 173
 // hostconfig and config structures.
174
-func verifyPlatformContainerSettings(daemon *Daemon, daemonCfg *config.Config, hostConfig *containertypes.HostConfig, update bool) (warnings []string, err error) {
174
+func verifyPlatformContainerSettings(daemon *Daemon, daemonCfg *configStore, hostConfig *containertypes.HostConfig, update bool) (warnings []string, err error) {
175 175
 	if hostConfig == nil {
176 176
 		return nil, nil
177 177
 	}
... ...
@@ -556,10 +556,6 @@ func (daemon *Daemon) setupSeccompProfile(*config.Config) error {
556 556
 	return nil
557 557
 }
558 558
 
559
-func (daemon *Daemon) loadRuntimes() error {
560
-	return nil
561
-}
562
-
563 559
 func setupResolvConf(config *config.Config) {}
564 560
 
565 561
 func getSysInfo(*config.Config) *sysinfo.SysInfo {
... ...
@@ -25,7 +25,7 @@ import (
25 25
 // fails. If the remove succeeds, the container name is released, and
26 26
 // network links are removed.
27 27
 func (daemon *Daemon) ContainerRm(name string, config *types.ContainerRmConfig) error {
28
-	return daemon.containerRm(daemon.config(), name, config)
28
+	return daemon.containerRm(&daemon.config().Config, name, config)
29 29
 }
30 30
 
31 31
 func (daemon *Daemon) containerRm(cfg *config.Config, name string, opts *types.ContainerRmConfig) error {
... ...
@@ -252,7 +252,7 @@ func (daemon *Daemon) ContainerExecStart(ctx context.Context, name string, optio
252 252
 		p.Cwd = "/"
253 253
 	}
254 254
 
255
-	daemonCfg := daemon.config()
255
+	daemonCfg := &daemon.config().Config
256 256
 	if err := daemon.execSetPlatformOpt(ctx, daemonCfg, ec, p); err != nil {
257 257
 		return err
258 258
 	}
... ...
@@ -9,7 +9,6 @@ import (
9 9
 	"github.com/containerd/containerd/pkg/apparmor"
10 10
 	containertypes "github.com/docker/docker/api/types/container"
11 11
 	"github.com/docker/docker/container"
12
-	"github.com/docker/docker/daemon/config"
13 12
 	specs "github.com/opencontainers/runtime-spec/specs-go"
14 13
 	"gotest.tools/v3/assert"
15 14
 )
... ...
@@ -50,7 +49,7 @@ func TestExecSetPlatformOptAppArmor(t *testing.T) {
50 50
 		},
51 51
 	}
52 52
 
53
-	cfg := &config.Config{}
53
+	cfg := &configStore{}
54 54
 	d := &Daemon{}
55 55
 	d.configStore.Store(cfg)
56 56
 
... ...
@@ -83,7 +82,7 @@ func TestExecSetPlatformOptAppArmor(t *testing.T) {
83 83
 				ec := &container.ExecConfig{Container: c, Privileged: execPrivileged}
84 84
 				p := &specs.Process{}
85 85
 
86
-				err := d.execSetPlatformOpt(context.Background(), cfg, ec, p)
86
+				err := d.execSetPlatformOpt(context.Background(), &cfg.Config, ec, p)
87 87
 				assert.NilError(t, err)
88 88
 				assert.Equal(t, p.ApparmorProfile, tc.expectedProfile)
89 89
 			})
... ...
@@ -64,14 +64,14 @@ func (daemon *Daemon) SystemInfo() *types.Info {
64 64
 
65 65
 	daemon.fillContainerStates(v)
66 66
 	daemon.fillDebugInfo(v)
67
-	daemon.fillAPIInfo(v, cfg)
67
+	daemon.fillAPIInfo(v, &cfg.Config)
68 68
 	// Retrieve platform specific info
69
-	daemon.fillPlatformInfo(v, sysInfo, cfg)
69
+	daemon.fillPlatformInfo(v, sysInfo, &cfg.Config)
70 70
 	daemon.fillDriverInfo(v)
71
-	daemon.fillPluginsInfo(v, cfg)
72
-	daemon.fillSecurityOptions(v, sysInfo, cfg)
71
+	daemon.fillPluginsInfo(v, &cfg.Config)
72
+	daemon.fillSecurityOptions(v, sysInfo, &cfg.Config)
73 73
 	daemon.fillLicense(v)
74
-	daemon.fillDefaultAddressPools(v, cfg)
74
+	daemon.fillDefaultAddressPools(v, &cfg.Config)
75 75
 
76 76
 	return v
77 77
 }
... ...
@@ -117,7 +117,7 @@ func (daemon *Daemon) SystemVersion() types.Version {
117 117
 
118 118
 	v.Platform.Name = dockerversion.PlatformName
119 119
 
120
-	daemon.fillPlatformVersion(&v, cfg)
120
+	daemon.fillPlatformVersion(&v, &cfg.Config)
121 121
 	return v
122 122
 }
123 123
 
... ...
@@ -38,22 +38,22 @@ func (daemon *Daemon) fillPlatformInfo(v *types.Info, sysInfo *sysinfo.SysInfo,
38 38
 		v.CPUSet = sysInfo.Cpuset
39 39
 		v.PidsLimit = sysInfo.PidsLimit
40 40
 	}
41
-	v.Runtimes = cfg.GetAllRuntimes()
41
+	v.Runtimes = make(map[string]types.Runtime)
42
+	for n, r := range cfg.Runtimes {
43
+		v.Runtimes[n] = types.Runtime{
44
+			Path: r.Path,
45
+			Args: append([]string(nil), r.Args...),
46
+		}
47
+	}
42 48
 	v.DefaultRuntime = cfg.DefaultRuntime
43 49
 	v.RuncCommit.ID = "N/A"
44 50
 	v.ContainerdCommit.ID = "N/A"
45 51
 	v.InitCommit.ID = "N/A"
46 52
 
47
-	if rt := cfg.GetRuntime(v.DefaultRuntime); rt != nil {
48
-		if rv, err := exec.Command(rt.Path, "--version").Output(); err == nil {
49
-			if _, _, commit, err := parseRuntimeVersion(string(rv)); err != nil {
50
-				logrus.Warnf("failed to parse %s version: %v", rt.Path, err)
51
-			} else {
52
-				v.RuncCommit.ID = commit
53
-			}
54
-		} else {
55
-			logrus.Warnf("failed to retrieve %s version: %v", rt.Path, err)
56
-		}
53
+	if _, _, commit, err := parseDefaultRuntimeVersion(cfg); err != nil {
54
+		logrus.Warnf(err.Error())
55
+	} else {
56
+		v.RuncCommit.ID = commit
57 57
 	}
58 58
 
59 59
 	if rv, err := daemon.containerd.Version(context.Background()); err == nil {
... ...
@@ -177,23 +177,16 @@ func (daemon *Daemon) fillPlatformVersion(v *types.Version, cfg *config.Config)
177 177
 		})
178 178
 	}
179 179
 
180
-	defaultRuntime := cfg.DefaultRuntime
181
-	if rt := cfg.GetRuntime(defaultRuntime); rt != nil {
182
-		if rv, err := exec.Command(rt.Path, "--version").Output(); err == nil {
183
-			if _, ver, commit, err := parseRuntimeVersion(string(rv)); err != nil {
184
-				logrus.Warnf("failed to parse %s version: %v", rt.Path, err)
185
-			} else {
186
-				v.Components = append(v.Components, types.ComponentVersion{
187
-					Name:    defaultRuntime,
188
-					Version: ver,
189
-					Details: map[string]string{
190
-						"GitCommit": commit,
191
-					},
192
-				})
193
-			}
194
-		} else {
195
-			logrus.Warnf("failed to retrieve %s version: %v", rt.Path, err)
196
-		}
180
+	if _, ver, commit, err := parseDefaultRuntimeVersion(cfg); err != nil {
181
+		logrus.Warnf(err.Error())
182
+	} else {
183
+		v.Components = append(v.Components, types.ComponentVersion{
184
+			Name:    cfg.DefaultRuntime,
185
+			Version: ver,
186
+			Details: map[string]string{
187
+				"GitCommit": commit,
188
+			},
189
+		})
197 190
 	}
198 191
 
199 192
 	if initBinary, err := cfg.LookupInitPath(); err != nil {
... ...
@@ -318,7 +311,7 @@ func parseInitVersion(v string) (version string, commit string, err error) {
318 318
 //	runc version 1.0.0-rc5+dev
319 319
 //	commit: 69663f0bd4b60df09991c08812a60108003fa340
320 320
 //	spec: 1.0.0
321
-func parseRuntimeVersion(v string) (runtime string, version string, commit string, err error) {
321
+func parseRuntimeVersion(v string) (runtime, version, commit string, err error) {
322 322
 	lines := strings.Split(strings.TrimSpace(v), "\n")
323 323
 	for _, line := range lines {
324 324
 		if strings.Contains(line, "version") {
... ...
@@ -338,6 +331,21 @@ func parseRuntimeVersion(v string) (runtime string, version string, commit strin
338 338
 	return runtime, version, commit, err
339 339
 }
340 340
 
341
+func parseDefaultRuntimeVersion(cfg *config.Config) (runtime, version, commit string, err error) {
342
+	if rt, ok := cfg.Runtimes[cfg.DefaultRuntime]; ok {
343
+		rv, err := exec.Command(rt.Path, "--version").Output()
344
+		if err != nil {
345
+			return "", "", "", fmt.Errorf("failed to retrieve %s version: %w", rt.Path, err)
346
+		}
347
+		runtime, version, commit, err := parseRuntimeVersion(string(rv))
348
+		if err != nil {
349
+			return "", "", "", fmt.Errorf("failed to parse %s version: %w", rt.Path, err)
350
+		}
351
+		return runtime, version, commit, err
352
+	}
353
+	return "", "", "", nil
354
+}
355
+
341 356
 func cgroupNamespacesEnabled(sysInfo *sysinfo.SysInfo, cfg *config.Config) bool {
342 357
 	return sysInfo.CgroupNamespaces && containertypes.CgroupnsMode(cfg.CgroupNamespaceMode).IsPrivate()
343 358
 }
... ...
@@ -41,7 +41,7 @@ func (daemon *Daemon) ContainerInspectCurrent(ctx context.Context, name string,
41 41
 
42 42
 	ctr.Lock()
43 43
 
44
-	base, err := daemon.getInspectData(daemon.config(), ctr)
44
+	base, err := daemon.getInspectData(&daemon.config().Config, ctr)
45 45
 	if err != nil {
46 46
 		ctr.Unlock()
47 47
 		return nil, err
... ...
@@ -106,7 +106,7 @@ func (daemon *Daemon) containerInspect120(name string) (*v1p20.ContainerJSON, er
106 106
 	ctr.Lock()
107 107
 	defer ctr.Unlock()
108 108
 
109
-	base, err := daemon.getInspectData(daemon.config(), ctr)
109
+	base, err := daemon.getInspectData(&daemon.config().Config, ctr)
110 110
 	if err != nil {
111 111
 		return nil, err
112 112
 	}
... ...
@@ -29,7 +29,7 @@ func (daemon *Daemon) containerInspectPre120(ctx context.Context, name string) (
29 29
 	ctr.Lock()
30 30
 	defer ctr.Unlock()
31 31
 
32
-	base, err := daemon.getInspectData(daemon.config(), ctr)
32
+	base, err := daemon.getInspectData(&daemon.config().Config, ctr)
33 33
 	if err != nil {
34 34
 		return nil, err
35 35
 	}
... ...
@@ -5,7 +5,6 @@ import (
5 5
 
6 6
 	containertypes "github.com/docker/docker/api/types/container"
7 7
 	"github.com/docker/docker/container"
8
-	"github.com/docker/docker/daemon/config"
9 8
 	"gotest.tools/v3/assert"
10 9
 	is "gotest.tools/v3/assert/cmp"
11 10
 )
... ...
@@ -24,13 +23,13 @@ func TestGetInspectData(t *testing.T) {
24 24
 	if d.UsesSnapshotter() {
25 25
 		t.Skip("does not apply to containerd snapshotters, which don't have RWLayer set")
26 26
 	}
27
-	cfg := &config.Config{}
27
+	cfg := &configStore{}
28 28
 	d.configStore.Store(cfg)
29 29
 
30
-	_, err := d.getInspectData(cfg, c)
30
+	_, err := d.getInspectData(&cfg.Config, c)
31 31
 	assert.Check(t, is.ErrorContains(err, "RWLayer of container inspect-me is unexpectedly nil"))
32 32
 
33 33
 	c.Dead = true
34
-	_, err = d.getInspectData(cfg, c)
34
+	_, err = d.getInspectData(&cfg.Config, c)
35 35
 	assert.Check(t, err)
36 36
 }
... ...
@@ -102,7 +102,7 @@ func (daemon *Daemon) handleContainerExit(c *container.Container, e *libcontaine
102 102
 	} else {
103 103
 		c.SetStopped(&exitStatus)
104 104
 		if !c.HasBeenManuallyRestarted {
105
-			defer daemon.autoRemove(cfg, c)
105
+			defer daemon.autoRemove(&cfg.Config, c)
106 106
 		}
107 107
 	}
108 108
 	defer c.Unlock() // needs to be called before autoRemove
... ...
@@ -131,7 +131,7 @@ func (daemon *Daemon) handleContainerExit(c *container.Container, e *libcontaine
131 131
 				daemon.setStateCounter(c)
132 132
 				c.CheckpointTo(daemon.containersReplica)
133 133
 				c.Unlock()
134
-				defer daemon.autoRemove(cfg, c)
134
+				defer daemon.autoRemove(&cfg.Config, c)
135 135
 				if err != restartmanager.ErrRestartCanceled {
136 136
 					logrus.Errorf("restartmanger wait error: %+v", err)
137 137
 				}
... ...
@@ -161,7 +161,7 @@ func (daemon *Daemon) startIngressWorker() {
161 161
 			select {
162 162
 			case r := <-ingressJobsChannel:
163 163
 				if r.create != nil {
164
-					daemon.setupIngress(daemon.config(), r.create, r.ip, ingressID)
164
+					daemon.setupIngress(&daemon.config().Config, r.create, r.ip, ingressID)
165 165
 					ingressID = r.create.ID
166 166
 				} else {
167 167
 					daemon.releaseIngress(ingressID)
... ...
@@ -278,13 +278,13 @@ func (daemon *Daemon) WaitForDetachment(ctx context.Context, networkName, networ
278 278
 
279 279
 // CreateManagedNetwork creates an agent network.
280 280
 func (daemon *Daemon) CreateManagedNetwork(create clustertypes.NetworkCreateRequest) error {
281
-	_, err := daemon.createNetwork(daemon.config(), create.NetworkCreateRequest, create.ID, true)
281
+	_, err := daemon.createNetwork(&daemon.config().Config, create.NetworkCreateRequest, create.ID, true)
282 282
 	return err
283 283
 }
284 284
 
285 285
 // CreateNetwork creates a network with the given name, driver and other optional parameters
286 286
 func (daemon *Daemon) CreateNetwork(create types.NetworkCreateRequest) (*types.NetworkCreateResponse, error) {
287
-	return daemon.createNetwork(daemon.config(), create, "", false)
287
+	return daemon.createNetwork(&daemon.config().Config, create, "", false)
288 288
 }
289 289
 
290 290
 func (daemon *Daemon) createNetwork(cfg *config.Config, create types.NetworkCreateRequest, id string, agent bool) (*types.NetworkCreateResponse, error) {
... ...
@@ -489,7 +489,7 @@ func inSlice(slice []string, s string) bool {
489 489
 }
490 490
 
491 491
 // withMounts sets the container's mounts
492
-func withMounts(daemon *Daemon, daemonCfg *dconfig.Config, c *container.Container) coci.SpecOpts {
492
+func withMounts(daemon *Daemon, daemonCfg *configStore, c *container.Container) coci.SpecOpts {
493 493
 	return func(ctx context.Context, _ coci.Client, _ *containers.Container, s *coci.Spec) (err error) {
494 494
 		if err := daemon.setupContainerMountsRoot(c); err != nil {
495 495
 			return err
... ...
@@ -1019,23 +1019,23 @@ func WithUser(c *container.Container) coci.SpecOpts {
1019 1019
 	}
1020 1020
 }
1021 1021
 
1022
-func (daemon *Daemon) createSpec(ctx context.Context, daemonCfg *dconfig.Config, c *container.Container) (retSpec *specs.Spec, err error) {
1022
+func (daemon *Daemon) createSpec(ctx context.Context, daemonCfg *configStore, c *container.Container) (retSpec *specs.Spec, err error) {
1023 1023
 	var (
1024 1024
 		opts []coci.SpecOpts
1025 1025
 		s    = oci.DefaultSpec()
1026 1026
 	)
1027 1027
 	opts = append(opts,
1028
-		withCommonOptions(daemon, daemonCfg, c),
1029
-		withCgroups(daemon, daemonCfg, c),
1028
+		withCommonOptions(daemon, &daemonCfg.Config, c),
1029
+		withCgroups(daemon, &daemonCfg.Config, c),
1030 1030
 		WithResources(c),
1031 1031
 		WithSysctls(c),
1032 1032
 		WithDevices(daemon, c),
1033
-		withRlimits(daemon, daemonCfg, c),
1033
+		withRlimits(daemon, &daemonCfg.Config, c),
1034 1034
 		WithNamespaces(daemon, c),
1035 1035
 		WithCapabilities(c),
1036 1036
 		WithSeccomp(daemon, c),
1037 1037
 		withMounts(daemon, daemonCfg, c),
1038
-		withLibnetwork(daemon, daemonCfg, c),
1038
+		withLibnetwork(daemon, &daemonCfg.Config, c),
1039 1039
 		WithApparmor(c),
1040 1040
 		WithSelinux(c),
1041 1041
 		WithOOMScore(&c.HostConfig.OomScoreAdj),
... ...
@@ -1069,7 +1069,7 @@ func (daemon *Daemon) createSpec(ctx context.Context, daemonCfg *dconfig.Config,
1069 1069
 		opts = append(opts, coci.WithReadonlyPaths(c.HostConfig.ReadonlyPaths))
1070 1070
 	}
1071 1071
 	if daemonCfg.Rootless {
1072
-		opts = append(opts, withRootless(daemon, daemonCfg))
1072
+		opts = append(opts, withRootless(daemon, &daemonCfg.Config))
1073 1073
 	}
1074 1074
 
1075 1075
 	var snapshotter, snapshotKey string
... ...
@@ -82,7 +82,7 @@ func TestTmpfsDevShmNoDupMount(t *testing.T) {
82 82
 	d := setupFakeDaemon(t, c)
83 83
 	defer cleanupFakeContainer(c)
84 84
 
85
-	_, err := d.createSpec(context.TODO(), &config.Config{}, c)
85
+	_, err := d.createSpec(context.TODO(), &configStore{}, c)
86 86
 	assert.Check(t, err)
87 87
 }
88 88
 
... ...
@@ -101,7 +101,7 @@ func TestIpcPrivateVsReadonly(t *testing.T) {
101 101
 	d := setupFakeDaemon(t, c)
102 102
 	defer cleanupFakeContainer(c)
103 103
 
104
-	s, err := d.createSpec(context.TODO(), &config.Config{}, c)
104
+	s, err := d.createSpec(context.TODO(), &configStore{}, c)
105 105
 	assert.Check(t, err)
106 106
 
107 107
 	// Find the /dev/shm mount in ms, check it does not have ro
... ...
@@ -131,7 +131,7 @@ func TestSysctlOverride(t *testing.T) {
131 131
 	defer cleanupFakeContainer(c)
132 132
 
133 133
 	// Ensure that the implicit sysctl is set correctly.
134
-	s, err := d.createSpec(context.TODO(), &config.Config{}, c)
134
+	s, err := d.createSpec(context.TODO(), &configStore{}, c)
135 135
 	assert.NilError(t, err)
136 136
 	assert.Equal(t, s.Hostname, "foobar")
137 137
 	assert.Equal(t, s.Linux.Sysctl["kernel.domainname"], c.Config.Domainname)
... ...
@@ -147,14 +147,14 @@ func TestSysctlOverride(t *testing.T) {
147 147
 	assert.Assert(t, c.HostConfig.Sysctls["kernel.domainname"] != c.Config.Domainname)
148 148
 	c.HostConfig.Sysctls["net.ipv4.ip_unprivileged_port_start"] = "1024"
149 149
 
150
-	s, err = d.createSpec(context.TODO(), &config.Config{}, c)
150
+	s, err = d.createSpec(context.TODO(), &configStore{}, c)
151 151
 	assert.NilError(t, err)
152 152
 	assert.Equal(t, s.Hostname, "foobar")
153 153
 	assert.Equal(t, s.Linux.Sysctl["kernel.domainname"], c.HostConfig.Sysctls["kernel.domainname"])
154 154
 	assert.Equal(t, s.Linux.Sysctl["net.ipv4.ip_unprivileged_port_start"], c.HostConfig.Sysctls["net.ipv4.ip_unprivileged_port_start"])
155 155
 
156 156
 	// Ensure the ping_group_range is not set on a daemon with user-namespaces enabled
157
-	s, err = d.createSpec(context.TODO(), &config.Config{RemappedRoot: "dummy:dummy"}, c)
157
+	s, err = d.createSpec(context.TODO(), &configStore{Config: config.Config{RemappedRoot: "dummy:dummy"}}, c)
158 158
 	assert.NilError(t, err)
159 159
 	_, ok := s.Linux.Sysctl["net.ipv4.ping_group_range"]
160 160
 	assert.Assert(t, !ok)
... ...
@@ -162,7 +162,7 @@ func TestSysctlOverride(t *testing.T) {
162 162
 	// Ensure the ping_group_range is set on a container in "host" userns mode
163 163
 	// on a daemon with user-namespaces enabled
164 164
 	c.HostConfig.UsernsMode = "host"
165
-	s, err = d.createSpec(context.TODO(), &config.Config{RemappedRoot: "dummy:dummy"}, c)
165
+	s, err = d.createSpec(context.TODO(), &configStore{Config: config.Config{RemappedRoot: "dummy:dummy"}}, c)
166 166
 	assert.NilError(t, err)
167 167
 	assert.Equal(t, s.Linux.Sysctl["net.ipv4.ping_group_range"], "0 2147483647")
168 168
 }
... ...
@@ -182,7 +182,7 @@ func TestSysctlOverrideHost(t *testing.T) {
182 182
 	defer cleanupFakeContainer(c)
183 183
 
184 184
 	// Ensure that the implicit sysctl is not set
185
-	s, err := d.createSpec(context.TODO(), &config.Config{}, c)
185
+	s, err := d.createSpec(context.TODO(), &configStore{}, c)
186 186
 	assert.NilError(t, err)
187 187
 	assert.Equal(t, s.Linux.Sysctl["net.ipv4.ip_unprivileged_port_start"], "")
188 188
 	assert.Equal(t, s.Linux.Sysctl["net.ipv4.ping_group_range"], "")
... ...
@@ -190,7 +190,7 @@ func TestSysctlOverrideHost(t *testing.T) {
190 190
 	// Set an explicit sysctl.
191 191
 	c.HostConfig.Sysctls["net.ipv4.ip_unprivileged_port_start"] = "1024"
192 192
 
193
-	s, err = d.createSpec(context.TODO(), &config.Config{}, c)
193
+	s, err = d.createSpec(context.TODO(), &configStore{}, c)
194 194
 	assert.NilError(t, err)
195 195
 	assert.Equal(t, s.Linux.Sysctl["net.ipv4.ip_unprivileged_port_start"], c.HostConfig.Sysctls["net.ipv4.ip_unprivileged_port_start"])
196 196
 }
... ...
@@ -28,7 +28,7 @@ const (
28 28
 	credentialSpecFileLocation     = "CredentialSpecs"
29 29
 )
30 30
 
31
-func (daemon *Daemon) createSpec(ctx context.Context, daemonCfg *config.Config, c *container.Container) (*specs.Spec, error) {
31
+func (daemon *Daemon) createSpec(ctx context.Context, daemonCfg *configStore, c *container.Container) (*specs.Spec, error) {
32 32
 	img, err := daemon.imageService.GetImage(ctx, string(c.ImageID), imagetypes.GetImageOpts{})
33 33
 	if err != nil {
34 34
 		return nil, err
... ...
@@ -143,7 +143,7 @@ func (daemon *Daemon) createSpec(ctx context.Context, daemonCfg *config.Config,
143 143
 		return nil, errors.Wrapf(err, "container %s", c.ID)
144 144
 	}
145 145
 
146
-	dnsSearch := daemon.getDNSSearchSettings(daemonCfg, c)
146
+	dnsSearch := daemon.getDNSSearchSettings(&daemonCfg.Config, c)
147 147
 
148 148
 	// Get endpoints for the libnetwork allocated networks to the container
149 149
 	var epList []string
... ...
@@ -56,7 +56,7 @@ func (daemon *Daemon) ContainersPrune(ctx context.Context, pruneFilters filters.
56 56
 		return nil, err
57 57
 	}
58 58
 
59
-	cfg := daemon.config()
59
+	cfg := &daemon.config().Config
60 60
 	allContainers := daemon.List()
61 61
 	for _, c := range allContainers {
62 62
 		select {
... ...
@@ -72,11 +72,13 @@ func (tx *reloadTxn) Rollback() error {
72 72
 func (daemon *Daemon) Reload(conf *config.Config) error {
73 73
 	daemon.configReload.Lock()
74 74
 	defer daemon.configReload.Unlock()
75
-	copied, err := copystructure.Copy(daemon.config())
75
+	copied, err := copystructure.Copy(daemon.config().Config)
76 76
 	if err != nil {
77 77
 		return err
78 78
 	}
79
-	newCfg := copied.(*config.Config)
79
+	newCfg := &configStore{
80
+		Config: copied.(config.Config),
81
+	}
80 82
 
81 83
 	attributes := map[string]string{}
82 84
 
... ...
@@ -91,7 +93,7 @@ func (daemon *Daemon) Reload(conf *config.Config) error {
91 91
 	// executing any registered rollback functions.
92 92
 
93 93
 	var txn reloadTxn
94
-	for _, reload := range []func(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error{
94
+	for _, reload := range []func(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error{
95 95
 		daemon.reloadPlatform,
96 96
 		daemon.reloadDebug,
97 97
 		daemon.reloadMaxConcurrentDownloadsAndUploads,
... ...
@@ -115,7 +117,7 @@ func (daemon *Daemon) Reload(conf *config.Config) error {
115 115
 		*config.Config
116 116
 		config.Proxies `json:"proxies"`
117 117
 	}{
118
-		Config: newCfg,
118
+		Config: &newCfg.Config,
119 119
 		Proxies: config.Proxies{
120 120
 			HTTPProxy:  config.MaskCredentials(newCfg.HTTPProxy),
121 121
 			HTTPSProxy: config.MaskCredentials(newCfg.HTTPSProxy),
... ...
@@ -141,7 +143,7 @@ func marshalAttributeSlice(v []string) string {
141 141
 
142 142
 // reloadDebug updates configuration with Debug option
143 143
 // and updates the passed attributes
144
-func (daemon *Daemon) reloadDebug(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error {
144
+func (daemon *Daemon) reloadDebug(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error {
145 145
 	// update corresponding configuration
146 146
 	if conf.IsValueSet("debug") {
147 147
 		newCfg.Debug = conf.Debug
... ...
@@ -153,7 +155,7 @@ func (daemon *Daemon) reloadDebug(txn *reloadTxn, newCfg, conf *config.Config, a
153 153
 
154 154
 // reloadMaxConcurrentDownloadsAndUploads updates configuration with max concurrent
155 155
 // download and upload options and updates the passed attributes
156
-func (daemon *Daemon) reloadMaxConcurrentDownloadsAndUploads(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error {
156
+func (daemon *Daemon) reloadMaxConcurrentDownloadsAndUploads(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error {
157 157
 	// We always "reset" as the cost is lightweight and easy to maintain.
158 158
 	newCfg.MaxConcurrentDownloads = config.DefaultMaxConcurrentDownloads
159 159
 	newCfg.MaxConcurrentUploads = config.DefaultMaxConcurrentUploads
... ...
@@ -184,7 +186,7 @@ func (daemon *Daemon) reloadMaxConcurrentDownloadsAndUploads(txn *reloadTxn, new
184 184
 
185 185
 // reloadMaxDownloadAttempts updates configuration with max concurrent
186 186
 // download attempts when a connection is lost and updates the passed attributes
187
-func (daemon *Daemon) reloadMaxDownloadAttempts(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error {
187
+func (daemon *Daemon) reloadMaxDownloadAttempts(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error {
188 188
 	// We always "reset" as the cost is lightweight and easy to maintain.
189 189
 	newCfg.MaxDownloadAttempts = config.DefaultDownloadAttempts
190 190
 	if conf.IsValueSet("max-download-attempts") && conf.MaxDownloadAttempts != 0 {
... ...
@@ -199,7 +201,7 @@ func (daemon *Daemon) reloadMaxDownloadAttempts(txn *reloadTxn, newCfg, conf *co
199 199
 
200 200
 // reloadShutdownTimeout updates configuration with daemon shutdown timeout option
201 201
 // and updates the passed attributes
202
-func (daemon *Daemon) reloadShutdownTimeout(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error {
202
+func (daemon *Daemon) reloadShutdownTimeout(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error {
203 203
 	// update corresponding configuration
204 204
 	if conf.IsValueSet("shutdown-timeout") {
205 205
 		newCfg.ShutdownTimeout = conf.ShutdownTimeout
... ...
@@ -213,7 +215,7 @@ func (daemon *Daemon) reloadShutdownTimeout(txn *reloadTxn, newCfg, conf *config
213 213
 
214 214
 // reloadLabels updates configuration with engine labels
215 215
 // and updates the passed attributes
216
-func (daemon *Daemon) reloadLabels(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error {
216
+func (daemon *Daemon) reloadLabels(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error {
217 217
 	// update corresponding configuration
218 218
 	if conf.IsValueSet("labels") {
219 219
 		newCfg.Labels = conf.Labels
... ...
@@ -226,7 +228,7 @@ func (daemon *Daemon) reloadLabels(txn *reloadTxn, newCfg, conf *config.Config,
226 226
 
227 227
 // reloadRegistryConfig updates the configuration with registry options
228 228
 // and updates the passed attributes.
229
-func (daemon *Daemon) reloadRegistryConfig(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error {
229
+func (daemon *Daemon) reloadRegistryConfig(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error {
230 230
 	// Update corresponding configuration.
231 231
 	if conf.IsValueSet("allow-nondistributable-artifacts") {
232 232
 		newCfg.ServiceOptions.AllowNondistributableArtifacts = conf.AllowNondistributableArtifacts
... ...
@@ -253,7 +255,7 @@ func (daemon *Daemon) reloadRegistryConfig(txn *reloadTxn, newCfg, conf *config.
253 253
 
254 254
 // reloadLiveRestore updates configuration with live restore option
255 255
 // and updates the passed attributes
256
-func (daemon *Daemon) reloadLiveRestore(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error {
256
+func (daemon *Daemon) reloadLiveRestore(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error {
257 257
 	// update corresponding configuration
258 258
 	if conf.IsValueSet("live-restore") {
259 259
 		newCfg.LiveRestoreEnabled = conf.LiveRestoreEnabled
... ...
@@ -265,7 +267,7 @@ func (daemon *Daemon) reloadLiveRestore(txn *reloadTxn, newCfg, conf *config.Con
265 265
 }
266 266
 
267 267
 // reloadNetworkDiagnosticPort updates the network controller starting the diagnostic if the config is valid
268
-func (daemon *Daemon) reloadNetworkDiagnosticPort(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error {
268
+func (daemon *Daemon) reloadNetworkDiagnosticPort(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error {
269 269
 	txn.OnCommit(func() error {
270 270
 		if conf == nil || daemon.netController == nil || !conf.IsValueSet("network-diagnostic-port") ||
271 271
 			conf.NetworkDiagnosticPort < 1 || conf.NetworkDiagnosticPort > 65535 {
... ...
@@ -284,7 +286,7 @@ func (daemon *Daemon) reloadNetworkDiagnosticPort(txn *reloadTxn, newCfg, conf *
284 284
 }
285 285
 
286 286
 // reloadFeatures updates configuration with enabled/disabled features
287
-func (daemon *Daemon) reloadFeatures(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error {
287
+func (daemon *Daemon) reloadFeatures(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error {
288 288
 	// update corresponding configuration
289 289
 	// note that we allow features option to be entirely unset
290 290
 	newCfg.Features = conf.Features
... ...
@@ -27,7 +27,7 @@ func newDaemonForReloadT(t *testing.T, cfg *config.Config) *Daemon {
27 27
 	var err error
28 28
 	daemon.registryService, err = registry.NewService(registry.ServiceOptions{})
29 29
 	assert.Assert(t, err)
30
-	daemon.configStore.Store(cfg)
30
+	daemon.configStore.Store(&configStore{Config: *cfg})
31 31
 	return daemon
32 32
 }
33 33
 
... ...
@@ -11,15 +11,19 @@ import (
11 11
 
12 12
 // reloadPlatform updates configuration with platform specific options
13 13
 // and updates the passed attributes
14
-func (daemon *Daemon) reloadPlatform(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error {
14
+func (daemon *Daemon) reloadPlatform(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error {
15 15
 	if conf.DefaultRuntime != "" {
16 16
 		newCfg.DefaultRuntime = conf.DefaultRuntime
17 17
 	}
18 18
 	if conf.IsValueSet("runtimes") {
19
-		newCfg.Runtimes = conf.Runtimes
20
-		txn.OnCommit(func() error { return daemon.initRuntimes(newCfg) })
19
+		newCfg.Config.Runtimes = conf.Runtimes
20
+	}
21
+	configureRuntimes(&newCfg.Config)
22
+	var err error
23
+	newCfg.Runtimes, err = setupRuntimes(&newCfg.Config)
24
+	if err != nil {
25
+		return err
21 26
 	}
22
-	configureRuntimes(newCfg)
23 27
 
24 28
 	if conf.IsValueSet("default-shm-size") {
25 29
 		newCfg.ShmSize = conf.ShmSize
... ...
@@ -35,7 +39,7 @@ func (daemon *Daemon) reloadPlatform(txn *reloadTxn, newCfg, conf *config.Config
35 35
 
36 36
 	// Update attributes
37 37
 	var runtimeList bytes.Buffer
38
-	for name, rt := range newCfg.Runtimes {
38
+	for name, rt := range newCfg.Config.Runtimes {
39 39
 		if runtimeList.Len() > 0 {
40 40
 			runtimeList.WriteRune(' ')
41 41
 		}
... ...
@@ -4,6 +4,6 @@ import "github.com/docker/docker/daemon/config"
4 4
 
5 5
 // reloadPlatform updates configuration with platform specific options
6 6
 // and updates the passed attributes
7
-func (daemon *Daemon) reloadPlatform(txn *reloadTxn, newCfg, conf *config.Config, attributes map[string]string) error {
7
+func (daemon *Daemon) reloadPlatform(txn *reloadTxn, newCfg *configStore, conf *config.Config, attributes map[string]string) error {
8 8
 	return nil
9 9
 }
... ...
@@ -6,7 +6,6 @@ import (
6 6
 
7 7
 	containertypes "github.com/docker/docker/api/types/container"
8 8
 	"github.com/docker/docker/container"
9
-	"github.com/docker/docker/daemon/config"
10 9
 )
11 10
 
12 11
 // ContainerRestart stops and starts a container. It attempts to
... ...
@@ -31,7 +30,7 @@ func (daemon *Daemon) ContainerRestart(ctx context.Context, name string, options
31 31
 // container. When stopping, wait for the given duration in seconds to
32 32
 // gracefully stop, before forcefully terminating the container. If
33 33
 // given a negative duration, wait forever for a graceful stop.
34
-func (daemon *Daemon) containerRestart(ctx context.Context, daemonCfg *config.Config, container *container.Container, options containertypes.StopOptions) error {
34
+func (daemon *Daemon) containerRestart(ctx context.Context, daemonCfg *configStore, container *container.Container, options containertypes.StopOptions) error {
35 35
 	// Determine isolation. If not specified in the hostconfig, use daemon default.
36 36
 	actualIsolation := container.HostConfig.Isolation
37 37
 	if containertypes.Isolation.IsDefault(actualIsolation) {
... ...
@@ -4,18 +4,24 @@ package daemon
4 4
 
5 5
 import (
6 6
 	"bytes"
7
+	"crypto/sha256"
8
+	"encoding/base32"
7 9
 	"encoding/json"
8 10
 	"fmt"
11
+	"io"
9 12
 	"os"
10 13
 	"os/exec"
11 14
 	"path/filepath"
12 15
 	"strings"
13 16
 
17
+	"github.com/containerd/containerd/plugin"
14 18
 	v2runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
15 19
 	"github.com/docker/docker/api/types"
16 20
 	"github.com/docker/docker/daemon/config"
17 21
 	"github.com/docker/docker/errdefs"
18 22
 	"github.com/docker/docker/libcontainerd/shimopts"
23
+	"github.com/docker/docker/pkg/ioutils"
24
+	"github.com/docker/docker/pkg/system"
19 25
 	"github.com/opencontainers/runtime-spec/specs-go/features"
20 26
 	"github.com/pkg/errors"
21 27
 	"github.com/sirupsen/logrus"
... ...
@@ -23,10 +29,21 @@ import (
23 23
 
24 24
 const (
25 25
 	defaultRuntimeName = "runc"
26
-
27
-	linuxShimV2 = "io.containerd.runc.v2"
28 26
 )
29 27
 
28
+type shimConfig struct {
29
+	Shim     string
30
+	Opts     interface{}
31
+	Features *features.Features
32
+
33
+	// Check if the ShimConfig is valid given the current state of the system.
34
+	PreflightCheck func() error
35
+}
36
+
37
+type runtimes struct {
38
+	configured map[string]*shimConfig
39
+}
40
+
30 41
 func configureRuntimes(conf *config.Config) {
31 42
 	if conf.DefaultRuntime == "" {
32 43
 		conf.DefaultRuntime = config.StockRuntimeName
... ...
@@ -34,13 +51,13 @@ func configureRuntimes(conf *config.Config) {
34 34
 	if conf.Runtimes == nil {
35 35
 		conf.Runtimes = make(map[string]types.Runtime)
36 36
 	}
37
-	conf.Runtimes[config.LinuxV2RuntimeName] = types.Runtime{Path: defaultRuntimeName, ShimConfig: defaultV2ShimConfig(conf, defaultRuntimeName)}
37
+	conf.Runtimes[config.LinuxV2RuntimeName] = types.Runtime{Path: defaultRuntimeName}
38 38
 	conf.Runtimes[config.StockRuntimeName] = conf.Runtimes[config.LinuxV2RuntimeName]
39 39
 }
40 40
 
41
-func defaultV2ShimConfig(conf *config.Config, runtimePath string) *types.ShimConfig {
42
-	return &types.ShimConfig{
43
-		Binary: linuxShimV2,
41
+func defaultV2ShimConfig(conf *config.Config, runtimePath string) *shimConfig {
42
+	shim := &shimConfig{
43
+		Shim: plugin.RuntimeRuncV2,
44 44
 		Opts: &v2runcoptions.Options{
45 45
 			BinaryName:    runtimePath,
46 46
 			Root:          filepath.Join(conf.ExecRoot, "runtime-"+defaultRuntimeName),
... ...
@@ -48,138 +65,142 @@ func defaultV2ShimConfig(conf *config.Config, runtimePath string) *types.ShimCon
48 48
 			NoPivotRoot:   os.Getenv("DOCKER_RAMDISK") != "",
49 49
 		},
50 50
 	}
51
+
52
+	var featuresStderr bytes.Buffer
53
+	featuresCmd := exec.Command(runtimePath, "features")
54
+	featuresCmd.Stderr = &featuresStderr
55
+	if featuresB, err := featuresCmd.Output(); err != nil {
56
+		logrus.WithError(err).Warnf("Failed to run %v: %q", featuresCmd.Args, featuresStderr.String())
57
+	} else {
58
+		var features features.Features
59
+		if jsonErr := json.Unmarshal(featuresB, &features); jsonErr != nil {
60
+			logrus.WithError(err).Warnf("Failed to unmarshal the output of %v as a JSON", featuresCmd.Args)
61
+		} else {
62
+			shim.Features = &features
63
+		}
64
+	}
65
+
66
+	return shim
51 67
 }
52 68
 
53
-func (daemon *Daemon) loadRuntimes() error {
54
-	return daemon.initRuntimes(daemon.config())
69
+func runtimeScriptsDir(cfg *config.Config) string {
70
+	return filepath.Join(cfg.Root, "runtimes")
55 71
 }
56 72
 
57
-func (daemon *Daemon) initRuntimes(cfg *config.Config) (err error) {
58
-	runtimeDir := filepath.Join(cfg.Root, "runtimes")
59
-	runtimeOldDir := runtimeDir + "-old"
60
-	// Remove old temp directory if any
61
-	os.RemoveAll(runtimeOldDir)
62
-	tmpDir, err := os.MkdirTemp(cfg.Root, "gen-runtimes")
63
-	if err != nil {
64
-		return errors.Wrap(err, "failed to get temp dir to generate runtime scripts")
73
+// initRuntimesDir creates a fresh directory where we'll store the runtime
74
+// scripts (i.e. in order to support runtimeArgs).
75
+func initRuntimesDir(cfg *config.Config) error {
76
+	runtimeDir := runtimeScriptsDir(cfg)
77
+	if err := os.RemoveAll(runtimeDir); err != nil {
78
+		return err
65 79
 	}
66
-	defer func() {
67
-		if err != nil {
68
-			if err1 := os.RemoveAll(tmpDir); err1 != nil {
69
-				logrus.WithError(err1).WithField("dir", tmpDir).
70
-					Warn("failed to remove tmp dir")
71
-			}
72
-			return
73
-		}
80
+	return system.MkdirAll(runtimeDir, 0700)
81
+}
74 82
 
75
-		if err = os.Rename(runtimeDir, runtimeOldDir); err != nil {
76
-			logrus.WithError(err).WithField("dir", runtimeDir).
77
-				Warn("failed to rename runtimes dir to old. Will try to removing it")
78
-			if err = os.RemoveAll(runtimeDir); err != nil {
79
-				logrus.WithError(err).WithField("dir", runtimeDir).
80
-					Warn("failed to remove old runtimes dir")
81
-				return
82
-			}
83
-		}
84
-		if err = os.Rename(tmpDir, runtimeDir); err != nil {
85
-			err = errors.Wrap(err, "failed to setup runtimes dir, new containers may not start")
86
-			return
87
-		}
88
-		if err = os.RemoveAll(runtimeOldDir); err != nil {
89
-			logrus.WithError(err).WithField("dir", runtimeOldDir).
90
-				Warn("failed to remove old runtimes dir")
91
-		}
92
-	}()
83
+func setupRuntimes(cfg *config.Config) (runtimes, error) {
84
+	newrt := runtimes{
85
+		configured: make(map[string]*shimConfig),
86
+	}
93 87
 
94
-	for name := range cfg.Runtimes {
95
-		rt := cfg.Runtimes[name]
88
+	dir := runtimeScriptsDir(cfg)
89
+	for name, rt := range cfg.Runtimes {
90
+		var c *shimConfig
96 91
 		if rt.Path == "" && rt.Type == "" {
97
-			return errors.Errorf("runtime %s: either a runtimeType or a path must be configured", name)
92
+			return runtimes{}, errors.Errorf("runtime %s: either a runtimeType or a path must be configured", name)
98 93
 		}
99 94
 		if rt.Path != "" {
100 95
 			if rt.Type != "" {
101
-				return errors.Errorf("runtime %s: cannot configure both path and runtimeType for the same runtime", name)
96
+				return runtimes{}, errors.Errorf("runtime %s: cannot configure both path and runtimeType for the same runtime", name)
102 97
 			}
103 98
 			if len(rt.Options) > 0 {
104
-				return errors.Errorf("runtime %s: options cannot be used with a path runtime", name)
99
+				return runtimes{}, errors.Errorf("runtime %s: options cannot be used with a path runtime", name)
105 100
 			}
106 101
 
107
-			if len(rt.Args) > 0 {
108
-				script := filepath.Join(tmpDir, name)
109
-				content := fmt.Sprintf("#!/bin/sh\n%s %s $@\n", rt.Path, strings.Join(rt.Args, " "))
110
-				if err := os.WriteFile(script, []byte(content), 0700); err != nil {
111
-					return err
102
+			binaryName := rt.Path
103
+			needsWrapper := len(rt.Args) > 0
104
+			if needsWrapper {
105
+				var err error
106
+				binaryName, err = wrapRuntime(dir, name, rt.Path, rt.Args)
107
+				if err != nil {
108
+					return runtimes{}, err
112 109
 				}
113 110
 			}
114
-			rt.ShimConfig = defaultV2ShimConfig(cfg, daemon.rewriteRuntimePath(cfg, name, rt.Path, rt.Args))
115
-			var featuresStderr bytes.Buffer
116
-			featuresCmd := exec.Command(rt.Path, append(rt.Args, "features")...)
117
-			featuresCmd.Stderr = &featuresStderr
118
-			if featuresB, err := featuresCmd.Output(); err != nil {
119
-				logrus.WithError(err).Warnf("Failed to run %v: %q", featuresCmd.Args, featuresStderr.String())
120
-			} else {
121
-				var features features.Features
122
-				if jsonErr := json.Unmarshal(featuresB, &features); jsonErr != nil {
123
-					logrus.WithError(err).Warnf("Failed to unmarshal the output of %v as a JSON", featuresCmd.Args)
124
-				} else {
125
-					rt.Features = &features
111
+			c = defaultV2ShimConfig(cfg, binaryName)
112
+			if needsWrapper {
113
+				path := rt.Path
114
+				c.PreflightCheck = func() error {
115
+					// Check that the runtime path actually exists so that we can return a well known error.
116
+					_, err := exec.LookPath(path)
117
+					return errors.Wrap(err, "error while looking up the specified runtime path")
126 118
 				}
127 119
 			}
128 120
 		} else {
129 121
 			if len(rt.Args) > 0 {
130
-				return errors.Errorf("runtime %s: args cannot be used with a runtimeType runtime", name)
122
+				return runtimes{}, errors.Errorf("runtime %s: args cannot be used with a runtimeType runtime", name)
131 123
 			}
132 124
 			// Unlike implicit runtimes, there is no restriction on configuring a shim by path.
133
-			rt.ShimConfig = &types.ShimConfig{Binary: rt.Type}
125
+			c = &shimConfig{Shim: rt.Type}
134 126
 			if len(rt.Options) > 0 {
135 127
 				// It has to be a pointer type or there'll be a panic in containerd/typeurl when we try to start the container.
136
-				rt.ShimConfig.Opts, err = shimopts.Generate(rt.Type, rt.Options)
128
+				var err error
129
+				c.Opts, err = shimopts.Generate(rt.Type, rt.Options)
137 130
 				if err != nil {
138
-					return errors.Wrapf(err, "runtime %v", name)
131
+					return runtimes{}, errors.Wrapf(err, "runtime %v", name)
139 132
 				}
140 133
 			}
141 134
 		}
142
-		cfg.Runtimes[name] = rt
135
+		newrt.configured[name] = c
143 136
 	}
144
-	return nil
137
+
138
+	return newrt, nil
145 139
 }
146 140
 
147
-// rewriteRuntimePath is used for runtimes which have custom arguments supplied.
148
-// This is needed because the containerd API only calls the OCI runtime binary, there is no options for extra arguments.
149
-// To support this case, the daemon wraps the specified runtime in a script that passes through those arguments.
150
-func (daemon *Daemon) rewriteRuntimePath(cfg *config.Config, name, p string, args []string) string {
151
-	if len(args) == 0 {
152
-		return p
141
+// A non-standard Base32 encoding which lacks vowels to avoid accidentally
142
+// spelling naughty words. Don't use this to encode any data which requires
143
+// compatibility with anything outside of the currently-running process.
144
+var base32Disemvoweled = base32.NewEncoding("0123456789BCDFGHJKLMNPQRSTVWXYZ-")
145
+
146
+// wrapRuntime writes a shell script to dir which will execute binary with args
147
+// concatenated to the script's argv. This is needed because the
148
+// io.containerd.runc.v2 shim has no options for passing extra arguments to the
149
+// runtime binary.
150
+func wrapRuntime(dir, name, binary string, args []string) (string, error) {
151
+	var wrapper bytes.Buffer
152
+	sum := sha256.New()
153
+	_, _ = fmt.Fprintf(io.MultiWriter(&wrapper, sum), "#!/bin/sh\n%s %s $@\n", binary, strings.Join(args, " "))
154
+	// Generate a consistent name for the wrapper script derived from the
155
+	// contents so that multiple wrapper scripts can coexist with the same
156
+	// base name. The existing scripts might still be referenced by running
157
+	// containers.
158
+	suffix := base32Disemvoweled.EncodeToString(sum.Sum(nil))
159
+	scriptPath := filepath.Join(dir, name+"."+suffix)
160
+	if err := ioutils.AtomicWriteFile(scriptPath, wrapper.Bytes(), 0700); err != nil {
161
+		return "", err
153 162
 	}
154
-
155
-	return filepath.Join(cfg.Root, "runtimes", name)
163
+	return scriptPath, nil
156 164
 }
157 165
 
158
-func (daemon *Daemon) getRuntime(cfg *config.Config, name string) (shim string, opts interface{}, err error) {
159
-	rt := cfg.GetRuntime(name)
160
-	if rt == nil {
161
-		if !config.IsPermissibleC8dRuntimeName(name) {
162
-			return "", nil, errdefs.InvalidParameter(errors.Errorf("unknown or invalid runtime name: %s", name))
166
+func (r *runtimes) Get(name string) (string, interface{}, error) {
167
+	rt := r.configured[name]
168
+	if rt != nil {
169
+		if rt.PreflightCheck != nil {
170
+			if err := rt.PreflightCheck(); err != nil {
171
+				return "", nil, err
172
+			}
163 173
 		}
164
-		return name, nil, nil
174
+		return rt.Shim, rt.Opts, nil
165 175
 	}
166 176
 
167
-	if len(rt.Args) > 0 {
168
-		// Check that the path of the runtime which the script wraps actually exists so
169
-		// that we can return a well known error which references the configured path
170
-		// instead of the wrapper script's.
171
-		if _, err := exec.LookPath(rt.Path); err != nil {
172
-			return "", nil, errors.Wrap(err, "error while looking up the specified runtime path")
173
-		}
177
+	if !config.IsPermissibleC8dRuntimeName(name) {
178
+		return "", nil, errdefs.InvalidParameter(errors.Errorf("unknown or invalid runtime name: %s", name))
174 179
 	}
180
+	return name, nil, nil
181
+}
175 182
 
176
-	if rt.ShimConfig == nil {
177
-		// Should never happen as daemon.initRuntimes always sets
178
-		// ShimConfig and config reloading is synchronized.
179
-		err := errdefs.System(errors.Errorf("BUG: runtime %s: rt.ShimConfig == nil", name))
180
-		logrus.Error(err)
181
-		return "", nil, err
183
+func (r *runtimes) Features(name string) *features.Features {
184
+	rt := r.configured[name]
185
+	if rt != nil {
186
+		return rt.Features
182 187
 	}
183
-
184
-	return rt.ShimConfig.Binary, rt.ShimConfig.Opts, nil
188
+	return nil
185 189
 }
... ...
@@ -3,8 +3,9 @@
3 3
 package daemon
4 4
 
5 5
 import (
6
+	"io/fs"
6 7
 	"os"
7
-	"path/filepath"
8
+	"strings"
8 9
 	"testing"
9 10
 
10 11
 	"github.com/containerd/containerd/plugin"
... ...
@@ -88,11 +89,9 @@ func TestInitRuntimes_InvalidConfigs(t *testing.T) {
88 88
 			assert.NilError(t, err)
89 89
 			cfg.Root = t.TempDir()
90 90
 			cfg.Runtimes["myruntime"] = tt.runtime
91
-			d := &Daemon{}
92
-			d.configStore.Store(cfg)
93
-			assert.Assert(t, os.Mkdir(filepath.Join(d.config().Root, "runtimes"), 0700))
91
+			assert.Assert(t, initRuntimesDir(cfg))
94 92
 
95
-			err = d.initRuntimes(d.config())
93
+			_, err = setupRuntimes(cfg)
96 94
 			assert.Check(t, is.ErrorContains(err, tt.expectErr))
97 95
 		})
98 96
 	}
... ...
@@ -127,7 +126,6 @@ func TestGetRuntime(t *testing.T) {
127 127
 	assert.NilError(t, err)
128 128
 
129 129
 	cfg.Root = t.TempDir()
130
-	assert.Assert(t, os.Mkdir(filepath.Join(cfg.Root, "runtimes"), 0700))
131 130
 	cfg.Runtimes = map[string]types.Runtime{
132 131
 		configuredRtName:         configuredRuntime,
133 132
 		rtWithArgsName:           rtWithArgs,
... ...
@@ -136,41 +134,43 @@ func TestGetRuntime(t *testing.T) {
136 136
 		configuredShimByPathName: configuredShimByPath,
137 137
 	}
138 138
 	configureRuntimes(cfg)
139
+	assert.NilError(t, initRuntimesDir(cfg))
140
+	runtimes, err := setupRuntimes(cfg)
141
+	assert.NilError(t, err)
139 142
 
140
-	d := &Daemon{}
141
-	d.configStore.Store(cfg)
142
-	assert.Assert(t, d.loadRuntimes())
143
-
144
-	stockRuntime, ok := cfg.Runtimes[config.StockRuntimeName]
143
+	stockRuntime, ok := runtimes.configured[config.StockRuntimeName]
145 144
 	assert.Assert(t, ok, "stock runtime could not be found (test needs to be updated)")
145
+	stockRuntime.Features = nil
146 146
 
147
-	configdOpts := *stockRuntime.ShimConfig.Opts.(*v2runcoptions.Options)
147
+	configdOpts := *stockRuntime.Opts.(*v2runcoptions.Options)
148 148
 	configdOpts.BinaryName = configuredRuntime.Path
149
+	wantConfigdRuntime := &shimConfig{
150
+		Shim: stockRuntime.Shim,
151
+		Opts: &configdOpts,
152
+	}
149 153
 
150 154
 	for _, tt := range []struct {
151 155
 		name, runtime string
152
-		wantShim      string
153
-		wantOpts      interface{}
156
+		want          *shimConfig
154 157
 	}{
155 158
 		{
156
-			name:     "StockRuntime",
157
-			runtime:  config.StockRuntimeName,
158
-			wantShim: stockRuntime.ShimConfig.Binary,
159
-			wantOpts: stockRuntime.ShimConfig.Opts,
159
+			name:    "StockRuntime",
160
+			runtime: config.StockRuntimeName,
161
+			want:    stockRuntime,
160 162
 		},
161 163
 		{
162
-			name:     "ShimName",
163
-			runtime:  "io.containerd.my-shim.v42",
164
-			wantShim: "io.containerd.my-shim.v42",
164
+			name:    "ShimName",
165
+			runtime: "io.containerd.my-shim.v42",
166
+			want:    &shimConfig{Shim: "io.containerd.my-shim.v42"},
165 167
 		},
166 168
 		{
167 169
 			// containerd is pretty loose about the format of runtime names. Perhaps too
168 170
 			// loose. The only requirements are that the name contain a dot and (depending
169 171
 			// on the containerd version) not start with a dot. It does not enforce any
170 172
 			// particular format of the dot-delimited components of the name.
171
-			name:     "VersionlessShimName",
172
-			runtime:  "io.containerd.my-shim",
173
-			wantShim: "io.containerd.my-shim",
173
+			name:    "VersionlessShimName",
174
+			runtime: "io.containerd.my-shim",
175
+			want:    &shimConfig{Shim: "io.containerd.my-shim"},
174 176
 		},
175 177
 		{
176 178
 			name:    "IllformedShimName",
... ...
@@ -193,50 +193,152 @@ func TestGetRuntime(t *testing.T) {
193 193
 			runtime: "my/io.containerd.runc.v2",
194 194
 		},
195 195
 		{
196
-			name:     "ConfiguredRuntime",
197
-			runtime:  configuredRtName,
198
-			wantShim: stockRuntime.ShimConfig.Binary,
199
-			wantOpts: &configdOpts,
196
+			name:    "ConfiguredRuntime",
197
+			runtime: configuredRtName,
198
+			want:    wantConfigdRuntime,
200 199
 		},
201 200
 		{
202
-			name:     "RuntimeWithArgs",
203
-			runtime:  rtWithArgsName,
204
-			wantShim: stockRuntime.ShimConfig.Binary,
205
-			wantOpts: defaultV2ShimConfig(
206
-				d.config(),
207
-				d.rewriteRuntimePath(
208
-					d.config(),
209
-					rtWithArgsName,
210
-					rtWithArgs.Path,
211
-					rtWithArgs.Args)).Opts,
212
-		},
213
-		{
214
-			name:     "ShimWithOpts",
215
-			runtime:  shimWithOptsName,
216
-			wantShim: shimWithOpts.Type,
217
-			wantOpts: &v2runcoptions.Options{IoUid: 42},
201
+			name:    "ShimWithOpts",
202
+			runtime: shimWithOptsName,
203
+			want: &shimConfig{
204
+				Shim: shimWithOpts.Type,
205
+				Opts: &v2runcoptions.Options{IoUid: 42},
206
+			},
218 207
 		},
219 208
 		{
220
-			name:     "ShimAlias",
221
-			runtime:  shimAliasName,
222
-			wantShim: shimAlias.Type,
209
+			name:    "ShimAlias",
210
+			runtime: shimAliasName,
211
+			want:    &shimConfig{Shim: shimAlias.Type},
223 212
 		},
224 213
 		{
225
-			name:     "ConfiguredShimByPath",
226
-			runtime:  configuredShimByPathName,
227
-			wantShim: configuredShimByPath.Type,
214
+			name:    "ConfiguredShimByPath",
215
+			runtime: configuredShimByPathName,
216
+			want:    &shimConfig{Shim: configuredShimByPath.Type},
228 217
 		},
229 218
 	} {
230 219
 		tt := tt
231 220
 		t.Run(tt.name, func(t *testing.T) {
232
-			gotShim, gotOpts, err := d.getRuntime(cfg, tt.runtime)
233
-			assert.Check(t, is.Equal(gotShim, tt.wantShim))
234
-			assert.Check(t, is.DeepEqual(gotOpts, tt.wantOpts))
235
-			if tt.wantShim != "" {
221
+			shim, opts, err := runtimes.Get(tt.runtime)
222
+			if tt.want != nil {
236 223
 				assert.Check(t, err)
224
+				got := &shimConfig{Shim: shim, Opts: opts}
225
+				assert.Check(t, is.DeepEqual(got, tt.want))
237 226
 			} else {
238
-				assert.Check(t, errdefs.IsInvalidParameter(err))
227
+				assert.Check(t, is.Equal(shim, ""))
228
+				assert.Check(t, is.Nil(opts))
229
+				assert.Check(t, errdefs.IsInvalidParameter(err), "[%T] %[1]v", err)
239 230
 			}
240 231
 		})
241 232
 	}
233
+	t.Run("RuntimeWithArgs", func(t *testing.T) {
234
+		shim, opts, err := runtimes.Get(rtWithArgsName)
235
+		assert.Check(t, err)
236
+		assert.Check(t, is.Equal(shim, stockRuntime.Shim))
237
+		runcopts, ok := opts.(*v2runcoptions.Options)
238
+		if assert.Check(t, ok, "runtimes.Get() opts = type %T, want *v2runcoptions.Options", opts) {
239
+			wrapper, err := os.ReadFile(runcopts.BinaryName)
240
+			if assert.Check(t, err) {
241
+				assert.Check(t, is.Contains(string(wrapper),
242
+					strings.Join(append([]string{rtWithArgs.Path}, rtWithArgs.Args...), " ")))
243
+			}
244
+		}
245
+	})
246
+}
247
+
248
+func TestGetRuntime_PreflightCheck(t *testing.T) {
249
+	cfg, err := config.New()
250
+	assert.NilError(t, err)
251
+
252
+	cfg.Root = t.TempDir()
253
+	cfg.Runtimes = map[string]types.Runtime{
254
+		"path-only": {
255
+			Path: "/usr/local/bin/file-not-found",
256
+		},
257
+		"with-args": {
258
+			Path: "/usr/local/bin/file-not-found",
259
+			Args: []string{"--arg"},
260
+		},
261
+	}
262
+	assert.NilError(t, initRuntimesDir(cfg))
263
+	runtimes, err := setupRuntimes(cfg)
264
+	assert.NilError(t, err, "runtime paths should not be validated during setupRuntimes()")
265
+
266
+	t.Run("PathOnly", func(t *testing.T) {
267
+		_, _, err := runtimes.Get("path-only")
268
+		assert.NilError(t, err, "custom runtimes without wrapper scripts should not have pre-flight checks")
269
+	})
270
+	t.Run("WithArgs", func(t *testing.T) {
271
+		_, _, err := runtimes.Get("with-args")
272
+		assert.ErrorIs(t, err, fs.ErrNotExist)
273
+	})
274
+}
275
+
276
+// TestRuntimeWrapping checks that reloading runtime config does not delete or
277
+// modify existing wrapper scripts, which could break lifecycle management of
278
+// existing containers.
279
+func TestRuntimeWrapping(t *testing.T) {
280
+	cfg, err := config.New()
281
+	assert.NilError(t, err)
282
+	cfg.Root = t.TempDir()
283
+	cfg.Runtimes = map[string]types.Runtime{
284
+		"change-args": {
285
+			Path: "/bin/true",
286
+			Args: []string{"foo", "bar"},
287
+		},
288
+		"dupe": {
289
+			Path: "/bin/true",
290
+			Args: []string{"foo", "bar"},
291
+		},
292
+		"change-path": {
293
+			Path: "/bin/true",
294
+			Args: []string{"baz"},
295
+		},
296
+		"drop-args": {
297
+			Path: "/bin/true",
298
+			Args: []string{"some", "arguments"},
299
+		},
300
+		"goes-away": {
301
+			Path: "/bin/true",
302
+			Args: []string{"bye"},
303
+		},
304
+	}
305
+	assert.NilError(t, initRuntimesDir(cfg))
306
+	rt, err := setupRuntimes(cfg)
307
+	assert.Check(t, err)
308
+
309
+	type WrapperInfo struct{ BinaryName, Content string }
310
+	wrappers := make(map[string]WrapperInfo)
311
+	for name := range cfg.Runtimes {
312
+		_, opts, err := rt.Get(name)
313
+		if assert.Check(t, err, "rt.Get(%q)", name) {
314
+			binary := opts.(*v2runcoptions.Options).BinaryName
315
+			content, err := os.ReadFile(binary)
316
+			assert.Check(t, err, "could not read wrapper script contents for runtime %q", binary)
317
+			wrappers[name] = WrapperInfo{BinaryName: binary, Content: string(content)}
318
+		}
319
+	}
320
+
321
+	cfg.Runtimes["change-args"] = types.Runtime{
322
+		Path: cfg.Runtimes["change-args"].Path,
323
+		Args: []string{"baz", "quux"},
324
+	}
325
+	cfg.Runtimes["change-path"] = types.Runtime{
326
+		Path: "/bin/false",
327
+		Args: cfg.Runtimes["change-path"].Args,
328
+	}
329
+	cfg.Runtimes["drop-args"] = types.Runtime{
330
+		Path: cfg.Runtimes["drop-args"].Path,
331
+	}
332
+	delete(cfg.Runtimes, "goes-away")
333
+
334
+	_, err = setupRuntimes(cfg)
335
+	assert.Check(t, err)
336
+
337
+	for name, info := range wrappers {
338
+		t.Run(name, func(t *testing.T) {
339
+			content, err := os.ReadFile(info.BinaryName)
340
+			assert.NilError(t, err)
341
+			assert.DeepEqual(t, info.Content, string(content))
342
+		})
343
+	}
242 344
 }
... ...
@@ -6,6 +6,16 @@ import (
6 6
 	"github.com/docker/docker/daemon/config"
7 7
 )
8 8
 
9
-func (daemon *Daemon) getRuntime(cfg *config.Config, name string) (shim string, opts interface{}, err error) {
9
+type runtimes struct{}
10
+
11
+func (r *runtimes) Get(name string) (string, interface{}, error) {
10 12
 	return "", nil, errors.New("not implemented")
11 13
 }
14
+
15
+func initRuntimesDir(*config.Config) error {
16
+	return nil
17
+}
18
+
19
+func setupRuntimes(*config.Config) (runtimes, error) {
20
+	return runtimes{}, nil
21
+}
... ...
@@ -9,7 +9,6 @@ import (
9 9
 	"github.com/docker/docker/api/types"
10 10
 	containertypes "github.com/docker/docker/api/types/container"
11 11
 	"github.com/docker/docker/container"
12
-	"github.com/docker/docker/daemon/config"
13 12
 	"github.com/docker/docker/errdefs"
14 13
 	"github.com/docker/docker/libcontainerd"
15 14
 	"github.com/pkg/errors"
... ...
@@ -57,7 +56,7 @@ func (daemon *Daemon) ContainerStart(ctx context.Context, name string, hostConfi
57 57
 		if hostConfig != nil {
58 58
 			logrus.Warn("DEPRECATED: Setting host configuration options when the container starts is deprecated and has been removed in Docker 1.12")
59 59
 			oldNetworkMode := ctr.HostConfig.NetworkMode
60
-			if err := daemon.setSecurityOptions(daemonCfg, ctr, hostConfig); err != nil {
60
+			if err := daemon.setSecurityOptions(&daemonCfg.Config, ctr, hostConfig); err != nil {
61 61
 				return errdefs.InvalidParameter(err)
62 62
 			}
63 63
 			if err := daemon.mergeAndVerifyLogConfig(&hostConfig.LogConfig); err != nil {
... ...
@@ -91,7 +90,7 @@ func (daemon *Daemon) ContainerStart(ctx context.Context, name string, hostConfi
91 91
 	// Adapt for old containers in case we have updates in this function and
92 92
 	// old containers never have chance to call the new function in create stage.
93 93
 	if hostConfig != nil {
94
-		if err := daemon.adaptContainerSettings(daemonCfg, ctr.HostConfig, false); err != nil {
94
+		if err := daemon.adaptContainerSettings(&daemonCfg.Config, ctr.HostConfig, false); err != nil {
95 95
 			return errdefs.InvalidParameter(err)
96 96
 		}
97 97
 	}
... ...
@@ -102,7 +101,7 @@ func (daemon *Daemon) ContainerStart(ctx context.Context, name string, hostConfi
102 102
 // container needs, such as storage and networking, as well as links
103 103
 // between containers. The container is left waiting for a signal to
104 104
 // begin running.
105
-func (daemon *Daemon) containerStart(ctx context.Context, daemonCfg *config.Config, container *container.Container, checkpoint string, checkpointDir string, resetRestartManager bool) (retErr error) {
105
+func (daemon *Daemon) containerStart(ctx context.Context, daemonCfg *configStore, container *container.Container, checkpoint string, checkpointDir string, resetRestartManager bool) (retErr error) {
106 106
 	start := time.Now()
107 107
 	container.Lock()
108 108
 	defer container.Unlock()
... ...
@@ -138,7 +137,7 @@ func (daemon *Daemon) containerStart(ctx context.Context, daemonCfg *config.Conf
138 138
 			// if containers AutoRemove flag is set, remove it after clean up
139 139
 			if container.HostConfig.AutoRemove {
140 140
 				container.Unlock()
141
-				if err := daemon.containerRm(daemonCfg, container.ID, &types.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil {
141
+				if err := daemon.containerRm(&daemonCfg.Config, container.ID, &types.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil {
142 142
 					logrus.Errorf("can't remove container %s: %v", container.ID, err)
143 143
 				}
144 144
 				container.Lock()
... ...
@@ -150,7 +149,7 @@ func (daemon *Daemon) containerStart(ctx context.Context, daemonCfg *config.Conf
150 150
 		return err
151 151
 	}
152 152
 
153
-	if err := daemon.initializeNetworking(daemonCfg, container); err != nil {
153
+	if err := daemon.initializeNetworking(&daemonCfg.Config, container); err != nil {
154 154
 		return err
155 155
 	}
156 156
 
... ...
@@ -4,21 +4,20 @@ package daemon // import "github.com/docker/docker/daemon"
4 4
 
5 5
 import (
6 6
 	"github.com/docker/docker/container"
7
-	"github.com/docker/docker/daemon/config"
8 7
 )
9 8
 
10 9
 // getLibcontainerdCreateOptions callers must hold a lock on the container
11
-func (daemon *Daemon) getLibcontainerdCreateOptions(daemonCfg *config.Config, container *container.Container) (string, interface{}, error) {
10
+func (daemon *Daemon) getLibcontainerdCreateOptions(daemonCfg *configStore, container *container.Container) (string, interface{}, error) {
12 11
 	// Ensure a runtime has been assigned to this container
13 12
 	if container.HostConfig.Runtime == "" {
14 13
 		container.HostConfig.Runtime = daemonCfg.DefaultRuntime
15 14
 		container.CheckpointTo(daemon.containersReplica)
16 15
 	}
17 16
 
18
-	binary, opts, err := daemon.getRuntime(daemonCfg, container.HostConfig.Runtime)
17
+	shim, opts, err := daemonCfg.Runtimes.Get(container.HostConfig.Runtime)
19 18
 	if err != nil {
20 19
 		return "", nil, setExitCodeFromError(container.SetExitCode, err)
21 20
 	}
22 21
 
23
-	return binary, opts, nil
22
+	return shim, opts, nil
24 23
 }
... ...
@@ -3,11 +3,10 @@ package daemon // import "github.com/docker/docker/daemon"
3 3
 import (
4 4
 	"github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
5 5
 	"github.com/docker/docker/container"
6
-	"github.com/docker/docker/daemon/config"
7 6
 	"github.com/docker/docker/pkg/system"
8 7
 )
9 8
 
10
-func (daemon *Daemon) getLibcontainerdCreateOptions(*config.Config, *container.Container) (string, interface{}, error) {
9
+func (daemon *Daemon) getLibcontainerdCreateOptions(*configStore, *container.Container) (string, interface{}, error) {
11 10
 	if system.ContainerdRuntimeSupported() {
12 11
 		opts := &options.Options{}
13 12
 		return "io.containerd.runhcs.v1", opts, nil