Browse code

Take storage version for kube/os as config params

Start using versioned defaults in OpenShift config flow

Clayton Coleman authored on 2015/04/13 23:25:18
Showing 14 changed files
... ...
@@ -5,6 +5,7 @@ import (
5 5
 	"fmt"
6 6
 	"io"
7 7
 
8
+	etcdclient "github.com/coreos/go-etcd/etcd"
8 9
 	"github.com/golang/glog"
9 10
 	"github.com/spf13/cobra"
10 11
 
... ...
@@ -92,7 +93,12 @@ func (o OverwriteBootstrapPolicyOptions) OverwriteBootstrapPolicy() error {
92 92
 		return utilerrors.NewAggregate(err)
93 93
 	}
94 94
 
95
-	etcdHelper, err := etcd.NewOpenShiftEtcdHelper(masterConfig.EtcdClientInfo)
95
+	// Connect and setup etcd interfaces
96
+	etcdClient, err := etcd.GetAndTestEtcdClient(masterConfig.EtcdClientInfo)
97
+	if err != nil {
98
+		return err
99
+	}
100
+	etcdHelper, err := newEtcdHelper(etcdClient, masterConfig.EtcdStorageConfig.OpenShiftStorageVersion)
96 101
 	if err != nil {
97 102
 		return err
98 103
 	}
... ...
@@ -169,3 +175,12 @@ func OverwriteBootstrapPolicy(etcdHelper tools.EtcdHelper, masterNamespace, poli
169 169
 		return nil
170 170
 	})
171 171
 }
172
+
173
+// newEtcdHelper returns an EtcdHelper for the provided storage version.
174
+func newEtcdHelper(client *etcdclient.Client, version string) (oshelper tools.EtcdHelper, err error) {
175
+	interfaces, err := latest.InterfacesFor(version)
176
+	if err != nil {
177
+		return tools.EtcdHelper{}, err
178
+	}
179
+	return tools.NewEtcdHelper(client, interfaces.Codec), nil
180
+}
... ...
@@ -47,8 +47,13 @@ type MasterConfig struct {
47 47
 	// CORSAllowedOrigins
48 48
 	CORSAllowedOrigins []string
49 49
 
50
+	// EtcdStorageConfig contains information about how API resources are
51
+	// stored in Etcd. These values are only relevant when etcd is the
52
+	// backing store for the cluster.
53
+	EtcdStorageConfig EtcdStorageConfig
50 54
 	// EtcdClientInfo contains information about how to connect to etcd
51 55
 	EtcdClientInfo EtcdConnectionInfo
56
+
52 57
 	// KubeletClientInfo contains information about how to connect to kubelets
53 58
 	KubeletClientInfo KubeletConnectionInfo
54 59
 
... ...
@@ -58,7 +63,7 @@ type MasterConfig struct {
58 58
 	EtcdConfig *EtcdConfig
59 59
 	// OAuthConfig, if present start the /oauth endpoint in this process
60 60
 	OAuthConfig *OAuthConfig
61
-	// AssetConfig, if present start the asset serverin this process
61
+	// AssetConfig, if present start the asset server in this process
62 62
 	AssetConfig *AssetConfig
63 63
 	// DNSConfig, if present start the DNS server in this process
64 64
 	DNSConfig *DNSConfig
... ...
@@ -117,6 +122,17 @@ type EtcdConnectionInfo struct {
117 117
 	ClientCert CertInfo
118 118
 }
119 119
 
120
+type EtcdStorageConfig struct {
121
+	// KubernetesStorageVersion is the API version that Kube resources in etcd should be
122
+	// serialized to. This value should *not* be advanced until all clients in the
123
+	// cluster that read from etcd have code that allows them to read the new version.
124
+	KubernetesStorageVersion string
125
+	// OpenShiftStorageVersion is the API version that OS resources in etcd should be
126
+	// serialized to. This value should *not* be advanced until all clients in the
127
+	// cluster that read from etcd have code that allows them to read the new version.
128
+	OpenShiftStorageVersion string
129
+}
130
+
120 131
 type ServingInfo struct {
121 132
 	// BindAddress is the ip:port to serve on
122 133
 	BindAddress string
... ...
@@ -2,11 +2,26 @@ package v1
2 2
 
3 3
 import (
4 4
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/conversion"
5
+
5 6
 	newer "github.com/openshift/origin/pkg/cmd/server/api"
6 7
 )
7 8
 
8 9
 func init() {
9
-	err := newer.Scheme.AddConversionFuncs(
10
+	err := newer.Scheme.AddDefaultingFuncs(
11
+		func(obj *EtcdStorageConfig) {
12
+			if len(obj.KubernetesStorageVersion) == 0 {
13
+				obj.KubernetesStorageVersion = "v1beta3"
14
+			}
15
+			if len(obj.OpenShiftStorageVersion) == 0 {
16
+				obj.OpenShiftStorageVersion = "v1beta1"
17
+			}
18
+		},
19
+	)
20
+	if err != nil {
21
+		// If one of the conversion functions is malformed, detect it immediately.
22
+		panic(err)
23
+	}
24
+	err = newer.Scheme.AddConversionFuncs(
10 25
 		func(in *ServingInfo, out *newer.ServingInfo, s conversion.Scope) error {
11 26
 			out.BindAddress = in.BindAddress
12 27
 			out.ClientCA = in.ClientCA
... ...
@@ -46,6 +46,10 @@ type MasterConfig struct {
46 46
 	// CORSAllowedOrigins
47 47
 	CORSAllowedOrigins []string `json:"corsAllowedOrigins"`
48 48
 
49
+	// EtcdStorageConfig contains information about how API resources are
50
+	// stored in Etcd. These values are only relevant when etcd is the
51
+	// backing store for the cluster.
52
+	EtcdStorageConfig EtcdStorageConfig `json:"etcdStorageConfig"`
49 53
 	// EtcdClientInfo contains information about how to connect to etcd
50 54
 	EtcdClientInfo EtcdConnectionInfo `json:"etcdClientInfo"`
51 55
 	// KubeletClientInfo contains information about how to connect to kubelets
... ...
@@ -57,7 +61,7 @@ type MasterConfig struct {
57 57
 	EtcdConfig *EtcdConfig `json:"etcdConfig"`
58 58
 	// OAuthConfig, if present start the /oauth endpoint in this process
59 59
 	OAuthConfig *OAuthConfig `json:"oauthConfig"`
60
-	// AssetConfig, if present start the asset serverin this process
60
+	// AssetConfig, if present start the asset server in this process
61 61
 	AssetConfig *AssetConfig `json:"assetConfig"`
62 62
 	// DNSConfig, if present start the DNS server in this process
63 63
 	DNSConfig *DNSConfig `json:"dnsConfig"`
... ...
@@ -115,6 +119,17 @@ type EtcdConnectionInfo struct {
115 115
 	CertInfo `json:",inline"`
116 116
 }
117 117
 
118
+type EtcdStorageConfig struct {
119
+	// KubernetesStorageVersion is the API version that Kube resources in etcd should be
120
+	// serialized to. This value should *not* be advanced until all clients in the
121
+	// cluster that read from etcd have code that allows them to read the new version.
122
+	KubernetesStorageVersion string `json:"kubernetesStorageVersion"`
123
+	// OpenShiftStorageVersion is the API version that OS resources in etcd should be
124
+	// serialized to. This value should *not* be advanced until all clients in the
125
+	// cluster that read from etcd have code that allows them to read the new version.
126
+	OpenShiftStorageVersion string `json:"openShiftStorageVersion"`
127
+}
128
+
118 129
 type ServingInfo struct {
119 130
 	// BindAddress is the ip:port to serve on
120 131
 	BindAddress string `json:"bindAddress"`
... ...
@@ -56,6 +56,7 @@ func ValidateMasterConfig(config *api.MasterConfig) fielderrors.ValidationErrorL
56 56
 		// Validate the etcdClientInfo by itself
57 57
 		allErrs = append(allErrs, ValidateEtcdConnectionInfo(config.EtcdClientInfo, nil).Prefix("etcdClientInfo")...)
58 58
 	}
59
+	allErrs = append(allErrs, ValidateEtcdStorageConfig(config.EtcdStorageConfig).Prefix("etcdStorageConfig")...)
59 60
 
60 61
 	allErrs = append(allErrs, ValidateImageConfig(config.ImageConfig).Prefix("imageConfig")...)
61 62
 
... ...
@@ -79,6 +80,19 @@ func ValidateMasterConfig(config *api.MasterConfig) fielderrors.ValidationErrorL
79 79
 	return allErrs
80 80
 }
81 81
 
82
+func ValidateEtcdStorageConfig(config api.EtcdStorageConfig) fielderrors.ValidationErrorList {
83
+	allErrs := fielderrors.ValidationErrorList{}
84
+
85
+	if len(config.KubernetesStorageVersion) == 0 {
86
+		allErrs = append(allErrs, fielderrors.NewFieldRequired("kubernetesStorageVersion"))
87
+	}
88
+	if len(config.OpenShiftStorageVersion) == 0 {
89
+		allErrs = append(allErrs, fielderrors.NewFieldRequired("openShiftStorageVersion"))
90
+	}
91
+
92
+	return allErrs
93
+}
94
+
82 95
 func ValidateAssetConfig(config *api.AssetConfig) fielderrors.ValidationErrorList {
83 96
 	allErrs := fielderrors.ValidationErrorList{}
84 97
 
... ...
@@ -9,7 +9,6 @@ import (
9 9
 
10 10
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
11 11
 
12
-	"github.com/openshift/origin/pkg/api/latest"
13 12
 	configapi "github.com/openshift/origin/pkg/cmd/server/api"
14 13
 )
15 14
 
... ...
@@ -68,9 +67,9 @@ func RunEtcd(etcdServerConfig *configapi.EtcdConfig) {
68 68
 	}()
69 69
 }
70 70
 
71
-// GetAndTestEtcdClient creates an etcd client based on the provided config and waits
72
-// until etcd server is reachable. It errors out and exits if the server cannot
73
-// be reached for a certain amount of time.
71
+// GetAndTestEtcdClient creates an etcd client based on the provided config. It will attempt to
72
+// connect to the etcd server and block until the server responds at least once, or return an
73
+// error if the server never responded.
74 74
 func GetAndTestEtcdClient(etcdClientInfo configapi.EtcdConnectionInfo) (*etcdclient.Client, error) {
75 75
 	var etcdClient *etcdclient.Client
76 76
 
... ...
@@ -96,7 +95,6 @@ func GetAndTestEtcdClient(etcdClientInfo configapi.EtcdConnectionInfo) (*etcdcli
96 96
 	}
97 97
 
98 98
 	for i := 0; ; i++ {
99
-		// TODO: make sure this works with etcd2 (root key may not exist)
100 99
 		_, err := etcdClient.Get("/", false, false)
101 100
 		if err == nil || tools.IsEtcdNotFound(err) {
102 101
 			break
... ...
@@ -109,20 +107,3 @@ func GetAndTestEtcdClient(etcdClientInfo configapi.EtcdConnectionInfo) (*etcdcli
109 109
 
110 110
 	return etcdClient, nil
111 111
 }
112
-
113
-// NewOpenShiftEtcdHelper returns an EtcdHelper for the provided arguments or an error if the version
114
-// is incorrect.
115
-func NewOpenShiftEtcdHelper(etcdClientInfo configapi.EtcdConnectionInfo) (helper tools.EtcdHelper, err error) {
116
-	// Connect and setup etcd interfaces
117
-	client, err := GetAndTestEtcdClient(etcdClientInfo)
118
-	if err != nil {
119
-		return tools.EtcdHelper{}, err
120
-	}
121
-
122
-	version := latest.Version
123
-	interfaces, err := latest.InterfacesFor(version)
124
-	if err != nil {
125
-		return helper, err
126
-	}
127
-	return tools.NewEtcdHelper(client, interfaces.Codec), nil
128
-}
... ...
@@ -8,7 +8,6 @@ import (
8 8
 
9 9
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/admission"
10 10
 	kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api"
11
-	klatest "github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
12 11
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
13 12
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/auth/authorizer"
14 13
 	kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client"
... ...
@@ -49,7 +48,7 @@ func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextM
49 49
 	if err != nil {
50 50
 		return nil, err
51 51
 	}
52
-	ketcdHelper, err := master.NewEtcdHelper(etcdClient, klatest.Version)
52
+	ketcdHelper, err := master.NewEtcdHelper(etcdClient, options.EtcdStorageConfig.KubernetesStorageVersion)
53 53
 	if err != nil {
54 54
 		return nil, fmt.Errorf("Error setting up Kubernetes server storage: %v", err)
55 55
 	}
... ...
@@ -34,7 +34,11 @@ type AuthConfig struct {
34 34
 }
35 35
 
36 36
 func BuildAuthConfig(options configapi.MasterConfig) (*AuthConfig, error) {
37
-	etcdHelper, err := etcd.NewOpenShiftEtcdHelper(options.EtcdClientInfo)
37
+	client, err := etcd.GetAndTestEtcdClient(options.EtcdClientInfo)
38
+	if err != nil {
39
+		return nil, err
40
+	}
41
+	etcdHelper, err := NewEtcdHelper(client, options.EtcdStorageConfig.OpenShiftStorageVersion)
38 42
 	if err != nil {
39 43
 		return nil, fmt.Errorf("Error setting up server storage: %v", err)
40 44
 	}
... ...
@@ -27,7 +27,6 @@ import (
27 27
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
28 28
 	kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client"
29 29
 	kmaster "github.com/GoogleCloudPlatform/kubernetes/pkg/master"
30
-	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
31 30
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
32 31
 	utilerrs "github.com/GoogleCloudPlatform/kubernetes/pkg/util/errors"
33 32
 
... ...
@@ -840,19 +839,6 @@ func (c *MasterConfig) ensureCORSAllowedOrigins() []*regexp.Regexp {
840 840
 	return allowedOriginRegexps
841 841
 }
842 842
 
843
-// NewEtcdHelper returns an EtcdHelper for the provided arguments or an error if the version
844
-// is incorrect.
845
-func NewEtcdHelper(version string, client *etcdclient.Client) (helper tools.EtcdHelper, err error) {
846
-	if len(version) == 0 {
847
-		version = latest.Version
848
-	}
849
-	interfaces, err := latest.InterfacesFor(version)
850
-	if err != nil {
851
-		return helper, err
852
-	}
853
-	return tools.NewEtcdHelper(client, interfaces.Codec), nil
854
-}
855
-
856 843
 // env returns an environment variable, or the defaultValue if it is not set.
857 844
 func env(key string, defaultValue string) string {
858 845
 	val := os.Getenv(key)
... ...
@@ -5,6 +5,8 @@ import (
5 5
 	"fmt"
6 6
 	"net/http"
7 7
 
8
+	etcdclient "github.com/coreos/go-etcd/etcd"
9
+
8 10
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/admission"
9 11
 	kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api"
10 12
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
... ...
@@ -27,12 +29,11 @@ import (
27 27
 	"github.com/openshift/origin/pkg/authorization/rulevalidation"
28 28
 	osclient "github.com/openshift/origin/pkg/client"
29 29
 	configapi "github.com/openshift/origin/pkg/cmd/server/api"
30
+	"github.com/openshift/origin/pkg/cmd/server/etcd"
31
+	"github.com/openshift/origin/pkg/cmd/util/variable"
30 32
 	accesstokenregistry "github.com/openshift/origin/pkg/oauth/registry/oauthaccesstoken"
31 33
 	accesstokenetcd "github.com/openshift/origin/pkg/oauth/registry/oauthaccesstoken/etcd"
32 34
 	projectauth "github.com/openshift/origin/pkg/project/auth"
33
-
34
-	"github.com/openshift/origin/pkg/cmd/server/etcd"
35
-	"github.com/openshift/origin/pkg/cmd/util/variable"
36 35
 )
37 36
 
38 37
 const (
... ...
@@ -92,8 +93,11 @@ type MasterConfig struct {
92 92
 }
93 93
 
94 94
 func BuildMasterConfig(options configapi.MasterConfig) (*MasterConfig, error) {
95
-
96
-	etcdHelper, err := etcd.NewOpenShiftEtcdHelper(options.EtcdClientInfo)
95
+	client, err := etcd.GetAndTestEtcdClient(options.EtcdClientInfo)
96
+	if err != nil {
97
+		return nil, err
98
+	}
99
+	etcdHelper, err := NewEtcdHelper(client, options.EtcdStorageConfig.OpenShiftStorageVersion)
97 100
 	if err != nil {
98 101
 		return nil, fmt.Errorf("Error setting up server storage: %v", err)
99 102
 	}
... ...
@@ -308,3 +312,12 @@ func (c *MasterConfig) DeploymentImageChangeControllerClient() *osclient.Client
308 308
 func (c *MasterConfig) OriginNamespaceControllerClients() (*osclient.Client, *kclient.Client) {
309 309
 	return c.OSClient, c.KubernetesClient
310 310
 }
311
+
312
+// NewEtcdHelper returns an EtcdHelper for the provided storage version.
313
+func NewEtcdHelper(client *etcdclient.Client, version string) (oshelper tools.EtcdHelper, err error) {
314
+	interfaces, err := latest.InterfacesFor(version)
315
+	if err != nil {
316
+		return tools.EtcdHelper{}, err
317
+	}
318
+	return tools.NewEtcdHelper(client, interfaces.Codec), nil
319
+}
... ...
@@ -205,7 +205,17 @@ func (args MasterArgs) BuildSerializeableMasterConfig() (*configapi.MasterConfig
205 205
 		}
206 206
 	}
207 207
 
208
-	return config, nil
208
+	// Roundtrip the config to v1 and back to ensure proper defaults are set.
209
+	ext, err := configapi.Scheme.ConvertToVersion(config, "v1")
210
+	if err != nil {
211
+		return nil, err
212
+	}
213
+	internal, err := configapi.Scheme.ConvertToVersion(ext, "")
214
+	if err != nil {
215
+		return nil, err
216
+	}
217
+
218
+	return internal.(*configapi.MasterConfig), nil
209 219
 }
210 220
 
211 221
 func (args MasterArgs) BuildSerializeableOAuthConfig() (*configapi.OAuthConfig, error) {
... ...
@@ -121,7 +121,17 @@ func (args NodeArgs) BuildSerializeableNodeConfig() (*configapi.NodeConfig, erro
121 121
 		config.ServingInfo.ClientCA = admin.DefaultKubeletClientCAFile(args.CertArgs.CertDir)
122 122
 	}
123 123
 
124
-	return config, nil
124
+	// Roundtrip the config to v1 and back to ensure proper defaults are set.
125
+	ext, err := configapi.Scheme.ConvertToVersion(config, "v1")
126
+	if err != nil {
127
+		return nil, err
128
+	}
129
+	internal, err := configapi.Scheme.ConvertToVersion(ext, "")
130
+	if err != nil {
131
+		return nil, err
132
+	}
133
+
134
+	return internal.(*configapi.NodeConfig), nil
125 135
 }
126 136
 
127 137
 // defaultHostname returns the default hostname for this system.
... ...
@@ -16,6 +16,7 @@ import (
16 16
 	"github.com/openshift/origin/pkg/client"
17 17
 	"github.com/openshift/origin/pkg/cmd/server/admin"
18 18
 	"github.com/openshift/origin/pkg/cmd/server/etcd"
19
+	"github.com/openshift/origin/pkg/cmd/server/origin"
19 20
 	"github.com/openshift/origin/pkg/cmd/util/tokencmd"
20 21
 	testutil "github.com/openshift/origin/test/util"
21 22
 )
... ...
@@ -94,7 +95,12 @@ func TestOverwritePolicyCommand(t *testing.T) {
94 94
 		t.Errorf("unexpected error: %v", err)
95 95
 	}
96 96
 
97
-	etcdHelper, err := etcd.NewOpenShiftEtcdHelper(masterConfig.EtcdClientInfo)
97
+	etcdClient, err := etcd.GetAndTestEtcdClient(masterConfig.EtcdClientInfo)
98
+	if err != nil {
99
+		t.Errorf("unexpected error: %v", err)
100
+	}
101
+
102
+	etcdHelper, err := origin.NewEtcdHelper(etcdClient, masterConfig.EtcdStorageConfig.OpenShiftStorageVersion)
98 103
 	if err != nil {
99 104
 		t.Errorf("unexpected error: %v", err)
100 105
 	}
... ...
@@ -14,6 +14,7 @@ import (
14 14
 	authapi "github.com/openshift/origin/pkg/auth/api"
15 15
 	"github.com/openshift/origin/pkg/auth/userregistry/identitymapper"
16 16
 	"github.com/openshift/origin/pkg/cmd/server/etcd"
17
+	"github.com/openshift/origin/pkg/cmd/server/origin"
17 18
 	"github.com/openshift/origin/pkg/user/api"
18 19
 	identityregistry "github.com/openshift/origin/pkg/user/registry/identity"
19 20
 	identityetcd "github.com/openshift/origin/pkg/user/registry/identity/etcd"
... ...
@@ -78,9 +79,14 @@ func TestUserInitialization(t *testing.T) {
78 78
 		t.Fatalf("unexpected error: %v", err)
79 79
 	}
80 80
 
81
-	etcdHelper, err := etcd.NewOpenShiftEtcdHelper(masterConfig.EtcdClientInfo)
81
+	etcdClient, err := etcd.GetAndTestEtcdClient(masterConfig.EtcdClientInfo)
82 82
 	if err != nil {
83
-		t.Fatalf("unexpected error: %v", err)
83
+		t.Errorf("unexpected error: %v", err)
84
+	}
85
+
86
+	etcdHelper, err := origin.NewEtcdHelper(etcdClient, masterConfig.EtcdStorageConfig.OpenShiftStorageVersion)
87
+	if err != nil {
88
+		t.Errorf("unexpected error: %v", err)
84 89
 	}
85 90
 
86 91
 	userRegistry := userregistry.NewRegistry(useretcd.NewREST(etcdHelper))