Browse code

Add an admission controller to block illegal service endpoints

Add an admission controller to allow blocking service endpoints in
particular CIDR ranges (by default, the ClusterNetworkCIDR and
ServiceNetworkCIDR) and give the endpoints controller service account
permission to bypass it.

Dan Winship authored on 2016/06/21 22:04:06
Showing 9 changed files
... ...
@@ -26,7 +26,7 @@ This command launches an instance of the Kubernetes apiserver (kube\-apiserver).
26 26
 
27 27
 .PP
28 28
 \fB\-\-admission\-control\fP="AlwaysAdmit"
29
-    Ordered list of plug\-ins to do admission control of resources into cluster. Comma\-delimited list of: AlwaysAdmit, AlwaysDeny, AlwaysPullImages, BuildByStrategy, BuildDefaults, BuildOverrides, ClusterResourceOverride, ClusterResourceQuota, DenyEscalatingExec, DenyExecOnPrivileged, ExternalIPRanger, ImageLimitRange, InitialResources, JenkinsBootstrapper, LimitPodHardAntiAffinityTopology, LimitRanger, NamespaceAutoProvision, NamespaceExists, NamespaceLifecycle, OriginNamespaceLifecycle, OriginPodNodeEnvironment, OriginResourceQuota, PersistentVolumeLabel, PodNodeConstraints, PodSecurityPolicy, ProjectRequestLimit, ResourceQuota, RunOnceDuration, SCCExecRestrictions, SecurityContextConstraint, SecurityContextDeny, ServiceAccount
29
+    Ordered list of plug\-ins to do admission control of resources into cluster. Comma\-delimited list of: AlwaysAdmit, AlwaysDeny, AlwaysPullImages, BuildByStrategy, BuildDefaults, BuildOverrides, ClusterResourceOverride, ClusterResourceQuota, DenyEscalatingExec, DenyExecOnPrivileged, ExternalIPRanger, ImageLimitRange, InitialResources, JenkinsBootstrapper, LimitPodHardAntiAffinityTopology, LimitRanger, NamespaceAutoProvision, NamespaceExists, NamespaceLifecycle, OriginNamespaceLifecycle, OriginPodNodeEnvironment, OriginResourceQuota, PersistentVolumeLabel, PodNodeConstraints, PodSecurityPolicy, ProjectRequestLimit, ResourceQuota, RestrictedEndpointsAdmission, RunOnceDuration, SCCExecRestrictions, SecurityContextConstraint, SecurityContextDeny, ServiceAccount
30 30
 
31 31
 .PP
32 32
 \fB\-\-admission\-control\-config\-file\fP=""
... ...
@@ -10,4 +10,6 @@ const (
10 10
 	NodeMetricsResource = "nodes/metrics"
11 11
 	NodeStatsResource   = "nodes/stats"
12 12
 	NodeLogResource     = "nodes/log"
13
+
14
+	RestrictedEndpointsResource = "endpoints/restricted"
13 15
 )
... ...
@@ -676,6 +676,12 @@ func init() {
676 676
 					Verbs:     sets.NewString("get", "list", "create", "update", "delete"),
677 677
 					Resources: sets.NewString("endpoints"),
678 678
 				},
679
+				// Permission for RestrictedEndpointsAdmission
680
+				{
681
+					APIGroups: []string{kapi.GroupName},
682
+					Verbs:     sets.NewString("create"),
683
+					Resources: sets.NewString("endpoints/restricted"),
684
+				},
679 685
 			},
680 686
 		},
681 687
 	)
... ...
@@ -307,6 +307,7 @@ var (
307 307
 		"OriginPodNodeEnvironment",
308 308
 		overrideapi.PluginName,
309 309
 		serviceadmit.ExternalIPPluginName,
310
+		serviceadmit.RestrictedEndpointsPluginName,
310 311
 		"LimitRanger",
311 312
 		"ServiceAccount",
312 313
 		"SecurityContextConstraint",
... ...
@@ -335,6 +336,7 @@ var (
335 335
 		"OriginPodNodeEnvironment",
336 336
 		overrideapi.PluginName,
337 337
 		serviceadmit.ExternalIPPluginName,
338
+		serviceadmit.RestrictedEndpointsPluginName,
338 339
 		"LimitRanger",
339 340
 		"ServiceAccount",
340 341
 		"SecurityContextConstraint",
... ...
@@ -451,13 +453,22 @@ func newAdmissionChain(pluginNames []string, admissionConfigFilename string, plu
451 451
 
452 452
 		case serviceadmit.ExternalIPPluginName:
453 453
 			// this needs to be moved upstream to be part of core config
454
-			reject, admit, err := serviceadmit.ParseCIDRRules(options.NetworkConfig.ExternalIPNetworkCIDRs)
454
+			reject, admit, err := serviceadmit.ParseRejectAdmitCIDRRules(options.NetworkConfig.ExternalIPNetworkCIDRs)
455 455
 			if err != nil {
456 456
 				// should have been caught with validation
457 457
 				return nil, err
458 458
 			}
459 459
 			plugins = append(plugins, serviceadmit.NewExternalIPRanger(reject, admit))
460 460
 
461
+		case serviceadmit.RestrictedEndpointsPluginName:
462
+			// we need to set some customer parameters, so create by hand
463
+			restrictedNetworks, err := serviceadmit.ParseSimpleCIDRRules([]string{options.NetworkConfig.ClusterNetworkCIDR, options.NetworkConfig.ServiceNetworkCIDR})
464
+			if err != nil {
465
+				// should have been caught with validation
466
+				return nil, err
467
+			}
468
+			plugins = append(plugins, serviceadmit.NewRestrictedEndpointsAdmission(restrictedNetworks))
469
+
461 470
 		case saadmit.PluginName:
462 471
 			// we need to set some custom parameters on the service account admission controller, so create that one by hand
463 472
 			saAdmitter := saadmit.NewServiceAccount(kubeClientSet)
... ...
@@ -53,7 +53,7 @@ func (d *ServiceExternalIPs) Check() types.DiagnosticResult {
53 53
 
54 54
 	admit, reject := []*net.IPNet{}, []*net.IPNet{}
55 55
 	if cidrs := masterConfig.NetworkConfig.ExternalIPNetworkCIDRs; cidrs != nil {
56
-		reject, admit, err = admission.ParseCIDRRules(cidrs)
56
+		reject, admit, err = admission.ParseRejectAdmitCIDRRules(cidrs)
57 57
 		if err != nil {
58 58
 			r.Error("DH2007", err, fmt.Sprintf("Could not parse master config NetworkConfig.ExternalIPNetworkCIDRs: (%[1]T) %[1]v", err))
59 59
 			return r
60 60
new file mode 100644
... ...
@@ -0,0 +1,117 @@
0
+package admission
1
+
2
+import (
3
+	"fmt"
4
+	"io"
5
+	"net"
6
+	"reflect"
7
+
8
+	authorizationapi "github.com/openshift/origin/pkg/authorization/api"
9
+	"github.com/openshift/origin/pkg/authorization/authorizer"
10
+	"github.com/openshift/origin/pkg/client"
11
+	oadmission "github.com/openshift/origin/pkg/cmd/server/admission"
12
+
13
+	kadmission "k8s.io/kubernetes/pkg/admission"
14
+	kapi "k8s.io/kubernetes/pkg/api"
15
+	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
16
+)
17
+
18
+const RestrictedEndpointsPluginName = "RestrictedEndpointsAdmission"
19
+
20
+func init() {
21
+	kadmission.RegisterPlugin("RestrictedEndpointsAdmission", func(client clientset.Interface, config io.Reader) (kadmission.Interface, error) {
22
+		return NewRestrictedEndpointsAdmission(nil), nil
23
+	})
24
+}
25
+
26
+type restrictedEndpointsAdmission struct {
27
+	*kadmission.Handler
28
+
29
+	client             client.Interface
30
+	authorizer         authorizer.Authorizer
31
+	restrictedNetworks []*net.IPNet
32
+}
33
+
34
+var _ = oadmission.WantsAuthorizer(&restrictedEndpointsAdmission{})
35
+
36
+// ParseSimpleCIDRRules parses a list of CIDR strings
37
+func ParseSimpleCIDRRules(rules []string) (networks []*net.IPNet, err error) {
38
+	for _, s := range rules {
39
+		_, cidr, err := net.ParseCIDR(s)
40
+		if err != nil {
41
+			return nil, err
42
+		}
43
+		networks = append(networks, cidr)
44
+	}
45
+	return networks, nil
46
+}
47
+
48
+// NewRestrictedEndpointsAdmission creates a new endpoints admission plugin.
49
+func NewRestrictedEndpointsAdmission(restrictedNetworks []*net.IPNet) *restrictedEndpointsAdmission {
50
+	return &restrictedEndpointsAdmission{
51
+		Handler:            kadmission.NewHandler(kadmission.Create, kadmission.Update),
52
+		restrictedNetworks: restrictedNetworks,
53
+	}
54
+}
55
+
56
+func (r *restrictedEndpointsAdmission) SetAuthorizer(a authorizer.Authorizer) {
57
+	r.authorizer = a
58
+}
59
+
60
+func (r *restrictedEndpointsAdmission) findRestrictedIP(ep *kapi.Endpoints) string {
61
+	for _, subset := range ep.Subsets {
62
+		for _, addr := range subset.Addresses {
63
+			ip := net.ParseIP(addr.IP)
64
+			if ip == nil {
65
+				continue
66
+			}
67
+			for _, net := range r.restrictedNetworks {
68
+				if net.Contains(ip) {
69
+					return addr.IP
70
+				}
71
+			}
72
+		}
73
+	}
74
+	return ""
75
+}
76
+
77
+func (r *restrictedEndpointsAdmission) checkAccess(attr kadmission.Attributes) (bool, error) {
78
+	ctx := kapi.WithUser(kapi.WithNamespace(kapi.NewContext(), attr.GetNamespace()), attr.GetUserInfo())
79
+	authzAttr := authorizer.DefaultAuthorizationAttributes{
80
+		Verb:         "create",
81
+		Resource:     authorizationapi.RestrictedEndpointsResource,
82
+		APIGroup:     kapi.GroupName,
83
+		ResourceName: attr.GetName(),
84
+	}
85
+	allow, _, err := r.authorizer.Authorize(ctx, authzAttr)
86
+	return allow, err
87
+}
88
+
89
+// Admit determines if the endpoints object should be admitted
90
+func (r *restrictedEndpointsAdmission) Admit(a kadmission.Attributes) error {
91
+	if a.GetResource().GroupResource() != kapi.Resource("endpoints") {
92
+		return nil
93
+	}
94
+	ep, ok := a.GetObject().(*kapi.Endpoints)
95
+	if !ok {
96
+		return nil
97
+	}
98
+	old, ok := a.GetOldObject().(*kapi.Endpoints)
99
+	if ok && reflect.DeepEqual(ep.Subsets, old.Subsets) {
100
+		return nil
101
+	}
102
+
103
+	restrictedIP := r.findRestrictedIP(ep)
104
+	if restrictedIP == "" {
105
+		return nil
106
+	}
107
+
108
+	allow, err := r.checkAccess(a)
109
+	if err != nil {
110
+		return err
111
+	}
112
+	if !allow {
113
+		return kadmission.NewForbidden(a, fmt.Errorf("endpoint address %s is not allowed", restrictedIP))
114
+	}
115
+	return nil
116
+}
... ...
@@ -28,9 +28,9 @@ type externalIPRanger struct {
28 28
 
29 29
 var _ kadmission.Interface = &externalIPRanger{}
30 30
 
31
-// ParseCIDRRules calculates a blacklist and whitelist from a list of string CIDR rules (treating
31
+// ParseRejectAdmitCIDRRules calculates a blacklist and whitelist from a list of string CIDR rules (treating
32 32
 // a leading ! as a negation). Returns an error if any rule is invalid.
33
-func ParseCIDRRules(rules []string) (reject, admit []*net.IPNet, err error) {
33
+func ParseRejectAdmitCIDRRules(rules []string) (reject, admit []*net.IPNet, err error) {
34 34
 	for _, s := range rules {
35 35
 		negate := false
36 36
 		if strings.HasPrefix(s, "!") {
37 37
new file mode 100644
... ...
@@ -0,0 +1,127 @@
0
+// +build integration
1
+
2
+package integration
3
+
4
+import (
5
+	"testing"
6
+
7
+	kapi "k8s.io/kubernetes/pkg/api"
8
+	kclient "k8s.io/kubernetes/pkg/client/unversioned"
9
+
10
+	configapi "github.com/openshift/origin/pkg/cmd/server/api"
11
+	"github.com/openshift/origin/pkg/cmd/server/bootstrappolicy"
12
+	testutil "github.com/openshift/origin/test/util"
13
+	testserver "github.com/openshift/origin/test/util/server"
14
+)
15
+
16
+const (
17
+	clusterNetworkCIDR = "10.128.0.0/14"
18
+	serviceNetworkCIDR = "172.30.0.0/16"
19
+)
20
+
21
+var exampleAddresses = map[string]string{
22
+	"cluster":  "10.128.0.2",
23
+	"service":  "172.30.0.2",
24
+	"external": "1.2.3.4",
25
+}
26
+
27
+func testOne(t *testing.T, client *kclient.Client, namespace, addrType string, success bool) *kapi.Endpoints {
28
+	testEndpoint := &kapi.Endpoints{}
29
+	testEndpoint.GenerateName = "test"
30
+	testEndpoint.Subsets = []kapi.EndpointSubset{
31
+		{
32
+			Addresses: []kapi.EndpointAddress{
33
+				{
34
+					IP: exampleAddresses[addrType],
35
+				},
36
+			},
37
+			Ports: []kapi.EndpointPort{
38
+				{
39
+					Port:     9999,
40
+					Protocol: kapi.ProtocolTCP,
41
+				},
42
+			},
43
+		},
44
+	}
45
+
46
+	ep, err := client.Endpoints(namespace).Create(testEndpoint)
47
+	if err != nil && success {
48
+		t.Fatalf("unexpected error creating %s network endpoint: %v", addrType, err)
49
+	} else if err == nil && !success {
50
+		t.Fatalf("unexpected success creating %s network endpoint", addrType)
51
+	}
52
+	return ep
53
+}
54
+
55
+func TestEndpointAdmission(t *testing.T) {
56
+	testutil.RequireEtcd(t)
57
+	masterConfig, err := testserver.DefaultMasterOptions()
58
+	if err != nil {
59
+		t.Fatalf("error creating config: %v", err)
60
+	}
61
+	masterConfig.KubernetesMasterConfig.AdmissionConfig.PluginConfig = map[string]configapi.AdmissionPluginConfig{
62
+		"RestrictedEndpointsAdmission": {
63
+			Configuration: &configapi.DefaultAdmissionConfig{},
64
+		},
65
+	}
66
+	masterConfig.NetworkConfig.ClusterNetworkCIDR = clusterNetworkCIDR
67
+	masterConfig.NetworkConfig.ServiceNetworkCIDR = serviceNetworkCIDR
68
+
69
+	kubeConfigFile, err := testserver.StartConfiguredMaster(masterConfig)
70
+	if err != nil {
71
+		t.Fatalf("error starting server: %v", err)
72
+	}
73
+	clusterAdminKubeClient, err := testutil.GetClusterAdminKubeClient(kubeConfigFile)
74
+	if err != nil {
75
+		t.Fatalf("error getting kube client: %v", err)
76
+	}
77
+	clusterAdminOSClient, err := testutil.GetClusterAdminClient(kubeConfigFile)
78
+	if err != nil {
79
+		t.Fatalf("error getting client: %v", err)
80
+	}
81
+	clientConfig, err := testutil.GetClusterAdminClientConfig(kubeConfigFile)
82
+	if err != nil {
83
+		t.Fatalf("error getting client config: %v", err)
84
+	}
85
+
86
+	// Cluster admin
87
+	testOne(t, clusterAdminKubeClient, "default", "cluster", true)
88
+	testOne(t, clusterAdminKubeClient, "default", "service", true)
89
+	testOne(t, clusterAdminKubeClient, "default", "external", true)
90
+
91
+	// Endpoint controller service account
92
+	_, serviceAccountClient, _, err := testutil.GetClientForServiceAccount(clusterAdminKubeClient, *clientConfig, bootstrappolicy.DefaultOpenShiftInfraNamespace, bootstrappolicy.InfraEndpointControllerServiceAccountName)
93
+	if err != nil {
94
+		t.Fatalf("error getting endpoint controller service account: %v", err)
95
+	}
96
+	testOne(t, serviceAccountClient, "default", "cluster", true)
97
+	testOne(t, serviceAccountClient, "default", "service", true)
98
+	testOne(t, serviceAccountClient, "default", "external", true)
99
+
100
+	// Project admin
101
+	_, err = testserver.CreateNewProject(clusterAdminOSClient, *clientConfig, "myproject", "myadmin")
102
+	if err != nil {
103
+		t.Fatalf("error creating project: %v", err)
104
+	}
105
+	_, projectAdminClient, _, err := testutil.GetClientForUser(*clientConfig, "myadmin")
106
+	if err != nil {
107
+		t.Fatalf("error getting project admin client: %v", err)
108
+	}
109
+
110
+	testOne(t, projectAdminClient, "myproject", "cluster", false)
111
+	testOne(t, projectAdminClient, "myproject", "service", false)
112
+	testOne(t, projectAdminClient, "myproject", "external", true)
113
+
114
+	// User without restricted endpoint permission can't modify IPs but can still do other modifications
115
+	ep := testOne(t, clusterAdminKubeClient, "myproject", "cluster", true)
116
+	ep.Annotations = map[string]string{"foo": "bar"}
117
+	ep, err = projectAdminClient.Endpoints("myproject").Update(ep)
118
+	if err != nil {
119
+		t.Fatalf("unexpected error updating endpoint annotation: %v", err)
120
+	}
121
+	ep.Subsets[0].Addresses[0].IP = exampleAddresses["service"]
122
+	ep, err = projectAdminClient.Endpoints("myproject").Update(ep)
123
+	if err == nil {
124
+		t.Fatalf("unexpected success modifying endpoint")
125
+	}
126
+}
... ...
@@ -2276,6 +2276,13 @@ items:
2276 2276
     - get
2277 2277
     - list
2278 2278
     - update
2279
+  - apiGroups:
2280
+    - ""
2281
+    attributeRestrictions: null
2282
+    resources:
2283
+    - endpoints/restricted
2284
+    verbs:
2285
+    - create
2279 2286
 - apiVersion: v1
2280 2287
   kind: ClusterRole
2281 2288
   metadata: