Browse code

Transition to Kube 1.0 DNS schema

Our schema will be:

# A records for each svc (portalIP or headless)
<service>.<namespace>.svc.cluster.local
# A records for each endpoint (same as headless)
<service>.<namespace>.endpoints.cluster.local

SRV records are returned correctly for unique port/name combinations.

Clayton Coleman authored on 2015/05/13 13:14:50
Showing 7 changed files
... ...
@@ -51,8 +51,8 @@ function exectest() {
51 51
   echo "Running $1..."
52 52
 
53 53
   result=1
54
-  if [ ! -z "${VERBOSE-}" ]; then
55
-    out=$("${testexec}" -test.v=true -v ${VERBOSE-} -test.run="^$1$" "${@:2}" 2>&1)
54
+  if [ -n "${VERBOSE-}" ]; then
55
+    "${testexec}" -test.v -test.run="^$1$" "${@:2}" 2>&1
56 56
     result=$?
57 57
   else
58 58
     out=$("${testexec}" -test.run="^$1$" "${@:2}" 2>&1)
... ...
@@ -569,7 +569,6 @@ func authenticationHandlerFilter(handler http.Handler, authenticator authenticat
569 569
 			http.Error(w, "Unauthorized", http.StatusUnauthorized)
570 570
 			return
571 571
 		}
572
-		glog.V(5).Infof("user %v -> %v", user, req.URL)
573 572
 
574 573
 		ctx, ok := contextMapper.Get(req)
575 574
 		if !ok {
... ...
@@ -59,6 +59,7 @@ func BindMasterArgs(args *MasterArgs, flags *pflag.FlagSet, prefix string) {
59 59
 	flags.Var(&args.MasterPublicAddr, prefix+"public-master", "The master address for use by public clients, if different (host, host:port, or URL). Defaults to same as --master.")
60 60
 	flags.Var(&args.EtcdAddr, prefix+"etcd", "The address of the etcd server (host, host:port, or URL). If specified, no built-in etcd will be started.")
61 61
 	flags.Var(&args.PortalNet, prefix+"portal-net", "A CIDR notation IP range from which to assign portal IPs. This must not overlap with any IP ranges assigned to nodes for pods.")
62
+	flags.Var(&args.DNSBindAddr, prefix+"dns", "The address to listen for DNS requests on.")
62 63
 
63 64
 	flags.StringVar(&args.EtcdDir, prefix+"etcd-dir", "openshift.local.etcd", "The etcd data directory.")
64 65
 
... ...
@@ -74,7 +75,7 @@ func NewDefaultMasterArgs() *MasterArgs {
74 74
 		EtcdAddr:         flagtypes.Addr{Value: "0.0.0.0:4001", DefaultScheme: "https", DefaultPort: 4001}.Default(),
75 75
 		PortalNet:        flagtypes.DefaultIPNet("172.30.0.0/16"),
76 76
 		MasterPublicAddr: flagtypes.Addr{Value: "localhost:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default(),
77
-		DNSBindAddr:      flagtypes.Addr{Value: "0.0.0.0:53", DefaultScheme: "http", DefaultPort: 53, AllowPrefix: true}.Default(),
77
+		DNSBindAddr:      flagtypes.Addr{Value: "0.0.0.0:53", DefaultScheme: "tcp", DefaultPort: 53, AllowPrefix: true}.Default(),
78 78
 
79 79
 		ConfigDir: &util.StringFlag{},
80 80
 
... ...
@@ -68,7 +68,7 @@ func NewDefaultNodeArgs() *NodeArgs {
68 68
 
69 69
 		MasterCertDir: "openshift.local.config/master/certificates",
70 70
 
71
-		ClusterDomain: cmdutil.Env("OPENSHIFT_DNS_DOMAIN", "local"),
71
+		ClusterDomain: cmdutil.Env("OPENSHIFT_DNS_DOMAIN", "cluster.local"),
72 72
 		ClusterDNS:    dnsIP,
73 73
 
74 74
 		NetworkPluginName: "",
... ...
@@ -12,8 +12,8 @@ import (
12 12
 // NewServerDefaults returns the default SkyDNS server configuration for a DNS server.
13 13
 func NewServerDefaults() (*server.Config, error) {
14 14
 	config := &server.Config{
15
-		Domain: "local.",
16
-		Local:  "openshift.default.local.",
15
+		Domain: "cluster.local.",
16
+		Local:  "openshift.default.svc.cluster.local.",
17 17
 	}
18 18
 	return config, server.SetDefaults(config)
19 19
 }
... ...
@@ -39,8 +39,11 @@ func ListenAndServe(config *server.Config, client *client.Client, etcdclient *et
39 39
 }
40 40
 
41 41
 func openshiftFallback(name string, exact bool) (string, bool) {
42
-	if name == "openshift.default" {
43
-		return "kubernetes.default.", true
42
+	if name == "openshift.default.svc" {
43
+		return "kubernetes.default.svc.", true
44
+	}
45
+	if name == "openshift.default.endpoints" {
46
+		return "kubernetes.default.endpoints.", true
44 47
 	}
45 48
 	return "", false
46 49
 }
... ...
@@ -2,7 +2,6 @@ package dns
2 2
 
3 3
 import (
4 4
 	"fmt"
5
-	"log"
6 5
 	"strings"
7 6
 
8 7
 	kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api"
... ...
@@ -48,16 +47,39 @@ func NewServiceResolver(config *server.Config, accessor ServiceAccessor, endpoin
48 48
 
49 49
 // Records implements the SkyDNS Backend interface and returns standard records for
50 50
 // a name.
51
+//
52
+// The standard pattern is <prefix>.<service_name>.<namespace>.(svc|endpoints).<base>
53
+//
54
+// * prefix may be any series of prefix values
55
+// * service_name and namespace must locate a real service
56
+// * svc indicates standard service rules apply (portalIP or endpoints as A records)
57
+//   * reverse lookup of IP is only possible for portalIP
58
+//   * SRV records are returned for each host+port combination as:
59
+//     _<port_name>._<port_protocol>.<dns>
60
+//     _<port_name>.<endpoint_id>.<dns>
61
+//   * endpoint_id is "portal" when portalIP is set
62
+// * endpoints always returns each individual endpoint as A records
63
+//
51 64
 func (b *ServiceResolver) Records(name string, exact bool) ([]msg.Service, error) {
52 65
 	if !strings.HasSuffix(name, b.base) {
53 66
 		return nil, nil
54 67
 	}
55
-	log.Printf("serving records for %s %t", name, exact)
56 68
 	prefix := strings.Trim(strings.TrimSuffix(name, b.base), ".")
57 69
 	segments := strings.Split(prefix, ".")
58
-	switch c := len(segments); {
59
-	case c >= 2:
60
-		svc, err := b.accessor.Services(segments[c-1]).Get(segments[c-2])
70
+	for i, j := 0, len(segments)-1; i < j; i, j = i+1, j-1 {
71
+		segments[i], segments[j] = segments[j], segments[i]
72
+	}
73
+	if len(segments) == 0 {
74
+		return nil, nil
75
+	}
76
+
77
+	switch segments[0] {
78
+	case "svc", "endpoints":
79
+		if len(segments) < 3 {
80
+			return nil, nil
81
+		}
82
+		namespace, name := segments[1], segments[2]
83
+		svc, err := b.accessor.Services(namespace).Get(name)
61 84
 		if err != nil {
62 85
 			if errors.IsNotFound(err) && b.fallback != nil {
63 86
 				if fallback, ok := b.fallback(prefix, exact); ok {
... ...
@@ -66,47 +88,142 @@ func (b *ServiceResolver) Records(name string, exact bool) ([]msg.Service, error
66 66
 			}
67 67
 			return nil, err
68 68
 		}
69
-		if svc.Spec.PortalIP == kapi.PortalIPNone {
70
-			endpoints, err := b.endpoints.Endpoints(segments[c-1]).Get(segments[c-2])
71
-			if err != nil {
72
-				return nil, err
69
+
70
+		// no portalIP and not headless, no DNS
71
+		if len(svc.Spec.PortalIP) == 0 {
72
+			return nil, nil
73
+		}
74
+
75
+		// if has a portal IP and looking at svc
76
+		if svc.Spec.PortalIP != kapi.PortalIPNone && segments[0] == "svc" {
77
+			if len(svc.Spec.Ports) == 0 {
78
+				return nil, nil
73 79
 			}
74
-			services := make([]msg.Service, 0, len(endpoints.Subsets)*4)
75
-			for _, s := range endpoints.Subsets {
76
-				for _, a := range s.Addresses {
77
-					for _, p := range s.Ports {
78
-						services = append(services, msg.Service{
79
-							Host: a.IP,
80
-							Port: p.Port,
81
-
82
-							Priority: 10,
83
-							Weight:   10,
84
-							Ttl:      30,
85
-
86
-							Text: "",
87
-							Key:  msg.Path(name),
88
-						})
89
-					}
80
+			services := []msg.Service{}
81
+			for _, p := range svc.Spec.Ports {
82
+				port := p.Port
83
+				if port == 0 {
84
+					port = p.TargetPort.IntVal
85
+				}
86
+				if port == 0 {
87
+					continue
88
+				}
89
+				if len(p.Protocol) == 0 {
90
+					p.Protocol = kapi.ProtocolTCP
91
+				}
92
+				portName := p.Name
93
+				if len(portName) == 0 {
94
+					portName = fmt.Sprintf("unknown-port-%d", port)
90 95
 				}
96
+				srvName := fmt.Sprintf("%s.portal.%s", portName, name)
97
+				keyName := fmt.Sprintf("_%s._%s.%s", portName, p.Protocol, name)
98
+				services = append(services,
99
+					msg.Service{
100
+						Host: svc.Spec.PortalIP,
101
+						Port: port,
102
+
103
+						Priority: 10,
104
+						Weight:   10,
105
+						Ttl:      30,
106
+
107
+						Text: "",
108
+						Key:  msg.Path(srvName),
109
+					},
110
+					msg.Service{
111
+						Host: srvName,
112
+						Port: port,
113
+
114
+						Priority: 10,
115
+						Weight:   10,
116
+						Ttl:      30,
117
+
118
+						Text: "",
119
+						Key:  msg.Path(keyName),
120
+					},
121
+				)
91 122
 			}
92 123
 			return services, nil
93 124
 		}
94
-		if len(svc.Spec.PortalIP) == 0 || len(svc.Spec.Ports) == 0 {
95
-			return nil, nil
125
+
126
+		// return endpoints
127
+		endpoints, err := b.endpoints.Endpoints(namespace).Get(name)
128
+		if err != nil {
129
+			return nil, err
130
+		}
131
+		targets := make(map[string]int)
132
+		services := make([]msg.Service, 0, len(endpoints.Subsets)*4)
133
+		count := 1
134
+		for _, s := range endpoints.Subsets {
135
+			for _, a := range s.Addresses {
136
+				shortName := ""
137
+				if a.TargetRef != nil {
138
+					name := fmt.Sprintf("%s-%s", a.TargetRef.Name, a.TargetRef.Namespace)
139
+					if c, ok := targets[name]; ok {
140
+						shortName = fmt.Sprintf("e%d", c)
141
+					} else {
142
+						shortName = fmt.Sprintf("e%d", count)
143
+						targets[name] = count
144
+						count++
145
+					}
146
+				} else {
147
+					shortName = fmt.Sprintf("e%d", count)
148
+					count++
149
+				}
150
+				hadPort := false
151
+				for _, p := range s.Ports {
152
+					port := p.Port
153
+					if port == 0 {
154
+						continue
155
+					}
156
+					hadPort = true
157
+					if len(p.Protocol) == 0 {
158
+						p.Protocol = kapi.ProtocolTCP
159
+					}
160
+					portName := p.Name
161
+					if len(portName) == 0 {
162
+						portName = fmt.Sprintf("unknown-port-%d", port)
163
+					}
164
+					srvName := fmt.Sprintf("%s.%s.%s", portName, shortName, name)
165
+					services = append(services, msg.Service{
166
+						Host: a.IP,
167
+						Port: port,
168
+
169
+						Priority: 10,
170
+						Weight:   10,
171
+						Ttl:      30,
172
+
173
+						Text: "",
174
+						Key:  msg.Path(srvName),
175
+					})
176
+					keyName := fmt.Sprintf("_%s._%s.%s", portName, p.Protocol, name)
177
+					services = append(services, msg.Service{
178
+						Host: srvName,
179
+						Port: port,
180
+
181
+						Priority: 10,
182
+						Weight:   10,
183
+						Ttl:      30,
184
+
185
+						Text: "",
186
+						Key:  msg.Path(keyName),
187
+					})
188
+				}
189
+
190
+				if !hadPort {
191
+					services = append(services, msg.Service{
192
+						Host: a.IP,
193
+
194
+						Priority: 10,
195
+						Weight:   10,
196
+						Ttl:      30,
197
+
198
+						Text: "",
199
+						Key:  msg.Path(name),
200
+					})
201
+				}
202
+			}
96 203
 		}
97
-		return []msg.Service{
98
-			{
99
-				Host: svc.Spec.PortalIP,
100
-				Port: svc.Spec.Ports[0].Port,
101
-
102
-				Priority: 10,
103
-				Weight:   10,
104
-				Ttl:      30,
105
-
106
-				Text: "",
107
-				Key:  msg.Path(name),
108
-			},
109
-		}, nil
204
+		return services, nil
110 205
 	}
111 206
 	return nil, nil
112 207
 }
... ...
@@ -128,7 +245,7 @@ func (b *ServiceResolver) ReverseRecord(name string) (*msg.Service, error) {
128 128
 		port = svc.Spec.Ports[0].Port
129 129
 	}
130 130
 	return &msg.Service{
131
-		Host: fmt.Sprintf("%s.%s.%s", svc.Name, svc.Namespace, b.base),
131
+		Host: fmt.Sprintf("%s.%s.svc.%s", svc.Name, svc.Namespace, b.base),
132 132
 		Port: port,
133 133
 
134 134
 		Priority: 10,
... ...
@@ -16,19 +16,19 @@ import (
16 16
 )
17 17
 
18 18
 func TestDNS(t *testing.T) {
19
-	masterConfig, clientFile, err := testutil.StartTestAllInOne()
19
+	masterConfig, clientFile, err := testutil.StartTestMaster()
20 20
 	if err != nil {
21 21
 		t.Fatalf("unexpected error: %v", err)
22 22
 	}
23 23
 
24
+	localIP := net.ParseIP("127.0.0.1")
24 25
 	var masterIP net.IP
25
-
26 26
 	// verify service DNS entry is visible
27 27
 	stop := make(chan struct{})
28 28
 	util.Until(func() {
29 29
 		m1 := &dns.Msg{
30 30
 			MsgHdr:   dns.MsgHdr{Id: dns.Id(), RecursionDesired: true},
31
-			Question: []dns.Question{{"kubernetes.default.local.", dns.TypeA, dns.ClassINET}},
31
+			Question: []dns.Question{{"kubernetes.default.svc.cluster.local.", dns.TypeA, dns.ClassINET}},
32 32
 		}
33 33
 		in, err := dns.Exchange(m1, masterConfig.DNSConfig.BindAddress)
34 34
 		if err != nil {
... ...
@@ -78,7 +78,9 @@ func TestDNS(t *testing.T) {
78 78
 			},
79 79
 			Subsets: []kapi.EndpointSubset{{
80 80
 				Addresses: []kapi.EndpointAddress{{IP: "172.0.0.1"}},
81
-				Ports:     []kapi.EndpointPort{{Port: 2345}},
81
+				Ports: []kapi.EndpointPort{
82
+					{Port: 2345},
83
+				},
82 84
 			}},
83 85
 		}); err != nil {
84 86
 			t.Fatalf("unexpected error: %v", err)
... ...
@@ -87,28 +89,94 @@ func TestDNS(t *testing.T) {
87 87
 	}
88 88
 	headlessIP := net.ParseIP("172.0.0.1")
89 89
 
90
+	if _, err := client.Services(kapi.NamespaceDefault).Create(&kapi.Service{
91
+		ObjectMeta: kapi.ObjectMeta{
92
+			Name: "headless2",
93
+		},
94
+		Spec: kapi.ServiceSpec{
95
+			PortalIP: kapi.PortalIPNone,
96
+			Ports:    []kapi.ServicePort{{Port: 443}},
97
+		},
98
+	}); err != nil {
99
+		t.Fatalf("unexpected error: %v", err)
100
+	}
101
+	if _, err := client.Endpoints(kapi.NamespaceDefault).Create(&kapi.Endpoints{
102
+		ObjectMeta: kapi.ObjectMeta{
103
+			Name: "headless2",
104
+		},
105
+		Subsets: []kapi.EndpointSubset{{
106
+			Addresses: []kapi.EndpointAddress{{IP: "172.0.0.2"}},
107
+			Ports: []kapi.EndpointPort{
108
+				{Port: 2345, Name: "other"},
109
+				{Port: 2346, Name: "http"},
110
+			},
111
+		}},
112
+	}); err != nil {
113
+		t.Fatalf("unexpected error: %v", err)
114
+	}
115
+	headless2IP := net.ParseIP("172.0.0.2")
116
+
90 117
 	// verify recursive DNS lookup is visible when expected
91 118
 	tests := []struct {
92 119
 		dnsQuestionName   string
93 120
 		recursionExpected bool
94 121
 		retry             bool
95
-		expect            *net.IP
122
+		expect            []*net.IP
123
+		srv               []*dns.SRV
96 124
 	}{
97
-		{
98
-			dnsQuestionName:   "foo.kubernetes.default.local.",
99
-			recursionExpected: false,
100
-			expect:            &masterIP,
125
+		{ // wildcard resolution of a service works
126
+			dnsQuestionName: "foo.kubernetes.default.svc.cluster.local.",
127
+			expect:          []*net.IP{&masterIP},
101 128
 		},
102
-		{
103
-			dnsQuestionName:   "openshift.default.local.",
104
-			recursionExpected: false,
105
-			expect:            &masterIP,
129
+		{ // resolving endpoints of a service works
130
+			dnsQuestionName: "kubernetes.default.endpoints.cluster.local.",
131
+			expect:          []*net.IP{&localIP},
106 132
 		},
107
-		{
108
-			dnsQuestionName:   "headless.default.local.",
109
-			recursionExpected: false,
110
-			retry:             true,
111
-			expect:            &headlessIP,
133
+		{ // openshift override works
134
+			dnsQuestionName: "openshift.default.svc.cluster.local.",
135
+			expect:          []*net.IP{&masterIP},
136
+		},
137
+		{ // headless service
138
+			dnsQuestionName: "headless.default.svc.cluster.local.",
139
+			expect:          []*net.IP{&headlessIP},
140
+		},
141
+		{ // specific port of a headless service
142
+			dnsQuestionName: "unknown-port-2345.e1.headless.default.svc.cluster.local.",
143
+			expect:          []*net.IP{&headlessIP},
144
+		},
145
+		{ // SRV record for that service
146
+			dnsQuestionName: "headless.default.svc.cluster.local.",
147
+			srv: []*dns.SRV{
148
+				&dns.SRV{
149
+					Target: "unknown-port-2345.e1.headless.",
150
+					Port:   2345,
151
+				},
152
+			},
153
+		},
154
+		{ // the SRV record resolves to the IP
155
+			dnsQuestionName: "unknown-port-2345.e1.headless.default.svc.cluster.local.",
156
+			expect:          []*net.IP{&headlessIP},
157
+		},
158
+		{ // headless 2 service
159
+			dnsQuestionName: "headless2.default.svc.cluster.local.",
160
+			expect:          []*net.IP{&headless2IP},
161
+		},
162
+		{ // SRV records for that service
163
+			dnsQuestionName: "headless2.default.svc.cluster.local.",
164
+			srv: []*dns.SRV{
165
+				&dns.SRV{
166
+					Target: "http.e1.headless2.",
167
+					Port:   2346,
168
+				},
169
+				&dns.SRV{
170
+					Target: "other.e1.headless2.",
171
+					Port:   2345,
172
+				},
173
+			},
174
+		},
175
+		{ // the SRV record resolves to the IP
176
+			dnsQuestionName: "other.e1.headless2.default.svc.cluster.local.",
177
+			expect:          []*net.IP{&headless2IP},
112 178
 		},
113 179
 		{
114 180
 			dnsQuestionName:   "www.google.com.",
... ...
@@ -116,40 +184,77 @@ func TestDNS(t *testing.T) {
116 116
 		},
117 117
 	}
118 118
 	for i, tc := range tests {
119
-		stop := make(chan struct{})
119
+		qType := dns.TypeA
120
+		if tc.srv != nil {
121
+			qType = dns.TypeSRV
122
+		}
123
+		m1 := &dns.Msg{
124
+			MsgHdr:   dns.MsgHdr{Id: dns.Id(), RecursionDesired: true},
125
+			Question: []dns.Question{{tc.dnsQuestionName, qType, dns.ClassINET}},
126
+		}
127
+		ch := make(chan struct{})
128
+		count := 0
120 129
 		util.Until(func() {
121
-			m1 := &dns.Msg{
122
-				MsgHdr:   dns.MsgHdr{Id: dns.Id(), RecursionDesired: true},
123
-				Question: []dns.Question{{tc.dnsQuestionName, dns.TypeA, dns.ClassINET}},
130
+			count++
131
+			if count > 100 {
132
+				t.Errorf("%d: failed after max iterations", i)
133
+				close(ch)
134
+				return
124 135
 			}
125 136
 			in, err := dns.Exchange(m1, masterConfig.DNSConfig.BindAddress)
126 137
 			if err != nil {
127
-				t.Logf("%d: unexpected error: %v", i, err)
128 138
 				return
129 139
 			}
130
-			if !tc.recursionExpected && len(in.Answer) != 1 {
131
-				if !tc.retry {
132
-					close(stop)
133
-					t.Errorf("%d: did not resolve or unexpected forward resolution: %#v", i, in)
140
+			switch {
141
+			case tc.srv != nil:
142
+				if len(in.Answer) != len(tc.srv) {
143
+					t.Logf("%d: incorrect number of answers: %#v", i, in)
144
+					return
134 145
 				}
146
+			case tc.recursionExpected:
147
+				if len(in.Answer) == 0 {
148
+					t.Errorf("%d: expected forward resolution: %#v", i, in)
149
+				}
150
+				close(ch)
135 151
 				return
136
-			} else if tc.recursionExpected && len(in.Answer) == 0 {
137
-				t.Errorf("%d: expected forward resolution: %#v", i, in)
138
-				return
152
+			default:
153
+				if len(in.Answer) != len(tc.expect) {
154
+					t.Logf("%d: did not resolve or unexpected forward resolution: %#v", i, in)
155
+					return
156
+				}
139 157
 			}
140
-			if a, ok := in.Answer[0].(*dns.A); ok {
141
-				if a.A == nil {
142
-					t.Errorf("expected an A record with an IP: %#v", a)
143
-				} else {
144
-					if tc.expect != nil && tc.expect.String() != a.A.String() {
145
-						t.Errorf("A record has a different IP than the test case: %v / %v", a.A, *tc.expect)
158
+			for _, answer := range in.Answer {
159
+				switch a := answer.(type) {
160
+				case *dns.A:
161
+					matches := false
162
+					if a.A != nil {
163
+						for _, expect := range tc.expect {
164
+							if a.A.String() == expect.String() {
165
+								matches = true
166
+								break
167
+							}
168
+						}
169
+					}
170
+					if !matches {
171
+						t.Errorf("A record does not match any expected answer: %v", a.A)
172
+					}
173
+				case *dns.SRV:
174
+					matches := false
175
+					for _, expect := range tc.srv {
176
+						if expect.Port == a.Port && expect.Target == a.Target {
177
+							matches = true
178
+							break
179
+						}
180
+					}
181
+					if !matches {
182
+						t.Errorf("SRV record does not match any expected answer: %#v", a)
146 183
 					}
184
+				default:
185
+					t.Errorf("expected an A or SRV record: %#v", in)
147 186
 				}
148
-			} else {
149
-				t.Errorf("expected an A record: %#v", in)
150 187
 			}
151 188
 			t.Log(in)
152
-			close(stop)
153
-		}, 50*time.Millisecond, stop)
189
+			close(ch)
190
+		}, 50*time.Millisecond, ch)
154 191
 	}
155 192
 }