Browse code

Add ingress load balancer

Ingress load balancer is achieved via a service sandbox which acts as
the proxy to translate incoming node port requests and mapping that to a
service entry. Once the right service is identified, the same internal
loadbalancer implementation is used to load balance to the right backend
instance.

Signed-off-by: Jana Radhakrishnan <mrjana@docker.com>

Jana Radhakrishnan authored on 2016/05/31 15:55:51
Showing 11 changed files
... ...
@@ -167,18 +167,25 @@ func (ep *endpoint) addToCluster() error {
167 167
 
168 168
 	c := n.getController()
169 169
 	if !ep.isAnonymous() && ep.Iface().Address() != nil {
170
+		var ingressPorts []*PortConfig
170 171
 		if ep.svcID != "" {
171
-			if err := c.addServiceBinding(ep.svcName, ep.svcID, n.ID(), ep.ID(), ep.virtualIP, ep.Iface().Address().IP); err != nil {
172
+			// Gossip ingress ports only in ingress network.
173
+			if n.ingress {
174
+				ingressPorts = ep.ingressPorts
175
+			}
176
+
177
+			if err := c.addServiceBinding(ep.svcName, ep.svcID, n.ID(), ep.ID(), ep.virtualIP, ingressPorts, ep.Iface().Address().IP); err != nil {
172 178
 				return err
173 179
 			}
174 180
 		}
175 181
 
176 182
 		buf, err := proto.Marshal(&EndpointRecord{
177
-			Name:        ep.Name(),
178
-			ServiceName: ep.svcName,
179
-			ServiceID:   ep.svcID,
180
-			VirtualIP:   ep.virtualIP.String(),
181
-			EndpointIP:  ep.Iface().Address().IP.String(),
183
+			Name:         ep.Name(),
184
+			ServiceName:  ep.svcName,
185
+			ServiceID:    ep.svcID,
186
+			VirtualIP:    ep.virtualIP.String(),
187
+			IngressPorts: ingressPorts,
188
+			EndpointIP:   ep.Iface().Address().IP.String(),
182 189
 		})
183 190
 
184 191
 		if err != nil {
... ...
@@ -208,7 +215,12 @@ func (ep *endpoint) deleteFromCluster() error {
208 208
 	c := n.getController()
209 209
 	if !ep.isAnonymous() {
210 210
 		if ep.svcID != "" && ep.Iface().Address() != nil {
211
-			if err := c.rmServiceBinding(ep.svcName, ep.svcID, n.ID(), ep.ID(), ep.virtualIP, ep.Iface().Address().IP); err != nil {
211
+			var ingressPorts []*PortConfig
212
+			if n.ingress {
213
+				ingressPorts = ep.ingressPorts
214
+			}
215
+
216
+			if err := c.rmServiceBinding(ep.svcName, ep.svcID, n.ID(), ep.ID(), ep.virtualIP, ingressPorts, ep.Iface().Address().IP); err != nil {
212 217
 				return err
213 218
 			}
214 219
 		}
... ...
@@ -362,6 +374,7 @@ func (c *controller) handleEpTableEvent(ev events.Event) {
362 362
 	svcID := epRec.ServiceID
363 363
 	vip := net.ParseIP(epRec.VirtualIP)
364 364
 	ip := net.ParseIP(epRec.EndpointIP)
365
+	ingressPorts := epRec.IngressPorts
365 366
 
366 367
 	if name == "" || ip == nil {
367 368
 		logrus.Errorf("Invalid endpoint name/ip received while handling service table event %s", value)
... ...
@@ -370,7 +383,7 @@ func (c *controller) handleEpTableEvent(ev events.Event) {
370 370
 
371 371
 	if isAdd {
372 372
 		if svcID != "" {
373
-			if err := c.addServiceBinding(svcName, svcID, nid, eid, vip, ip); err != nil {
373
+			if err := c.addServiceBinding(svcName, svcID, nid, eid, vip, ingressPorts, ip); err != nil {
374 374
 				logrus.Errorf("Failed adding service binding for value %s: %v", value, err)
375 375
 				return
376 376
 			}
... ...
@@ -379,7 +392,7 @@ func (c *controller) handleEpTableEvent(ev events.Event) {
379 379
 		n.addSvcRecords(name, ip, nil, true)
380 380
 	} else {
381 381
 		if svcID != "" {
382
-			if err := c.rmServiceBinding(svcName, svcID, nid, eid, vip, ip); err != nil {
382
+			if err := c.rmServiceBinding(svcName, svcID, nid, eid, vip, ingressPorts, ip); err != nil {
383 383
 				logrus.Errorf("Failed adding service binding for value %s: %v", value, err)
384 384
 				return
385 385
 			}
... ...
@@ -10,6 +10,7 @@
10 10
 
11 11
 	It has these top-level messages:
12 12
 		EndpointRecord
13
+		PortConfig
13 14
 */
14 15
 package libnetwork
15 16
 
... ...
@@ -35,32 +36,113 @@ var _ = math.Inf
35 35
 // is compatible with the proto package it is being compiled against.
36 36
 const _ = proto.GoGoProtoPackageIsVersion1
37 37
 
38
+type PortConfig_Protocol int32
39
+
40
+const (
41
+	ProtocolTCP PortConfig_Protocol = 0
42
+	ProtocolUDP PortConfig_Protocol = 1
43
+)
44
+
45
+var PortConfig_Protocol_name = map[int32]string{
46
+	0: "TCP",
47
+	1: "UDP",
48
+}
49
+var PortConfig_Protocol_value = map[string]int32{
50
+	"TCP": 0,
51
+	"UDP": 1,
52
+}
53
+
54
+func (x PortConfig_Protocol) String() string {
55
+	return proto.EnumName(PortConfig_Protocol_name, int32(x))
56
+}
57
+func (PortConfig_Protocol) EnumDescriptor() ([]byte, []int) { return fileDescriptorAgent, []int{1, 0} }
58
+
59
+// EndpointRecord specifies all the endpoint specific information that
60
+// needs to gossiped to nodes participating in the network.
38 61
 type EndpointRecord struct {
39
-	Name        string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
62
+	// Name of the endpoint
63
+	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
64
+	// Service name of the service to which this endpoint belongs.
40 65
 	ServiceName string `protobuf:"bytes,2,opt,name=service_name,json=serviceName,proto3" json:"service_name,omitempty"`
41
-	ServiceID   string `protobuf:"bytes,3,opt,name=service_id,json=serviceId,proto3" json:"service_id,omitempty"`
42
-	VirtualIP   string `protobuf:"bytes,4,opt,name=virtual_ip,json=virtualIp,proto3" json:"virtual_ip,omitempty"`
43
-	EndpointIP  string `protobuf:"bytes,5,opt,name=endpoint_ip,json=endpointIp,proto3" json:"endpoint_ip,omitempty"`
66
+	// Service ID of the service to which this endpoint belongs.
67
+	ServiceID string `protobuf:"bytes,3,opt,name=service_id,json=serviceId,proto3" json:"service_id,omitempty"`
68
+	// Virtual IP of the service to which this endpoint belongs.
69
+	VirtualIP string `protobuf:"bytes,4,opt,name=virtual_ip,json=virtualIp,proto3" json:"virtual_ip,omitempty"`
70
+	// IP assigned to this endpoint.
71
+	EndpointIP string `protobuf:"bytes,5,opt,name=endpoint_ip,json=endpointIp,proto3" json:"endpoint_ip,omitempty"`
72
+	// IngressPorts exposed by the service to which this endpoint belongs.
73
+	IngressPorts []*PortConfig `protobuf:"bytes,6,rep,name=ingress_ports,json=ingressPorts" json:"ingress_ports,omitempty"`
44 74
 }
45 75
 
46 76
 func (m *EndpointRecord) Reset()                    { *m = EndpointRecord{} }
47 77
 func (*EndpointRecord) ProtoMessage()               {}
48 78
 func (*EndpointRecord) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{0} }
49 79
 
80
+func (m *EndpointRecord) GetIngressPorts() []*PortConfig {
81
+	if m != nil {
82
+		return m.IngressPorts
83
+	}
84
+	return nil
85
+}
86
+
87
+// PortConfig specifies an exposed port which can be
88
+// addressed using the given name. This can be later queried
89
+// using a service discovery api or a DNS SRV query. The node
90
+// port specifies a port that can be used to address this
91
+// service external to the cluster by sending a connection
92
+// request to this port to any node on the cluster.
93
+type PortConfig struct {
94
+	// Name for the port. If provided the port information can
95
+	// be queried using the name as in a DNS SRV query.
96
+	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
97
+	// Protocol for the port which is exposed.
98
+	Protocol PortConfig_Protocol `protobuf:"varint,2,opt,name=protocol,proto3,enum=libnetwork.PortConfig_Protocol" json:"protocol,omitempty"`
99
+	// The port which the application is exposing and is bound to.
100
+	Port uint32 `protobuf:"varint,3,opt,name=port,proto3" json:"port,omitempty"`
101
+	// NodePort specifies the port on which the service is
102
+	// exposed on all nodes on the cluster. If not specified an
103
+	// arbitrary port in the node port range is allocated by the
104
+	// system. If specified it should be within the node port
105
+	// range and it should be available.
106
+	NodePort uint32 `protobuf:"varint,4,opt,name=node_port,json=nodePort,proto3" json:"node_port,omitempty"`
107
+}
108
+
109
+func (m *PortConfig) Reset()                    { *m = PortConfig{} }
110
+func (*PortConfig) ProtoMessage()               {}
111
+func (*PortConfig) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{1} }
112
+
50 113
 func init() {
51 114
 	proto.RegisterType((*EndpointRecord)(nil), "libnetwork.EndpointRecord")
115
+	proto.RegisterType((*PortConfig)(nil), "libnetwork.PortConfig")
116
+	proto.RegisterEnum("libnetwork.PortConfig_Protocol", PortConfig_Protocol_name, PortConfig_Protocol_value)
52 117
 }
53 118
 func (this *EndpointRecord) GoString() string {
54 119
 	if this == nil {
55 120
 		return "nil"
56 121
 	}
57
-	s := make([]string, 0, 9)
122
+	s := make([]string, 0, 10)
58 123
 	s = append(s, "&libnetwork.EndpointRecord{")
59 124
 	s = append(s, "Name: "+fmt.Sprintf("%#v", this.Name)+",\n")
60 125
 	s = append(s, "ServiceName: "+fmt.Sprintf("%#v", this.ServiceName)+",\n")
61 126
 	s = append(s, "ServiceID: "+fmt.Sprintf("%#v", this.ServiceID)+",\n")
62 127
 	s = append(s, "VirtualIP: "+fmt.Sprintf("%#v", this.VirtualIP)+",\n")
63 128
 	s = append(s, "EndpointIP: "+fmt.Sprintf("%#v", this.EndpointIP)+",\n")
129
+	if this.IngressPorts != nil {
130
+		s = append(s, "IngressPorts: "+fmt.Sprintf("%#v", this.IngressPorts)+",\n")
131
+	}
132
+	s = append(s, "}")
133
+	return strings.Join(s, "")
134
+}
135
+func (this *PortConfig) GoString() string {
136
+	if this == nil {
137
+		return "nil"
138
+	}
139
+	s := make([]string, 0, 8)
140
+	s = append(s, "&libnetwork.PortConfig{")
141
+	s = append(s, "Name: "+fmt.Sprintf("%#v", this.Name)+",\n")
142
+	s = append(s, "Protocol: "+fmt.Sprintf("%#v", this.Protocol)+",\n")
143
+	s = append(s, "Port: "+fmt.Sprintf("%#v", this.Port)+",\n")
144
+	s = append(s, "NodePort: "+fmt.Sprintf("%#v", this.NodePort)+",\n")
64 145
 	s = append(s, "}")
65 146
 	return strings.Join(s, "")
66 147
 }
... ...
@@ -134,6 +216,57 @@ func (m *EndpointRecord) MarshalTo(data []byte) (int, error) {
134 134
 		i = encodeVarintAgent(data, i, uint64(len(m.EndpointIP)))
135 135
 		i += copy(data[i:], m.EndpointIP)
136 136
 	}
137
+	if len(m.IngressPorts) > 0 {
138
+		for _, msg := range m.IngressPorts {
139
+			data[i] = 0x32
140
+			i++
141
+			i = encodeVarintAgent(data, i, uint64(msg.Size()))
142
+			n, err := msg.MarshalTo(data[i:])
143
+			if err != nil {
144
+				return 0, err
145
+			}
146
+			i += n
147
+		}
148
+	}
149
+	return i, nil
150
+}
151
+
152
+func (m *PortConfig) Marshal() (data []byte, err error) {
153
+	size := m.Size()
154
+	data = make([]byte, size)
155
+	n, err := m.MarshalTo(data)
156
+	if err != nil {
157
+		return nil, err
158
+	}
159
+	return data[:n], nil
160
+}
161
+
162
+func (m *PortConfig) MarshalTo(data []byte) (int, error) {
163
+	var i int
164
+	_ = i
165
+	var l int
166
+	_ = l
167
+	if len(m.Name) > 0 {
168
+		data[i] = 0xa
169
+		i++
170
+		i = encodeVarintAgent(data, i, uint64(len(m.Name)))
171
+		i += copy(data[i:], m.Name)
172
+	}
173
+	if m.Protocol != 0 {
174
+		data[i] = 0x10
175
+		i++
176
+		i = encodeVarintAgent(data, i, uint64(m.Protocol))
177
+	}
178
+	if m.Port != 0 {
179
+		data[i] = 0x18
180
+		i++
181
+		i = encodeVarintAgent(data, i, uint64(m.Port))
182
+	}
183
+	if m.NodePort != 0 {
184
+		data[i] = 0x20
185
+		i++
186
+		i = encodeVarintAgent(data, i, uint64(m.NodePort))
187
+	}
137 188
 	return i, nil
138 189
 }
139 190
 
... ...
@@ -187,6 +320,31 @@ func (m *EndpointRecord) Size() (n int) {
187 187
 	if l > 0 {
188 188
 		n += 1 + l + sovAgent(uint64(l))
189 189
 	}
190
+	if len(m.IngressPorts) > 0 {
191
+		for _, e := range m.IngressPorts {
192
+			l = e.Size()
193
+			n += 1 + l + sovAgent(uint64(l))
194
+		}
195
+	}
196
+	return n
197
+}
198
+
199
+func (m *PortConfig) Size() (n int) {
200
+	var l int
201
+	_ = l
202
+	l = len(m.Name)
203
+	if l > 0 {
204
+		n += 1 + l + sovAgent(uint64(l))
205
+	}
206
+	if m.Protocol != 0 {
207
+		n += 1 + sovAgent(uint64(m.Protocol))
208
+	}
209
+	if m.Port != 0 {
210
+		n += 1 + sovAgent(uint64(m.Port))
211
+	}
212
+	if m.NodePort != 0 {
213
+		n += 1 + sovAgent(uint64(m.NodePort))
214
+	}
190 215
 	return n
191 216
 }
192 217
 
... ...
@@ -213,6 +371,20 @@ func (this *EndpointRecord) String() string {
213 213
 		`ServiceID:` + fmt.Sprintf("%v", this.ServiceID) + `,`,
214 214
 		`VirtualIP:` + fmt.Sprintf("%v", this.VirtualIP) + `,`,
215 215
 		`EndpointIP:` + fmt.Sprintf("%v", this.EndpointIP) + `,`,
216
+		`IngressPorts:` + strings.Replace(fmt.Sprintf("%v", this.IngressPorts), "PortConfig", "PortConfig", 1) + `,`,
217
+		`}`,
218
+	}, "")
219
+	return s
220
+}
221
+func (this *PortConfig) String() string {
222
+	if this == nil {
223
+		return "nil"
224
+	}
225
+	s := strings.Join([]string{`&PortConfig{`,
226
+		`Name:` + fmt.Sprintf("%v", this.Name) + `,`,
227
+		`Protocol:` + fmt.Sprintf("%v", this.Protocol) + `,`,
228
+		`Port:` + fmt.Sprintf("%v", this.Port) + `,`,
229
+		`NodePort:` + fmt.Sprintf("%v", this.NodePort) + `,`,
216 230
 		`}`,
217 231
 	}, "")
218 232
 	return s
... ...
@@ -399,6 +571,173 @@ func (m *EndpointRecord) Unmarshal(data []byte) error {
399 399
 			}
400 400
 			m.EndpointIP = string(data[iNdEx:postIndex])
401 401
 			iNdEx = postIndex
402
+		case 6:
403
+			if wireType != 2 {
404
+				return fmt.Errorf("proto: wrong wireType = %d for field IngressPorts", wireType)
405
+			}
406
+			var msglen int
407
+			for shift := uint(0); ; shift += 7 {
408
+				if shift >= 64 {
409
+					return ErrIntOverflowAgent
410
+				}
411
+				if iNdEx >= l {
412
+					return io.ErrUnexpectedEOF
413
+				}
414
+				b := data[iNdEx]
415
+				iNdEx++
416
+				msglen |= (int(b) & 0x7F) << shift
417
+				if b < 0x80 {
418
+					break
419
+				}
420
+			}
421
+			if msglen < 0 {
422
+				return ErrInvalidLengthAgent
423
+			}
424
+			postIndex := iNdEx + msglen
425
+			if postIndex > l {
426
+				return io.ErrUnexpectedEOF
427
+			}
428
+			m.IngressPorts = append(m.IngressPorts, &PortConfig{})
429
+			if err := m.IngressPorts[len(m.IngressPorts)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
430
+				return err
431
+			}
432
+			iNdEx = postIndex
433
+		default:
434
+			iNdEx = preIndex
435
+			skippy, err := skipAgent(data[iNdEx:])
436
+			if err != nil {
437
+				return err
438
+			}
439
+			if skippy < 0 {
440
+				return ErrInvalidLengthAgent
441
+			}
442
+			if (iNdEx + skippy) > l {
443
+				return io.ErrUnexpectedEOF
444
+			}
445
+			iNdEx += skippy
446
+		}
447
+	}
448
+
449
+	if iNdEx > l {
450
+		return io.ErrUnexpectedEOF
451
+	}
452
+	return nil
453
+}
454
+func (m *PortConfig) Unmarshal(data []byte) error {
455
+	l := len(data)
456
+	iNdEx := 0
457
+	for iNdEx < l {
458
+		preIndex := iNdEx
459
+		var wire uint64
460
+		for shift := uint(0); ; shift += 7 {
461
+			if shift >= 64 {
462
+				return ErrIntOverflowAgent
463
+			}
464
+			if iNdEx >= l {
465
+				return io.ErrUnexpectedEOF
466
+			}
467
+			b := data[iNdEx]
468
+			iNdEx++
469
+			wire |= (uint64(b) & 0x7F) << shift
470
+			if b < 0x80 {
471
+				break
472
+			}
473
+		}
474
+		fieldNum := int32(wire >> 3)
475
+		wireType := int(wire & 0x7)
476
+		if wireType == 4 {
477
+			return fmt.Errorf("proto: PortConfig: wiretype end group for non-group")
478
+		}
479
+		if fieldNum <= 0 {
480
+			return fmt.Errorf("proto: PortConfig: illegal tag %d (wire type %d)", fieldNum, wire)
481
+		}
482
+		switch fieldNum {
483
+		case 1:
484
+			if wireType != 2 {
485
+				return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType)
486
+			}
487
+			var stringLen uint64
488
+			for shift := uint(0); ; shift += 7 {
489
+				if shift >= 64 {
490
+					return ErrIntOverflowAgent
491
+				}
492
+				if iNdEx >= l {
493
+					return io.ErrUnexpectedEOF
494
+				}
495
+				b := data[iNdEx]
496
+				iNdEx++
497
+				stringLen |= (uint64(b) & 0x7F) << shift
498
+				if b < 0x80 {
499
+					break
500
+				}
501
+			}
502
+			intStringLen := int(stringLen)
503
+			if intStringLen < 0 {
504
+				return ErrInvalidLengthAgent
505
+			}
506
+			postIndex := iNdEx + intStringLen
507
+			if postIndex > l {
508
+				return io.ErrUnexpectedEOF
509
+			}
510
+			m.Name = string(data[iNdEx:postIndex])
511
+			iNdEx = postIndex
512
+		case 2:
513
+			if wireType != 0 {
514
+				return fmt.Errorf("proto: wrong wireType = %d for field Protocol", wireType)
515
+			}
516
+			m.Protocol = 0
517
+			for shift := uint(0); ; shift += 7 {
518
+				if shift >= 64 {
519
+					return ErrIntOverflowAgent
520
+				}
521
+				if iNdEx >= l {
522
+					return io.ErrUnexpectedEOF
523
+				}
524
+				b := data[iNdEx]
525
+				iNdEx++
526
+				m.Protocol |= (PortConfig_Protocol(b) & 0x7F) << shift
527
+				if b < 0x80 {
528
+					break
529
+				}
530
+			}
531
+		case 3:
532
+			if wireType != 0 {
533
+				return fmt.Errorf("proto: wrong wireType = %d for field Port", wireType)
534
+			}
535
+			m.Port = 0
536
+			for shift := uint(0); ; shift += 7 {
537
+				if shift >= 64 {
538
+					return ErrIntOverflowAgent
539
+				}
540
+				if iNdEx >= l {
541
+					return io.ErrUnexpectedEOF
542
+				}
543
+				b := data[iNdEx]
544
+				iNdEx++
545
+				m.Port |= (uint32(b) & 0x7F) << shift
546
+				if b < 0x80 {
547
+					break
548
+				}
549
+			}
550
+		case 4:
551
+			if wireType != 0 {
552
+				return fmt.Errorf("proto: wrong wireType = %d for field NodePort", wireType)
553
+			}
554
+			m.NodePort = 0
555
+			for shift := uint(0); ; shift += 7 {
556
+				if shift >= 64 {
557
+					return ErrIntOverflowAgent
558
+				}
559
+				if iNdEx >= l {
560
+					return io.ErrUnexpectedEOF
561
+				}
562
+				b := data[iNdEx]
563
+				iNdEx++
564
+				m.NodePort |= (uint32(b) & 0x7F) << shift
565
+				if b < 0x80 {
566
+					break
567
+				}
568
+			}
402 569
 		default:
403 570
 			iNdEx = preIndex
404 571
 			skippy, err := skipAgent(data[iNdEx:])
... ...
@@ -526,20 +865,29 @@ var (
526 526
 )
527 527
 
528 528
 var fileDescriptorAgent = []byte{
529
-	// 228 bytes of a gzipped FileDescriptorProto
530
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0x4e, 0x4c, 0x4f, 0xcd,
531
-	0x2b, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xca, 0xc9, 0x4c, 0xca, 0x4b, 0x2d, 0x29,
532
-	0xcf, 0x2f, 0xca, 0x96, 0x12, 0x49, 0xcf, 0x4f, 0xcf, 0x07, 0x0b, 0xeb, 0x83, 0x58, 0x10, 0x15,
533
-	0x4a, 0x57, 0x18, 0xb9, 0xf8, 0x5c, 0xf3, 0x52, 0x0a, 0xf2, 0x33, 0xf3, 0x4a, 0x82, 0x52, 0x93,
534
-	0xf3, 0x8b, 0x52, 0x84, 0x84, 0xb8, 0x58, 0xf2, 0x12, 0x73, 0x53, 0x25, 0x18, 0x15, 0x18, 0x35,
535
-	0x38, 0x83, 0xc0, 0x6c, 0x21, 0x45, 0x2e, 0x9e, 0xe2, 0xd4, 0xa2, 0xb2, 0xcc, 0xe4, 0xd4, 0x78,
536
-	0xb0, 0x1c, 0x13, 0x58, 0x8e, 0x1b, 0x2a, 0xe6, 0x07, 0x52, 0xa2, 0xc3, 0xc5, 0x05, 0x53, 0x92,
537
-	0x99, 0x22, 0xc1, 0x0c, 0x52, 0xe0, 0xc4, 0xfb, 0xe8, 0x9e, 0x3c, 0x67, 0x30, 0x44, 0xd4, 0xd3,
538
-	0x25, 0x88, 0x13, 0xaa, 0xc0, 0x33, 0x05, 0xa4, 0xba, 0x2c, 0xb3, 0xa8, 0xa4, 0x34, 0x31, 0x27,
539
-	0x3e, 0xb3, 0x40, 0x82, 0x05, 0xa1, 0x3a, 0x0c, 0x22, 0xea, 0x19, 0x10, 0xc4, 0x09, 0x55, 0xe0,
540
-	0x59, 0x20, 0xa4, 0xcf, 0xc5, 0x9d, 0x0a, 0x75, 0x24, 0x48, 0x39, 0x2b, 0x58, 0x39, 0x1f, 0x50,
541
-	0x39, 0x17, 0xcc, 0xed, 0x40, 0xf5, 0x5c, 0x30, 0x25, 0x9e, 0x05, 0x4e, 0x12, 0x37, 0x1e, 0xca,
542
-	0x31, 0x7c, 0x78, 0x28, 0xc7, 0xd8, 0xf0, 0x48, 0x8e, 0xf1, 0x04, 0x10, 0x5f, 0x00, 0xe2, 0x07,
543
-	0x40, 0x9c, 0xc4, 0x06, 0xf6, 0xb7, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0xae, 0x11, 0xc5, 0x8d,
544
-	0x28, 0x01, 0x00, 0x00,
529
+	// 370 bytes of a gzipped FileDescriptorProto
530
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x6c, 0x90, 0xbf, 0x4e, 0x32, 0x41,
531
+	0x14, 0xc5, 0x59, 0xe0, 0x23, 0xec, 0x5d, 0x96, 0x8f, 0x4c, 0x8c, 0xd9, 0x60, 0xb2, 0x20, 0x15,
532
+	0x85, 0x59, 0x12, 0x2c, 0xe9, 0x00, 0x8b, 0x6d, 0xcc, 0x66, 0xfc, 0xd3, 0x12, 0x60, 0xc7, 0xcd,
533
+	0x44, 0x9c, 0xd9, 0xcc, 0xae, 0xd8, 0x5a, 0x1a, 0x3b, 0x1f, 0xc0, 0xca, 0x97, 0xb1, 0xb4, 0xb0,
534
+	0xb0, 0x32, 0xc2, 0x13, 0xf8, 0x08, 0xce, 0x0c, 0xbb, 0x12, 0x13, 0x8a, 0x9b, 0xdc, 0x9c, 0xf3,
535
+	0xbb, 0x37, 0x27, 0x07, 0xac, 0x69, 0x44, 0x58, 0xea, 0xc5, 0x82, 0xa7, 0x1c, 0xc1, 0x82, 0xce,
536
+	0x18, 0x49, 0xef, 0xb8, 0xb8, 0x6e, 0xee, 0x45, 0x3c, 0xe2, 0x5a, 0xee, 0xa9, 0x6d, 0x43, 0x74,
537
+	0x9e, 0x8a, 0x50, 0x3f, 0x61, 0x61, 0xcc, 0x29, 0x4b, 0x31, 0x99, 0x73, 0x11, 0x22, 0x04, 0x65,
538
+	0x36, 0xbd, 0x21, 0x8e, 0xd1, 0x36, 0xba, 0x26, 0xd6, 0x3b, 0x3a, 0x84, 0x5a, 0x42, 0xc4, 0x92,
539
+	0xce, 0xc9, 0x44, 0x7b, 0x45, 0xed, 0x59, 0x99, 0x76, 0xaa, 0x90, 0x23, 0x80, 0x1c, 0xa1, 0xa1,
540
+	0x53, 0x52, 0xc0, 0xd0, 0x5e, 0x7f, 0xb6, 0xcc, 0xb3, 0x8d, 0xea, 0x8f, 0xb1, 0x99, 0x01, 0x7e,
541
+	0xa8, 0xe8, 0x25, 0x15, 0xe9, 0xed, 0x74, 0x31, 0xa1, 0xb1, 0x53, 0xde, 0xd2, 0x97, 0x1b, 0xd5,
542
+	0x0f, 0xb0, 0x99, 0x01, 0x7e, 0x8c, 0x7a, 0x60, 0x91, 0x2c, 0xa4, 0xc2, 0xff, 0x69, 0xbc, 0x2e,
543
+	0x71, 0xc8, 0xb3, 0x4b, 0x1e, 0x72, 0x44, 0x1e, 0x0c, 0xc0, 0xa6, 0x2c, 0x12, 0x24, 0x49, 0x26,
544
+	0x31, 0x17, 0x69, 0xe2, 0x54, 0xda, 0xa5, 0xae, 0xd5, 0xdf, 0xf7, 0xb6, 0x85, 0x78, 0x81, 0x34,
545
+	0x46, 0x9c, 0x5d, 0xd1, 0x08, 0xd7, 0x32, 0x58, 0x49, 0x49, 0xe7, 0xdd, 0x00, 0xd8, 0x9a, 0x3b,
546
+	0xfb, 0x18, 0x40, 0x55, 0xf7, 0x37, 0xe7, 0x0b, 0xdd, 0x45, 0xbd, 0xdf, 0xda, 0xfd, 0xda, 0x0b,
547
+	0x32, 0x0c, 0xff, 0x1e, 0xa8, 0x87, 0x2a, 0x94, 0xee, 0xc8, 0xc6, 0x7a, 0x47, 0x07, 0x60, 0x32,
548
+	0x1e, 0x12, 0x9d, 0x56, 0xd7, 0x61, 0xe3, 0xaa, 0x12, 0xd4, 0xa7, 0xce, 0x18, 0xaa, 0xf9, 0x1b,
549
+	0xe4, 0x40, 0xe9, 0x7c, 0x14, 0x34, 0x0a, 0xcd, 0xff, 0x8f, 0xcf, 0x6d, 0x2b, 0x97, 0xa5, 0xa4,
550
+	0x9c, 0x8b, 0x71, 0xd0, 0x30, 0xfe, 0x3a, 0x52, 0x6a, 0x96, 0x1f, 0x5e, 0xdc, 0xc2, 0xd0, 0xf9,
551
+	0x58, 0xb9, 0x85, 0xef, 0x95, 0x6b, 0xdc, 0xaf, 0x5d, 0xe3, 0x55, 0xce, 0x9b, 0x9c, 0x2f, 0x39,
552
+	0xb3, 0x8a, 0x8e, 0x76, 0xfc, 0x13, 0x00, 0x00, 0xff, 0xff, 0xca, 0xbb, 0xca, 0xdf, 0x3c, 0x02,
553
+	0x00, 0x00,
545 554
 }
... ...
@@ -11,10 +11,56 @@ option (gogoproto.gostring_all) = true;
11 11
 option (gogoproto.sizer_all) = true;
12 12
 option (gogoproto.goproto_stringer_all) = false;
13 13
 
14
+// EndpointRecord specifies all the endpoint specific information that
15
+// needs to gossiped to nodes participating in the network.
14 16
 message EndpointRecord {
17
+	// Name of the endpoint
15 18
 	string name = 1;
19
+
20
+	// Service name of the service to which this endpoint belongs.
16 21
 	string service_name = 2;
22
+
23
+	// Service ID of the service to which this endpoint belongs.
17 24
 	string service_id = 3 [(gogoproto.customname) = "ServiceID"];
25
+
26
+	// Virtual IP of the service to which this endpoint belongs.
18 27
 	string virtual_ip = 4 [(gogoproto.customname) = "VirtualIP"];
28
+
29
+	// IP assigned to this endpoint.
19 30
 	string endpoint_ip = 5 [(gogoproto.customname) = "EndpointIP"];
20
-}
21 31
\ No newline at end of file
32
+
33
+	// IngressPorts exposed by the service to which this endpoint belongs.
34
+	repeated PortConfig ingress_ports = 6;
35
+}
36
+
37
+// PortConfig specifies an exposed port which can be
38
+// addressed using the given name. This can be later queried
39
+// using a service discovery api or a DNS SRV query. The node
40
+// port specifies a port that can be used to address this
41
+// service external to the cluster by sending a connection
42
+// request to this port to any node on the cluster.
43
+message PortConfig {
44
+	enum Protocol {
45
+		option (gogoproto.goproto_enum_prefix) = false;
46
+
47
+		TCP = 0 [(gogoproto.enumvalue_customname) = "ProtocolTCP"];
48
+		UDP = 1 [(gogoproto.enumvalue_customname) = "ProtocolUDP"];
49
+	}
50
+
51
+	// Name for the port. If provided the port information can
52
+	// be queried using the name as in a DNS SRV query.
53
+	string name = 1;
54
+
55
+	// Protocol for the port which is exposed.
56
+	Protocol protocol = 2;
57
+
58
+	// The port which the application is exposing and is bound to.
59
+	uint32 port = 3;
60
+
61
+	// NodePort specifies the port on which the service is
62
+	// exposed on all nodes on the cluster. If not specified an
63
+	// arbitrary port in the node port range is allocated by the
64
+	// system. If specified it should be within the node port
65
+	// range and it should be available.
66
+	uint32 node_port = 4;
67
+}
... ...
@@ -136,6 +136,7 @@ type controller struct {
136 136
 	nmap            map[string]*netWatch
137 137
 	serviceBindings map[string]*service
138 138
 	defOsSbox       osl.Sandbox
139
+	ingressSandbox  *sandbox
139 140
 	sboxOnce        sync.Once
140 141
 	agent           *agent
141 142
 	sync.Mutex
... ...
@@ -623,9 +624,7 @@ func (c *controller) NetworkByID(id string) (Network, error) {
623 623
 }
624 624
 
625 625
 // NewSandbox creates a new sandbox for the passed container id
626
-func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (Sandbox, error) {
627
-	var err error
628
-
626
+func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (sBox Sandbox, err error) {
629 627
 	if containerID == "" {
630 628
 		return nil, types.BadRequestErrorf("invalid container ID")
631 629
 	}
... ...
@@ -662,11 +661,29 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (S
662 662
 			controller:  c,
663 663
 		}
664 664
 	}
665
+	sBox = sb
665 666
 
666 667
 	heap.Init(&sb.endpoints)
667 668
 
668 669
 	sb.processOptions(options...)
669 670
 
671
+	c.Lock()
672
+	if sb.ingress && c.ingressSandbox != nil {
673
+		return nil, fmt.Errorf("ingress sandbox already present")
674
+	}
675
+
676
+	c.ingressSandbox = sb
677
+	c.Unlock()
678
+	defer func() {
679
+		if err != nil {
680
+			c.Lock()
681
+			if sb.ingress {
682
+				c.ingressSandbox = nil
683
+			}
684
+			c.Unlock()
685
+		}
686
+	}()
687
+
670 688
 	if err = sb.setupResolutionFiles(); err != nil {
671 689
 		return nil, err
672 690
 	}
... ...
@@ -70,6 +70,7 @@ type endpoint struct {
70 70
 	svcID             string
71 71
 	svcName           string
72 72
 	virtualIP         net.IP
73
+	ingressPorts      []*PortConfig
73 74
 	dbIndex           uint64
74 75
 	dbExists          bool
75 76
 	sync.Mutex
... ...
@@ -95,6 +96,7 @@ func (ep *endpoint) MarshalJSON() ([]byte, error) {
95 95
 	epMap["svcName"] = ep.svcName
96 96
 	epMap["svcID"] = ep.svcID
97 97
 	epMap["virtualIP"] = ep.virtualIP.String()
98
+	epMap["ingressPorts"] = ep.ingressPorts
98 99
 
99 100
 	return json.Marshal(epMap)
100 101
 }
... ...
@@ -192,6 +194,11 @@ func (ep *endpoint) UnmarshalJSON(b []byte) (err error) {
192 192
 		ep.virtualIP = net.ParseIP(vip.(string))
193 193
 	}
194 194
 
195
+	pc, _ := json.Marshal(epMap["ingressPorts"])
196
+	var ingressPorts []*PortConfig
197
+	json.Unmarshal(pc, &ingressPorts)
198
+	ep.ingressPorts = ingressPorts
199
+
195 200
 	ma, _ := json.Marshal(epMap["myAliases"])
196 201
 	var myAliases []string
197 202
 	json.Unmarshal(ma, &myAliases)
... ...
@@ -220,6 +227,9 @@ func (ep *endpoint) CopyTo(o datastore.KVObject) error {
220 220
 	dstEp.svcID = ep.svcID
221 221
 	dstEp.virtualIP = ep.virtualIP
222 222
 
223
+	dstEp.ingressPorts = make([]*PortConfig, len(ep.ingressPorts))
224
+	copy(dstEp.ingressPorts, ep.ingressPorts)
225
+
223 226
 	if ep.iface != nil {
224 227
 		dstEp.iface = &endpointInterface{}
225 228
 		ep.iface.CopyTo(dstEp.iface)
... ...
@@ -899,11 +909,12 @@ func CreateOptionAlias(name string, alias string) EndpointOption {
899 899
 }
900 900
 
901 901
 // CreateOptionService function returns an option setter for setting service binding configuration
902
-func CreateOptionService(name, id string, vip net.IP) EndpointOption {
902
+func CreateOptionService(name, id string, vip net.IP, ingressPorts []*PortConfig) EndpointOption {
903 903
 	return func(ep *endpoint) {
904 904
 		ep.svcName = name
905 905
 		ep.svcID = id
906 906
 		ep.virtualIP = vip
907
+		ep.ingressPorts = ingressPorts
907 908
 	}
908 909
 }
909 910
 
... ...
@@ -7,10 +7,13 @@ import (
7 7
 	"encoding/binary"
8 8
 	"fmt"
9 9
 	"net"
10
+	"os/exec"
11
+	"strings"
10 12
 	"sync"
11 13
 	"syscall"
12 14
 	"unsafe"
13 15
 
16
+	"github.com/Sirupsen/logrus"
14 17
 	"github.com/vishvananda/netlink/nl"
15 18
 	"github.com/vishvananda/netns"
16 19
 )
... ...
@@ -55,6 +58,10 @@ func (f *ipvsFlags) Len() int {
55 55
 func setup() {
56 56
 	ipvsOnce.Do(func() {
57 57
 		var err error
58
+		if out, err := exec.Command("modprobe", "-va", "ip_vs").CombinedOutput(); err != nil {
59
+			logrus.Warnf("Running modprobe nf_nat failed with message: `%s`, error: %v", strings.TrimSpace(string(out)), err)
60
+		}
61
+
58 62
 		ipvsFamily, err = getIPVSFamily()
59 63
 		if err != nil {
60 64
 			panic("could not get ipvs family")
... ...
@@ -185,6 +185,7 @@ type network struct {
185 185
 	drvOnce      *sync.Once
186 186
 	internal     bool
187 187
 	inDelete     bool
188
+	ingress      bool
188 189
 	driverTables []string
189 190
 	sync.Mutex
190 191
 }
... ...
@@ -326,6 +327,7 @@ func (n *network) CopyTo(o datastore.KVObject) error {
326 326
 	dstN.drvOnce = n.drvOnce
327 327
 	dstN.internal = n.internal
328 328
 	dstN.inDelete = n.inDelete
329
+	dstN.ingress = n.ingress
329 330
 
330 331
 	// copy labels
331 332
 	if dstN.labels == nil {
... ...
@@ -432,6 +434,7 @@ func (n *network) MarshalJSON() ([]byte, error) {
432 432
 	}
433 433
 	netMap["internal"] = n.internal
434 434
 	netMap["inDelete"] = n.inDelete
435
+	netMap["ingress"] = n.ingress
435 436
 	return json.Marshal(netMap)
436 437
 }
437 438
 
... ...
@@ -522,6 +525,9 @@ func (n *network) UnmarshalJSON(b []byte) (err error) {
522 522
 	if v, ok := netMap["inDelete"]; ok {
523 523
 		n.inDelete = v.(bool)
524 524
 	}
525
+	if v, ok := netMap["ingress"]; ok {
526
+		n.ingress = v.(bool)
527
+	}
525 528
 	// Reconcile old networks with the recently added `--ipv6` flag
526 529
 	if !n.enableIPv6 {
527 530
 		n.enableIPv6 = len(n.ipamV6Info) > 0
... ...
@@ -553,6 +559,14 @@ func NetworkOptionGeneric(generic map[string]interface{}) NetworkOption {
553 553
 	}
554 554
 }
555 555
 
556
+// NetworkOptionIngress returns an option setter to indicate if a network is
557
+// an ingress network.
558
+func NetworkOptionIngress() NetworkOption {
559
+	return func(n *network) {
560
+		n.ingress = true
561
+	}
562
+}
563
+
556 564
 // NetworkOptionPersist returns an option setter to set persistence policy for a network
557 565
 func NetworkOptionPersist(persist bool) NetworkOption {
558 566
 	return func(n *network) {
... ...
@@ -84,6 +84,7 @@ type sandbox struct {
84 84
 	dbExists      bool
85 85
 	isStub        bool
86 86
 	inDelete      bool
87
+	ingress       bool
87 88
 	sync.Mutex
88 89
 }
89 90
 
... ...
@@ -1013,6 +1014,14 @@ func OptionPortMapping(portBindings []types.PortBinding) SandboxOption {
1013 1013
 	}
1014 1014
 }
1015 1015
 
1016
+// OptionIngress function returns an option setter for marking a
1017
+// sandbox as the controller's ingress sandbox.
1018
+func OptionIngress() SandboxOption {
1019
+	return func(sb *sandbox) {
1020
+		sb.ingress = true
1021
+	}
1022
+}
1023
+
1016 1024
 func (eh epHeap) Len() int { return len(eh) }
1017 1025
 
1018 1026
 func (eh epHeap) Less(i, j int) bool {
... ...
@@ -19,6 +19,10 @@ type service struct {
19 19
 	// Map of loadbalancers for the service one-per attached
20 20
 	// network. It is keyed with network ID.
21 21
 	loadBalancers map[string]*loadBalancer
22
+
23
+	// List of ingress ports exposed by the service
24
+	ingressPorts []*PortConfig
25
+
22 26
 	sync.Mutex
23 27
 }
24 28
 
... ...
@@ -29,4 +33,7 @@ type loadBalancer struct {
29 29
 	// Map of backend IPs backing this loadbalancer on this
30 30
 	// network. It is keyed with endpoint ID.
31 31
 	backEnds map[string]net.IP
32
+
33
+	// Back pointer to service to which the loadbalancer belongs.
34
+	service *service
32 35
 }
... ...
@@ -2,6 +2,8 @@ package libnetwork
2 2
 
3 3
 import (
4 4
 	"fmt"
5
+	"io"
6
+	"io/ioutil"
5 7
 	"net"
6 8
 	"os"
7 9
 	"os/exec"
... ...
@@ -13,6 +15,7 @@ import (
13 13
 	"github.com/docker/docker/pkg/reexec"
14 14
 	"github.com/docker/libnetwork/iptables"
15 15
 	"github.com/docker/libnetwork/ipvs"
16
+	"github.com/gogo/protobuf/proto"
16 17
 	"github.com/vishvananda/netlink/nl"
17 18
 	"github.com/vishvananda/netns"
18 19
 )
... ...
@@ -21,15 +24,16 @@ func init() {
21 21
 	reexec.Register("fwmarker", fwMarker)
22 22
 }
23 23
 
24
-func newService(name string, id string) *service {
24
+func newService(name string, id string, ingressPorts []*PortConfig) *service {
25 25
 	return &service{
26 26
 		name:          name,
27 27
 		id:            id,
28
+		ingressPorts:  ingressPorts,
28 29
 		loadBalancers: make(map[string]*loadBalancer),
29 30
 	}
30 31
 }
31 32
 
32
-func (c *controller) addServiceBinding(name, sid, nid, eid string, vip net.IP, ip net.IP) error {
33
+func (c *controller) addServiceBinding(name, sid, nid, eid string, vip net.IP, ingressPorts []*PortConfig, ip net.IP) error {
33 34
 	var (
34 35
 		s          *service
35 36
 		addService bool
... ...
@@ -45,7 +49,7 @@ func (c *controller) addServiceBinding(name, sid, nid, eid string, vip net.IP, i
45 45
 	if !ok {
46 46
 		// Create a new service if we are seeing this service
47 47
 		// for the first time.
48
-		s = newService(name, sid)
48
+		s = newService(name, sid, ingressPorts)
49 49
 		c.serviceBindings[sid] = s
50 50
 	}
51 51
 	c.Unlock()
... ...
@@ -60,6 +64,7 @@ func (c *controller) addServiceBinding(name, sid, nid, eid string, vip net.IP, i
60 60
 			vip:      vip,
61 61
 			fwMark:   fwMarkCtr,
62 62
 			backEnds: make(map[string]net.IP),
63
+			service:  s,
63 64
 		}
64 65
 
65 66
 		fwMarkCtrMu.Lock()
... ...
@@ -91,13 +96,13 @@ func (c *controller) addServiceBinding(name, sid, nid, eid string, vip net.IP, i
91 91
 	// Add loadbalancer service and backend in all sandboxes in
92 92
 	// the network only if vip is valid.
93 93
 	if len(vip) != 0 {
94
-		n.(*network).addLBBackend(ip, vip, lb.fwMark, addService)
94
+		n.(*network).addLBBackend(ip, vip, lb.fwMark, ingressPorts, addService)
95 95
 	}
96 96
 
97 97
 	return nil
98 98
 }
99 99
 
100
-func (c *controller) rmServiceBinding(name, sid, nid, eid string, vip net.IP, ip net.IP) error {
100
+func (c *controller) rmServiceBinding(name, sid, nid, eid string, vip net.IP, ingressPorts []*PortConfig, ip net.IP) error {
101 101
 	var rmService bool
102 102
 
103 103
 	n, err := c.NetworkByID(nid)
... ...
@@ -151,14 +156,14 @@ func (c *controller) rmServiceBinding(name, sid, nid, eid string, vip net.IP, ip
151 151
 	// Remove loadbalancer service(if needed) and backend in all
152 152
 	// sandboxes in the network only if the vip is valid.
153 153
 	if len(vip) != 0 {
154
-		n.(*network).rmLBBackend(ip, vip, lb.fwMark, rmService)
154
+		n.(*network).rmLBBackend(ip, vip, lb.fwMark, ingressPorts, rmService)
155 155
 	}
156 156
 
157 157
 	return nil
158 158
 }
159 159
 
160 160
 // Get all loadbalancers on this network that is currently discovered
161
-// on this node..
161
+// on this node.
162 162
 func (n *network) connectedLoadbalancers() []*loadBalancer {
163 163
 	c := n.getController()
164 164
 
... ...
@@ -178,7 +183,29 @@ func (n *network) connectedLoadbalancers() []*loadBalancer {
178 178
 // Populate all loadbalancers on the network that the passed endpoint
179 179
 // belongs to, into this sandbox.
180 180
 func (sb *sandbox) populateLoadbalancers(ep *endpoint) {
181
+	var gwIP net.IP
182
+
181 183
 	n := ep.getNetwork()
184
+	eIP := ep.Iface().Address()
185
+
186
+	if sb.ingress {
187
+		// For the ingress sandbox if this is not gateway
188
+		// endpoint do nothing.
189
+		if ep != sb.getGatewayEndpoint() {
190
+			return
191
+		}
192
+
193
+		// This is the gateway endpoint. Now get the ingress
194
+		// network and plumb the loadbalancers.
195
+		gwIP = ep.Iface().Address().IP
196
+		for _, ep := range sb.getConnectedEndpoints() {
197
+			if !ep.endpointInGWNetwork() {
198
+				n = ep.getNetwork()
199
+				eIP = ep.Iface().Address()
200
+			}
201
+		}
202
+	}
203
+
182 204
 	for _, lb := range n.connectedLoadbalancers() {
183 205
 		// Skip if vip is not valid.
184 206
 		if len(lb.vip) == 0 {
... ...
@@ -187,7 +214,8 @@ func (sb *sandbox) populateLoadbalancers(ep *endpoint) {
187 187
 
188 188
 		addService := true
189 189
 		for _, ip := range lb.backEnds {
190
-			sb.addLBBackend(ip, lb.vip, lb.fwMark, addService)
190
+			sb.addLBBackend(ip, lb.vip, lb.fwMark, lb.service.ingressPorts,
191
+				eIP, gwIP, addService)
191 192
 			addService = false
192 193
 		}
193 194
 	}
... ...
@@ -196,11 +224,16 @@ func (sb *sandbox) populateLoadbalancers(ep *endpoint) {
196 196
 // Add loadbalancer backend to all sandboxes which has a connection to
197 197
 // this network. If needed add the service as well, as specified by
198 198
 // the addService bool.
199
-func (n *network) addLBBackend(ip, vip net.IP, fwMark uint32, addService bool) {
199
+func (n *network) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, addService bool) {
200 200
 	n.WalkEndpoints(func(e Endpoint) bool {
201 201
 		ep := e.(*endpoint)
202 202
 		if sb, ok := ep.getSandbox(); ok {
203
-			sb.addLBBackend(ip, vip, fwMark, addService)
203
+			var gwIP net.IP
204
+			if ep := sb.getGatewayEndpoint(); ep != nil {
205
+				gwIP = ep.Iface().Address().IP
206
+			}
207
+
208
+			sb.addLBBackend(ip, vip, fwMark, ingressPorts, ep.Iface().Address(), gwIP, addService)
204 209
 		}
205 210
 
206 211
 		return false
... ...
@@ -210,11 +243,16 @@ func (n *network) addLBBackend(ip, vip net.IP, fwMark uint32, addService bool) {
210 210
 // Remove loadbalancer backend from all sandboxes which has a
211 211
 // connection to this network. If needed remove the service entry as
212 212
 // well, as specified by the rmService bool.
213
-func (n *network) rmLBBackend(ip, vip net.IP, fwMark uint32, rmService bool) {
213
+func (n *network) rmLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, rmService bool) {
214 214
 	n.WalkEndpoints(func(e Endpoint) bool {
215 215
 		ep := e.(*endpoint)
216 216
 		if sb, ok := ep.getSandbox(); ok {
217
-			sb.rmLBBackend(ip, vip, fwMark, rmService)
217
+			var gwIP net.IP
218
+			if ep := sb.getGatewayEndpoint(); ep != nil {
219
+				gwIP = ep.Iface().Address().IP
220
+			}
221
+
222
+			sb.rmLBBackend(ip, vip, fwMark, ingressPorts, ep.Iface().Address(), gwIP, rmService)
218 223
 		}
219 224
 
220 225
 		return false
... ...
@@ -222,7 +260,7 @@ func (n *network) rmLBBackend(ip, vip net.IP, fwMark uint32, rmService bool) {
222 222
 }
223 223
 
224 224
 // Add loadbalancer backend into one connected sandbox.
225
-func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, addService bool) {
225
+func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, gwIP net.IP, addService bool) {
226 226
 	i, err := ipvs.New(sb.Key())
227 227
 	if err != nil {
228 228
 		logrus.Errorf("Failed to create a ipvs handle for sbox %s: %v", sb.Key(), err)
... ...
@@ -237,8 +275,17 @@ func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, addService bool)
237 237
 	}
238 238
 
239 239
 	if addService {
240
-		logrus.Debugf("Creating service for vip %s fwMark %d", vip, fwMark)
241
-		if err := invokeFWMarker(sb.Key(), vip, fwMark, false); err != nil {
240
+		var iPorts []*PortConfig
241
+		if sb.ingress {
242
+			iPorts = ingressPorts
243
+			if err := programIngress(gwIP, iPorts, false); err != nil {
244
+				logrus.Errorf("Failed to add ingress: %v", err)
245
+				return
246
+			}
247
+		}
248
+
249
+		logrus.Debugf("Creating service for vip %s fwMark %d ingressPorts %#v", vip, fwMark, iPorts)
250
+		if err := invokeFWMarker(sb.Key(), vip, fwMark, iPorts, eIP, false); err != nil {
242 251
 			logrus.Errorf("Failed to add firewall mark rule in sbox %s: %v", sb.Key(), err)
243 252
 			return
244 253
 		}
... ...
@@ -264,7 +311,7 @@ func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, addService bool)
264 264
 }
265 265
 
266 266
 // Remove loadbalancer backend from one connected sandbox.
267
-func (sb *sandbox) rmLBBackend(ip, vip net.IP, fwMark uint32, rmService bool) {
267
+func (sb *sandbox) rmLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, gwIP net.IP, rmService bool) {
268 268
 	i, err := ipvs.New(sb.Key())
269 269
 	if err != nil {
270 270
 		logrus.Errorf("Failed to create a ipvs handle for sbox %s: %v", sb.Key(), err)
... ...
@@ -295,16 +342,68 @@ func (sb *sandbox) rmLBBackend(ip, vip net.IP, fwMark uint32, rmService bool) {
295 295
 			return
296 296
 		}
297 297
 
298
-		if err := invokeFWMarker(sb.Key(), vip, fwMark, true); err != nil {
298
+		var iPorts []*PortConfig
299
+		if sb.ingress {
300
+			iPorts = ingressPorts
301
+			if err := programIngress(gwIP, iPorts, true); err != nil {
302
+				logrus.Errorf("Failed to delete ingress: %v", err)
303
+				return
304
+			}
305
+		}
306
+
307
+		if err := invokeFWMarker(sb.Key(), vip, fwMark, iPorts, eIP, true); err != nil {
299 308
 			logrus.Errorf("Failed to add firewall mark rule in sbox %s: %v", sb.Key(), err)
300 309
 			return
301 310
 		}
302 311
 	}
303 312
 }
304 313
 
314
+func programIngress(gwIP net.IP, ingressPorts []*PortConfig, isDelete bool) error {
315
+	addDelOpt := "-A"
316
+	if isDelete {
317
+		addDelOpt = "-D"
318
+	}
319
+
320
+	for _, iPort := range ingressPorts {
321
+		rule := strings.Fields(fmt.Sprintf("-t nat %s PREROUTING -p %s --dport %d -j DNAT --to-destination %s:%d",
322
+			addDelOpt, strings.ToLower(PortConfig_Protocol_name[int32(iPort.Protocol)]), iPort.NodePort, gwIP, iPort.NodePort))
323
+		if err := iptables.RawCombinedOutput(rule...); err != nil {
324
+			return fmt.Errorf("setting up rule failed, %v: %v", rule, err)
325
+		}
326
+	}
327
+
328
+	return nil
329
+}
330
+
305 331
 // Invoke fwmarker reexec routine to mark vip destined packets with
306 332
 // the passed firewall mark.
307
-func invokeFWMarker(path string, vip net.IP, fwMark uint32, isDelete bool) error {
333
+func invokeFWMarker(path string, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, isDelete bool) error {
334
+	var ingressPortsFile string
335
+	if len(ingressPorts) != 0 {
336
+		f, err := ioutil.TempFile("", "port_configs")
337
+		if err != nil {
338
+			return err
339
+		}
340
+
341
+		buf, err := proto.Marshal(&EndpointRecord{
342
+			IngressPorts: ingressPorts,
343
+		})
344
+
345
+		n, err := f.Write(buf)
346
+		if err != nil {
347
+			f.Close()
348
+			return err
349
+		}
350
+
351
+		if n < len(buf) {
352
+			f.Close()
353
+			return io.ErrShortWrite
354
+		}
355
+
356
+		ingressPortsFile = f.Name()
357
+		f.Close()
358
+	}
359
+
308 360
 	addDelOpt := "-A"
309 361
 	if isDelete {
310 362
 		addDelOpt = "-D"
... ...
@@ -312,13 +411,15 @@ func invokeFWMarker(path string, vip net.IP, fwMark uint32, isDelete bool) error
312 312
 
313 313
 	cmd := &exec.Cmd{
314 314
 		Path:   reexec.Self(),
315
-		Args:   append([]string{"fwmarker"}, path, vip.String(), fmt.Sprintf("%d", fwMark), addDelOpt),
315
+		Args:   append([]string{"fwmarker"}, path, vip.String(), fmt.Sprintf("%d", fwMark), addDelOpt, ingressPortsFile, eIP.IP.String()),
316 316
 		Stdout: os.Stdout,
317 317
 		Stderr: os.Stderr,
318 318
 	}
319
+
319 320
 	if err := cmd.Run(); err != nil {
320 321
 		return fmt.Errorf("reexec failed: %v", err)
321 322
 	}
323
+
322 324
 	return nil
323 325
 }
324 326
 
... ...
@@ -327,11 +428,29 @@ func fwMarker() {
327 327
 	runtime.LockOSThread()
328 328
 	defer runtime.UnlockOSThread()
329 329
 
330
-	if len(os.Args) < 5 {
330
+	if len(os.Args) < 7 {
331 331
 		logrus.Error("invalid number of arguments..")
332 332
 		os.Exit(1)
333 333
 	}
334 334
 
335
+	var ingressPorts []*PortConfig
336
+	if os.Args[5] != "" {
337
+		buf, err := ioutil.ReadFile(os.Args[5])
338
+		if err != nil {
339
+			logrus.Errorf("Failed to read ports config file: %v", err)
340
+			os.Exit(6)
341
+		}
342
+
343
+		var epRec EndpointRecord
344
+		err = proto.Unmarshal(buf, &epRec)
345
+		if err != nil {
346
+			logrus.Errorf("Failed to unmarshal ports config data: %v", err)
347
+			os.Exit(7)
348
+		}
349
+
350
+		ingressPorts = epRec.IngressPorts
351
+	}
352
+
335 353
 	vip := os.Args[2]
336 354
 	fwMark, err := strconv.ParseUint(os.Args[3], 10, 32)
337 355
 	if err != nil {
... ...
@@ -340,6 +459,17 @@ func fwMarker() {
340 340
 	}
341 341
 	addDelOpt := os.Args[4]
342 342
 
343
+	rules := [][]string{}
344
+	for _, iPort := range ingressPorts {
345
+		rule := strings.Fields(fmt.Sprintf("-t nat %s PREROUTING -p %s --dport %d -j REDIRECT --to-port %d",
346
+			addDelOpt, strings.ToLower(PortConfig_Protocol_name[int32(iPort.Protocol)]), iPort.NodePort, iPort.Port))
347
+		rules = append(rules, rule)
348
+
349
+		rule = strings.Fields(fmt.Sprintf("-t mangle %s PREROUTING -p %s --dport %d -j MARK --set-mark %d",
350
+			addDelOpt, strings.ToLower(PortConfig_Protocol_name[int32(iPort.Protocol)]), iPort.NodePort, fwMark))
351
+		rules = append(rules, rule)
352
+	}
353
+
343 354
 	ns, err := netns.GetFromPath(os.Args[1])
344 355
 	if err != nil {
345 356
 		logrus.Errorf("failed get network namespace %q: %v", os.Args[1], err)
... ...
@@ -352,9 +482,27 @@ func fwMarker() {
352 352
 		os.Exit(4)
353 353
 	}
354 354
 
355
+	if len(ingressPorts) != 0 && addDelOpt == "-A" {
356
+		ruleParams := strings.Fields(fmt.Sprintf("-m ipvs --ipvs -j SNAT --to-source %s", os.Args[6]))
357
+		if !iptables.Exists("nat", "POSTROUTING", ruleParams...) {
358
+			rule := append(strings.Fields("-t nat -A POSTROUTING"), ruleParams...)
359
+			rules = append(rules, rule)
360
+
361
+			err := ioutil.WriteFile("/proc/sys/net/ipv4/vs/conntrack", []byte{'1', '\n'}, 0644)
362
+			if err != nil {
363
+				logrus.Errorf("Failed to write to /proc/sys/net/ipv4/vs/conntrack: %v", err)
364
+				os.Exit(8)
365
+			}
366
+		}
367
+	}
368
+
355 369
 	rule := strings.Fields(fmt.Sprintf("-t mangle %s OUTPUT -d %s/32 -j MARK --set-mark %d", addDelOpt, vip, fwMark))
356
-	if err := iptables.RawCombinedOutputNative(rule...); err != nil {
357
-		logrus.Errorf("setting up rule failed, %v: %v", rule, err)
358
-		os.Exit(5)
370
+	rules = append(rules, rule)
371
+
372
+	for _, rule := range rules {
373
+		if err := iptables.RawCombinedOutputNative(rule...); err != nil {
374
+			logrus.Errorf("setting up rule failed, %v: %v", rule, err)
375
+			os.Exit(5)
376
+		}
359 377
 	}
360 378
 }
... ...
@@ -7,11 +7,11 @@ import (
7 7
 	"net"
8 8
 )
9 9
 
10
-func (c *controller) addServiceBinding(name, sid, nid, eid string, vip net.IP, ip net.IP) error {
10
+func (c *controller) addServiceBinding(name, sid, nid, eid string, vip net.IP, ingressPorts []*PortConfig, ip net.IP) error {
11 11
 	return fmt.Errorf("not supported")
12 12
 }
13 13
 
14
-func (c *controller) rmServiceBinding(name, sid, nid, eid string, vip net.IP, ip net.IP) error {
14
+func (c *controller) rmServiceBinding(name, sid, nid, eid string, vip net.IP, ingressPorts []*PortConfig, ip net.IP) error {
15 15
 	return fmt.Errorf("not supported")
16 16
 }
17 17