Browse code

Delay port redirect until packet reaches container

With port redirect in the ingress path happening before ipvs in the
ingess sandbox, there is a chance of 5-tuple collision in the ipvs
connection table for two entirely different services have different
PublishedPorts but the same TargetPort. To disambiguate the ipvs
connection table, delay the port redirect from PublishedPort to
TargetPort until after the loadbalancing has happened in ipvs. To be
specific, perform the redirect after the packet enters the real backend
container namespace.

Signed-off-by: Jana Radhakrishnan <mrjana@docker.com>

Jana Radhakrishnan authored on 2016/09/22 04:15:14
Showing 1 changed files
... ...
@@ -26,6 +26,7 @@ import (
26 26
 
27 27
 func init() {
28 28
 	reexec.Register("fwmarker", fwMarker)
29
+	reexec.Register("redirecter", redirecter)
29 30
 }
30 31
 
31 32
 func newService(name string, id string, ingressPorts []*PortConfig, aliases []string) *service {
... ...
@@ -275,6 +276,12 @@ func (sb *sandbox) populateLoadbalancers(ep *endpoint) {
275 275
 	n := ep.getNetwork()
276 276
 	eIP := ep.Iface().Address()
277 277
 
278
+	if n.ingress {
279
+		if err := addRedirectRules(sb.Key(), eIP, ep.ingressPorts); err != nil {
280
+			logrus.Errorf("Failed to add redirect rules for ep %s: %v", ep.Name(), err)
281
+		}
282
+	}
283
+
278 284
 	if sb.ingress {
279 285
 		// For the ingress sandbox if this is not gateway
280 286
 		// endpoint do nothing.
... ...
@@ -390,7 +397,7 @@ func (sb *sandbox) addLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*P
390 390
 		}
391 391
 
392 392
 		logrus.Debugf("Creating service for vip %s fwMark %d ingressPorts %#v", vip, fwMark, ingressPorts)
393
-		if err := invokeFWMarker(sb.Key(), vip, fwMark, ingressPorts, filteredPorts, eIP, false); err != nil {
393
+		if err := invokeFWMarker(sb.Key(), vip, fwMark, ingressPorts, eIP, false); err != nil {
394 394
 			logrus.Errorf("Failed to add firewall mark rule in sbox %s: %v", sb.Key(), err)
395 395
 			return
396 396
 		}
... ...
@@ -461,7 +468,7 @@ func (sb *sandbox) rmLBBackend(ip, vip net.IP, fwMark uint32, ingressPorts []*Po
461 461
 			}
462 462
 		}
463 463
 
464
-		if err := invokeFWMarker(sb.Key(), vip, fwMark, ingressPorts, filteredPorts, eIP, true); err != nil {
464
+		if err := invokeFWMarker(sb.Key(), vip, fwMark, ingressPorts, eIP, true); err != nil {
465 465
 			logrus.Errorf("Failed to add firewall mark rule in sbox %s: %v", sb.Key(), err)
466 466
 		}
467 467
 	}
... ...
@@ -755,11 +762,8 @@ func readPortsFromFile(fileName string) ([]*PortConfig, error) {
755 755
 
756 756
 // Invoke fwmarker reexec routine to mark vip destined packets with
757 757
 // the passed firewall mark.
758
-func invokeFWMarker(path string, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, filteredPorts []*PortConfig, eIP *net.IPNet, isDelete bool) error {
759
-	var (
760
-		ingressPortsFile  string
761
-		filteredPortsFile string
762
-	)
758
+func invokeFWMarker(path string, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, isDelete bool) error {
759
+	var ingressPortsFile string
763 760
 
764 761
 	if len(ingressPorts) != 0 {
765 762
 		var err error
... ...
@@ -767,14 +771,8 @@ func invokeFWMarker(path string, vip net.IP, fwMark uint32, ingressPorts []*Port
767 767
 		if err != nil {
768 768
 			return err
769 769
 		}
770
-	}
771 770
 
772
-	if len(filteredPorts) != 0 {
773
-		var err error
774
-		filteredPortsFile, err = writePortsToFile(filteredPorts)
775
-		if err != nil {
776
-			return err
777
-		}
771
+		defer os.Remove(ingressPortsFile)
778 772
 	}
779 773
 
780 774
 	addDelOpt := "-A"
... ...
@@ -784,7 +782,7 @@ func invokeFWMarker(path string, vip net.IP, fwMark uint32, ingressPorts []*Port
784 784
 
785 785
 	cmd := &exec.Cmd{
786 786
 		Path:   reexec.Self(),
787
-		Args:   append([]string{"fwmarker"}, path, vip.String(), fmt.Sprintf("%d", fwMark), addDelOpt, ingressPortsFile, filteredPortsFile, eIP.String()),
787
+		Args:   append([]string{"fwmarker"}, path, vip.String(), fmt.Sprintf("%d", fwMark), addDelOpt, ingressPortsFile, eIP.String()),
788 788
 		Stdout: os.Stdout,
789 789
 		Stderr: os.Stderr,
790 790
 	}
... ...
@@ -801,13 +799,12 @@ func fwMarker() {
801 801
 	runtime.LockOSThread()
802 802
 	defer runtime.UnlockOSThread()
803 803
 
804
-	if len(os.Args) < 8 {
804
+	if len(os.Args) < 7 {
805 805
 		logrus.Error("invalid number of arguments..")
806 806
 		os.Exit(1)
807 807
 	}
808 808
 
809 809
 	var ingressPorts []*PortConfig
810
-	var filteredPorts []*PortConfig
811 810
 	if os.Args[5] != "" {
812 811
 		var err error
813 812
 		ingressPorts, err = readPortsFromFile(os.Args[5])
... ...
@@ -817,15 +814,6 @@ func fwMarker() {
817 817
 		}
818 818
 	}
819 819
 
820
-	if os.Args[6] != "" {
821
-		var err error
822
-		filteredPorts, err = readPortsFromFile(os.Args[6])
823
-		if err != nil {
824
-			logrus.Errorf("Failed reading filtered ports file: %v", err)
825
-			os.Exit(7)
826
-		}
827
-	}
828
-
829 820
 	vip := os.Args[2]
830 821
 	fwMark, err := strconv.ParseUint(os.Args[3], 10, 32)
831 822
 	if err != nil {
... ...
@@ -835,12 +823,6 @@ func fwMarker() {
835 835
 	addDelOpt := os.Args[4]
836 836
 
837 837
 	rules := [][]string{}
838
-	for _, iPort := range filteredPorts {
839
-		rule := strings.Fields(fmt.Sprintf("-t nat %s PREROUTING -p %s --dport %d -j REDIRECT --to-port %d",
840
-			addDelOpt, strings.ToLower(PortConfig_Protocol_name[int32(iPort.Protocol)]), iPort.PublishedPort, iPort.TargetPort))
841
-		rules = append(rules, rule)
842
-	}
843
-
844 838
 	for _, iPort := range ingressPorts {
845 839
 		rule := strings.Fields(fmt.Sprintf("-t mangle %s PREROUTING -p %s --dport %d -j MARK --set-mark %d",
846 840
 			addDelOpt, strings.ToLower(PortConfig_Protocol_name[int32(iPort.Protocol)]), iPort.PublishedPort, fwMark))
... ...
@@ -860,9 +842,9 @@ func fwMarker() {
860 860
 	}
861 861
 
862 862
 	if addDelOpt == "-A" {
863
-		eIP, subnet, err := net.ParseCIDR(os.Args[7])
863
+		eIP, subnet, err := net.ParseCIDR(os.Args[6])
864 864
 		if err != nil {
865
-			logrus.Errorf("Failed to parse endpoint IP %s: %v", os.Args[7], err)
865
+			logrus.Errorf("Failed to parse endpoint IP %s: %v", os.Args[6], err)
866 866
 			os.Exit(9)
867 867
 		}
868 868
 
... ...
@@ -889,3 +871,82 @@ func fwMarker() {
889 889
 		}
890 890
 	}
891 891
 }
892
+
893
+func addRedirectRules(path string, eIP *net.IPNet, ingressPorts []*PortConfig) error {
894
+	var ingressPortsFile string
895
+
896
+	if len(ingressPorts) != 0 {
897
+		var err error
898
+		ingressPortsFile, err = writePortsToFile(ingressPorts)
899
+		if err != nil {
900
+			return err
901
+		}
902
+		defer os.Remove(ingressPortsFile)
903
+	}
904
+
905
+	cmd := &exec.Cmd{
906
+		Path:   reexec.Self(),
907
+		Args:   append([]string{"redirecter"}, path, eIP.String(), ingressPortsFile),
908
+		Stdout: os.Stdout,
909
+		Stderr: os.Stderr,
910
+	}
911
+
912
+	if err := cmd.Run(); err != nil {
913
+		return fmt.Errorf("reexec failed: %v", err)
914
+	}
915
+
916
+	return nil
917
+}
918
+
919
+// Redirecter reexec function.
920
+func redirecter() {
921
+	runtime.LockOSThread()
922
+	defer runtime.UnlockOSThread()
923
+
924
+	if len(os.Args) < 4 {
925
+		logrus.Error("invalid number of arguments..")
926
+		os.Exit(1)
927
+	}
928
+
929
+	var ingressPorts []*PortConfig
930
+	if os.Args[3] != "" {
931
+		var err error
932
+		ingressPorts, err = readPortsFromFile(os.Args[3])
933
+		if err != nil {
934
+			logrus.Errorf("Failed reading ingress ports file: %v", err)
935
+			os.Exit(2)
936
+		}
937
+	}
938
+
939
+	eIP, _, err := net.ParseCIDR(os.Args[2])
940
+	if err != nil {
941
+		logrus.Errorf("Failed to parse endpoint IP %s: %v", os.Args[2], err)
942
+		os.Exit(3)
943
+	}
944
+
945
+	rules := [][]string{}
946
+	for _, iPort := range ingressPorts {
947
+		rule := strings.Fields(fmt.Sprintf("-t nat -A PREROUTING -d %s -p %s --dport %d -j REDIRECT --to-port %d",
948
+			eIP.String(), strings.ToLower(PortConfig_Protocol_name[int32(iPort.Protocol)]), iPort.PublishedPort, iPort.TargetPort))
949
+		rules = append(rules, rule)
950
+	}
951
+
952
+	ns, err := netns.GetFromPath(os.Args[1])
953
+	if err != nil {
954
+		logrus.Errorf("failed get network namespace %q: %v", os.Args[1], err)
955
+		os.Exit(4)
956
+	}
957
+	defer ns.Close()
958
+
959
+	if err := netns.Set(ns); err != nil {
960
+		logrus.Errorf("setting into container net ns %v failed, %v", os.Args[1], err)
961
+		os.Exit(5)
962
+	}
963
+
964
+	for _, rule := range rules {
965
+		if err := iptables.RawCombinedOutputNative(rule...); err != nil {
966
+			logrus.Errorf("setting up rule failed, %v: %v", rule, err)
967
+			os.Exit(5)
968
+		}
969
+	}
970
+}