Browse code

libnetwork: fwmarker without reexec

Signed-off-by: Cory Snider <csnider@mirantis.com>

Cory Snider authored on 2022/11/01 07:04:50
Showing 1 changed files
... ...
@@ -25,7 +25,6 @@ import (
25 25
 )
26 26
 
27 27
 func init() {
28
-	reexec.Register("fwmarker", fwMarker)
29 28
 	reexec.Register("redirector", redirector)
30 29
 }
31 30
 
... ...
@@ -141,7 +140,7 @@ func (n *network) addLBBackend(ip net.IP, lb *loadBalancer) {
141 141
 		}
142 142
 
143 143
 		logrus.Debugf("Creating service for vip %s fwMark %d ingressPorts %#v in sbox %.7s (%.7s)", lb.vip, lb.fwMark, lb.service.ingressPorts, sb.ID(), sb.ContainerID())
144
-		if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, false, n.loadBalancerMode); err != nil {
144
+		if err := sb.configureFWMark(lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, false, n.loadBalancerMode); err != nil {
145 145
 			logrus.Errorf("Failed to add firewall mark rule in sbox %.7s (%.7s): %v", sb.ID(), sb.ContainerID(), err)
146 146
 			return
147 147
 		}
... ...
@@ -240,7 +239,7 @@ func (n *network) rmLBBackend(ip net.IP, lb *loadBalancer, rmService bool, fullR
240 240
 			}
241 241
 		}
242 242
 
243
-		if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, true, n.loadBalancerMode); err != nil {
243
+		if err := sb.configureFWMark(lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, true, n.loadBalancerMode); err != nil {
244 244
 			logrus.Errorf("Failed to delete firewall mark rule in sbox %.7s (%.7s): %v", sb.ID(), sb.ContainerID(), err)
245 245
 		}
246 246
 
... ...
@@ -381,7 +380,7 @@ func programIngress(gwIP net.IP, ingressPorts []*PortConfig, isDelete bool) erro
381 381
 		}
382 382
 
383 383
 		path := filepath.Join("/proc/sys/net/ipv4/conf", oifName, "route_localnet")
384
-		if err := os.WriteFile(path, []byte{'1', '\n'}, 0644); err != nil { //nolint:gosec // gosec complains about perms here, which must be 0644 in this case
384
+		if err := os.WriteFile(path, []byte{'1', '\n'}, 0o644); err != nil { //nolint:gosec // gosec complains about perms here, which must be 0644 in this case
385 385
 			return fmt.Errorf("could not write to %s: %v", path, err)
386 386
 		}
387 387
 
... ...
@@ -578,122 +577,59 @@ func readPortsFromFile(fileName string) ([]*PortConfig, error) {
578 578
 	return epRec.IngressPorts, nil
579 579
 }
580 580
 
581
-// Invoke fwmarker reexec routine to mark vip destined packets with
582
-// the passed firewall mark.
583
-func invokeFWMarker(path string, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, isDelete bool, lbMode string) error {
584
-	var ingressPortsFile string
585
-
586
-	if len(ingressPorts) != 0 {
587
-		var err error
588
-		ingressPortsFile, err = writePortsToFile(ingressPorts)
589
-		if err != nil {
590
-			return err
591
-		}
592
-
593
-		defer os.Remove(ingressPortsFile)
594
-	}
581
+// configureFWMark configures the sandbox firewall to mark vip destined packets
582
+// with the firewall mark fwMark.
583
+func (sb *sandbox) configureFWMark(vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, isDelete bool, lbMode string) error {
584
+	// TODO IPv6 support
585
+	iptable := iptables.GetIptable(iptables.IPv4)
595 586
 
587
+	fwMarkStr := strconv.FormatUint(uint64(fwMark), 10)
596 588
 	addDelOpt := "-A"
597 589
 	if isDelete {
598 590
 		addDelOpt = "-D"
599 591
 	}
600 592
 
601
-	cmd := &exec.Cmd{
602
-		Path:   reexec.Self(),
603
-		Args:   append([]string{"fwmarker"}, path, vip.String(), strconv.FormatUint(uint64(fwMark), 10), addDelOpt, ingressPortsFile, eIP.String(), lbMode),
604
-		Stdout: os.Stdout,
605
-		Stderr: os.Stderr,
606
-	}
607
-
608
-	if err := cmd.Run(); err != nil {
609
-		return fmt.Errorf("reexec failed: %v", err)
610
-	}
611
-
612
-	return nil
613
-}
614
-
615
-// Firewall marker reexec function.
616
-func fwMarker() {
617
-	// TODO IPv6 support
618
-	iptable := iptables.GetIptable(iptables.IPv4)
619
-	runtime.LockOSThread()
620
-	defer runtime.UnlockOSThread()
621
-
622
-	if len(os.Args) < 8 {
623
-		logrus.Error("invalid number of arguments..")
624
-		os.Exit(1)
625
-	}
626
-
627
-	var ingressPorts []*PortConfig
628
-	if os.Args[5] != "" {
629
-		var err error
630
-		ingressPorts, err = readPortsFromFile(os.Args[5])
631
-		if err != nil {
632
-			logrus.Errorf("Failed reading ingress ports file: %v", err)
633
-			os.Exit(2)
634
-		}
635
-	}
636
-
637
-	vip := os.Args[2]
638
-	fwMark := os.Args[3]
639
-	if _, err := strconv.ParseUint(fwMark, 10, 32); err != nil {
640
-		logrus.Errorf("bad fwmark value(%s) passed: %v", fwMark, err)
641
-		os.Exit(3)
642
-	}
643
-	addDelOpt := os.Args[4]
644
-
645 593
 	rules := make([][]string, 0, len(ingressPorts))
646 594
 	for _, iPort := range ingressPorts {
647 595
 		var (
648 596
 			protocol      = strings.ToLower(PortConfig_Protocol_name[int32(iPort.Protocol)])
649 597
 			publishedPort = strconv.FormatUint(uint64(iPort.PublishedPort), 10)
650 598
 		)
651
-		rule := []string{"-t", "mangle", addDelOpt, "PREROUTING", "-p", protocol, "--dport", publishedPort, "-j", "MARK", "--set-mark", fwMark}
599
+		rule := []string{"-t", "mangle", addDelOpt, "PREROUTING", "-p", protocol, "--dport", publishedPort, "-j", "MARK", "--set-mark", fwMarkStr}
652 600
 		rules = append(rules, rule)
653 601
 	}
654 602
 
655
-	ns, err := netns.GetFromPath(os.Args[1])
656
-	if err != nil {
657
-		logrus.Errorf("failed get network namespace %q: %v", os.Args[1], err)
658
-		os.Exit(4)
659
-	}
660
-	defer ns.Close()
603
+	var innerErr error
604
+	err := sb.ExecFunc(func() {
605
+		if !isDelete && lbMode == loadBalancerModeNAT {
606
+			subnet := net.IPNet{IP: eIP.IP.Mask(eIP.Mask), Mask: eIP.Mask}
607
+			ruleParams := []string{"-m", "ipvs", "--ipvs", "-d", subnet.String(), "-j", "SNAT", "--to-source", eIP.IP.String()}
608
+			if !iptable.Exists("nat", "POSTROUTING", ruleParams...) {
609
+				rule := append([]string{"-t", "nat", "-A", "POSTROUTING"}, ruleParams...)
610
+				rules = append(rules, rule)
661 611
 
662
-	if err := netns.Set(ns); err != nil {
663
-		logrus.Errorf("setting into container net ns %v failed, %v", os.Args[1], err)
664
-		os.Exit(5)
665
-	}
666
-
667
-	lbMode := os.Args[7]
668
-	if addDelOpt == "-A" && lbMode == loadBalancerModeNAT {
669
-		eIP, subnet, err := net.ParseCIDR(os.Args[6])
670
-		if err != nil {
671
-			logrus.Errorf("Failed to parse endpoint IP %s: %v", os.Args[6], err)
672
-			os.Exit(6)
673
-		}
674
-
675
-		ruleParams := []string{"-m", "ipvs", "--ipvs", "-d", subnet.String(), "-j", "SNAT", "--to-source", eIP.String()}
676
-		if !iptable.Exists("nat", "POSTROUTING", ruleParams...) {
677
-			rule := append([]string{"-t", "nat", "-A", "POSTROUTING"}, ruleParams...)
678
-			rules = append(rules, rule)
679
-
680
-			err := os.WriteFile("/proc/sys/net/ipv4/vs/conntrack", []byte{'1', '\n'}, 0644)
681
-			if err != nil {
682
-				logrus.Errorf("Failed to write to /proc/sys/net/ipv4/vs/conntrack: %v", err)
683
-				os.Exit(7)
612
+				err := os.WriteFile("/proc/sys/net/ipv4/vs/conntrack", []byte{'1', '\n'}, 0644)
613
+				if err != nil {
614
+					innerErr = err
615
+					return
616
+				}
684 617
 			}
685 618
 		}
686
-	}
687 619
 
688
-	rule := []string{"-t", "mangle", addDelOpt, "INPUT", "-d", vip + "/32", "-j", "MARK", "--set-mark", fwMark}
689
-	rules = append(rules, rule)
620
+		rule := []string{"-t", "mangle", addDelOpt, "INPUT", "-d", vip.String() + "/32", "-j", "MARK", "--set-mark", fwMarkStr}
621
+		rules = append(rules, rule)
690 622
 
691
-	for _, rule := range rules {
692
-		if err := iptable.RawCombinedOutputNative(rule...); err != nil {
693
-			logrus.Errorf("set up rule failed, %v: %v", rule, err)
694
-			os.Exit(8)
623
+		for _, rule := range rules {
624
+			if err := iptable.RawCombinedOutputNative(rule...); err != nil {
625
+				innerErr = fmt.Errorf("set up rule failed, %v: %w", rule, err)
626
+				return
627
+			}
695 628
 		}
629
+	})
630
+	if err != nil {
631
+		return err
696 632
 	}
633
+	return innerErr
697 634
 }
698 635
 
699 636
 func addRedirectRules(path string, eIP *net.IPNet, ingressPorts []*PortConfig) error {