Signed-off-by: Cory Snider <csnider@mirantis.com>
| ... | ... |
@@ -25,7 +25,6 @@ import ( |
| 25 | 25 |
) |
| 26 | 26 |
|
| 27 | 27 |
func init() {
|
| 28 |
- reexec.Register("fwmarker", fwMarker)
|
|
| 29 | 28 |
reexec.Register("redirector", redirector)
|
| 30 | 29 |
} |
| 31 | 30 |
|
| ... | ... |
@@ -141,7 +140,7 @@ func (n *network) addLBBackend(ip net.IP, lb *loadBalancer) {
|
| 141 | 141 |
} |
| 142 | 142 |
|
| 143 | 143 |
logrus.Debugf("Creating service for vip %s fwMark %d ingressPorts %#v in sbox %.7s (%.7s)", lb.vip, lb.fwMark, lb.service.ingressPorts, sb.ID(), sb.ContainerID())
|
| 144 |
- if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, false, n.loadBalancerMode); err != nil {
|
|
| 144 |
+ if err := sb.configureFWMark(lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, false, n.loadBalancerMode); err != nil {
|
|
| 145 | 145 |
logrus.Errorf("Failed to add firewall mark rule in sbox %.7s (%.7s): %v", sb.ID(), sb.ContainerID(), err)
|
| 146 | 146 |
return |
| 147 | 147 |
} |
| ... | ... |
@@ -240,7 +239,7 @@ func (n *network) rmLBBackend(ip net.IP, lb *loadBalancer, rmService bool, fullR |
| 240 | 240 |
} |
| 241 | 241 |
} |
| 242 | 242 |
|
| 243 |
- if err := invokeFWMarker(sb.Key(), lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, true, n.loadBalancerMode); err != nil {
|
|
| 243 |
+ if err := sb.configureFWMark(lb.vip, lb.fwMark, lb.service.ingressPorts, eIP, true, n.loadBalancerMode); err != nil {
|
|
| 244 | 244 |
logrus.Errorf("Failed to delete firewall mark rule in sbox %.7s (%.7s): %v", sb.ID(), sb.ContainerID(), err)
|
| 245 | 245 |
} |
| 246 | 246 |
|
| ... | ... |
@@ -381,7 +380,7 @@ func programIngress(gwIP net.IP, ingressPorts []*PortConfig, isDelete bool) erro |
| 381 | 381 |
} |
| 382 | 382 |
|
| 383 | 383 |
path := filepath.Join("/proc/sys/net/ipv4/conf", oifName, "route_localnet")
|
| 384 |
- if err := os.WriteFile(path, []byte{'1', '\n'}, 0644); err != nil { //nolint:gosec // gosec complains about perms here, which must be 0644 in this case
|
|
| 384 |
+ if err := os.WriteFile(path, []byte{'1', '\n'}, 0o644); err != nil { //nolint:gosec // gosec complains about perms here, which must be 0644 in this case
|
|
| 385 | 385 |
return fmt.Errorf("could not write to %s: %v", path, err)
|
| 386 | 386 |
} |
| 387 | 387 |
|
| ... | ... |
@@ -578,122 +577,59 @@ func readPortsFromFile(fileName string) ([]*PortConfig, error) {
|
| 578 | 578 |
return epRec.IngressPorts, nil |
| 579 | 579 |
} |
| 580 | 580 |
|
| 581 |
-// Invoke fwmarker reexec routine to mark vip destined packets with |
|
| 582 |
-// the passed firewall mark. |
|
| 583 |
-func invokeFWMarker(path string, vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, isDelete bool, lbMode string) error {
|
|
| 584 |
- var ingressPortsFile string |
|
| 585 |
- |
|
| 586 |
- if len(ingressPorts) != 0 {
|
|
| 587 |
- var err error |
|
| 588 |
- ingressPortsFile, err = writePortsToFile(ingressPorts) |
|
| 589 |
- if err != nil {
|
|
| 590 |
- return err |
|
| 591 |
- } |
|
| 592 |
- |
|
| 593 |
- defer os.Remove(ingressPortsFile) |
|
| 594 |
- } |
|
| 581 |
+// configureFWMark configures the sandbox firewall to mark vip destined packets |
|
| 582 |
+// with the firewall mark fwMark. |
|
| 583 |
+func (sb *sandbox) configureFWMark(vip net.IP, fwMark uint32, ingressPorts []*PortConfig, eIP *net.IPNet, isDelete bool, lbMode string) error {
|
|
| 584 |
+ // TODO IPv6 support |
|
| 585 |
+ iptable := iptables.GetIptable(iptables.IPv4) |
|
| 595 | 586 |
|
| 587 |
+ fwMarkStr := strconv.FormatUint(uint64(fwMark), 10) |
|
| 596 | 588 |
addDelOpt := "-A" |
| 597 | 589 |
if isDelete {
|
| 598 | 590 |
addDelOpt = "-D" |
| 599 | 591 |
} |
| 600 | 592 |
|
| 601 |
- cmd := &exec.Cmd{
|
|
| 602 |
- Path: reexec.Self(), |
|
| 603 |
- Args: append([]string{"fwmarker"}, path, vip.String(), strconv.FormatUint(uint64(fwMark), 10), addDelOpt, ingressPortsFile, eIP.String(), lbMode),
|
|
| 604 |
- Stdout: os.Stdout, |
|
| 605 |
- Stderr: os.Stderr, |
|
| 606 |
- } |
|
| 607 |
- |
|
| 608 |
- if err := cmd.Run(); err != nil {
|
|
| 609 |
- return fmt.Errorf("reexec failed: %v", err)
|
|
| 610 |
- } |
|
| 611 |
- |
|
| 612 |
- return nil |
|
| 613 |
-} |
|
| 614 |
- |
|
| 615 |
-// Firewall marker reexec function. |
|
| 616 |
-func fwMarker() {
|
|
| 617 |
- // TODO IPv6 support |
|
| 618 |
- iptable := iptables.GetIptable(iptables.IPv4) |
|
| 619 |
- runtime.LockOSThread() |
|
| 620 |
- defer runtime.UnlockOSThread() |
|
| 621 |
- |
|
| 622 |
- if len(os.Args) < 8 {
|
|
| 623 |
- logrus.Error("invalid number of arguments..")
|
|
| 624 |
- os.Exit(1) |
|
| 625 |
- } |
|
| 626 |
- |
|
| 627 |
- var ingressPorts []*PortConfig |
|
| 628 |
- if os.Args[5] != "" {
|
|
| 629 |
- var err error |
|
| 630 |
- ingressPorts, err = readPortsFromFile(os.Args[5]) |
|
| 631 |
- if err != nil {
|
|
| 632 |
- logrus.Errorf("Failed reading ingress ports file: %v", err)
|
|
| 633 |
- os.Exit(2) |
|
| 634 |
- } |
|
| 635 |
- } |
|
| 636 |
- |
|
| 637 |
- vip := os.Args[2] |
|
| 638 |
- fwMark := os.Args[3] |
|
| 639 |
- if _, err := strconv.ParseUint(fwMark, 10, 32); err != nil {
|
|
| 640 |
- logrus.Errorf("bad fwmark value(%s) passed: %v", fwMark, err)
|
|
| 641 |
- os.Exit(3) |
|
| 642 |
- } |
|
| 643 |
- addDelOpt := os.Args[4] |
|
| 644 |
- |
|
| 645 | 593 |
rules := make([][]string, 0, len(ingressPorts)) |
| 646 | 594 |
for _, iPort := range ingressPorts {
|
| 647 | 595 |
var ( |
| 648 | 596 |
protocol = strings.ToLower(PortConfig_Protocol_name[int32(iPort.Protocol)]) |
| 649 | 597 |
publishedPort = strconv.FormatUint(uint64(iPort.PublishedPort), 10) |
| 650 | 598 |
) |
| 651 |
- rule := []string{"-t", "mangle", addDelOpt, "PREROUTING", "-p", protocol, "--dport", publishedPort, "-j", "MARK", "--set-mark", fwMark}
|
|
| 599 |
+ rule := []string{"-t", "mangle", addDelOpt, "PREROUTING", "-p", protocol, "--dport", publishedPort, "-j", "MARK", "--set-mark", fwMarkStr}
|
|
| 652 | 600 |
rules = append(rules, rule) |
| 653 | 601 |
} |
| 654 | 602 |
|
| 655 |
- ns, err := netns.GetFromPath(os.Args[1]) |
|
| 656 |
- if err != nil {
|
|
| 657 |
- logrus.Errorf("failed get network namespace %q: %v", os.Args[1], err)
|
|
| 658 |
- os.Exit(4) |
|
| 659 |
- } |
|
| 660 |
- defer ns.Close() |
|
| 603 |
+ var innerErr error |
|
| 604 |
+ err := sb.ExecFunc(func() {
|
|
| 605 |
+ if !isDelete && lbMode == loadBalancerModeNAT {
|
|
| 606 |
+ subnet := net.IPNet{IP: eIP.IP.Mask(eIP.Mask), Mask: eIP.Mask}
|
|
| 607 |
+ ruleParams := []string{"-m", "ipvs", "--ipvs", "-d", subnet.String(), "-j", "SNAT", "--to-source", eIP.IP.String()}
|
|
| 608 |
+ if !iptable.Exists("nat", "POSTROUTING", ruleParams...) {
|
|
| 609 |
+ rule := append([]string{"-t", "nat", "-A", "POSTROUTING"}, ruleParams...)
|
|
| 610 |
+ rules = append(rules, rule) |
|
| 661 | 611 |
|
| 662 |
- if err := netns.Set(ns); err != nil {
|
|
| 663 |
- logrus.Errorf("setting into container net ns %v failed, %v", os.Args[1], err)
|
|
| 664 |
- os.Exit(5) |
|
| 665 |
- } |
|
| 666 |
- |
|
| 667 |
- lbMode := os.Args[7] |
|
| 668 |
- if addDelOpt == "-A" && lbMode == loadBalancerModeNAT {
|
|
| 669 |
- eIP, subnet, err := net.ParseCIDR(os.Args[6]) |
|
| 670 |
- if err != nil {
|
|
| 671 |
- logrus.Errorf("Failed to parse endpoint IP %s: %v", os.Args[6], err)
|
|
| 672 |
- os.Exit(6) |
|
| 673 |
- } |
|
| 674 |
- |
|
| 675 |
- ruleParams := []string{"-m", "ipvs", "--ipvs", "-d", subnet.String(), "-j", "SNAT", "--to-source", eIP.String()}
|
|
| 676 |
- if !iptable.Exists("nat", "POSTROUTING", ruleParams...) {
|
|
| 677 |
- rule := append([]string{"-t", "nat", "-A", "POSTROUTING"}, ruleParams...)
|
|
| 678 |
- rules = append(rules, rule) |
|
| 679 |
- |
|
| 680 |
- err := os.WriteFile("/proc/sys/net/ipv4/vs/conntrack", []byte{'1', '\n'}, 0644)
|
|
| 681 |
- if err != nil {
|
|
| 682 |
- logrus.Errorf("Failed to write to /proc/sys/net/ipv4/vs/conntrack: %v", err)
|
|
| 683 |
- os.Exit(7) |
|
| 612 |
+ err := os.WriteFile("/proc/sys/net/ipv4/vs/conntrack", []byte{'1', '\n'}, 0644)
|
|
| 613 |
+ if err != nil {
|
|
| 614 |
+ innerErr = err |
|
| 615 |
+ return |
|
| 616 |
+ } |
|
| 684 | 617 |
} |
| 685 | 618 |
} |
| 686 |
- } |
|
| 687 | 619 |
|
| 688 |
- rule := []string{"-t", "mangle", addDelOpt, "INPUT", "-d", vip + "/32", "-j", "MARK", "--set-mark", fwMark}
|
|
| 689 |
- rules = append(rules, rule) |
|
| 620 |
+ rule := []string{"-t", "mangle", addDelOpt, "INPUT", "-d", vip.String() + "/32", "-j", "MARK", "--set-mark", fwMarkStr}
|
|
| 621 |
+ rules = append(rules, rule) |
|
| 690 | 622 |
|
| 691 |
- for _, rule := range rules {
|
|
| 692 |
- if err := iptable.RawCombinedOutputNative(rule...); err != nil {
|
|
| 693 |
- logrus.Errorf("set up rule failed, %v: %v", rule, err)
|
|
| 694 |
- os.Exit(8) |
|
| 623 |
+ for _, rule := range rules {
|
|
| 624 |
+ if err := iptable.RawCombinedOutputNative(rule...); err != nil {
|
|
| 625 |
+ innerErr = fmt.Errorf("set up rule failed, %v: %w", rule, err)
|
|
| 626 |
+ return |
|
| 627 |
+ } |
|
| 695 | 628 |
} |
| 629 |
+ }) |
|
| 630 |
+ if err != nil {
|
|
| 631 |
+ return err |
|
| 696 | 632 |
} |
| 633 |
+ return innerErr |
|
| 697 | 634 |
} |
| 698 | 635 |
|
| 699 | 636 |
func addRedirectRules(path string, eIP *net.IPNet, ingressPorts []*PortConfig) error {
|