@@ -38,19 +38,14 @@ add_ovs_flows() {

     fi

     # from container

-    ovs-ofctl -O OpenFlow13 add-flow br0 "table=2, priority=100, in_port=${ovs_port}, arp, nw_src=${ipaddr}, arp_sha=${macaddr}, actions=load:${tenant_id}->NXM_NX_REG0[], goto_table:5"

-    ovs-ofctl -O OpenFlow13 add-flow br0 "table=2, priority=100, in_port=${ovs_port}, ip, nw_src=${ipaddr}, actions=load:${tenant_id}->NXM_NX_REG0[], goto_table:3"

+    ovs-ofctl -O OpenFlow13 add-flow br0 "table=20, priority=100, in_port=${ovs_port}, arp, nw_src=${ipaddr}, arp_sha=${macaddr}, actions=load:${tenant_id}->NXM_NX_REG0[], goto_table:30"

+    ovs-ofctl -O OpenFlow13 add-flow br0 "table=20, priority=100, in_port=${ovs_port}, ip, nw_src=${ipaddr}, actions=load:${tenant_id}->NXM_NX_REG0[], goto_table:30"

     # arp request/response to container (not isolated)

-    ovs-ofctl -O OpenFlow13 add-flow br0 "table=6, priority=100, arp, nw_dst=${ipaddr}, actions=output:${ovs_port}"

+    ovs-ofctl -O OpenFlow13 add-flow br0 "table=40, priority=100, arp, nw_dst=${ipaddr}, actions=output:${ovs_port}"

     # IP to container

-    if [ $tenant_id = "0" ]; then

-	ovs-ofctl -O OpenFlow13 add-flow br0 "table=7, priority=100, ip, nw_dst=${ipaddr}, actions=output:${ovs_port}"

-    else

-	ovs-ofctl -O OpenFlow13 add-flow br0 "table=7, priority=100, reg0=0, ip, nw_dst=${ipaddr}, actions=output:${ovs_port}"

-	ovs-ofctl -O OpenFlow13 add-flow br0 "table=7, priority=100, reg0=${tenant_id}, ip, nw_dst=${ipaddr}, actions=output:${ovs_port}"

-    fi

+    ovs-ofctl -O OpenFlow13 add-flow br0 "table=70, priority=100, ip, nw_dst=${ipaddr}, actions=load:${tenant_id}->NXM_NX_REG1[], load:${ovs_port}->NXM_NX_REG2[], goto_table:80"

     # Pod ingress == OVS bridge egress

     # linux-htb used here since that's the Kubernetes default traffic shaper too

@@ -25,7 +25,7 @@ import (

 const (

 	// rule versioning; increment each time flow rules change

-	VERSION        = 1

+	VERSION        = 2

 	VERSION_TABLE  = "table=253"

 	VERSION_ACTION = "actions=note:"

@@ -36,16 +36,21 @@ const (

 	VXLAN_PORT = "4789"

 )

-func getPluginVersion(multitenant bool) []string {

+func (plugin *OsdnNode) getPluginVersion() []string {

 	if VERSION > 254 {

 		panic("Version too large!")

 	}

 	version := fmt.Sprintf("%02X", VERSION)

-	if multitenant {

-		return []string{"01", version}

-	}

-	// single-tenant

-	return []string{"00", version}

+	pluginId := ""

+	switch plugin.policy.(type) {

+	case *singleTenantPlugin:

+		pluginId = "00"

+	case *multiTenantPlugin:

+		pluginId = "01"

+	default:

+		panic("Not an OpenShift-SDN plugin")

+	}

+	return []string{pluginId, version}

 }

 func (plugin *OsdnNode) getLocalSubnet() (string, error) {

@@ -133,7 +138,7 @@ func (plugin *OsdnNode) alreadySetUp(localSubnetGatewayCIDR, clusterNetworkCIDR

 		// OVS note action format hex bytes separated by '.'; first

 		// byte is plugin type (multi-tenant/single-tenant) and second

 		// byte is flow rule version

-		expected := getPluginVersion(plugin.multitenant)

+		expected := plugin.getPluginVersion()

 		existing := strings.Split(flow[idx+len(VERSION_ACTION):], ".")

 		if len(existing) >= 2 && existing[0] == expected[0] && existing[1] == expected[1] {

 			found = true

@@ -232,64 +237,68 @@ func (plugin *OsdnNode) SetupSDN() (bool, error) {

 	otx := plugin.ovs.NewTransaction()

 	// Table 0: initial dispatch based on in_port

 	// vxlan0

-	otx.AddFlow("table=0, priority=200, in_port=1, arp, nw_src=%s, nw_dst=%s, actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:1", clusterNetworkCIDR, localSubnetCIDR)

-	otx.AddFlow("table=0, priority=200, in_port=1, ip, nw_src=%s, nw_dst=%s, actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:1", clusterNetworkCIDR, localSubnetCIDR)

+	otx.AddFlow("table=0, priority=200, in_port=1, arp, nw_src=%s, nw_dst=%s, actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:10", clusterNetworkCIDR, localSubnetCIDR)

+	otx.AddFlow("table=0, priority=200, in_port=1, ip, nw_src=%s, nw_dst=%s, actions=move:NXM_NX_TUN_ID[0..31]->NXM_NX_REG0[],goto_table:10", clusterNetworkCIDR, localSubnetCIDR)

 	otx.AddFlow("table=0, priority=150, in_port=1, actions=drop")

 	// tun0

-	otx.AddFlow("table=0, priority=200, in_port=2, arp, nw_src=%s, nw_dst=%s, actions=goto_table:5", localSubnetGateway, clusterNetworkCIDR)

-	otx.AddFlow("table=0, priority=200, in_port=2, ip, actions=goto_table:5")

+	otx.AddFlow("table=0, priority=200, in_port=2, arp, nw_src=%s, nw_dst=%s, actions=goto_table:30", localSubnetGateway, clusterNetworkCIDR)

+	otx.AddFlow("table=0, priority=200, in_port=2, ip, actions=goto_table:30")

 	otx.AddFlow("table=0, priority=150, in_port=2, actions=drop")

 	// else, from a container

-	otx.AddFlow("table=0, priority=100, arp, actions=goto_table:2")

-	otx.AddFlow("table=0, priority=100, ip, actions=goto_table:2")

+	otx.AddFlow("table=0, priority=100, arp, actions=goto_table:20")

+	otx.AddFlow("table=0, priority=100, ip, actions=goto_table:20")

 	otx.AddFlow("table=0, priority=0, actions=drop")

-	// Table 1: VXLAN ingress filtering; filled in by AddHostSubnetRules()

-	// eg, "table=1, priority=100, tun_src=${remote_node_ip}, actions=goto_table:5"

-	otx.AddFlow("table=1, priority=0, actions=drop")

+	// Table 10: VXLAN ingress filtering; filled in by AddHostSubnetRules()

+	// eg, "table=10, priority=100, tun_src=${remote_node_ip}, actions=goto_table:30"

+	otx.AddFlow("table=10, priority=0, actions=drop")

-	// Table 2: from OpenShift container; validate IP/MAC, assign tenant-id; filled in by openshift-sdn-ovs

-	// eg, "table=2, priority=100, in_port=${ovs_port}, arp, nw_src=${ipaddr}, arp_sha=${macaddr}, actions=load:${tenant_id}->NXM_NX_REG0[], goto_table:5"

-	//     "table=2, priority=100, in_port=${ovs_port}, ip, nw_src=${ipaddr}, actions=load:${tenant_id}->NXM_NX_REG0[], goto_table:3"

+	// Table 20: from OpenShift container; validate IP/MAC, assign tenant-id; filled in by openshift-sdn-ovs

+	// eg, "table=20, priority=100, in_port=${ovs_port}, arp, nw_src=${ipaddr}, arp_sha=${macaddr}, actions=load:${tenant_id}->NXM_NX_REG0[], goto_table:30"

+	//     "table=20, priority=100, in_port=${ovs_port}, ip, nw_src=${ipaddr}, actions=load:${tenant_id}->NXM_NX_REG0[], goto_table:30"

 	// (${tenant_id} is always 0 for single-tenant)

-	otx.AddFlow("table=2, priority=0, actions=drop")

-

-	// Table 3: from OpenShift container; service vs non-service

-	otx.AddFlow("table=3, priority=100, ip, nw_dst=%s, actions=goto_table:4", serviceNetworkCIDR)

-	otx.AddFlow("table=3, priority=0, actions=goto_table:5")

-

-	// Table 4: from OpenShift container; service dispatch; filled in by AddServiceRules()

-	otx.AddFlow("table=4, priority=200, reg0=0, actions=output:2")

-	// eg, "table=4, priority=100, reg0=${tenant_id}, ${service_proto}, nw_dst=${service_ip}, tp_dst=${service_port}, actions=output:2"

-	otx.AddFlow("table=4, priority=0, actions=drop")

-

-	// Table 5: general routing

-	otx.AddFlow("table=5, priority=300, arp, nw_dst=%s, actions=output:2", localSubnetGateway)

-	otx.AddFlow("table=5, priority=300, ip, nw_dst=%s, actions=output:2", localSubnetGateway)

-	otx.AddFlow("table=5, priority=200, arp, nw_dst=%s, actions=goto_table:6", localSubnetCIDR)

-	otx.AddFlow("table=5, priority=200, ip, nw_dst=%s, actions=goto_table:7", localSubnetCIDR)

-	otx.AddFlow("table=5, priority=100, arp, nw_dst=%s, actions=goto_table:8", clusterNetworkCIDR)

-	otx.AddFlow("table=5, priority=100, ip, nw_dst=%s, actions=goto_table:8", clusterNetworkCIDR)

-	otx.AddFlow("table=5, priority=0, ip, actions=goto_table:9")

-	otx.AddFlow("table=5, priority=0, arp, actions=drop")

-

-	// Table 6: ARP to container, filled in by openshift-sdn-ovs

-	// eg, "table=6, priority=100, arp, nw_dst=${container_ip}, actions=output:${ovs_port}"

-	otx.AddFlow("table=6, priority=0, actions=drop")

-

-	// Table 7: IP to container; filled in by openshift-sdn-ovs

-	// eg, "table=7, priority=100, reg0=0, ip, nw_dst=${ipaddr}, actions=output:${ovs_port}"

-	// eg, "table=7, priority=100, reg0=${tenant_id}, ip, nw_dst=${ipaddr}, actions=output:${ovs_port}"

-	otx.AddFlow("table=7, priority=0, actions=drop")

-

-	// Table 8: to remote container; filled in by AddHostSubnetRules()

-	// eg, "table=8, priority=100, arp, nw_dst=${remote_subnet_cidr}, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31], set_field:${remote_node_ip}->tun_dst,output:1"

-	// eg, "table=8, priority=100, ip, nw_dst=${remote_subnet_cidr}, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31], set_field:${remote_node_ip}->tun_dst,output:1"

-	otx.AddFlow("table=8, priority=0, actions=drop")

-

-	// Table 9: egress network policy dispatch; edited by updateEgressNetworkPolicyRules()

-	// eg, "table=9, reg0=${tenant_id}, priority=2, ip, nw_dst=${external_cidr}, actions=drop

-	otx.AddFlow("table=9, priority=0, actions=output:2")

+	otx.AddFlow("table=20, priority=0, actions=drop")

+

+	// Table 30: general routing

+	otx.AddFlow("table=30, priority=300, arp, nw_dst=%s, actions=output:2", localSubnetGateway)

+	otx.AddFlow("table=30, priority=200, arp, nw_dst=%s, actions=goto_table:40", localSubnetCIDR)

+	otx.AddFlow("table=30, priority=100, arp, nw_dst=%s, actions=goto_table:50", clusterNetworkCIDR)

+	otx.AddFlow("table=30, priority=300, ip, nw_dst=%s, actions=output:2", localSubnetGateway)

+	otx.AddFlow("table=30, priority=100, ip, nw_dst=%s, actions=goto_table:60", serviceNetworkCIDR)

+	otx.AddFlow("table=30, priority=200, ip, nw_dst=%s, actions=goto_table:70", localSubnetCIDR)

+	otx.AddFlow("table=30, priority=100, ip, nw_dst=%s, actions=goto_table:90", clusterNetworkCIDR)

+	otx.AddFlow("table=30, priority=0, ip, actions=goto_table:100")

+	otx.AddFlow("table=30, priority=0, arp, actions=drop")

+

+	// Table 40: ARP to local container, filled in by openshift-sdn-ovs

+	// eg, "table=40, priority=100, arp, nw_dst=${container_ip}, actions=output:${ovs_port}"

+	otx.AddFlow("table=40, priority=0, actions=drop")

+

+	// Table 50: ARP to remote container; filled in by AddHostSubnetRules()

+	// eg, "table=50, priority=100, arp, nw_dst=${remote_subnet_cidr}, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31], set_field:${remote_node_ip}->tun_dst,output:1"

+	otx.AddFlow("table=50, priority=0, actions=drop")

+

+	// Table 60: IP to service: vnid/port mappings; filled in by AddServiceRules()

+	otx.AddFlow("table=60, priority=200, reg0=0, actions=output:2")

+	// eg, "table=60, priority=100, reg0=${tenant_id}, ${service_proto}, nw_dst=${service_ip}, tp_dst=${service_port}, actions=load:${tenant_id}->NXM_NX_REG1[], load:2->NXM_NX_REG2[], goto_table:80"

+	otx.AddFlow("table=60, priority=0, actions=drop")

+

+	// Table 70: IP to local container: vnid/port mappings; filled in by openshift-sdn-ovs

+	// eg, "table=70, priority=100, ip, nw_dst=${ipaddr}, actions=load:${tenant_id}->NXM_NX_REG1[], load:${ovs_port}->NXM_NX_REG2[], goto_table:80"

+	otx.AddFlow("table=70, priority=0, actions=drop")

+

+	// Table 80: IP policy enforcement; mostly managed by the osdnPolicy

+	otx.AddFlow("table=80, priority=300, ip, nw_src=%s/32, actions=output:NXM_NX_REG2[]", localSubnetGateway)

+	// eg, "table=80, priority=100, reg0=${tenant_id}, reg1=${tenant_id}, actions=output:NXM_NX_REG2[]"

+	otx.AddFlow("table=80, priority=0, actions=drop")

+

+	// Table 90: IP to remote container; filled in by AddHostSubnetRules()

+	// eg, "table=90, priority=100, ip, nw_dst=${remote_subnet_cidr}, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31], set_field:${remote_node_ip}->tun_dst,output:1"

+	otx.AddFlow("table=90, priority=0, actions=drop")

+

+	// Table 100: egress network policy dispatch; edited by UpdateEgressNetworkPolicy()

+	// eg, "table=100, reg0=${tenant_id}, priority=2, ip, nw_dst=${external_cidr}, actions=drop

+	otx.AddFlow("table=100, priority=0, actions=output:2")

 	err = otx.EndTransaction()

 	if err != nil {

@@ -322,7 +331,7 @@ func (plugin *OsdnNode) SetupSDN() (bool, error) {

 	// Table 253: rule version; note action is hex bytes separated by '.'

 	otx = plugin.ovs.NewTransaction()

-	pluginVersion := getPluginVersion(plugin.multitenant)

+	pluginVersion := plugin.getPluginVersion()

 	otx.AddFlow("%s, %s%s.%s", VERSION_TABLE, VERSION_ACTION, pluginVersion[0], pluginVersion[1])

 	err = otx.EndTransaction()

 	if err != nil {

@@ -340,27 +349,27 @@ func policyNames(policies []osapi.EgressNetworkPolicy) string {

 	return strings.Join(names, ", ")

 }

-func (plugin *OsdnNode) updateEgressNetworkPolicyRules(vnid uint32) error {

+func (plugin *OsdnNode) updateEgressNetworkPolicyRules(vnid uint32) {

 	otx := plugin.ovs.NewTransaction()

 	policies := plugin.egressPolicies[vnid]

-	namespaces := plugin.vnids.GetNamespaces(vnid)

+	namespaces := plugin.policy.GetNamespaces(vnid)

 	if len(policies) == 0 {

-		otx.DeleteFlows("table=9, reg0=%d", vnid)

+		otx.DeleteFlows("table=100, reg0=%d", vnid)

 	} else if vnid == 0 {

 		glog.Errorf("EgressNetworkPolicy in global network namespace is not allowed (%s); ignoring", policyNames(policies))

 	} else if len(namespaces) > 1 {

 		glog.Errorf("EgressNetworkPolicy not allowed in shared NetNamespace (%s); dropping all traffic", strings.Join(namespaces, ", "))

-		otx.DeleteFlows("table=9, reg0=%d", vnid)

-		otx.AddFlow("table=9, reg0=%d, priority=1, actions=drop", vnid)

+		otx.DeleteFlows("table=100, reg0=%d", vnid)

+		otx.AddFlow("table=100, reg0=%d, priority=1, actions=drop", vnid)

 	} else if len(policies) > 1 {

 		glog.Errorf("multiple EgressNetworkPolicies in same network namespace (%s) is not allowed; dropping all traffic", policyNames(policies))

-		otx.DeleteFlows("table=9, reg0=%d", vnid)

-		otx.AddFlow("table=9, reg0=%d, priority=1, actions=drop", vnid)

+		otx.DeleteFlows("table=100, reg0=%d", vnid)

+		otx.AddFlow("table=100, reg0=%d, priority=1, actions=drop", vnid)

 	} else /* vnid != 0 && len(policies) == 1 */ {

 		// Temporarily drop all outgoing traffic, to avoid race conditions while modifying the other rules

-		otx.AddFlow("table=9, reg0=%d, cookie=1, priority=65535, actions=drop", vnid)

-		otx.DeleteFlows("table=9, reg0=%d, cookie=0/1", vnid)

+		otx.AddFlow("table=100, reg0=%d, cookie=1, priority=65535, actions=drop", vnid)

+		otx.DeleteFlows("table=100, reg0=%d, cookie=0/1", vnid)

 		for i, rule := range policies[0].Spec.Egress {

 			priority := len(policies[0].Spec.Egress) - i

@@ -379,99 +388,77 @@ func (plugin *OsdnNode) updateEgressNetworkPolicyRules(vnid uint32) error {

 				dst = fmt.Sprintf(", nw_dst=%s", rule.To.CIDRSelector)

 			}

-			otx.AddFlow("table=9, reg0=%d, priority=%d, ip%s, actions=%s", vnid, priority, dst, action)

+			otx.AddFlow("table=100, reg0=%d, priority=%d, ip%s, actions=%s", vnid, priority, dst, action)

 		}

-		otx.DeleteFlows("table=9, reg0=%d, cookie=1/1", vnid)

+		otx.DeleteFlows("table=100, reg0=%d, cookie=1/1", vnid)

 	}

-	err := otx.EndTransaction()

-	if err != nil {

-		return fmt.Errorf("Error updating OVS flows for EgressNetworkPolicy: %v", err)

+	if err := otx.EndTransaction(); err != nil {

+		glog.Errorf("Error updating OVS flows for EgressNetworkPolicy: %v", err)

 	}

-	return nil

 }

-func (plugin *OsdnNode) AddHostSubnetRules(subnet *osapi.HostSubnet) error {

+func (plugin *OsdnNode) AddHostSubnetRules(subnet *osapi.HostSubnet) {

 	glog.Infof("AddHostSubnetRules for %s", hostSubnetToString(subnet))

 	otx := plugin.ovs.NewTransaction()

-	otx.AddFlow("table=1, priority=100, tun_src=%s, actions=goto_table:5", subnet.HostIP)

+	otx.AddFlow("table=10, priority=100, tun_src=%s, actions=goto_table:30", subnet.HostIP)

 	if vnid, ok := subnet.Annotations[osapi.FixedVnidHost]; ok {

-		otx.AddFlow("table=8, priority=100, arp, nw_dst=%s, actions=load:%s->NXM_NX_TUN_ID[0..31],set_field:%s->tun_dst,output:1", subnet.Subnet, vnid, subnet.HostIP)

-		otx.AddFlow("table=8, priority=100, ip, nw_dst=%s, actions=load:%s->NXM_NX_TUN_ID[0..31],set_field:%s->tun_dst,output:1", subnet.Subnet, vnid, subnet.HostIP)

+		otx.AddFlow("table=50, priority=100, arp, nw_dst=%s, actions=load:%s->NXM_NX_TUN_ID[0..31],set_field:%s->tun_dst,output:1", subnet.Subnet, vnid, subnet.HostIP)

+		otx.AddFlow("table=90, priority=100, ip, nw_dst=%s, actions=load:%s->NXM_NX_TUN_ID[0..31],set_field:%s->tun_dst,output:1", subnet.Subnet, vnid, subnet.HostIP)

 	} else {

-		otx.AddFlow("table=8, priority=100, arp, nw_dst=%s, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:%s->tun_dst,output:1", subnet.Subnet, subnet.HostIP)

-		otx.AddFlow("table=8, priority=100, ip, nw_dst=%s, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:%s->tun_dst,output:1", subnet.Subnet, subnet.HostIP)

+		otx.AddFlow("table=50, priority=100, arp, nw_dst=%s, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:%s->tun_dst,output:1", subnet.Subnet, subnet.HostIP)

+		otx.AddFlow("table=90, priority=100, ip, nw_dst=%s, actions=move:NXM_NX_REG0[]->NXM_NX_TUN_ID[0..31],set_field:%s->tun_dst,output:1", subnet.Subnet, subnet.HostIP)

 	}

-	err := otx.EndTransaction()

-	if err != nil {

-		return fmt.Errorf("Error adding OVS flows for subnet: %v, %v", subnet, err)

+	if err := otx.EndTransaction(); err != nil {

+		glog.Errorf("Error adding OVS flows for subnet %q: %v", subnet.Subnet, err)

 	}

-	return nil

 }

-func (plugin *OsdnNode) DeleteHostSubnetRules(subnet *osapi.HostSubnet) error {

+func (plugin *OsdnNode) DeleteHostSubnetRules(subnet *osapi.HostSubnet) {

 	glog.Infof("DeleteHostSubnetRules for %s", hostSubnetToString(subnet))

 	otx := plugin.ovs.NewTransaction()

-	otx.DeleteFlows("table=1, tun_src=%s", subnet.HostIP)

-	otx.DeleteFlows("table=8, ip, nw_dst=%s", subnet.Subnet)

-	otx.DeleteFlows("table=8, arp, nw_dst=%s", subnet.Subnet)

-	err := otx.EndTransaction()

-	if err != nil {

-		return fmt.Errorf("Error deleting OVS flows for subnet: %v, %v", subnet, err)

+	otx.DeleteFlows("table=10, tun_src=%s", subnet.HostIP)

+	otx.DeleteFlows("table=50, arp, nw_dst=%s", subnet.Subnet)

+	otx.DeleteFlows("table=90, ip, nw_dst=%s", subnet.Subnet)

+	if err := otx.EndTransaction(); err != nil {

+		glog.Errorf("Error deleting OVS flows for subnet %q: %v", subnet.Subnet, err)

 	}

-	return nil

 }

-func (plugin *OsdnNode) AddServiceRules(service *kapi.Service, netID uint32) error {

-	if !plugin.multitenant {

-		return nil

-	}

-

+func (plugin *OsdnNode) AddServiceRules(service *kapi.Service, netID uint32) {

 	glog.V(5).Infof("AddServiceRules for %v", service)

 	otx := plugin.ovs.NewTransaction()

 	for _, port := range service.Spec.Ports {

 		otx.AddFlow(generateAddServiceRule(netID, service.Spec.ClusterIP, port.Protocol, int(port.Port)))

-		err := otx.EndTransaction()

-		if err != nil {

-			return fmt.Errorf("Error adding OVS flows for service: %v, netid: %d, %v", service, netID, err)

+		if err := otx.EndTransaction(); err != nil {

+			glog.Errorf("Error adding OVS flows for service %v, netid %d: %v", service, netID, err)

 		}

 	}

-	return nil

 }

-func (plugin *OsdnNode) DeleteServiceRules(service *kapi.Service) error {

-	if !plugin.multitenant {

-		return nil

-	}

-

+func (plugin *OsdnNode) DeleteServiceRules(service *kapi.Service) {

 	glog.V(5).Infof("DeleteServiceRules for %v", service)

 	otx := plugin.ovs.NewTransaction()

 	for _, port := range service.Spec.Ports {

 		otx.DeleteFlows(generateDeleteServiceRule(service.Spec.ClusterIP, port.Protocol, int(port.Port)))

-		err := otx.EndTransaction()

-		if err != nil {

-			return fmt.Errorf("Error deleting OVS flows for service: %v, %v", service, err)

+		if err := otx.EndTransaction(); err != nil {

+			glog.Errorf("Error deleting OVS flows for service %v: %v", service, err)

 		}

 	}

-	return nil

 }

 func generateBaseServiceRule(IP string, protocol kapi.Protocol, port int) string {

-	return fmt.Sprintf("table=4, %s, nw_dst=%s, tp_dst=%d", strings.ToLower(string(protocol)), IP, port)

+	return fmt.Sprintf("table=60, %s, nw_dst=%s, tp_dst=%d", strings.ToLower(string(protocol)), IP, port)

 }

 func generateAddServiceRule(netID uint32, IP string, protocol kapi.Protocol, port int) string {

 	baseRule := generateBaseServiceRule(IP, protocol, port)

-	if netID == 0 {

-		return fmt.Sprintf("%s, priority=100, actions=output:2", baseRule)

-	} else {

-		return fmt.Sprintf("%s, priority=100, reg0=%d, actions=output:2", baseRule, netID)

-	}

+	return fmt.Sprintf("%s, priority=100, actions=load:%d->NXM_NX_REG1[], load:2->NXM_NX_REG2[], goto_table:80", baseRule, netID)

 }

 func generateDeleteServiceRule(IP string, protocol kapi.Protocol, port int) string {

@@ -19,7 +19,7 @@ func (plugin *OsdnNode) SetupEgressNetworkPolicy() error {

 	}

 	for _, policy := range policies.Items {

-		vnid, err := plugin.vnids.GetVNID(policy.Namespace)

+		vnid, err := plugin.policy.GetVNID(policy.Namespace)

 		if err != nil {

 			glog.Warningf("Could not find netid for namespace %q: %v", policy.Namespace, err)

 			continue

@@ -28,10 +28,7 @@ func (plugin *OsdnNode) SetupEgressNetworkPolicy() error {

 	}

 	for vnid := range plugin.egressPolicies {

-		err := plugin.updateEgressNetworkPolicyRules(vnid)

-		if err != nil {

-			return err

-		}

+		plugin.updateEgressNetworkPolicyRules(vnid)

 	}

 	go utilwait.Forever(plugin.watchEgressNetworkPolicies, 0)

@@ -42,7 +39,7 @@ func (plugin *OsdnNode) watchEgressNetworkPolicies() {

 	RunEventQueue(plugin.osClient, EgressNetworkPolicies, func(delta cache.Delta) error {

 		policy := delta.Object.(*osapi.EgressNetworkPolicy)

-		vnid, err := plugin.vnids.GetVNID(policy.Namespace)

+		vnid, err := plugin.policy.GetVNID(policy.Namespace)

 		if err != nil {

 			return fmt.Errorf("Could not find netid for namespace %q: %v", policy.Namespace, err)

 		}

@@ -59,15 +56,12 @@ func (plugin *OsdnNode) watchEgressNetworkPolicies() {

 		}

 		plugin.egressPolicies[vnid] = policies

-		err = plugin.updateEgressNetworkPolicyRules(vnid)

-		if err != nil {

-			return err

-		}

+		plugin.updateEgressNetworkPolicyRules(vnid)

 		return nil

 	})

 }

-func (plugin *OsdnNode) UpdateEgressNetworkPolicyVNID(namespace string, oldVnid, newVnid uint32) error {

+func (plugin *OsdnNode) UpdateEgressNetworkPolicyVNID(namespace string, oldVnid, newVnid uint32) {

 	var policy *osapi.EgressNetworkPolicy

 	policies := plugin.egressPolicies[oldVnid]

@@ -75,21 +69,13 @@ func (plugin *OsdnNode) UpdateEgressNetworkPolicyVNID(namespace string, oldVnid,

 		if oldPolicy.Namespace == namespace {

 			policy = &oldPolicy

 			plugin.egressPolicies[oldVnid] = append(policies[:i], policies[i+1:]...)

-			err := plugin.updateEgressNetworkPolicyRules(oldVnid)

-			if err != nil {

-				return err

-			}

+			plugin.updateEgressNetworkPolicyRules(oldVnid)

 			break

 		}

 	}

 	if policy != nil {

 		plugin.egressPolicies[newVnid] = append(plugin.egressPolicies[newVnid], *policy)

-		err := plugin.updateEgressNetworkPolicyRules(newVnid)

-		if err != nil {

-			return err

-		}

+		plugin.updateEgressNetworkPolicyRules(newVnid)

 	}

-

-	return nil

 }

new file mode 100644

@@ -0,0 +1,186 @@

+package plugin

+

+import (

+	"sync"

+

+	"github.com/golang/glog"

+

+	kapi "k8s.io/kubernetes/pkg/api"

+

+	osapi "github.com/openshift/origin/pkg/sdn/api"

+)

+

+type multiTenantPlugin struct {

+	node  *OsdnNode

+	vnids *nodeVNIDMap

+

+	vnidRefsLock sync.Mutex

+	vnidRefs     map[uint32]int

+}

+

+func NewMultiTenantPlugin() osdnPolicy {

+	return &multiTenantPlugin{

+		vnidRefs: make(map[uint32]int),

+	}

+}

+

+func (mp *multiTenantPlugin) Name() string {

+	return osapi.MultiTenantPluginName

+}

+

+func (mp *multiTenantPlugin) Start(node *OsdnNode) error {

+	mp.node = node

+	mp.vnids = newNodeVNIDMap(mp, node.osClient)

+	if err := mp.vnids.Start(); err != nil {

+		return err

+	}

+

+	otx := node.ovs.NewTransaction()

+	otx.AddFlow("table=80, priority=200, reg0=0, actions=output:NXM_NX_REG2[]")

+	otx.AddFlow("table=80, priority=200, reg1=0, actions=output:NXM_NX_REG2[]")

+	if err := otx.EndTransaction(); err != nil {

+		return err

+	}

+

+	if err := mp.node.SetupEgressNetworkPolicy(); err != nil {

+		return err

+	}

+

+	return nil

+}

+

+func (mp *multiTenantPlugin) updatePodNetwork(namespace string, oldNetID, netID uint32) {

+	// FIXME: this is racy; traffic coming from the pods gets switched to the new

+	// VNID before the service and firewall rules are updated to match. We need

+	// to do the updates as a single transaction (ovs-ofctl --bundle).

+

+	pods, err := mp.node.GetLocalPods(namespace)

+	if err != nil {

+		glog.Errorf("Could not get list of local pods in namespace %q: %v", namespace, err)

+	}

+	services, err := mp.node.kClient.Core().Services(namespace).List(kapi.ListOptions{})

+	if err != nil {

+		glog.Errorf("Could not get list of services in namespace %q: %v", namespace, err)

+		services = &kapi.ServiceList{}

+	}

+

+	movedVNIDRefs := 0

+

+	// Update OF rules for the existing/old pods in the namespace

+	for _, pod := range pods {

+		err = mp.node.UpdatePod(pod)

+		if err == nil {

+			movedVNIDRefs++

+		} else {

+			glog.Errorf("Could not update pod %q in namespace %q: %v", pod.Name, namespace, err)

+		}

+	}

+

+	// Update OF rules for the old services in the namespace

+	for _, svc := range services.Items {

+		if !kapi.IsServiceIPSet(&svc) {

+			continue

+		}

+

+		mp.node.DeleteServiceRules(&svc)

+		mp.node.AddServiceRules(&svc, netID)

+		movedVNIDRefs++

+	}

+

+	if movedVNIDRefs > 0 {

+		mp.moveVNIDRefs(movedVNIDRefs, oldNetID, netID)

+	}

+

+	// Update namespace references in egress firewall rules

+	mp.node.UpdateEgressNetworkPolicyVNID(namespace, oldNetID, netID)

+}

+

+func (mp *multiTenantPlugin) AddNetNamespace(netns *osapi.NetNamespace) {

+	mp.updatePodNetwork(netns.Name, 0, netns.NetID)

+}

+

+func (mp *multiTenantPlugin) UpdateNetNamespace(netns *osapi.NetNamespace, oldNetID uint32) {

+	mp.updatePodNetwork(netns.Name, oldNetID, netns.NetID)

+}

+

+func (mp *multiTenantPlugin) DeleteNetNamespace(netns *osapi.NetNamespace) {

+	mp.updatePodNetwork(netns.Name, netns.NetID, 0)

+}

+

+func (mp *multiTenantPlugin) GetVNID(namespace string) (uint32, error) {

+	return mp.vnids.WaitAndGetVNID(namespace)

+}

+

+func (mp *multiTenantPlugin) GetNamespaces(vnid uint32) []string {

+	return mp.vnids.GetNamespaces(vnid)

+}

+

+func (mp *multiTenantPlugin) RefVNID(vnid uint32) {

+	if vnid == 0 {

+		return

+	}

+

+	mp.vnidRefsLock.Lock()

+	defer mp.vnidRefsLock.Unlock()

+	mp.vnidRefs[vnid] += 1

+	if mp.vnidRefs[vnid] > 1 {

+		return

+	}

+	glog.V(5).Infof("RefVNID %d adding rule", vnid)

+

+	otx := mp.node.ovs.NewTransaction()

+	otx.AddFlow("table=80, priority=100, reg0=%d, reg1=%d, actions=output:NXM_NX_REG2[]", vnid, vnid)

+	if err := otx.EndTransaction(); err != nil {

+		glog.Errorf("Error adding OVS flow for VNID: %v", err)

+	}

+}

+

+func (mp *multiTenantPlugin) UnrefVNID(vnid uint32) {

+	if vnid == 0 {

+		return

+	}

+

+	mp.vnidRefsLock.Lock()

+	defer mp.vnidRefsLock.Unlock()

+	if mp.vnidRefs[vnid] == 0 {

+		glog.Warningf("refcounting error on vnid %d", vnid)

+		return

+	}

+	mp.vnidRefs[vnid] -= 1

+	if mp.vnidRefs[vnid] > 0 {

+		return

+	}

+	glog.V(5).Infof("UnrefVNID %d removing rule", vnid)

+

+	otx := mp.node.ovs.NewTransaction()

+	otx.DeleteFlows("table=80, reg0=%d, reg1=%d", vnid, vnid)

+	if err := otx.EndTransaction(); err != nil {

+		glog.Errorf("Error deleting OVS flow for VNID: %v", err)

+	}

+}

+

+func (mp *multiTenantPlugin) moveVNIDRefs(num int, oldVNID, newVNID uint32) {

+	glog.V(5).Infof("moveVNIDRefs %d -> %d", oldVNID, newVNID)

+

+	mp.vnidRefsLock.Lock()

+	defer mp.vnidRefsLock.Unlock()

+

+	otx := mp.node.ovs.NewTransaction()

+	if mp.vnidRefs[oldVNID] <= num {

+		otx.DeleteFlows("table=80, reg0=%d, reg1=%d", oldVNID, oldVNID)

+	}

+	if mp.vnidRefs[newVNID] == 0 {

+		otx.AddFlow("table=80, priority=100, reg0=%d, reg1=%d, actions=output:NXM_NX_REG2[]", newVNID, newVNID)

+	}

+	err := otx.EndTransaction()

+	if err != nil {

+		glog.Errorf("Error modifying OVS flows for VNID: %v", err)

+	}

+

+	mp.vnidRefs[oldVNID] -= num

+	if mp.vnidRefs[oldVNID] < 0 {

+		glog.Warningf("refcounting error on vnid %d", oldVNID)

+		mp.vnidRefs[oldVNID] = 0

+	}

+	mp.vnidRefs[newVNID] += num

+}

@@ -20,6 +20,7 @@ import (

 	docker "github.com/fsouza/go-dockerclient"

 	kapi "k8s.io/kubernetes/pkg/api"

+	"k8s.io/kubernetes/pkg/client/cache"

 	kclientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"

 	"k8s.io/kubernetes/pkg/fields"

 	knetwork "k8s.io/kubernetes/pkg/kubelet/network"

@@ -29,8 +30,23 @@ import (

 	kwait "k8s.io/kubernetes/pkg/util/wait"

 )

+type osdnPolicy interface {

+	Name() string

+	Start(node *OsdnNode) error

+

+	AddNetNamespace(netns *osapi.NetNamespace)

+	UpdateNetNamespace(netns *osapi.NetNamespace, oldNetID uint32)

+	DeleteNetNamespace(netns *osapi.NetNamespace)

+

+	GetVNID(namespace string) (uint32, error)

+	GetNamespaces(vnid uint32) []string

+

+	RefVNID(vnid uint32)

+	UnrefVNID(vnid uint32)

+}

+

 type OsdnNode struct {

-	multitenant        bool

+	policy             osdnPolicy

 	kClient            *kclientset.Clientset

 	osClient           *osclient.Client

 	ovs                *ovs.Interface

@@ -41,7 +57,6 @@ type OsdnNode struct {

 	hostName           string

 	podNetworkReady    chan struct{}

 	kubeletInitReady   chan struct{}

-	vnids              *nodeVNIDMap

 	iptablesSyncPeriod time.Duration

 	mtu                uint32

 	egressPolicies     map[uint32][]osapi.EgressNetworkPolicy

@@ -54,7 +69,14 @@ type OsdnNode struct {

 // Called by higher layers to create the plugin SDN node instance

 func NewNodePlugin(pluginName string, osClient *osclient.Client, kClient *kclientset.Clientset, hostname string, selfIP string, iptablesSyncPeriod time.Duration, mtu uint32) (*OsdnNode, error) {

-	if !osapi.IsOpenShiftNetworkPlugin(pluginName) {

+	var policy osdnPolicy

+	switch strings.ToLower(pluginName) {

+	case osapi.SingleTenantPluginName:

+		policy = NewSingleTenantPlugin()

+	case osapi.MultiTenantPluginName:

+		policy = NewMultiTenantPlugin()

+	default:

+		// Not an OpenShift plugin

 		return nil, nil

 	}

@@ -88,13 +110,12 @@ func NewNodePlugin(pluginName string, osClient *osclient.Client, kClient *kclien

 	}

 	plugin := &OsdnNode{

-		multitenant:        osapi.IsOpenShiftMultitenantNetworkPlugin(pluginName),

+		policy:             policy,

 		kClient:            kClient,

 		osClient:           osClient,

 		ovs:                ovsif,

 		localIP:            selfIP,

 		hostName:           hostname,

-		vnids:              newNodeVNIDMap(),

 		podNetworkReady:    make(chan struct{}),

 		kubeletInitReady:   make(chan struct{}),

 		iptablesSyncPeriod: iptablesSyncPeriod,

@@ -195,14 +216,10 @@ func (node *OsdnNode) Start() error {

 		return err

 	}

-	if node.multitenant {

-		if err = node.VnidStartNode(); err != nil {

-			return err

-		}

-		if err = node.SetupEgressNetworkPolicy(); err != nil {

-			return err

-		}

+	if err = node.policy.Start(node); err != nil {

+		return err

 	}

+	go kwait.Forever(node.watchServices, 0)

 	// Wait for kubelet to init the plugin so we get a knetwork.Host

 	log.V(5).Infof("Waiting for kubelet network plugin initialization")

@@ -217,7 +234,7 @@ func (node *OsdnNode) Start() error {

 		})

 	log.V(5).Infof("Creating and initializing openshift-sdn pod manager")

-	node.podManager, err = newPodManager(node.host, node.multitenant, node.localSubnetCIDR, node.networkInfo, node.kClient, node.vnids, node.mtu)

+	node.podManager, err = newPodManager(node.host, node.localSubnetCIDR, node.networkInfo, node.kClient, node.policy, node.mtu)

 	if err != nil {

 		return err

 	}

@@ -235,6 +252,10 @@ func (node *OsdnNode) Start() error {

 			err = node.UpdatePod(p)

 			if err != nil {

 				log.Warningf("Could not update pod %q: %s", p.Name, err)

+				continue

+			}

+			if vnid, err := node.policy.GetVNID(p.Namespace); err == nil {

+				node.policy.RefVNID(vnid)

 			}

 		}

 	}

@@ -295,3 +316,60 @@ func (node *OsdnNode) IsPodNetworkReady() error {

 		return fmt.Errorf("SDN pod network is not ready")

 	}

 }

+

+func isServiceChanged(oldsvc, newsvc *kapi.Service) bool {

+	if len(oldsvc.Spec.Ports) == len(newsvc.Spec.Ports) {

+		for i := range oldsvc.Spec.Ports {

+			if oldsvc.Spec.Ports[i].Protocol != newsvc.Spec.Ports[i].Protocol ||

+				oldsvc.Spec.Ports[i].Port != newsvc.Spec.Ports[i].Port {

+				return true

+			}

+		}

+		return false

+	}

+	return true

+}

+

+func (node *OsdnNode) watchServices() {

+	services := make(map[string]*kapi.Service)

+	RunEventQueue(node.kClient.CoreClient, Services, func(delta cache.Delta) error {

+		serv := delta.Object.(*kapi.Service)

+

+		// Ignore headless services

+		if !kapi.IsServiceIPSet(serv) {

+			return nil

+		}

+

+		log.V(5).Infof("Watch %s event for Service %q", delta.Type, serv.ObjectMeta.Name)

+		switch delta.Type {

+		case cache.Sync, cache.Added, cache.Updated:

+			oldsvc, exists := services[string(serv.UID)]

+			if exists {

+				if !isServiceChanged(oldsvc, serv) {

+					break

+				}

+				node.DeleteServiceRules(oldsvc)

+			}

+

+			netid, err := node.policy.GetVNID(serv.Namespace)

+			if err != nil {

+				return fmt.Errorf("skipped adding service rules for serviceEvent: %v, Error: %v", delta.Type, err)

+			}

+

+			node.AddServiceRules(serv, netid)

+			services[string(serv.UID)] = serv

+			if !exists {

+				node.policy.RefVNID(netid)

+			}

+		case cache.Deleted:

+			delete(services, string(serv.UID))

+			node.DeleteServiceRules(serv)

+

+			netid, err := node.policy.GetVNID(serv.Namespace)

+			if err == nil {

+				node.policy.UnrefVNID(netid)

+			}

+		}

+		return nil

+	})

+}

@@ -33,9 +33,8 @@ type podManager struct {

 	runningPods map[string]*kubehostport.RunningPod

 	// Live pod setup/teardown stuff not used in testing code

-	multitenant     bool

 	kClient         *kclientset.Clientset

-	vnids           *nodeVNIDMap

+	policy          osdnPolicy

 	ipamConfig      []byte

 	mtu             uint32

 	hostportHandler kubehostport.HostportHandler

@@ -43,11 +42,10 @@ type podManager struct {

 }

 // Creates a new live podManager; used by node code

-func newPodManager(host knetwork.Host, multitenant bool, localSubnetCIDR string, netInfo *NetworkInfo, kClient *kclientset.Clientset, vnids *nodeVNIDMap, mtu uint32) (*podManager, error) {

+func newPodManager(host knetwork.Host, localSubnetCIDR string, netInfo *NetworkInfo, kClient *kclientset.Clientset, policy osdnPolicy, mtu uint32) (*podManager, error) {

 	pm := newDefaultPodManager(host)

-	pm.multitenant = multitenant

 	pm.kClient = kClient

-	pm.vnids = vnids

+	pm.policy = policy

 	pm.mtu = mtu

 	pm.hostportHandler = kubehostport.NewHostportHandler()

 	pm.podHandler = pm

@@ -91,11 +91,9 @@ func (m *podManager) getPodConfig(req *cniserver.PodRequest) (*PodConfig, *kapi.

 	var err error

 	config := &PodConfig{}

-	if m.multitenant {

-		config.vnid, err = m.vnids.GetVNID(req.PodNamespace)

-		if err != nil {

-			return nil, nil, err

-		}

+	config.vnid, err = m.policy.GetVNID(req.PodNamespace)

+	if err != nil {

+		return nil, nil, err

 	}

 	pod, err := m.kClient.Pods(req.PodNamespace).Get(req.PodName)

@@ -449,6 +447,7 @@ func (m *podManager) setup(req *cniserver.PodRequest) (*cnitypes.Result, *kubeho

 		return nil, nil, err

 	}

+	m.policy.RefVNID(podConfig.vnid)

 	success = true

 	return ipamResult, newPod, nil

 }

@@ -523,6 +522,9 @@ func (m *podManager) teardown(req *cniserver.PodRequest) error {

 			return err

 		}

 	}

+	if vnid, err := m.policy.GetVNID(req.PodNamespace); err == nil {

+		m.policy.UnrefVNID(vnid)

+	}

 	if err := m.ipamDel(req.ContainerId); err != nil {

 		return err

new file mode 100644

@@ -0,0 +1,44 @@

+package plugin

+

+import (

+	osapi "github.com/openshift/origin/pkg/sdn/api"

+)

+

+type singleTenantPlugin struct{}

+

+func NewSingleTenantPlugin() osdnPolicy {

+	return &singleTenantPlugin{}

+}

+

+func (sp *singleTenantPlugin) Name() string {

+	return osapi.SingleTenantPluginName

+}

+

+func (sp *singleTenantPlugin) Start(node *OsdnNode) error {

+	otx := node.ovs.NewTransaction()

+	otx.AddFlow("table=80, priority=200, actions=output:NXM_NX_REG2[]")

+	return otx.EndTransaction()

+}

+

+func (sp *singleTenantPlugin) AddNetNamespace(netns *osapi.NetNamespace) {

+}

+

+func (sp *singleTenantPlugin) UpdateNetNamespace(netns *osapi.NetNamespace, oldNetID uint32) {

+}

+