@@ -104,6 +104,10 @@ type JoinInfo interface {

 	// SetGatewayIPv6 sets the default IPv6 gateway when a container joins the endpoint.

 	SetGatewayIPv6(net.IP) error

+	// AddStaticRoute adds a routes to the sandbox.

+	// It may be used in addtion to or instead of a default gateway (as above).

+	AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP, interfaceID int) error

+

 	// SetHostsPath sets the overriding /etc/hosts path to use for the container.

 	SetHostsPath(string) error

@@ -87,6 +87,7 @@ type testEndpoint struct {

 	gw6            net.IP

 	hostsPath      string

 	resolvConfPath string

+	routes         []types.StaticRoute

 }

 func (te *testEndpoint) Interfaces() []driverapi.InterfaceInfo {

@@ -157,6 +158,11 @@ func (te *testEndpoint) SetResolvConfPath(path string) error {

 	return nil

 }

+func (te *testEndpoint) AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP, interfaceID int) error {

+	te.routes = append(te.routes, types.StaticRoute{destination, routeType, nextHop, interfaceID})

+	return nil

+}

+

 func TestQueryEndpointInfo(t *testing.T) {

 	testQueryEndpointInfo(t, true)

 }

@@ -149,6 +149,11 @@ func (test *testEndpoint) SetNames(src string, dst string) error {

 	return nil

 }

+func (test *testEndpoint) AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP, interfaceID int) error {

+	//TODO

+	return nil

+}

+

 func (test *testEndpoint) ID() int {

 	return test.id

 }

@@ -293,6 +293,7 @@ func (ep *endpoint) Join(containerID string, options ...EndpointOption) (*Contai

 			SrcName: i.srcName,

 			DstName: i.dstPrefix,

 			Address: &i.addr,

+			Routes:  i.routes,

 		}

 		if i.addrv6.IP.To16() != nil {

 			iface.AddressIPv6 = &i.addrv6

@@ -302,6 +303,13 @@ func (ep *endpoint) Join(containerID string, options ...EndpointOption) (*Contai

 			return nil, err

 		}

 	}

+	// Set up non-interface routes.

+	for _, r := range ep.joinInfo.StaticRoutes {

+		err = sb.AddStaticRoute(r)

+		if err != nil {

+			return nil, err

+		}

+	}

 	err = sb.SetGateway(joinInfo.gw)

 	if err != nil {

@@ -360,6 +368,14 @@ func (ep *endpoint) Leave(containerID string, options ...EndpointOption) error {

 		}

 	}

+	// Remove non-interface routes.

+	for _, r := range ep.joinInfo.StaticRoutes {

+		err = sb.RemoveStaticRoute(r)

+		if err != nil {

+			logrus.Debugf("Remove route failed: %v", err)

+		}

+	}

+

 	ctrlr.sandboxRm(container.data.SandboxKey)

 	return err

@@ -46,6 +46,7 @@ type endpointInterface struct {

 	addrv6    net.IPNet

 	srcName   string

 	dstPrefix string

+	routes    []*net.IPNet

 }

 type endpointJoinInfo struct {

@@ -53,6 +54,7 @@ type endpointJoinInfo struct {

 	gw6            net.IP

 	hostsPath      string

 	resolvConfPath string

+	StaticRoutes   []*types.StaticRoute

 }

 func (ep *endpoint) Info() EndpointInfo {

@@ -149,6 +151,35 @@ func (ep *endpoint) InterfaceNames() []driverapi.InterfaceNameInfo {

 	return iList

 }

+func (ep *endpoint) AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP, interfaceID int) error {

+	ep.Lock()

+	defer ep.Unlock()

+

+	r := types.StaticRoute{destination, routeType, nextHop, interfaceID}

+

+	if routeType == types.NEXTHOP {

+		// If the route specifies a next-hop, then it's loosely routed (i.e. not bound to a particular interface).

+		ep.joinInfo.StaticRoutes = append(ep.joinInfo.StaticRoutes, &r)

+	} else {

+		// If the route doesn't specify a next-hop, it must be a connected route, bound to an interface.

+		if err := ep.addInterfaceRoute(&r); err != nil {

+			return err

+		}

+	}

+	return nil

+}

+

+func (ep *endpoint) addInterfaceRoute(route *types.StaticRoute) error {

+	for _, iface := range ep.iFaces {

+		if iface.id == route.InterfaceID {

+			iface.routes = append(iface.routes, route.Destination)

+			return nil

+		}

+	}

+	return types.BadRequestErrorf("Interface with ID %d doesn't exist.",

+		route.InterfaceID)

+}

+

 func (ep *endpoint) SandboxKey() string {

 	ep.Lock()

 	defer ep.Unlock()

@@ -19,6 +19,7 @@ func configureInterface(iface netlink.Link, settings *Interface) error {

 		{setInterfaceName, fmt.Sprintf("error renaming interface %q to %q", ifaceName, settings.DstName)},

 		{setInterfaceIP, fmt.Sprintf("error setting interface %q IP to %q", ifaceName, settings.Address)},

 		{setInterfaceIPv6, fmt.Sprintf("error setting interface %q IPv6 to %q", ifaceName, settings.AddressIPv6)},

+		{setInterfaceRoutes, fmt.Sprintf("error setting interface %q routes to %q", ifaceName, settings.Routes)},

 	}

 	for _, config := range ifaceConfigurators {

@@ -63,6 +64,78 @@ func programGateway(path string, gw net.IP) error {

 	})

 }

+// Program a route in to the namespace routing table.

+func programRoute(path string, dest *net.IPNet, nh net.IP) error {

+	runtime.LockOSThread()

+	defer runtime.UnlockOSThread()

+

+	origns, err := netns.Get()

+	if err != nil {

+		return err

+	}

+	defer origns.Close()

+

+	f, err := os.OpenFile(path, os.O_RDONLY, 0)

+	if err != nil {

+		return fmt.Errorf("failed get network namespace %q: %v", path, err)

+	}

+	defer f.Close()

+

+	nsFD := f.Fd()

+	if err = netns.Set(netns.NsHandle(nsFD)); err != nil {

+		return err

+	}

+	defer netns.Set(origns)

+

+	gwRoutes, err := netlink.RouteGet(nh)

+	if err != nil {

+		return fmt.Errorf("route for the next hop could not be found: %v", err)

+	}

+

+	return netlink.RouteAdd(&netlink.Route{

+		Scope:     netlink.SCOPE_UNIVERSE,

+		LinkIndex: gwRoutes[0].LinkIndex,

+		Gw:        gwRoutes[0].Gw,

+		Dst:       dest,

+	})

+}

+

+// Delete a route from the namespace routing table.

+func removeRoute(path string, dest *net.IPNet, nh net.IP) error {

+	runtime.LockOSThread()

+	defer runtime.UnlockOSThread()

+

+	origns, err := netns.Get()

+	if err != nil {

+		return err

+	}

+	defer origns.Close()

+

+	f, err := os.OpenFile(path, os.O_RDONLY, 0)

+	if err != nil {

+		return fmt.Errorf("failed get network namespace %q: %v", path, err)

+	}

+	defer f.Close()

+

+	nsFD := f.Fd()

+	if err = netns.Set(netns.NsHandle(nsFD)); err != nil {

+		return err

+	}

+	defer netns.Set(origns)

+

+	gwRoutes, err := netlink.RouteGet(nh)

+	if err != nil {

+		return fmt.Errorf("route for the next hop could not be found: %v", err)

+	}

+

+	return netlink.RouteDel(&netlink.Route{

+		Scope:     netlink.SCOPE_UNIVERSE,

+		LinkIndex: gwRoutes[0].LinkIndex,

+		Gw:        gwRoutes[0].Gw,

+		Dst:       dest,

+	})

+}

+

 func setInterfaceIP(iface netlink.Link, settings *Interface) error {

 	ipAddr := &netlink.Addr{IPNet: settings.Address, Label: ""}

 	return netlink.AddrAdd(iface, ipAddr)

@@ -79,3 +152,17 @@ func setInterfaceIPv6(iface netlink.Link, settings *Interface) error {

 func setInterfaceName(iface netlink.Link, settings *Interface) error {

 	return netlink.LinkSetName(iface, settings.DstName)

 }

+

+func setInterfaceRoutes(iface netlink.Link, settings *Interface) error {

+	for _, route := range settings.Routes {

+		err := netlink.RouteAdd(&netlink.Route{

+			Scope:     netlink.SCOPE_UNIVERSE,

+			LinkIndex: iface.Attrs().Index,

+			Dst:       route,

+		})

+		if err != nil {

+			return err

+		}

+	}

+	return nil

+}

@@ -8,6 +8,7 @@ import (

 	"sync"

 	"syscall"

+	"github.com/docker/libnetwork/types"

 	"github.com/vishvananda/netlink"

 	"github.com/vishvananda/netns"

 )

@@ -263,6 +264,35 @@ func (n *networkNamespace) SetGatewayIPv6(gw net.IP) error {

 	return err

 }

+func (n *networkNamespace) AddStaticRoute(r *types.StaticRoute) error {

+	err := programRoute(n.path, r.Destination, r.NextHop)

+	if err == nil {

+		n.Lock()

+		n.sinfo.StaticRoutes = append(n.sinfo.StaticRoutes, r)

+		n.Unlock()

+	}

+	return err

+}

+

+func (n *networkNamespace) RemoveStaticRoute(r *types.StaticRoute) error {

+	err := removeRoute(n.path, r.Destination, r.NextHop)

+	if err == nil {

+		n.Lock()

+		lastIndex := len(n.sinfo.StaticRoutes) - 1

+		for i, v := range n.sinfo.StaticRoutes {

+			if v == r {

+				// Overwrite the route we're removing with the last element

+				n.sinfo.StaticRoutes[i] = n.sinfo.StaticRoutes[lastIndex]

+				// Shorten the slice to trim the extra element

+				n.sinfo.StaticRoutes = n.sinfo.StaticRoutes[:lastIndex]

+				break

+			}

+		}

+		n.Unlock()

+	}

+	return err

+}

+

 func (n *networkNamespace) Interfaces() []*Interface {

 	n.Lock()

 	defer n.Unlock()

@@ -35,6 +35,12 @@ type Sandbox interface {

 	// Set default IPv6 gateway for the sandbox

 	SetGatewayIPv6(gw net.IP) error

+	// Add a static route to the sandbox.

+	AddStaticRoute(*types.StaticRoute) error

+

+	// Remove a static route from the sandbox.

+	RemoveStaticRoute(*types.StaticRoute) error

+

 	// Destroy the sandbox

 	Destroy() error

 }

@@ -51,7 +57,11 @@ type Info struct {

 	// IPv6 gateway for the sandbox.

 	GatewayIPv6 net.IP

-	// TODO: Add routes and ip tables etc.

+	// Additional static routes for the sandbox.  (Note that directly

+	// connected routes are stored on the particular interface they refer to.)

+	StaticRoutes []*types.StaticRoute

+

+	// TODO: Add ip tables etc.

 }

 // Interface represents the settings and identity of a network device. It is

@@ -74,15 +84,25 @@ type Interface struct {

 	// IPv6 address for the interface.

 	AddressIPv6 *net.IPNet

+

+	// IP routes for the interface.

+	Routes []*net.IPNet

 }

 // GetCopy returns a copy of this Interface structure

 func (i *Interface) GetCopy() *Interface {

+	copiedRoutes := make([]*net.IPNet, len(i.Routes))

+

+	for index := range i.Routes {

+		copiedRoutes[index] = types.GetIPNetCopy(i.Routes[index])

+	}

+

 	return &Interface{

 		SrcName:     i.SrcName,

 		DstName:     i.DstName,

 		Address:     types.GetIPNetCopy(i.Address),

 		AddressIPv6: types.GetIPNetCopy(i.AddressIPv6),

+		Routes:      copiedRoutes,

 	}

 }

@@ -108,6 +128,16 @@ func (i *Interface) Equal(o *Interface) bool {

 		return false

 	}

+	if len(i.Routes) != len(o.Routes) {

+		return false

+	}

+

+	for index := range i.Routes {

+		if !types.CompareIPNet(i.Routes[index], o.Routes[index]) {

+			return false

+		}

+	}

+

 	return true

 }

@@ -120,7 +150,15 @@ func (s *Info) GetCopy() *Info {

 	gw := types.GetIPCopy(s.Gateway)

 	gw6 := types.GetIPCopy(s.GatewayIPv6)

-	return &Info{Interfaces: list, Gateway: gw, GatewayIPv6: gw6}

+	routes := make([]*types.StaticRoute, len(s.StaticRoutes))

+	for i, r := range s.StaticRoutes {

+		routes[i] = r.GetCopy()

+	}

+

+	return &Info{Interfaces: list,

+		Gateway:      gw,

+		GatewayIPv6:  gw6,

+		StaticRoutes: routes}

 }

 // Equal checks if this instance of SandboxInfo is equal to the passed one

@@ -154,6 +192,17 @@ func (s *Info) Equal(o *Info) bool {

 		}

 	}

+	for index := range s.StaticRoutes {

+		ss := s.StaticRoutes[index]

+		oo := o.StaticRoutes[index]

+		if !types.CompareIPNet(ss.Destination, oo.Destination) {

+			return false

+		}

+		if !ss.NextHop.Equal(oo.NextHop) {

+			return false

+		}

+	}

+

 	return true

 }

@@ -63,6 +63,13 @@ func newInfo(t *testing.T) (*Info, error) {

 	intf1.AddressIPv6 = addrv6

 	intf1.AddressIPv6.IP = ip6

+	_, route, err := net.ParseCIDR("192.168.2.1/32")

+	if err != nil {

+		return nil, err

+	}

+

+	intf1.Routes = []*net.IPNet{route}

+

 	veth = &netlink.Veth{

 		LinkAttrs: netlink.LinkAttrs{Name: vethName3, TxQLen: 0},

 		PeerName:  vethName4}

@@ -92,6 +99,7 @@ func newInfo(t *testing.T) (*Info, error) {

 	intf2.AddressIPv6.IP = ip6

 	sinfo := &Info{Interfaces: []*Interface{intf1, intf2}}

+

 	sinfo.Gateway = net.ParseIP("192.168.1.1")

 	// sinfo.GatewayIPv6 = net.ParseIP("2001:DB8::1")

 	sinfo.GatewayIPv6 = net.ParseIP("fe80::1")

@@ -190,12 +190,14 @@ func getInterfaceList() []*Interface {

 			DstName:     "eth0",

 			Address:     netv4a,

 			AddressIPv6: netv6a,

+			Routes:      []*net.IPNet{netv4a, netv6a},

 		},

 		&Interface{

 			SrcName:     "veth7654321",

 			DstName:     "eth1",

 			Address:     netv4b,

 			AddressIPv6: netv6b,

+			Routes:      []*net.IPNet{netv4b, netv6b},

 		},

 	}

 }

@@ -184,6 +184,40 @@ func CompareIPNet(a, b *net.IPNet) bool {

 	return a.IP.Equal(b.IP) && bytes.Equal(a.Mask, b.Mask)

 }

+const (

+	// NEXTHOP indicates a StaticRoute with an IP next hop.

+	NEXTHOP = iota

+

+	// CONNECTED indicates a StaticRoute with a interface for directly connected peers.

+	CONNECTED

+)

+

+// StaticRoute is a statically-provisioned IP route.

+type StaticRoute struct {

+	Destination *net.IPNet

+

+	RouteType int // NEXT_HOP or CONNECTED

+

+	// NextHop will be resolved by the kernel (i.e. as a loose hop).

+	NextHop net.IP

+

+	// InterfaceID must refer to a defined interface on the

+	// Endpoint to which the routes are specified.  Routes specified this way

+	// are interpreted as directly connected to the specified interface (no

+	// next hop will be used).

+	InterfaceID int

+}

+

+// GetCopy returns a copy of this StaticRoute structure

+func (r *StaticRoute) GetCopy() *StaticRoute {

+	d := GetIPNetCopy(r.Destination)

+	nh := GetIPCopy(r.NextHop)

+	return &StaticRoute{Destination: d,

+		RouteType:   r.RouteType,

+		NextHop:     nh,

+		InterfaceID: r.InterfaceID}

+}

+

 /******************************

  * Well-known Error Interfaces

  ******************************/