Browse code

libnet/drvapi: make NetworkAllocate optional

This method is only used by the cnmallocator to allocate Swarm-scoped
network resources. Its only concrete implementation is in the ovmanager.
Other network drivers are implementing it too to adhere to the
driverapi.Driver interface, but they all return a 'not implemented'
error.

Extract this method into a separate interface, and add a dedicated
RegisterNetworkAllocator to the driver registry. Update the cnmallocator
to load 'network allocators' instead of 'drivers'.

Signed-off-by: Albin Kerouanton <albinker@gmail.com>

Albin Kerouanton authored on 2025/09/01 07:14:38
Showing 18 changed files
... ...
@@ -15,7 +15,6 @@ import (
15 15
 	"github.com/moby/moby/v2/daemon/libnetwork/ipams/defaultipam"
16 16
 	remoteipam "github.com/moby/moby/v2/daemon/libnetwork/ipams/remote"
17 17
 	"github.com/moby/moby/v2/daemon/libnetwork/netlabel"
18
-	"github.com/moby/moby/v2/daemon/libnetwork/scope"
19 18
 	"github.com/moby/moby/v2/pkg/plugingetter"
20 19
 	"github.com/moby/swarmkit/v2/api"
21 20
 	"github.com/moby/swarmkit/v2/manager/allocator/networkallocator"
... ...
@@ -81,7 +80,7 @@ type network struct {
81 81
 }
82 82
 
83 83
 type networkDriver struct {
84
-	driver driverapi.Driver // driver is nil when isNodeLocal == true
84
+	driver driverapi.NetworkAllocator // driver is nil when isNodeLocal == true
85 85
 	name   string
86 86
 	// isNodeLocal indicates whether that driver is locally-managed or requires
87 87
 	// global resources allocation.
... ...
@@ -786,24 +785,25 @@ func (na *cnmNetworkAllocator) resolveDriver(n *api.Network) (*networkDriver, er
786 786
 		return &networkDriver{name: dName, isNodeLocal: true}, nil
787 787
 	}
788 788
 
789
-	d, drvCap := na.networkRegistry.Driver(dName)
790
-	if d == nil {
791
-		err := na.loadDriver(dName)
792
-		if err != nil {
793
-			return nil, err
789
+	if na.networkRegistry.HasDriverOrNwAllocator(dName) {
790
+		if nwAlloc := na.networkRegistry.NetworkAllocator(dName); nwAlloc != nil {
791
+			return &networkDriver{driver: nwAlloc, name: dName}, nil
794 792
 		}
793
+		return &networkDriver{name: dName, isNodeLocal: true}, nil
794
+	}
795 795
 
796
-		d, drvCap = na.networkRegistry.Driver(dName)
797
-		if d == nil {
798
-			return nil, fmt.Errorf("could not resolve network driver %s", dName)
796
+	if err := na.loadDriver(dName); err != nil {
797
+		return nil, err
798
+	}
799
+
800
+	if na.networkRegistry.HasDriverOrNwAllocator(dName) {
801
+		if nwAlloc := na.networkRegistry.NetworkAllocator(dName); nwAlloc != nil {
802
+			return &networkDriver{driver: nwAlloc, name: dName}, nil
799 803
 		}
804
+		return &networkDriver{name: dName, isNodeLocal: true}, nil
800 805
 	}
801 806
 
802
-	return &networkDriver{
803
-		driver:      d,
804
-		name:        dName,
805
-		isNodeLocal: drvCap.DataScope == scope.Local,
806
-	}, nil
807
+	return nil, fmt.Errorf("network driver %s not found", dName)
807 808
 }
808 809
 
809 810
 func (na *cnmNetworkAllocator) loadDriver(name string) error {
... ...
@@ -512,6 +512,10 @@ func (c *Controller) RegisterDriver(networkType string, driver driverapi.Driver,
512 512
 	return nil
513 513
 }
514 514
 
515
+func (c *Controller) RegisterNetworkAllocator(_ string, _ driverapi.NetworkAllocator) error {
516
+	return nil
517
+}
518
+
515 519
 // XXX  This should be made driver agnostic.  See comment below.
516 520
 const overlayDSROptionString = "dsr"
517 521
 
... ...
@@ -13,16 +13,6 @@ const NetworkPluginEndpointType = "NetworkDriver"
13 13
 
14 14
 // Driver is an interface that every plugin driver needs to implement.
15 15
 type Driver interface {
16
-	// NetworkAllocate invokes the driver method to allocate network
17
-	// specific resources passing network id and network specific config.
18
-	// It returns a key,value pair of network specific driver allocations
19
-	// to the caller.
20
-	NetworkAllocate(nid string, options map[string]string, ipV4Data, ipV6Data []IPAMData) (map[string]string, error)
21
-
22
-	// NetworkFree invokes the driver method to free network specific resources
23
-	// associated with a given network id.
24
-	NetworkFree(nid string) error
25
-
26 16
 	// CreateNetwork invokes the driver method to create a network
27 17
 	// passing the network id and network specific config. The
28 18
 	// config mechanism will eventually be replaced with labels
... ...
@@ -64,6 +54,23 @@ type Driver interface {
64 64
 	IsBuiltIn() bool
65 65
 }
66 66
 
67
+// NetworkAllocator is a special kind of network driver used by cnmallocator to
68
+// allocate resources inside a Swarm cluster.
69
+type NetworkAllocator interface {
70
+	// NetworkAllocate invokes the driver method to allocate network
71
+	// specific resources passing network id and network specific config.
72
+	// It returns a key,value pair of network specific driver allocations
73
+	// to the caller.
74
+	NetworkAllocate(nid string, options map[string]string, ipV4Data, ipV6Data []IPAMData) (map[string]string, error)
75
+
76
+	// NetworkFree invokes the driver method to free network specific resources
77
+	// associated with a given network id.
78
+	NetworkFree(nid string) error
79
+
80
+	// IsBuiltIn returns true if it is a built-in driver
81
+	IsBuiltIn() bool
82
+}
83
+
67 84
 // TableWatcher is an optional interface for a network driver.
68 85
 type TableWatcher interface {
69 86
 	// EventNotify notifies the driver when a CRUD operation has
... ...
@@ -187,6 +194,7 @@ type JoinInfo interface {
187 187
 // Registerer provides a way for network drivers to be dynamically registered.
188 188
 type Registerer interface {
189 189
 	RegisterDriver(name string, driver Driver, capability Capability) error
190
+	RegisterNetworkAllocator(name string, driver NetworkAllocator) error
190 191
 }
191 192
 
192 193
 // Capability represents the high level capabilities of the drivers which libnetwork can make use of
... ...
@@ -689,14 +689,6 @@ func (d *driver) getNetworks() []*bridgeNetwork {
689 689
 	return ls
690 690
 }
691 691
 
692
-func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
693
-	return nil, types.NotImplementedErrorf("not implemented")
694
-}
695
-
696
-func (d *driver) NetworkFree(id string) error {
697
-	return types.NotImplementedErrorf("not implemented")
698
-}
699
-
700 692
 func (d *driver) GetSkipGwAlloc(opts options.Generic) (ipv4, ipv6 bool, _ error) {
701 693
 	// The network doesn't exist yet, so use a dummy id that's long enough to be
702 694
 	// truncated to a short-id (12 characters) and used in the bridge device name.
... ...
@@ -23,14 +23,6 @@ func Register(r driverapi.Registerer) error {
23 23
 	})
24 24
 }
25 25
 
26
-func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
27
-	return nil, types.NotImplementedErrorf("not implemented")
28
-}
29
-
30
-func (d *driver) NetworkFree(id string) error {
31
-	return types.NotImplementedErrorf("not implemented")
32
-}
33
-
34 26
 func (d *driver) CreateNetwork(ctx context.Context, id string, option map[string]any, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
35 27
 	d.Lock()
36 28
 	defer d.Unlock()
... ...
@@ -9,7 +9,6 @@ import (
9 9
 	"github.com/moby/moby/v2/daemon/libnetwork/datastore"
10 10
 	"github.com/moby/moby/v2/daemon/libnetwork/driverapi"
11 11
 	"github.com/moby/moby/v2/daemon/libnetwork/scope"
12
-	"github.com/moby/moby/v2/daemon/libnetwork/types"
13 12
 )
14 13
 
15 14
 const (
... ...
@@ -76,14 +75,6 @@ func Register(r driverapi.Registerer, store *datastore.Store, config map[string]
76 76
 	})
77 77
 }
78 78
 
79
-func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
80
-	return nil, types.NotImplementedErrorf("not implemented")
81
-}
82
-
83
-func (d *driver) NetworkFree(id string) error {
84
-	return types.NotImplementedErrorf("not implemented")
85
-}
86
-
87 79
 func (d *driver) EndpointOperInfo(nid, eid string) (map[string]any, error) {
88 80
 	return make(map[string]any), nil
89 81
 }
... ...
@@ -31,6 +31,11 @@ func (dt *driverTester) RegisterDriver(name string, drv driverapi.Driver, capabi
31 31
 	return nil
32 32
 }
33 33
 
34
+func (dt *driverTester) RegisterNetworkAllocator(name string, _ driverapi.NetworkAllocator) error {
35
+	dt.t.Fatalf("Unexpected call to RegisterNetworkAllocator for %q", name)
36
+	return nil
37
+}
38
+
34 39
 func TestIpvlanRegister(t *testing.T) {
35 40
 	if err := Register(&driverTester{t: t}, storeutils.NewTempStore(t), nil); err != nil {
36 41
 		t.Fatal(err)
... ...
@@ -9,7 +9,6 @@ import (
9 9
 	"github.com/moby/moby/v2/daemon/libnetwork/datastore"
10 10
 	"github.com/moby/moby/v2/daemon/libnetwork/driverapi"
11 11
 	"github.com/moby/moby/v2/daemon/libnetwork/scope"
12
-	"github.com/moby/moby/v2/daemon/libnetwork/types"
13 12
 )
14 13
 
15 14
 const (
... ...
@@ -70,14 +69,6 @@ func Register(r driverapi.Registerer, store *datastore.Store, _ map[string]any)
70 70
 	})
71 71
 }
72 72
 
73
-func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
74
-	return nil, types.NotImplementedErrorf("not implemented")
75
-}
76
-
77
-func (d *driver) NetworkFree(id string) error {
78
-	return types.NotImplementedErrorf("not implemented")
79
-}
80
-
81 73
 func (d *driver) EndpointOperInfo(nid, eid string) (map[string]any, error) {
82 74
 	return make(map[string]any), nil
83 75
 }
... ...
@@ -31,6 +31,11 @@ func (dt *driverTester) RegisterDriver(name string, drv driverapi.Driver, capabi
31 31
 	return nil
32 32
 }
33 33
 
34
+func (dt *driverTester) RegisterNetworkAllocator(name string, _ driverapi.NetworkAllocator) error {
35
+	dt.t.Fatalf("Unexpected call to RegisterNetworkAllocator for %q", name)
36
+	return nil
37
+}
38
+
34 39
 func TestMacvlanRegister(t *testing.T) {
35 40
 	if err := Register(&driverTester{t: t}, storeutils.NewTempStore(t), nil); err != nil {
36 41
 		t.Fatal(err)
... ...
@@ -23,14 +23,6 @@ func Register(r driverapi.Registerer) error {
23 23
 	})
24 24
 }
25 25
 
26
-func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
27
-	return nil, types.NotImplementedErrorf("not implemented")
28
-}
29
-
30
-func (d *driver) NetworkFree(id string) error {
31
-	return types.NotImplementedErrorf("not implemented")
32
-}
33
-
34 26
 func (d *driver) CreateNetwork(ctx context.Context, id string, option map[string]any, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
35 27
 	d.Lock()
36 28
 	defer d.Unlock()
... ...
@@ -82,14 +82,6 @@ func init() {
82 82
 	runtime.LockOSThread()
83 83
 }
84 84
 
85
-func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
86
-	return nil, types.NotImplementedErrorf("not implemented")
87
-}
88
-
89
-func (d *driver) NetworkFree(id string) error {
90
-	return types.NotImplementedErrorf("not implemented")
91
-}
92
-
93 85
 func (d *driver) CreateNetwork(ctx context.Context, id string, option map[string]any, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
94 86
 	if id == "" {
95 87
 		return errors.New("invalid network id")
... ...
@@ -35,6 +35,11 @@ func (dt *driverTester) RegisterDriver(name string, drv driverapi.Driver, capabi
35 35
 	return nil
36 36
 }
37 37
 
38
+func (dt *driverTester) RegisterNetworkAllocator(name string, _ driverapi.NetworkAllocator) error {
39
+	dt.t.Fatalf("Unexpected call to RegisterNetworkAllocator for %q", name)
40
+	return nil
41
+}
42
+
38 43
 func TestOverlayInit(t *testing.T) {
39 44
 	if err := Register(&driverTester{t: t}, nil); err != nil {
40 45
 		t.Fatal(err)
... ...
@@ -13,8 +13,6 @@ import (
13 13
 	"github.com/moby/moby/v2/daemon/libnetwork/driverapi"
14 14
 	"github.com/moby/moby/v2/daemon/libnetwork/drivers/overlay/overlayutils"
15 15
 	"github.com/moby/moby/v2/daemon/libnetwork/netlabel"
16
-	"github.com/moby/moby/v2/daemon/libnetwork/scope"
17
-	"github.com/moby/moby/v2/daemon/libnetwork/types"
18 16
 )
19 17
 
20 18
 const (
... ...
@@ -48,10 +46,7 @@ type network struct {
48 48
 
49 49
 // Register registers a new instance of the overlay driver.
50 50
 func Register(r driverapi.Registerer) error {
51
-	return r.RegisterDriver(networkType, newDriver(), driverapi.Capability{
52
-		DataScope:         scope.Global,
53
-		ConnectivityScope: scope.Global,
54
-	})
51
+	return r.RegisterNetworkAllocator(networkType, newDriver())
55 52
 }
56 53
 
57 54
 func newDriver() *driver {
... ...
@@ -163,40 +158,6 @@ func (n *network) releaseVxlanID() {
163 163
 	}
164 164
 }
165 165
 
166
-func (d *driver) CreateNetwork(ctx context.Context, id string, option map[string]any, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
167
-	return types.NotImplementedErrorf("not implemented")
168
-}
169
-
170
-func (d *driver) DeleteNetwork(nid string) error {
171
-	return types.NotImplementedErrorf("not implemented")
172
-}
173
-
174
-func (d *driver) CreateEndpoint(_ context.Context, nid, eid string, ifInfo driverapi.InterfaceInfo, epOptions map[string]any) error {
175
-	return types.NotImplementedErrorf("not implemented")
176
-}
177
-
178
-func (d *driver) DeleteEndpoint(nid, eid string) error {
179
-	return types.NotImplementedErrorf("not implemented")
180
-}
181
-
182
-func (d *driver) EndpointOperInfo(nid, eid string) (map[string]any, error) {
183
-	return nil, types.NotImplementedErrorf("not implemented")
184
-}
185
-
186
-// Join method is invoked when a Sandbox is attached to an endpoint.
187
-func (d *driver) Join(_ context.Context, nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, _, _ map[string]any) error {
188
-	return types.NotImplementedErrorf("not implemented")
189
-}
190
-
191
-// Leave method is invoked when a Sandbox detaches from an endpoint.
192
-func (d *driver) Leave(nid, eid string) error {
193
-	return types.NotImplementedErrorf("not implemented")
194
-}
195
-
196
-func (d *driver) Type() string {
197
-	return networkType
198
-}
199
-
200 166
 func (d *driver) IsBuiltIn() bool {
201 167
 	return true
202 168
 }
... ...
@@ -64,6 +64,11 @@ func Register(r driverapi.Registerer, pg plugingetter.PluginGetter) error {
64 64
 		if err = r.RegisterDriver(name, d, *c); err != nil {
65 65
 			log.G(context.TODO()).Errorf("error registering driver for %s due to %v", name, err)
66 66
 		}
67
+		if c.DataScope == scope.Global {
68
+			if err := r.RegisterNetworkAllocator(name, d); err != nil {
69
+				log.G(context.TODO()).Errorf("error registering network allocator for %s due to %v", name, err)
70
+			}
71
+		}
67 72
 	}
68 73
 
69 74
 	// Unit test code is unaware of a true PluginStore. So we fall back to v1 plugins.
... ...
@@ -54,14 +54,6 @@ type network struct {
54 54
 	sync.Mutex
55 55
 }
56 56
 
57
-func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
58
-	return nil, types.NotImplementedErrorf("not implemented")
59
-}
60
-
61
-func (d *driver) NetworkFree(id string) error {
62
-	return types.NotImplementedErrorf("not implemented")
63
-}
64
-
65 57
 func (d *driver) CreateNetwork(ctx context.Context, id string, option map[string]any, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
66 58
 	var (
67 59
 		networkName   string
... ...
@@ -943,14 +943,6 @@ func (d *driver) Leave(nid, eid string) error {
943 943
 	return nil
944 944
 }
945 945
 
946
-func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
947
-	return nil, types.NotImplementedErrorf("not implemented")
948
-}
949
-
950
-func (d *driver) NetworkFree(id string) error {
951
-	return types.NotImplementedErrorf("not implemented")
952
-}
953
-
954 946
 func (d *driver) Type() string {
955 947
 	return d.name
956 948
 }
... ...
@@ -22,8 +22,9 @@ type Networks struct {
22 22
 	// Notify is called whenever a network driver is registered.
23 23
 	Notify driverapi.Registerer
24 24
 
25
-	mu      sync.Mutex
26
-	drivers map[string]driverData
25
+	mu       sync.Mutex
26
+	drivers  map[string]driverData
27
+	nwAllocs map[string]driverapi.NetworkAllocator
27 28
 }
28 29
 
29 30
 var _ driverapi.Registerer = (*Networks)(nil)
... ...
@@ -88,3 +89,51 @@ func (nr *Networks) RegisterDriver(ntype string, driver driverapi.Driver, capabi
88 88
 
89 89
 	return nil
90 90
 }
91
+
92
+// NetworkAllocator returns the NetworkAllocator registered under name, and its capability.
93
+func (nr *Networks) NetworkAllocator(name string) driverapi.NetworkAllocator {
94
+	nr.mu.Lock()
95
+	defer nr.mu.Unlock()
96
+
97
+	d := nr.nwAllocs[name]
98
+	return d
99
+}
100
+
101
+func (nr *Networks) RegisterNetworkAllocator(ntype string, nwAlloc driverapi.NetworkAllocator) error {
102
+	if strings.TrimSpace(ntype) == "" {
103
+		return errors.New("network type string cannot be empty")
104
+	}
105
+
106
+	nr.mu.Lock()
107
+	dd, ok := nr.nwAllocs[ntype]
108
+	nr.mu.Unlock()
109
+
110
+	if ok && dd.IsBuiltIn() {
111
+		return driverapi.ErrActiveRegistration(ntype)
112
+	}
113
+
114
+	if nr.Notify != nil {
115
+		if err := nr.Notify.RegisterNetworkAllocator(ntype, nwAlloc); err != nil {
116
+			return err
117
+		}
118
+	}
119
+
120
+	nr.mu.Lock()
121
+	defer nr.mu.Unlock()
122
+
123
+	if nr.nwAllocs == nil {
124
+		nr.nwAllocs = make(map[string]driverapi.NetworkAllocator)
125
+	}
126
+	nr.nwAllocs[ntype] = nwAlloc
127
+
128
+	return nil
129
+}
130
+
131
+func (nr *Networks) HasDriverOrNwAllocator(ntype string) bool {
132
+	nr.mu.Lock()
133
+	defer nr.mu.Unlock()
134
+
135
+	_, hasDriver := nr.drivers[ntype]
136
+	_, hasNwAlloc := nr.nwAllocs[ntype]
137
+	return hasDriver || hasNwAlloc
138
+}
... ...
@@ -823,11 +823,3 @@ func (b *badDriver) Type() string {
823 823
 func (b *badDriver) IsBuiltIn() bool {
824 824
 	return false
825 825
 }
826
-
827
-func (b *badDriver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
828
-	return nil, types.NotImplementedErrorf("not implemented")
829
-}
830
-
831
-func (b *badDriver) NetworkFree(id string) error {
832
-	return types.NotImplementedErrorf("not implemented")
833
-}