//go:build linux package overlay import ( "context" "errors" "fmt" "net/netip" "syscall" "github.com/containerd/log" "github.com/moby/moby/v2/daemon/libnetwork/internal/hashable" "github.com/moby/moby/v2/daemon/libnetwork/internal/setmatrix" "github.com/moby/moby/v2/daemon/libnetwork/osl" ) type peerEntry struct { eid string mac hashable.MACAddr vtep netip.Addr } func (p *peerEntry) isLocal() bool { return !p.vtep.IsValid() } type peerMap struct { mp setmatrix.SetMatrix[netip.Prefix, peerEntry] } func (pm *peerMap) Walk(f func(netip.Prefix, peerEntry)) { for _, peerAddr := range pm.mp.Keys() { entry, ok := pm.Get(peerAddr) if ok { f(peerAddr, entry) } } } func (pm *peerMap) Get(peerIP netip.Prefix) (peerEntry, bool) { c, _ := pm.mp.Get(peerIP) if len(c) == 0 { return peerEntry{}, false } return c[0], true } func (pm *peerMap) Add(eid string, peerIP netip.Prefix, peerMac hashable.MACAddr, vtep netip.Addr) (bool, int) { pEntry := peerEntry{ eid: eid, mac: peerMac, vtep: vtep, } b, i := pm.mp.Insert(peerIP, pEntry) if i != 1 { // Transient case, there is more than one endpoint that is using the same IP s, _ := pm.mp.String(peerIP) log.G(context.TODO()).Warnf("peerDbAdd transient condition - Key:%s cardinality:%d db state:%s", peerIP, i, s) } return b, i } func (pm *peerMap) Delete(eid string, peerIP netip.Prefix, peerMac hashable.MACAddr, vtep netip.Addr) (bool, int) { pEntry := peerEntry{ eid: eid, mac: peerMac, vtep: vtep, } b, i := pm.mp.Remove(peerIP, pEntry) if i != 0 { // Transient case, there is more than one endpoint that is using the same IP s, _ := pm.mp.String(peerIP) log.G(context.TODO()).Warnf("peerDbDelete transient condition - Key:%s cardinality:%d db state:%s", peerIP, i, s) } return b, i } // The overlay uses a lazy initialization approach, this means that when a network is created // and the driver registered the overlay does not allocate resources till the moment that a // sandbox is actually created. // At the moment of this call, that happens when a sandbox is initialized, is possible that // networkDB has already delivered some events of peers already available on remote nodes, // these peers are saved into the peerDB and this function is used to properly configure // the network sandbox with all those peers that got previously notified. // // The caller is responsible for ensuring that peerAdd and peerDelete are not // called concurrently with this function to guarantee consistency. func (n *network) initSandboxPeerDB() error { var errs []error n.peerdb.Walk(func(peerIP netip.Prefix, pEntry peerEntry) { if !pEntry.isLocal() { if err := n.addNeighbor(peerIP, pEntry.mac, pEntry.vtep); err != nil { errs = append(errs, fmt.Errorf("failed to add neighbor entries for %s: %w", peerIP, err)) } } }) return errors.Join(errs...) } // peerAdd adds a new entry to the peer database. // // Local peers are signified by an invalid vtep (i.e. netip.Addr{}). func (n *network) peerAdd(eid string, peerIP netip.Prefix, peerMac hashable.MACAddr, vtep netip.Addr) error { if eid == "" { return errors.New("invalid endpoint id") } inserted, dbEntries := n.peerdb.Add(eid, peerIP, peerMac, vtep) if !inserted { log.G(context.TODO()).Warnf("Entry already present in db: nid:%s eid:%s peerIP:%v peerMac:%v vtep:%v", n.id, eid, peerIP, peerMac, vtep) } if vtep.IsValid() { err := n.addNeighbor(peerIP, peerMac, vtep) if err != nil { if dbEntries > 1 && errors.As(err, &osl.NeighborSearchError{}) { // Conflicting neighbor entries are already programmed into the kernel and we are in the transient case. // Upon deletion if the active configuration is deleted the next one from the database will be restored. return nil } return fmt.Errorf("peer add operation failed: %w", err) } } return nil } // addNeighbor programs the kernel so the given peer is reachable through the VXLAN tunnel. func (n *network) addNeighbor(peerIP netip.Prefix, peerMac hashable.MACAddr, vtep netip.Addr) error { if n.sbox == nil { // We are hitting this case for all the events that are arriving before that the sandbox // is being created. The peer got already added into the database and the sandbox init will // call the peerDbUpdateSandbox that will configure all these peers from the database return nil } s := n.getSubnetforIP(peerIP) if s == nil { return fmt.Errorf("couldn't find the subnet %q in network %q", peerIP.String(), n.id) } if err := n.joinSandbox(s, false); err != nil { return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err) } if n.secure { if err := n.driver.setupEncryption(vtep); err != nil { log.G(context.TODO()).Warn(err) } } // Add neighbor entry for the peer IP if err := n.sbox.AddNeighbor(peerIP.Addr().AsSlice(), peerMac.AsSlice(), osl.WithLinkName(s.vxlanName)); err != nil { return fmt.Errorf("could not add neighbor entry into the sandbox: %w", err) } // Add fdb entry to the bridge for the peer mac if n.fdbCnt.Add(hashable.IPMACFrom(vtep, peerMac), 1) == 1 { if err := n.sbox.AddNeighbor(vtep.AsSlice(), peerMac.AsSlice(), osl.WithLinkName(s.vxlanName), osl.WithFamily(syscall.AF_BRIDGE)); err != nil { return fmt.Errorf("could not add fdb entry into the sandbox: %w", err) } } return nil } // peerDelete removes an entry from the peer database. // // Local peers are signified by an invalid vtep (i.e. netip.Addr{}). func (n *network) peerDelete(eid string, peerIP netip.Prefix, peerMac hashable.MACAddr, vtep netip.Addr) error { if eid == "" { return errors.New("invalid endpoint id") } logger := log.G(context.TODO()).WithFields(log.Fields{ "nid": n.id, "eid": eid, "ip": peerIP, "mac": peerMac, "vtep": vtep, }) deleted, dbEntries := n.peerdb.Delete(eid, peerIP, peerMac, vtep) if !deleted { logger.Warn("Peer entry was not in db") } if vtep.IsValid() { err := n.deleteNeighbor(peerIP, peerMac, vtep) if err != nil { if dbEntries > 0 && errors.As(err, &osl.NeighborSearchError{}) { // We fall in here if there is a transient state and if the neighbor that is being deleted // was never been configured into the kernel (we allow only 1 configuration at the time per mapping) return nil } logger.WithError(err).Warn("Peer delete operation failed") } } if dbEntries > 0 { // If there is still an entry into the database and the deletion went through without errors means that there is now no // configuration active in the kernel. // Restore one configuration for the ip directly from the database, note that is guaranteed that there is one peerEntry, ok := n.peerdb.Get(peerIP) if !ok { return fmt.Errorf("peerDelete: unable to restore a configuration: no entry for %v found in the database", peerIP) } err := n.addNeighbor(peerIP, peerEntry.mac, peerEntry.vtep) if err != nil { return fmt.Errorf("peer delete operation failed: %w", err) } } return nil } // deleteNeighbor removes programming from the kernel for the given peer to be // reachable through the VXLAN tunnel. It is the inverse of [driver.addNeighbor]. func (n *network) deleteNeighbor(peerIP netip.Prefix, peerMac hashable.MACAddr, vtep netip.Addr) error { if n.sbox == nil { return nil } if n.secure { if err := n.driver.removeEncryption(vtep); err != nil { log.G(context.TODO()).Warn(err) } } s := n.getSubnetforIP(peerIP) if s == nil { return fmt.Errorf("could not find the subnet %q in network %q", peerIP.String(), n.id) } // Remove fdb entry to the bridge for the peer mac if n.fdbCnt.Add(hashable.IPMACFrom(vtep, peerMac), -1) == 0 { if err := n.sbox.DeleteNeighbor(vtep.AsSlice(), peerMac.AsSlice(), osl.WithLinkName(s.vxlanName), osl.WithFamily(syscall.AF_BRIDGE)); err != nil { return fmt.Errorf("could not delete fdb entry in the sandbox: %w", err) } } // Delete neighbor entry for the peer IP if err := n.sbox.DeleteNeighbor(peerIP.Addr().AsSlice(), peerMac.AsSlice(), osl.WithLinkName(s.vxlanName)); err != nil { return fmt.Errorf("could not delete neighbor entry in the sandbox:%v", err) } return nil }