kube-vip/pkg/vip/address.go

// Some code in this file is copied or based on
// https://github.com/telekom/multi-networkpolicy-nftables/blob/f037e79605643e5a9f6debef55297a7170800c51/pkg/server/netfilterrules.go

package vip

import (
	"fmt"
	"hash/fnv"
	"math"
	"net"
	"slices"
	"strconv"
	"strings"
	"sync"

	log "log/slog"

	"github.com/google/nftables"
	"github.com/google/nftables/binaryutil"
	"github.com/google/nftables/expr"
	"github.com/pkg/errors"
	"github.com/vishvananda/netlink"
	"github.com/vishvananda/netlink/nl"
	"golang.org/x/sys/unix"
	v1 "k8s.io/api/core/v1"

	iptables "github.com/kube-vip/kube-vip/pkg/iptables"
	"github.com/kube-vip/kube-vip/pkg/kubevip"
	nfinternal "github.com/kube-vip/kube-vip/pkg/nftables"
	"github.com/kube-vip/kube-vip/pkg/utils"

	"github.com/kube-vip/kube-vip/pkg/networkinterface"
)

const (
	defaultValidLft = 60
	iptablesComment = "%s kube-vip load balancer IP"
	// Linux reserves IFA_PROTO values 0-3 for unspecified and kernel-generated
	// addresses; user-space protocols start at 4. This is only a safety floor,
	firstUserAddressProtocol = 4

	DefaultMaskIPv4 = 32
	DefaultMaskIPv6 = 128

	NoLifetime = 0

	vipIPVSMark uint32 = 10042
)

// Network is an interface that enable managing operations for a given IP
type Network interface {
	AddIP(precheck bool, skipDAD bool, minLifetime ...int) (bool, error)
	AddRoute(precheck bool) (bool, error)
	ReplaceRoute() error
	DeleteIP() (bool, error)
	DeleteRoute() error
	UpdateRoutes() (bool, error)
	IsSet() (*netlink.Addr, error)
	IP() string
	CIDR() string
	IPisLinkLocal() bool
	PrepareRoute() *netlink.Route
	RouteHash() string
	SetIP(ip string) error
	SetServicePorts(service *v1.Service)
	Interface() string
	IsDADFAIL() bool
	IsDNS() bool
	IsDDNS() bool
	DDNSHostName() string
	DNSName() string
	SetMask(mask string) error
	SetHasEndpoints(value bool)
	HasEndpoints() bool
	ARPName() string
	GetPossibleSubnets() string
	DHCPFamily() string
	IPVSMark() uint32
}

// network - This allows network configuration
type network struct {
	mu             sync.RWMutex
	address        *netlink.Addr
	link           *networkinterface.Link
	ports          []v1.ServicePort
	serviceName    string
	enableSecurity bool
	ignoreSecurity bool

	dnsName string
	isDDNS  bool

	forwardMethod string

	routeTable       int
	routingTableType int
	routingProtocol  int

	// reassertToggle alternates the inert realm attribute between two values
	// on every ReplaceRoute so each re-assertion is a real kernel change and
	// therefore a netlink event visible to listening routing daemons.
	reassertToggle bool

	ipvsEnabled bool

	hasEndpoints bool

	possibleSubnets string

	// used by DHCP to get address of proper family
	dhcpFamily string

	// use internal nftables implementation instead of iptables based one
	nftables bool

	// ipvsMark is used to mark IPVS connections for further processing
	ipvsMark uint32

	ipvsPort uint16

	// dadSkip marks the address with IFA_F_NODAD on every add:
	// anycast semantics, e.g. ECMP, must not use DAD
	dadSkip bool
}

// NewConfig will attempt to provide an interface to the kernel network configuration
func NewConfig(address string, iface string, loGlobalScope bool, subnet string, isDDNS bool,
	dhcpMode string, requireDualStack, isDualStack bool, tableID int, tableType int, routingProtocol int,
	dnsMode, forwardMethod, iptablesBackend string, ipvsEnabled bool, ipvsPort uint16, enableSecurity bool,
	intfMgr *networkinterface.Manager, nftables bool, skipDAD bool) ([]Network, error) {
	networks := []Network{}

	link, err := netlink.LinkByName(iface)
	if err != nil {
		return networks, errors.Wrapf(err, "could not get link for interface '%s'", iface)
	}

	networkLink := intfMgr.Get(link)

	ipvsMark := vipIPVSMark

	if utils.IsIP(address) {
		result := &network{
			link:             networkLink,
			routeTable:       tableID,
			routingTableType: tableType,
			routingProtocol:  routingProtocol,
			forwardMethod:    forwardMethod,
			ipvsEnabled:      ipvsEnabled,
			enableSecurity:   enableSecurity,
			possibleSubnets:  subnet,
			nftables:         nftables,
			ipvsMark:         ipvsMark,
			ipvsPort:         ipvsPort,
			dadSkip:          skipDAD,
		}

		subnet, err = SelectSubnet(address, subnet)
		if err != nil {
			return networks, fmt.Errorf("unable to select subnet for IP %q from %q: %w", address, subnet, err)
		}

		// Check if the subnet needs overriding
		cidr, err := utils.FormatIPWithSubnetMask(address, subnet)
		if err != nil {
			return networks, errors.Wrapf(err, "could not format address '%s' with subnetMask '%s'", address, subnet)
		}
		result.address, err = netlink.ParseAddr(cidr)
		if err != nil {
			return networks, errors.Wrapf(err, "could not parse address '%s'", address)
		}
		markKubeVIPAddress(result.address, result.routingProtocol)

		// set address as deprecated so it isn't used as source address according to RFC 3484
		result.address.PreferedLft = 0

		// Also set ValidLft so the netlink library actually sets them
		result.address.ValidLft = math.MaxInt

		if iface == "lo" && !loGlobalScope {
			// set host scope on loopback, otherwise global scope will be used by default
			result.address.Scope = unix.RT_SCOPE_HOST
		}

		networks = append(networks, result)
	} else {
		// try to resolve the address
		log.Debug("looking up host", "address", address, "dnsMode", dnsMode)
		ips, err := utils.LookupHost(address, dnsMode, requireDualStack)
		if (dnsMode == utils.DualFamily && isDDNS && isDualStack) || err != nil {
			// return early for ddns if no IP is allocated for the domain
			// when leader starts, should do get IP from DHCP for the domain
			if isDDNS {
				log.Info("isDDNS true", "dhcpMode", dhcpMode)
				if strings.EqualFold(dhcpMode, utils.IPv4Family) || strings.EqualFold(dhcpMode, utils.DualFamily) {
					result := &network{
						link:             networkLink,
						routeTable:       tableID,
						routingTableType: tableType,
						routingProtocol:  routingProtocol,
						forwardMethod:    forwardMethod,
						isDDNS:           isDDNS,
						dnsName:          address,
						ipvsEnabled:      ipvsEnabled,
						enableSecurity:   enableSecurity,
						possibleSubnets:  subnet,
						dhcpFamily:       utils.IPv4Family,
						nftables:         nftables,
						ipvsMark:         ipvsMark,
						ipvsPort:         ipvsPort,
						dadSkip:          skipDAD,
					}

					networks = append(networks, result)
				}

				if strings.EqualFold(dhcpMode, utils.IPv6Family) || strings.EqualFold(dhcpMode, utils.DualFamily) {
					result := &network{
						link:             networkLink,
						routeTable:       tableID,
						routingTableType: tableType,
						routingProtocol:  routingProtocol,
						forwardMethod:    forwardMethod,
						isDDNS:           isDDNS,
						dnsName:          address,
						ipvsEnabled:      ipvsEnabled,
						enableSecurity:   enableSecurity,
						possibleSubnets:  subnet,
						dhcpFamily:       utils.IPv6Family,
						nftables:         nftables,
						ipvsMark:         ipvsMark,
						ipvsPort:         ipvsPort,
						dadSkip:          skipDAD,
					}

					networks = append(networks, result)
				}

				return networks, nil
			}
			return nil, err
		}

		for _, ip := range ips {
			result := &network{
				link:             networkLink,
				routeTable:       tableID,
				routingTableType: tableType,
				routingProtocol:  routingProtocol,
				forwardMethod:    forwardMethod,
				isDDNS:           isDDNS,
				dnsName:          address,
				ipvsEnabled:      ipvsEnabled,
				enableSecurity:   enableSecurity,
				possibleSubnets:  subnet,
				nftables:         nftables,
				ipvsMark:         ipvsMark,
				ipvsPort:         ipvsPort,
				dadSkip:          skipDAD,
			}

			s, err := SelectSubnet(ip, subnet)
			if err != nil {
				return nil, fmt.Errorf("failed to select subnet: %w", err)
			}

			if result.address, err = netlink.ParseAddr(fmt.Sprintf("%s/%s", ip, s)); err != nil {
				return networks, err
			}
			markKubeVIPAddress(result.address, result.routingProtocol)
			// set ValidLft so that the VIP expires if the DNS entry is updated, otherwise it'll be refreshed by the DNS prober
			result.address.ValidLft = defaultValidLft

			// set address as deprecated so it isn't used as source address according to RFC 3484
			result.address.PreferedLft = 0

			result.dhcpFamily = strings.ToLower(utils.IPv6Family)
			if net.ParseIP(ip).To4() != nil {
				result.dhcpFamily = strings.ToLower(utils.IPv4Family)
			}

			networks = append(networks, result)
		}

	}

	return networks, nil
}

// ListRoutes returns all routes from selected table with selected protocol
func ListRoutes(table, protocol int) ([]netlink.Route, error) {
	route := &netlink.Route{
		Table:    table,
		Protocol: netlink.RouteProtocol(protocol),
	}
	routes, err := netlink.RouteListFiltered(nl.FAMILY_ALL, route, netlink.RT_FILTER_PROTOCOL|netlink.RT_FILTER_TABLE)
	if err != nil {
		return nil, fmt.Errorf("error getting routes from table [%d] with protocol [%d]: %w", table, protocol, err)
	}
	return routes, nil
}

// ListRoutesByDst returns all routes from selected table with selected destination IP
func ListRoutesByDst(table int, dst *net.IPNet) ([]netlink.Route, error) {
	route := &netlink.Route{
		Dst:   dst,
		Table: table,
	}
	routes, err := netlink.RouteListFiltered(nl.FAMILY_ALL, route, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_DST)
	if err != nil {
		return nil, fmt.Errorf("error getting routes from table [%d] with destination IP [%s]: %w", table, dst.String(), err)
	}
	return routes, nil
}

func (configurator *network) PrepareRoute() (route *netlink.Route) {
	configurator.mu.RLock()
	defer configurator.mu.RUnlock()
	_ = configurator.link.WithInterface(func(intf netlink.Link) error {
		route = configurator.prepareRoute(intf)
		return nil
	})
	return route
}

func (configurator *network) prepareRoute(intf netlink.Link) *netlink.Route {
	routeScope := netlink.SCOPE_UNIVERSE
	if configurator.routingTableType == unix.RTN_LOCAL {
		routeScope = netlink.SCOPE_LINK
	}
	route := &netlink.Route{
		Scope:     routeScope,
		Dst:       configurator.address.IPNet,
		LinkIndex: intf.Attrs().Index,
		Table:     configurator.routeTable,
		Type:      configurator.routingTableType,
		Protocol:  netlink.RouteProtocol(configurator.routingProtocol),
	}
	return route
}

func (configurator *network) RouteHash() string {
	r := configurator.PrepareRoute()
	return NetlinkHash(r)
}

func NetlinkHash(r *netlink.Route) string {
	h := fnv.New32a()
	h.Write([]byte(r.String()))

	return strconv.FormatUint(uint64(h.Sum32()), 16)
}

// AddRoute - Add an IP address to a route table
func (configurator *network) AddRoute(precheck bool) (added bool, err error) {
	configurator.mu.RLock()
	defer configurator.mu.RUnlock()
	err = configurator.link.WithInterface(func(intf netlink.Link) error {
		route := configurator.prepareRoute(intf)
		exists := false
		if precheck {
			var existsErr error
			exists, existsErr = configurator.routeExists(intf, route)
			if existsErr != nil {
				return errors.Wrap(existsErr, "failed to check route")
			}
		}
		if exists {
			return nil
		}
		if routeErr := netlink.RouteAdd(route); routeErr != nil {
			return errors.Wrap(routeErr, "failed to add route")
		}
		added = true
		return nil
	})
	return added, err
}

func (configurator *network) routeExists(intf netlink.Link, route *netlink.Route) (bool, error) {
	routes, err := netlink.RouteList(intf, netlink.FAMILY_ALL)
	if err != nil {
		return false, errors.Wrap(err, "failed to list routes")
	}

	for _, r := range routes {
		if r.Equal(*route) {
			return true, nil
		}
	}

	return false, nil
}

// ReplaceRoute - (re-)assert the route in the route table. The kernel emits
// no netlink notification for a no-op replace, so alternate an inert
// attribute (the routing realm) to make every re-assertion a visible event:
// routing daemons (e.g. FRR's zebra) can lose the original route event when
// the same-prefix interface address is processed in the same netlink batch,
// leaving the route in the kernel but never redistributed.
func (configurator *network) ReplaceRoute() error {
	configurator.mu.Lock()
	defer configurator.mu.Unlock()
	return configurator.link.WithInterface(func(intf netlink.Link) error {
		route := configurator.prepareRoute(intf)
		configurator.reassertToggle = !configurator.reassertToggle
		if configurator.reassertToggle {
			route.Realm = 1
		} else {
			route.Realm = 2
		}
		return netlink.RouteReplace(route)
	})
}

// DeleteRoute - Delete an IP address from a route table
func (configurator *network) DeleteRoute() error {
	route := configurator.PrepareRoute()
	return netlink.RouteDel(route)
}

// GetRoutes - Get an IP addresses from a route table
func (configurator *network) getRoutes() (*[]netlink.Route, error) {
	configurator.mu.RLock()
	defer configurator.mu.RUnlock()
	routes, err := ListRoutesByDst(configurator.routeTable, configurator.address.IPNet)
	if err != nil {
		return nil, fmt.Errorf("error getting routes: %w", err)
	}
	return &routes, nil
}

func (configurator *network) UpdateRoutes() (bool, error) {
	routes, err := configurator.getRoutes()
	if err != nil {
		return false, fmt.Errorf("error updating routes: %w", err)
	}
	isUpdated := false
	r := configurator.PrepareRoute()
	for _, route := range *routes {
		if route.Protocol == unix.RTPROT_BOOT &&
			(route.Type == r.Type || route.Type == unix.RTN_UNICAST) &&
			route.LinkIndex == r.LinkIndex && route.Scope == r.Scope {
			if err = netlink.RouteReplace(r); err != nil {
				return false, fmt.Errorf("error replacing route: %w", err)
			}
			isUpdated = true
		}
	}
	return isUpdated, nil
}

// shouldSkipDAD returns whether the address must carry the IFA_F_NODAD flag:
// either because configuration or because the caller requests it for this
// specific add (e.g. DADFAILED state recovery in ARP mode).
func (configurator *network) shouldSkipDAD(override bool) bool {
	return override || configurator.dadSkip
}

// AddIP - Add an IP address to the interface
// precheck: if true, check if the IP already exists before adding
// skipDAD: if true, set IFA_F_NODAD flag for IPv6 addresses to skip Duplicate Address Detection
func (configurator *network) AddIP(precheck bool, skipDAD bool, minLifetime ...int) (bool, error) {
	configurator.mu.Lock()
	defer configurator.mu.Unlock()
	var added bool
	err := configurator.link.WithInterface(func(intf netlink.Link) error {
		var addErr error
		added, addErr = configurator.addIP(intf, precheck, skipDAD, minLifetime...)
		return addErr
	})
	return added, err
}

func (configurator *network) addIP(intf netlink.Link, precheck bool, skipDAD bool, minLifetime ...int) (bool, error) {
	var existing *netlink.Addr
	var err error
	if precheck {
		if existing, err = configurator.isSet(intf); err != nil {
			return false, errors.Wrap(err, "could not check if address exists")
		}
	}

	lifetime := NoLifetime

	if len(minLifetime) > 0 {
		lifetime = minLifetime[0]
	}

	if existing != nil && existing.ValidLft > lifetime {
		return false, nil
	}

	// For IPv6 addresses, optionally set NODAD flag to skip Duplicate Address Detection (DAD)
	// This prevents DADFAILED loops when recovering from a previous DADFAILED state
	// The flag tells the kernel to skip DAD, which is safe when we're re-adding
	// an address that we know should be ours (e.g., after DADFAILED recovery).
	// We also allow to globally configure NODAD in case user knows they are running in an
	// environment where multiple nodes may advertise the same VIP (e.g., ECMP routing).
	if utils.IsIPv6(configurator.address.IP.String()) {
		if configurator.shouldSkipDAD(skipDAD) {
			configurator.address.Flags |= unix.IFA_F_NODAD
			log.Debug("Setting IFA_F_NODAD flag for IPv6 address to skip DAD", "ip", configurator.address.IP.String())
		} else {
			configurator.address.Flags &^= unix.IFA_F_NODAD
		}
	}

	log.Debug("replacing IP", "address", configurator.address)
	if err := netlink.AddrReplace(intf, configurator.address); err != nil {
		return false, errors.Wrap(err, fmt.Sprintf("could not add ip to device %q", intf.Attrs().Name))
	}

	if configurator.nftables {
		if err := configurator.configureNFTables(); err != nil {
			return true, errors.Wrap(err, "could not configure NFTables")
		}
	} else {
		if err := configurator.configureIPTables(); err != nil {
			return true, errors.Wrap(err, "could not configure IPTables")
		}
	}

	return true, nil
}

func (configurator *network) configureIPTables() error {
	if configurator.enableSecurity && !configurator.ignoreSecurity && len(configurator.ports) > 0 {
		if err := configurator.addIptablesRulesToLimitTrafficPorts(); err != nil {
			return errors.Wrap(err, "could not add iptables rules to limit traffic ports")
		}
	}

	// It seems that masquerading is only required with IPv4 for IPVS to work.
	if configurator.serviceName == "" && configurator.ipvsEnabled && configurator.forwardMethod == "masquerade" && configurator.address.IP.To4() != nil {
		if err := configurator.addIptablesRulesForMasquerade(); err != nil {
			return errors.Wrap(err, "could not add iptables rules for masquerade")
		}
	}

	return nil
}

func (configurator *network) configureNFTables() error {
	log.Debug("configure nftables", "security enabled", configurator.enableSecurity, "ignore security", configurator.ignoreSecurity,
		"ports", configurator.ports, "service-name", configurator.serviceName)

	opt := nftables.TableFamilyIPv4
	if utils.IsIPv6(configurator.address.IP.String()) {
		opt = nftables.TableFamilyIPv6
	}

	c, err := nfinternal.NewClient(opt)
	if err != nil {
		return fmt.Errorf("unable to create nftables client: %w", err)
	}

	comment := fmt.Sprintf(iptablesComment, "control-plane")
	if configurator.serviceName != "" {
		comment = fmt.Sprintf(iptablesComment, configurator.serviceName)
	}

	if configurator.enableSecurity && !configurator.ignoreSecurity && len(configurator.ports) > 0 {
		if err := configurator.addNftablesRulesToLimitTrafficPorts(c, comment); err != nil {
			return errors.Wrap(err, "could not add nftables rules to limit traffic ports")
		}
	}

	// It seems that masquerading is only required with IPv4 for IPVS to work.
	if configurator.serviceName == "" && configurator.ipvsEnabled && configurator.forwardMethod == "masquerade" && configurator.address.IP.To4() != nil {
		if err := configurator.addNftablesRulesForMasquerade(c, comment); err != nil {
			return errors.Wrap(err, "could not add nftables rules for masquerade")
		}
	}

	if err := c.Close(); err != nil {
		return fmt.Errorf("failed to close nftables client: %w", err)
	}

	return nil
}

func (configurator *network) addIptablesRulesToLimitTrafficPorts() error {
	vip := configurator.address.IP.String()

	opt := iptables.IPFamily(iptables.ProtocolIPv4)
	if utils.IsIPv6(vip) {
		opt = iptables.IPFamily(iptables.ProtocolIPv6)
	}

	ipt, err := iptables.New(opt)
	if err != nil {
		return errors.Wrap(err, "could not create iptables client")
	}
	comment := fmt.Sprintf(iptablesComment, configurator.serviceName)
	if err := insertCommonIPTablesRules(ipt, vip, comment); err != nil {
		return fmt.Errorf("could not add common iptables rules: %w", err)
	}
	log.Debug("add iptables rules", "vip", vip, "ports", configurator.ports)
	if err := configurator.insertIPTablesRulesForServicePorts(ipt, vip, comment); err != nil {
		return fmt.Errorf("could not add iptables rules for service ports: %v", err)
	}

	return nil
}

func (configurator *network) addNftablesRulesToLimitTrafficPorts(c *nfinternal.Client, comment string) error {
	if _, err := c.GetChain(nfinternal.TableFilter, iptables.ChainInput); err != nil {
		if errors.Is(err, nfinternal.ErrChainNotFound) {
			p := nftables.ChainPolicyAccept
			ch := &nftables.Chain{
				Table:    c.GetTable(nfinternal.TableFilter),
				Name:     iptables.ChainInput,
				Hooknum:  nftables.ChainHookInput,
				Type:     nftables.ChainTypeFilter,
				Priority: nftables.ChainPriorityFilter,
				Policy:   &p,
			}
			_ = c.AddChain(ch)
			c.Flush()
		} else {
			return fmt.Errorf("failed to get table: %s", nfinternal.TableFilter)
		}
	}

	vip := configurator.address.IP.String()
	firstRule, err := insertCommonNFTablesRules(c, vip, comment)
	if err != nil {
		return fmt.Errorf("could not add common nftables rules: %w", err)
	}

	if err := configurator.insertNFTablesRulesForServicePorts(c, vip, comment, firstRule.Handle); err != nil {
		return fmt.Errorf("could not add nftables rules for service ports: %v", err)
	}

	return nil
}

func (configurator *network) insertIPTablesRulesForServicePorts(ipt *iptables.IPTables, vip, comment string) error {
	isPortsRuleExisting := make([]bool, len(configurator.ports))

	// delete rules of ports that are not in the service
	rules, err := ipt.List(iptables.TableFilter, iptables.ChainInput)
	if err != nil {
		return fmt.Errorf("could not list iptables rules: %w", err)
	}
	for _, rule := range rules {
		// only handle rules with kube-vip comment
		if iptables.GetIPTablesRuleSpecification(rule, "--comment") != comment {
			continue
		}
		// if the rule is not for the vip, delete it
		if iptables.GetIPTablesRuleSpecification(rule, "-d") != vip {
			if err := ipt.Delete(iptables.TableFilter, iptables.ChainInput, rule); err != nil {
				return fmt.Errorf("could not delete iptables rule: %w", err)
			}
		}

		protocol := iptables.GetIPTablesRuleSpecification(rule, "-p")
		port := iptables.GetIPTablesRuleSpecification(rule, "--dport")
		// ignore DHCP client port
		if protocol == string(v1.ProtocolUDP) && port == dhcpClientPort {
			continue
		}
		// if the rule is for the vip, but its protocol and port are not in the service, delete it
		toBeDeleted := true
		for i, p := range configurator.ports {
			if string(p.Protocol) == protocol && strconv.Itoa(int(p.Port)) == port {
				// the rule is for the vip and its protocol and port are in the service, keep it and mark it as existing
				toBeDeleted = false
				isPortsRuleExisting[i] = true
			}
		}
		if toBeDeleted {
			if err := ipt.Delete(iptables.TableFilter, iptables.ChainInput, strings.Split(rule, "")...); err != nil {
				return fmt.Errorf("could not delete iptables rule: %w", err)
			}
		}
	}
	// add rules of ports that are not existing
	// iptables -A INPUT -d <vip> -p <protocol> --dport <port> -j ACCEPT -m comment —comment “<namespace/service-name> kube-vip load balancer IP”
	for i, ok := range isPortsRuleExisting {
		if !ok {
			if err := ipt.InsertUnique(iptables.TableFilter, iptables.ChainInput, 1, "-d", vip, "-p",
				string(configurator.ports[i].Protocol), "--dport", strconv.Itoa(int(configurator.ports[i].Port)),
				"-m", "comment", "--comment", comment, "-j", "ACCEPT"); err != nil {
				return fmt.Errorf("could not add iptables rule to accept the traffic to VIP %s for allowed "+
					"port %d: %v", vip, configurator.ports[i].Port, err)
			}
		}
	}

	return nil
}

func (configurator *network) insertNFTablesRulesForServicePorts(c *nfinternal.Client, vip, comment string, handle uint64) error {
	chain, err := c.GetChain(nfinternal.TableFilter, iptables.ChainInput)
	if err != nil {
		return fmt.Errorf("failed to get chain %q in table %q: %w", iptables.ChainInput, nfinternal.TableFilter, err)
	}

	portsTCP := []nftables.SetElement{}
	portsUDP := []nftables.SetElement{}
	portsSCTP := []nftables.SetElement{}
	for _, p := range configurator.ports {
		switch p.Protocol {
		case v1.ProtocolTCP:
			portsTCP = append(portsTCP, nftables.SetElement{
				Key: binaryutil.BigEndian.PutUint16(uint16(p.Port)), //nolint:gosec
			})
		case v1.ProtocolUDP:
			portsUDP = append(portsUDP, nftables.SetElement{
				Key: binaryutil.BigEndian.PutUint16(uint16(p.Port)), //nolint:gosec
			})
		case v1.ProtocolSCTP:
			portsSCTP = append(portsSCTP, nftables.SetElement{
				Key: binaryutil.BigEndian.PutUint16(uint16(p.Port)), //nolint:gosec
			})
		}
	}

	if err := addNFTPortRule(c, chain, vip, portsTCP, unix.IPPROTO_TCP, comment, handle); err != nil {
		return fmt.Errorf("failed to add TCP ports for VIP %q: %w", vip, err)
	}

	if err := addNFTPortRule(c, chain, vip, portsUDP, unix.IPPROTO_UDP, comment, handle); err != nil {
		return fmt.Errorf("failed to add UDP ports for VIP %q: %w", vip, err)
	}

	if err := addNFTPortRule(c, chain, vip, portsSCTP, unix.IPPROTO_SCTP, comment, handle); err != nil {
		return fmt.Errorf("failed to add SCTP ports for VIP %q: %w", vip, err)
	}

	return nil
}

func addNFTPortRule(c *nfinternal.Client, chain *nftables.Chain, vip string,
	ports []nftables.SetElement, protocol int, comment string, handle uint64) error {
	if len(ports) == 0 {
		return nil
	}

	protocolString := ""
	switch protocol {
	case unix.IPPROTO_UDP:
		protocolString = "UDP"
	case unix.IPPROTO_SCTP:
		protocolString = "SCTP"
	default:
		protocolString = "TCP"
	}

	setComment := fmt.Sprintf("%s - %s ports", comment, protocolString)

	set, err := c.NewSet(chain, setComment)
	if err != nil {
		return fmt.Errorf("failed to create set: %w", err)
	}

	if err := c.UpdateSet(set, ports); err != nil {
		return fmt.Errorf("failed to update set for TCP ports")
	}

	ip := net.ParseIP(vip)

	if ip.To4() != nil {
		ip = ip.To4()
	} else {
		ip = ip.To16()
	}

	r := &nftables.Rule{
		Table:    chain.Table,
		Chain:    chain,
		UserData: nfinternal.UserDataComment(setComment),
		Position: handle,
		Exprs: []expr.Any{
			&expr.Payload{
				DestRegister: 0x1,
				Base:         expr.PayloadBaseNetworkHeader,
				Offset:       c.GetDstOffset(),
				Len:          c.GetLen(),
			},
			&expr.Cmp{
				Op:       expr.CmpOpEq,
				Register: 0x1,
				Data:     []byte(ip),
			},
			&expr.Meta{
				Key:      expr.MetaKeyL4PROTO,
				Register: 1,
			},
			&expr.Cmp{
				Op:       expr.CmpOpEq,
				Register: 1,
				Data:     binaryutil.BigEndian.PutUint32(uint32(protocol)), //nolint:gosec
			},
			&expr.Payload{
				DestRegister: 1,
				Base:         expr.PayloadBaseTransportHeader,
				Offset:       2, // l4 offset
				Len:          2, // l4 offset
			},
			&expr.Lookup{
				SetName:        set.Name,
				SetID:          set.ID,
				SourceRegister: 0x1,
			},
			&expr.Counter{},
			&expr.Verdict{
				Kind: expr.VerdictAccept,
			},
		},
	}

	if _, err := c.AddUnique(r); err != nil {
		return fmt.Errorf("failed to create rule for ports: %w", err)
	}

	c.Flush()

	return nil
}

func insertCommonIPTablesRules(ipt *iptables.IPTables, vip, comment string) error {
	if err := ipt.InsertUnique(iptables.TableFilter, iptables.ChainInput, 1, "-d", vip, "-p",
		string(v1.ProtocolUDP), "--dport", dhcpClientPort, "-m", "comment", "--comment", comment, "-j", "ACCEPT"); err != nil {
		return fmt.Errorf("could not add iptables rule to accept the traffic to VIP %s for DHCP client port: %w", vip, err)
	}
	// add rule to drop the traffic to VIP that is not allowed
	// iptables -A INPUT -d <vip> -j DROP
	if err := ipt.InsertUnique(iptables.TableFilter, iptables.ChainInput, 2, "-d", vip, "-m",
		"comment", "--comment", comment, "-j", "DROP"); err != nil {
		return fmt.Errorf("could not add iptables rule to drop the traffic to VIP %s: %v", vip, err)
	}
	return nil
}

func insertCommonNFTablesRules(c *nfinternal.Client, vip, comment string) (*nftables.Rule, error) {
	ch, err := c.GetChain(nfinternal.TableFilter, iptables.ChainInput)
	if err != nil {
		return nil, fmt.Errorf("failed to get chain %q in table %q: %w", iptables.ChainInput, nfinternal.TableFilter, err)
	}

	ip := net.ParseIP(vip)

	port, err := strconv.ParseUint(dhcpClientPort, 10, 16)
	if err != nil {
		return nil, fmt.Errorf("failed to convert DHCP port to int: %w", err)
	}

	if ip.To4() != nil {
		ip = ip.To4()
	} else {
		ip = ip.To16()
	}

	r := &nftables.Rule{
		Table:    ch.Table,
		Chain:    ch,
		UserData: nfinternal.UserDataComment(fmt.Sprintf("%s - DHCP", comment)),
		Exprs: []expr.Any{
			&expr.Payload{
				DestRegister: 0x1,
				Base:         expr.PayloadBaseNetworkHeader,
				Offset:       c.GetDstOffset(),
				Len:          c.GetLen(),
			},
			&expr.Cmp{
				Op:       expr.CmpOpEq,
				Register: 0x1,
				Data:     []byte(ip),
			},
			&expr.Meta{
				Key:      expr.MetaKeyL4PROTO,
				Register: 1,
			},
			&expr.Cmp{
				Op:       expr.CmpOpEq,
				Register: 1,
				Data:     []byte{unix.IPPROTO_UDP},
			},
			&expr.Payload{
				OperationType: expr.PayloadLoad,
				Len:           2,
				Base:          expr.PayloadBaseTransportHeader,
				Offset:        2,
				DestRegister:  1,
			},
			&expr.Cmp{
				Op:       expr.CmpOpEq,
				Register: 1,
				Data:     binaryutil.BigEndian.PutUint16(uint16(port)),
			},
			&expr.Counter{},
			&expr.Verdict{
				Kind: expr.VerdictAccept,
			},
		},
	}

	firstRule, err := c.InsertUnique(r)
	if err != nil {
		return firstRule, fmt.Errorf("failed to insert DHCP rule: %w", err)
	}

	c.Flush()

	firstRule, err = c.FindRuleByComment(ch.Table, ch, fmt.Sprintf("%s - DHCP", comment))
	if err != nil {
		return nil, fmt.Errorf("initial rule not found: %w", err)
	}

	if firstRule == nil {
		return nil, fmt.Errorf("ninitial rule not present")
	}

	r2 := &nftables.Rule{
		Table:    ch.Table,
		Chain:    ch,
		Position: firstRule.Handle,
		UserData: nfinternal.UserDataComment(fmt.Sprintf("%s - drop", comment)),
		Exprs: []expr.Any{
			&expr.Payload{
				DestRegister: 1,
				Base:         expr.PayloadBaseNetworkHeader,
				Offset:       c.GetDstOffset(), // IPv4 destination address
				Len:          c.GetLen(),
			},
			&expr.Cmp{
				Op:       expr.CmpOpEq,
				Register: 1,
				Data:     []byte(ip),
			},

			&expr.Counter{},

			&expr.Verdict{
				Kind: expr.VerdictDrop,
			},
		},
	}

	if _, err := c.AddUnique(r2); err != nil {
		return firstRule, fmt.Errorf("failed to insert DROP rule: %w", err)
	}

	c.Flush()

	return firstRule, nil
}

func deleteCommonIPTablesRules(ipt *iptables.IPTables, vip, comment string) error {
	if err := ipt.DeleteIfExists(iptables.TableFilter, iptables.ChainInput, "-d", vip, "-p",
		string(v1.ProtocolUDP), "--dport", dhcpClientPort, "-m", "comment", "--comment", comment, "-j", "ACCEPT"); err != nil {
		return fmt.Errorf("could not delete iptables rule to accept the traffic to VIP %s for DHCP client port: %w", vip, err)
	}
	// add rule to drop the traffic to VIP that is not allowed
	// iptables -A INPUT -d <vip> -j DROP
	if err := ipt.DeleteIfExists(iptables.TableFilter, iptables.ChainInput, "-d", vip, "-m", "comment",
		"--comment", comment, "-j", "DROP"); err != nil {
		return fmt.Errorf("could not delete iptables rule to drop the traffic to VIP %s: %v", vip, err)
	}
	return nil
}

func deleteCommonNFTablesRules(c *nfinternal.Client, comment string) error {
	table := c.GetTable(iptables.TableFilter)
	chain, err := c.GetChain(table.Name, iptables.ChainInput)
	if err != nil {
		// if there is no chain there's nothing to delete
		if errors.Is(err, nfinternal.ErrChainNotFound) {
			return nil
		}
		return fmt.Errorf("failed to find chain %q: %w", iptables.ChainInput, err)
	}

	ruleDHCP, err := c.FindRuleByComment(table, chain, fmt.Sprintf("%s - DHCP", comment))
	if err != nil {
		return fmt.Errorf("failed to find the rule: %w", err)
	}

	if ruleDHCP == nil {
		return nil
	}

	if err := c.DeleteRule(ruleDHCP); err != nil {
		return fmt.Errorf("failed to delete common rule for DHCP: %w", err)
	}

	ruleDrop, err := c.FindRuleByComment(table, chain, fmt.Sprintf("%s - drop", comment))
	if err != nil {
		return fmt.Errorf("failed to find the rule: %w", err)
	}

	if ruleDrop == nil {
		return nil
	}

	if err := c.DeleteRule(ruleDrop); err != nil {
		return fmt.Errorf("failed to delete common rule for Drop action: %w", err)
	}

	c.Flush()

	return nil
}

func deleteNFTPortRule(c *nfinternal.Client, comment string) error {

	table := c.GetTable(iptables.TableFilter)
	chain, err := c.GetChain(table.Name, iptables.ChainInput)
	if err != nil {
		// if there is no chain there's nothing to delete
		if errors.Is(err, nfinternal.ErrChainNotFound) {
			return nil
		}
		return fmt.Errorf("failed to find chain %q: %w", iptables.ChainInput, err)
	}

	for _, t := range []string{"TCP", "UDP", "SCTP"} {
		rule, err := c.FindRuleByComment(table, chain, fmt.Sprintf("%s - %s ports", comment, t))
		if err != nil {
			log.Warn("failed to find the rule", "error", err)
		}

		if rule == nil {
			continue
		}

		if err := c.DeleteRule(rule); err != nil {
			return fmt.Errorf("failed to delete common rule for Drop action: %w", err)
		}

		if err := c.DeleteSet(rule.Table, fmt.Sprintf("%s - %s ports", comment, t)); err != nil {
			return fmt.Errorf("failed to delete ports set: %w", err)
		}
	}

	c.Flush()

	return nil
}

func (configurator *network) removeIptablesRuleToLimitTrafficPorts() error {
	ipt, err := iptables.New()
	if err != nil {
		return errors.Wrap(err, "could not create iptables client")
	}
	vip := configurator.address.IP.String()
	comment := fmt.Sprintf(iptablesComment, configurator.serviceName)

	if err := deleteCommonIPTablesRules(ipt, vip, comment); err != nil {
		return fmt.Errorf("could not delete common iptables rules: %w", err)
	}

	log.Debug("remove iptables rules", "vip", vip, "ports", configurator.ports)
	for _, port := range configurator.ports {
		// iptables -D INPUT -d  <VIP> -p <protocol> --dport <port> -j ACCEPT
		if err := ipt.DeleteIfExists(iptables.TableFilter, iptables.ChainInput, "-d", vip, "-p", string(port.Protocol),
			"--dport", strconv.Itoa(int(port.Port)), "-m", "comment", "--comment", comment, "-j", "ACCEPT"); err != nil {
			return fmt.Errorf("could not delete iptables rule to accept the traffic to VIP %s for allowed port %d: %v", vip, port.Port, err)
		}
	}

	return nil
}

func (configurator *network) removeNftablesRuleToLimitTrafficPorts(c *nfinternal.Client) error {

	vip := configurator.address.IP.String()
	comment := fmt.Sprintf(iptablesComment, configurator.serviceName)

	log.Debug("remove nftables common rules", "vip", vip)
	if err := deleteCommonNFTablesRules(c, comment); err != nil {
		return fmt.Errorf("could not delete common nftables rules: %w", err)
	}

	log.Debug("remove nftables port rules", "vip", vip, "ports", configurator.ports)
	if err := deleteNFTPortRule(c, comment); err != nil {
		return fmt.Errorf("faield to remove port rule: %w", err)
	}

	return nil
}

// DeleteIP - Remove an IP address from the interface
func (configurator *network) DeleteIP() (bool, error) {
	configurator.mu.Lock()
	defer configurator.mu.Unlock()
	var deleted bool
	err := configurator.link.WithInterface(func(intf netlink.Link) error {
		var deleteErr error
		deleted, deleteErr = configurator.deleteIP(intf)
		return deleteErr
	})
	return deleted, err
}

func (configurator *network) deleteIP(intf netlink.Link) (bool, error) {
	result, err := configurator.isSet(intf)
	if err != nil {
		return false, errors.Wrap(err, "ip check in DeleteIP failed")
	}

	// Nothing to delete
	if result == nil {
		return false, nil
	}

	if err = netlink.AddrDel(intf, configurator.address); err != nil {
		return false, errors.Wrap(err, "could not delete ip")
	}

	if configurator.nftables {
		vip := configurator.address.IP.String()

		opt := nftables.TableFamilyIPv4
		if utils.IsIPv6(vip) {
			opt = nftables.TableFamilyIPv6
		}
		c, err := nfinternal.NewClient(opt)
		if err != nil {
			return false, fmt.Errorf("unable to create nftables client: %w", err)
		}

		if configurator.enableSecurity && !configurator.ignoreSecurity {
			if err := configurator.removeNftablesRuleToLimitTrafficPorts(c); err != nil {
				log.Warn("could not remove nftables rules to limit traffic ports", "error", err)
			}
		}

		if configurator.serviceName == "" && configurator.ipvsEnabled && configurator.forwardMethod == "masquerade" && configurator.address.IP.To4() != nil {
			if err := configurator.removeNftablesRulesForMasquerade(c); err != nil {
				log.Warn("could not remove nftables masquerade rules", "error", err)
			}
		}

		if err := c.Close(); err != nil {
			return true, errors.Wrap(err, "failed to close nftables client")
		}
	} else {
		if configurator.enableSecurity && !configurator.ignoreSecurity {
			if err := configurator.removeIptablesRuleToLimitTrafficPorts(); err != nil {
				return true, errors.Wrap(err, "could not remove iptables rules to limit traffic ports")
			}
		}

		if configurator.serviceName == "" && configurator.ipvsEnabled && configurator.forwardMethod == "masquerade" && configurator.address.IP.To4() != nil {
			if err := configurator.removeIptablesRulesForMasquerade(); err != nil {
				return true, errors.Wrap(err, "could not remove iptables masquerade rules ")
			}
		}
	}

	return true, nil
}

func (configurator *network) addIptablesRulesForMasquerade() error {
	ver, err := iptables.GetVersion()
	if err != nil {
		return errors.Wrap(err, "could not get iptables version")
	}

	ipt, err := iptables.New(iptables.EnableNFTables(ver.BackendMode == "nft"))
	if err != nil {
		return errors.Wrap(err, "could not create iptables client")
	}

	vip := configurator.address.IP.String()
	comment := fmt.Sprintf(iptablesComment, vip)
	if err := addMasqueradeRuleForVIP(ipt, vip, comment); err != nil {
		return err
	}

	return nil
}

// TO DO: It seems it is not be possible to use google/nftables with IPVS due to lack of IPVS matcher in nft
func (configurator *network) addNftablesRulesForMasquerade(c *nfinternal.Client, comment string) error {
	vip := configurator.address.IP.String()
	cmt := fmt.Sprintf("%s - IPVS, VIP %s, MARK %d", comment, vip, configurator.ipvsMark)

	markChain := &nftables.Chain{
		Name:     "ipvs_prerouting",
		Table:    c.GetTable(iptables.TableMangle),
		Type:     nftables.ChainTypeFilter,
		Hooknum:  nftables.ChainHookPrerouting,
		Priority: nftables.ChainPriorityMangle,
	}

	log.Debug("adding IPVS rule", "table", markChain.Table.Name, "chain", markChain.Name, "comment", cmt)

	markChain = c.AddChain(markChain)

	ip := net.ParseIP(vip)

	if ip.To4() != nil {
		ip = ip.To4()
	} else {
		ip = ip.To16()
	}

	port := binaryutil.BigEndian.PutUint16(configurator.ipvsPort)

	mark := binaryutil.NativeEndian.PutUint32(configurator.ipvsMark)

	markRule := &nftables.Rule{
		Table:    markChain.Table,
		Chain:    markChain,
		UserData: nfinternal.UserDataComment(cmt),
		Exprs: []expr.Any{
			&expr.Payload{
				DestRegister: 1,
				Base:         expr.PayloadBaseNetworkHeader,
				Offset:       c.GetDstOffset(),
				Len:          c.GetLen(),
			},
			&expr.Cmp{
				Register: 1,
				Data:     ip,
			},
			&expr.Payload{
				DestRegister: 1,
				Base:         expr.PayloadBaseTransportHeader,
				Offset:       2,
				Len:          2,
			},
			&expr.Cmp{
				Register: 1,
				Data:     port,
			},
			&expr.Immediate{
				Register: 1,
				Data:     mark,
			},
			&expr.Meta{
				Key:            expr.MetaKeyMARK,
				Register:       1,
				SourceRegister: true,
			},
		},
	}

	if _, err := c.AddUnique(markRule); err != nil {
		return fmt.Errorf("failed to add mark rule for IPVS: %w", err)
	}

	chain, err := c.GetChain(iptables.TableNat, iptables.ChainPOSTROUTING)
	if err != nil {
		if errors.Is(err, nfinternal.ErrChainNotFound) {
			p := nftables.ChainPolicyAccept
			ch := &nftables.Chain{
				Table:    c.GetTable(iptables.TableNat),
				Name:     iptables.ChainPOSTROUTING,
				Hooknum:  nftables.ChainHookPostrouting,
				Type:     nftables.ChainTypeFilter,
				Priority: nftables.ChainPriorityMangle,
				Policy:   &p,
			}
			_ = c.AddChain(ch)
		}

		return fmt.Errorf("failed to get chain %s in table %s: %w", iptables.ChainPOSTROUTING, iptables.TableNat, err)
	}

	log.Debug("adding IPVS rule", "table", chain.Table.Name, "chain", chain.Name, "comment", cmt)

	rule := &nftables.Rule{
		Table:    chain.Table,
		Chain:    chain,
		UserData: nfinternal.UserDataComment(cmt),
		Exprs: []expr.Any{
			&expr.Meta{
				Key:      expr.MetaKeyMARK,
				Register: 1,
			},
			&expr.Cmp{
				Op:       expr.CmpOpEq,
				Register: 1,
				Data:     mark,
			},
			&expr.Masq{},
		},
	}

	if _, err := c.InsertUnique(rule); err != nil {
		return fmt.Errorf("failed to insert nftables rule %q: %w", comment, err)
	}

	c.Flush()

	return nil
}

// addIptablesRulesForMasquerade add iptables rules for MASQUERADE
// insert example
func (configurator *network) removeIptablesRulesForMasquerade() error {
	ver, err := iptables.GetVersion()
	if err != nil {
		return errors.Wrap(err, "could not get iptables version")
	}
	ipt, err := iptables.New(iptables.EnableNFTables(ver.BackendMode == "nft"))
	if err != nil {
		return errors.Wrap(err, "could not create iptables client")
	}
	vip := configurator.address.IP.String()
	comment := fmt.Sprintf(iptablesComment, vip)

	err = delMasqueradeRuleForVIP(ipt, vip, comment)
	if err != nil {
		return err
	}

	return nil
}

func (configurator *network) removeNftablesRulesForMasquerade(c *nfinternal.Client) error {
	chain, err := c.GetChain(iptables.TableNat, iptables.ChainPOSTROUTING)
	if err != nil {
		return fmt.Errorf("failed to get chain %s in table %s: %w", iptables.ChainPOSTROUTING, iptables.TableNat, err)
	}

	comment := fmt.Sprintf(iptablesComment, "control-plane")
	if configurator.serviceName != "" {
		comment = fmt.Sprintf(iptablesComment, configurator.serviceName)
	}
	cmt := fmt.Sprintf("%s - IPVS, VIP %s, MARK %d", comment, configurator.address.IP.String(), configurator.ipvsMark)

	r, err := c.FindRuleByComment(chain.Table, chain, cmt)
	if err != nil {
		return fmt.Errorf("failed to find rule %q for deletion: %w", cmt, err)
	}

	if err := c.DeleteRule(r); err != nil {
		return fmt.Errorf("failed to delete rule %q: %w", cmt, err)
	}

	markChain, err := c.GetChain(iptables.TableMangle, "ipvs_prerouting")
	if err != nil {
		return fmt.Errorf("failed to get chain %s in table %s: %w", "ipvs_prerouting", iptables.TableMangle, err)
	}

	markRule, err := c.FindRuleByComment(markChain.Table, markChain, cmt)
	if err != nil {
		return fmt.Errorf("failed to find rule %q for deletion: %w", cmt, err)
	}

	if err := c.DeleteRule(markRule); err != nil {
		return fmt.Errorf("failed to delete rule %q: %w", cmt, err)
	}

	c.Flush()

	existing, err := c.List(markChain.Table, markChain)
	if err != nil {
		return fmt.Errorf("failed to list rules %s in table %s: %w", "ipvs_prerouting", iptables.TableMangle, err)
	}

	if len(existing) == 0 {
		c.DeleteChain(markChain)
	}

	c.Flush()

	return nil
}

// TODO: investigate if adding "--vport <port>" would be better or not quite necessary
// After this rule is added, ipvs kernel module is also loaded
func addMasqueradeRuleForVIP(ipt *iptables.IPTables, vip, comment string) error {
	err := ipt.InsertUnique(iptables.TableNat, iptables.ChainPOSTROUTING,
		1, "-m", "ipvs", "--vaddr", vip, "-j", "MASQUERADE", "-m", "comment", "--comment", comment)
	if err != nil {
		return fmt.Errorf("could not add masquerade rule for VIP %s: %v", vip, err)
	}
	return nil
}

func delMasqueradeRuleForVIP(ipt *iptables.IPTables, vip, comment string) error {
	err := ipt.DeleteIfExists(iptables.TableNat, iptables.ChainPOSTROUTING,
		"-m", "ipvs", "--vaddr", vip, "-j", "MASQUERADE", "-m", "comment", "--comment", comment)
	if err != nil {
		return fmt.Errorf("could not del masquerade rule for VIP %s: %v", vip, err)
	}
	return nil
}

// IsDADFAIL - Returns true if the address is IPv6 and has DADFAILED flag
func (configurator *network) IsDADFAIL() bool {
	configurator.mu.RLock()
	defer configurator.mu.RUnlock()
	var dadFailed bool
	_ = configurator.link.WithInterface(func(intf netlink.Link) error {
		if configurator.address == nil || !utils.IsIPv6(configurator.address.IP.String()) {
			return nil
		}
		addresses, err := netlink.AddrList(intf, netlink.FAMILY_V6)
		if err != nil {
			return nil
		}
		for _, address := range addresses {
			if address.IP.Equal(configurator.address.IP) && addressHasDADFAILEDFlag(address) {
				dadFailed = true
				break
			}
		}
		return nil
	})
	return dadFailed
}

func addressHasDADFAILEDFlag(address netlink.Addr) bool {
	return address.Flags&unix.IFA_F_DADFAILED != 0
}

// isSet - Check to see if VIP is set
func (configurator *network) IsSet() (result *netlink.Addr, err error) {
	configurator.mu.RLock()
	defer configurator.mu.RUnlock()
	err = configurator.link.WithInterface(func(intf netlink.Link) error {
		result, err = configurator.isSet(intf)
		return err
	})
	return result, err
}

func (configurator *network) isSet(intf netlink.Link) (result *netlink.Addr, err error) {
	var addresses []netlink.Addr

	if configurator.address == nil {
		return nil, nil
	}

	if configurator.address.Mask == nil {
		return nil, nil
	}

	addresses, err = netlink.AddrList(intf, 0)
	if err != nil {
		err = errors.Wrap(err, "could not list addresses")

		return nil, err
	}

	for _, address := range addresses {
		if address.Equal(*configurator.address) {
			return &address, nil
		}
	}

	return nil, nil
}

// SetIP updates the IP that is used
func (configurator *network) SetIP(ip string) error {
	configurator.mu.Lock()
	defer configurator.mu.Unlock()
	return configurator.setIP(ip)
}

func (configurator *network) setIP(ip string) error {
	if strings.Contains("/", ip) {
		return fmt.Errorf("ip should not contain CIDR notation got: %s", ip)
	}

	if configurator.address == nil {
		log.Debug("possible", "subnets", configurator.possibleSubnets)
		subnet, err := SelectSubnet(ip, configurator.possibleSubnets)
		if err != nil {
			return fmt.Errorf("unable to select subnet for IP %q from %q: %w", ip, subnet, err)
		}

		// Check if the subnet needs overriding
		cidr, err := utils.FormatIPWithSubnetMask(ip, subnet)
		if err != nil {
			return errors.Wrapf(err, "2 could not format address %q with subnetMask %q", ip, subnet)
		}
		configurator.address, err = netlink.ParseAddr(cidr)
		if err != nil {
			return errors.Wrapf(err, "could not parse address %q", cidr)
		}
	}

	ones, _ := configurator.address.Mask.Size()
	cidr, err := utils.FormatIPWithSubnetMask(ip, strconv.Itoa(ones))
	if err != nil {
		return fmt.Errorf("could not format address '%s' with subnetMask '%s'", ip, strconv.Itoa(ones))
	}
	addr, err := netlink.ParseAddr(cidr)
	if err != nil {
		return err
	}
	if configurator.address != nil && configurator.dnsName != "" {
		addr.ValidLft = defaultValidLft
	} else {
		addr.ValidLft = math.MaxInt
	}

	// set address as deprecated so it isn't used as source address according to RFC 3484
	addr.PreferedLft = 0

	configurator.address = addr
	if configurator.routingProtocol != 0 {
		markKubeVIPAddress(configurator.address, configurator.routingProtocol)
	}
	return nil
}

// IsKubeVIPAddress reports whether an address carries kube-vip's configured
// protocol origin. IFA_PROTO is supported for both IPv4 and IPv6 on Linux 5.18+.
func IsKubeVIPAddress(address netlink.Addr, protocol int) bool {
	return protocol >= firstUserAddressProtocol && address.Protocol == protocol
}

// RetainedKubeVIPAddressKeys returns the tagged kernel addresses matching the
// supplied VIPs. Callers use the keys with CleanupKubeVIPAddresses after they
// determine which VIPs remain locally referenced.
func RetainedKubeVIPAddressKeys(protocol int, vips map[string]struct{}) (map[string]struct{}, error) {
	retained := make(map[string]struct{})
	err := forEachKubeVIPAddress(protocol, func(_ netlink.Link, address netlink.Addr) error {
		if _, retain := vips[address.IP.String()]; retain {
			retained[addressKey(address)] = struct{}{}
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	return retained, nil
}

// CleanupKubeVIPAddresses removes kube-vip addresses not reasserted by this
// process. The retained keys must come from successful AddrReplace operations.
func CleanupKubeVIPAddresses(protocol int, retained map[string]struct{}) (int, error) {
	removed := 0
	err := forEachKubeVIPAddress(protocol, func(link netlink.Link, address netlink.Addr) error {
		key := addressKey(address)
		if key == "" {
			return nil
		}
		if _, keep := retained[key]; keep {
			return nil
		}
		if err := netlink.AddrDel(link, &address); err != nil {
			return errors.Wrapf(err, "deleting kube-vip address %q from interface %q", address.IP, link.Attrs().Name)
		}
		removed++
		return nil
	})
	if err != nil {
		return removed, err
	}
	return removed, nil
}

func forEachKubeVIPAddress(protocol int, visit func(netlink.Link, netlink.Addr) error) error {
	links, err := netlink.LinkList()
	if err != nil {
		return errors.Wrap(err, "listing network links")
	}
	for _, link := range links {
		addresses, err := netlink.AddrList(link, netlink.FAMILY_ALL)
		if err != nil {
			return errors.Wrapf(err, "listing addresses on interface %q", link.Attrs().Name)
		}
		for _, address := range addresses {
			if IsKubeVIPAddress(address, protocol) {
				if err := visit(link, address); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func addressKey(address netlink.Addr) string {
	if address.LinkIndex <= 0 || address.IP == nil {
		return ""
	}
	prefixLength, _ := address.Mask.Size()
	return fmt.Sprintf("%d/%s/%d", address.LinkIndex, address.IP, prefixLength)
}

func markKubeVIPAddress(address *netlink.Addr, protocol int) {
	if address != nil {
		address.Protocol = protocol
	}
}

// SetServicePorts updates the service ports from the service
// If you want to limit traffic to the VIP to only the service ports, add service ports to the network firstly.
func (configurator *network) SetServicePorts(service *v1.Service) {
	configurator.mu.Lock()
	defer configurator.mu.Unlock()

	configurator.ports = service.Spec.Ports
	configurator.serviceName = service.Namespace + "/" + service.Name
	configurator.ignoreSecurity = service.Annotations[kubevip.ServiceSecurityIgnore] == "true"
}

// IP - return the IP Address
func (configurator *network) IP() string {
	configurator.mu.RLock()
	defer configurator.mu.RUnlock()

	if configurator.address == nil || configurator.address.IP == nil {
		return ""
	}

	return configurator.address.IP.String()
}

func (configurator *network) CIDR() string {
	configurator.mu.RLock()
	defer configurator.mu.RUnlock()

	if configurator.address == nil || configurator.address.IPNet == nil {
		return ""
	}

	return configurator.address.IPNet.String()
}

// IP - return the IP Address
func (configurator *network) IPisLinkLocal() bool {
	configurator.mu.RLock()
	defer configurator.mu.RUnlock()

	return configurator.address.IP.IsLinkLocalUnicast()
}

// DNSName return the configured dnsName when use DNS
func (configurator *network) DNSName() string {
	configurator.mu.RLock()
	defer configurator.mu.RUnlock()
	return configurator.dnsName
}

// IsDNS - when dnsName is configured
func (configurator *network) IsDNS() bool {
	configurator.mu.RLock()
	defer configurator.mu.RUnlock()
	return configurator.dnsName != ""
}

// IsDDNS - return true if use dynamic dns
func (configurator *network) IsDDNS() bool {
	configurator.mu.RLock()
	defer configurator.mu.RUnlock()
	return configurator.isDDNS
}

// DDNSHostName - return the hostname for dynamic dns
// when dDNSHostName is not empty, use DHCP to get IP for hostname: dDNSHostName
// it's expected that dynamic DNS should be configured so
// the fqdn for apiserver endpoint is dDNSHostName.{LocalDomain}
func (configurator *network) DDNSHostName() string {
	configurator.mu.RLock()
	defer configurator.mu.RUnlock()
	return getHostName(configurator.dnsName)
}

// Interface - return the Interface name
func (configurator *network) Interface() string {
	var name string
	_ = configurator.link.WithInterface(func(intf netlink.Link) error {
		name = intf.Attrs().Name
		return nil
	})
	return name
}

func GarbageCollect(adapter, address string, intfMgr *networkinterface.Manager) (found bool, err error) {
	// Get adapter
	link, err := netlink.LinkByName(adapter)
	if err != nil {
		return true, errors.Wrapf(err, "could not get link for interface '%s'", adapter)
	}

	l := intfMgr.Get(link)
	err = l.WithInterface(func(intf netlink.Link) error {
		addrs, listErr := netlink.AddrList(intf, netlink.FAMILY_ALL)
		if listErr != nil {
			return listErr
		}
		for _, existing := range addrs {
			if existing.IP.String() == address {
				found = true
				existing := existing
				if deleteErr := netlink.AddrDel(intf, &existing); deleteErr != nil {
					return errors.Wrap(deleteErr, "could not delete ip")
				}
			}
		}
		return nil
	})
	return found, err
}

func (configurator *network) SetMask(mask string) error {
	configurator.mu.Lock()
	defer configurator.mu.Unlock()
	selectedMask := mask
	var err error

	if mask == "" {
		return fmt.Errorf("no mask provided")
	}

	ip := ""
	if configurator.address != nil && configurator.address.IP != nil {
		ip = configurator.address.IP.String()
	}
	if ip != "" {
		selectedMask, err = SelectSubnet(ip, mask)
		if err != nil {
			return fmt.Errorf("failed to select mask %q: %w", mask, err)
		}
	} else if len(strings.Split(mask, ",")) > 1 {
		return fmt.Errorf("cannot select mask from %q when IP address is unknown", mask)
	}

	m, err := strconv.Atoi(selectedMask)
	if err != nil {
		return err
	}

	size := DefaultMaskIPv4
	family := utils.IPv4Family

	if ip != "" {
		if utils.IsIPv6(ip) {
			size = DefaultMaskIPv6
			family = utils.IPv6Family
		}

		if m > size {
			return fmt.Errorf("provided CIDR mask '%d' is greater than the highest mask value for the %s family (%d)", m, family, size)
		}
	}

	toSet := net.CIDRMask(m, size)
	if toSet == nil {
		return fmt.Errorf("failed to create mask /%d", m)
	}

	configurator.address.Mask = toSet
	return nil
}

func (configurator *network) SetHasEndpoints(value bool) {
	configurator.mu.Lock()
	defer configurator.mu.Unlock()
	ip := ""
	if configurator.address != nil && configurator.address.IP != nil {
		ip = configurator.address.IP.String()
	}
	log.Debug("setting HasEndpoints", "ip", ip, "value", value)
	configurator.hasEndpoints = value
}

func (configurator *network) HasEndpoints() bool {
	configurator.mu.RLock()
	defer configurator.mu.RUnlock()
	ip := ""
	if configurator.address != nil && configurator.address.IP != nil {
		ip = configurator.address.IP.String()
	}
	log.Debug("getting HasEndpoints", "ip", ip, "value", configurator.hasEndpoints)
	return configurator.hasEndpoints
}

func (configurator *network) ARPName() string {
	return fmt.Sprintf("%s-%s", configurator.CIDR(), configurator.Interface())
}

func (configurator *network) GetPossibleSubnets() string {
	configurator.mu.RLock()
	defer configurator.mu.RUnlock()
	return configurator.possibleSubnets
}

func (configurator *network) DHCPFamily() string {
	configurator.mu.RLock()
	defer configurator.mu.RUnlock()
	return configurator.dhcpFamily
}

func (configurator *network) IPVSMark() uint32 {
	configurator.mu.RLock()
	defer configurator.mu.RUnlock()
	return configurator.ipvsMark
}

// SelectSubnet formats an IP address with the appropriate CIDR based on the input.
// The input SubnetMasks can be "32,128" (dual-stack), "32", "128" (SingleStack).
func SelectSubnet(rawIP string, subnetMasks string) (string, error) {
	// Split the SubnetMasks input into DualStack or SingleStack
	// If the input is "32,128", it will be split into ["32", "128"]
	subnetMasksParts := strings.Split(subnetMasks, ",")
	if len(subnetMasksParts) == 0 {
		return "", fmt.Errorf("no subnetMasks provided got: %q", subnetMasks)
	} else if len(subnetMasksParts) > 2 {
		return "", fmt.Errorf("invalid subnetMasks provided got: %q", subnetMasks)
	}
	if slices.Contains(subnetMasksParts, "auto") {
		return "", fmt.Errorf("auto subnet discovery only works for services: %q", subnetMasks)
	}

	// Parse the raw IP address
	ip := net.ParseIP(rawIP)
	if ip == nil {
		return "", fmt.Errorf("invalid IP address: %s", rawIP)
	}
	if ip.To4() != nil {
		return subnetMasksParts[0], nil
	}
	if ip.To16() != nil {
		subnetMask := subnetMasksParts[0]
		if len(subnetMasksParts) == 2 {
			subnetMask = subnetMasksParts[1]
		}
		return subnetMask, nil
	}
	return "", fmt.Errorf("unable to select subnet mask for: IP %q and masks %q", rawIP, subnetMasks)
}