Unintentionally created duplicate functions that can't be used

Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>

.github/workflows/anchore-syft.yml

@@ -26,6 +26,6 @@ jobs:
         with:
           ref: ${{ github.ref_name }}
       - name: Anchore SBOM Action
-        uses: anchore/sbom-action@v0.18.0
+        uses: anchore/sbom-action@v0.20.4
         with:
           format: cyclonedx-json

.github/workflows/ci-pull-request.yaml

@@ -0,0 +1,51 @@
+name: For each PR
+on:
+  pull_request:
+jobs:
+  e2e-tests:
+    runs-on: ubuntu-latest
+    name: E2E tests
+    strategy:
+      matrix:
+        mode: ["arp", "rt", "bgp"]
+      fail-fast: true
+      max-parallel: 1
+    steps:
+      - name: Ensure fs wont cause issues
+        run: sudo sysctl fs.inotify.max_user_instances=8192 &&  sudo sysctl fs.inotify.max_user_watches=524288
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Build image locally
+        run: make dockerx86Local
+      - name: Run Manifest generation tests
+        run: make manifest-test
+      - name: Run ARP mode tests v1.29.0 onwards
+        run: make e2e-tests129-arp
+        if: matrix.mode== 'arp'
+      - name: Run RT mode tests v1.29.0 onwards
+        run: make e2e-tests129-rt
+        if: matrix.mode== 'rt'
+      - name: Get GoBGP binaries
+        run: make get-gobgp
+        if: matrix.mode== 'bgp'
+      - name: Run BGP mode tests v1.29.0 onwards
+        run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 make e2e-tests129-bgp
+        if: matrix.mode== 'bgp'
+  service-e2e-tests:
+    runs-on: ubuntu-latest
+    name: E2E service tests
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Build image with iptables
+        run: make dockerx86ActionIPTables
+      - name: Run tests
+        run: DOCKERTAG=action make service-tests

.github/workflows/ci.yaml

@@ -1,4 +1,4 @@
-name: For each commit and PR
+name: For each commit
 on:
   push:
   pull_request:
@@ -10,7 +10,7 @@ jobs:
       - name: Init
         run: sudo apt-get update && sudo apt-get install -y build-essential &&  sudo sysctl fs.inotify.max_user_instances=8192 &&  sudo sysctl fs.inotify.max_user_watches=524288
       - name: Install golangci-lint
-        run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.62.2
+        run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.64.8
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Install Go
@@ -43,40 +43,6 @@ jobs:
           go-version-file: go.mod
       - name: Run tests
         run: make integration-tests
-  e2e-tests:
-    runs-on: ubuntu-latest
-    name: E2E ARP tests
-    steps:
-      - name: Ensure fs wont cause issues
-        run: sudo sysctl fs.inotify.max_user_instances=8192 &&  sudo sysctl fs.inotify.max_user_watches=524288
-      - name: Checkout code
-        uses: actions/checkout@v4
-      - name: Install Go
-        uses: actions/setup-go@v5
-        with:
-          go-version-file: go.mod
-      - name: Build image locally
-        run: make dockerx86Local
-      - name: Run Manifest generation tests
-        run: make manifest-test
-#      - name: Run Control plane tests
-#        run: make e2e-tests
-      - name: Run Control plane tests v1.29.0 onwards
-        run: make e2e-tests129
-  service-e2e-tests:
-    runs-on: ubuntu-latest
-    name: E2E service tests
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      - name: Install Go
-        uses: actions/setup-go@v5
-        with:
-          go-version-file: go.mod
-      - name: Build image with iptables
-        run: make dockerx86ActionIPTables
-      - name: Run tests
-        run: DOCKERTAG=action make service-tests
   image-vul-check:
     runs-on: ubuntu-latest
     name: Image vulnerability scan

.github/workflows/release.yaml

@@ -27,6 +27,12 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Generate Metadata
+        uses: docker/metadata-action@v5.7.0
+        id: metadata
+        with:
+          labels: |
+            org.opencontainers.image.documentation=https://kube-vip.io/docs/
       - name: Build and push main branch
         id: docker_build
         uses: docker/build-push-action@v6
@@ -34,6 +40,7 @@ jobs:
           context: .
           platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
           push: ${{ github.event_name != 'pull_request' }}
+          labels: ${{ steps.metadata.outputs.labels }}
           tags: >-
             plndr/kube-vip:${{ github.ref_name }},
             plndr/kube-vip:latest,
@@ -47,6 +54,7 @@ jobs:
           file: Dockerfile_iptables
           platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
           push: ${{ github.event_name != 'pull_request' }}
+          labels: ${{ steps.metadata.outputs.labels }}
           tags: >-
             plndr/kube-vip-iptables:${{ github.ref_name }},
             plndr/kube-vip-iptables:latest,

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:experimental
 
-FROM golang:1.24.1-alpine3.20 as dev
+FROM golang:1.24.5-alpine3.22 as dev
 RUN apk add --no-cache git ca-certificates make
 RUN adduser -D appuser
 COPY . /src/

Dockerfile_iptables

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:experimental
 
-FROM golang:1.24.1-alpine3.20 as dev
+FROM golang:1.24.5-alpine3.22 as dev
 RUN apk add --no-cache git make
 RUN adduser -D appuser
 COPY . /src/
@@ -11,7 +11,7 @@ RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
     --mount=type=cache,sharing=locked,id=goroot,target=/root/.cache/go-build \
     CGO_ENABLED=0 GOOS=linux make build
 
-FROM alpine:3.21.3
+FROM alpine:3.22.1
 # Update pkgs and add iptables
 RUN apk upgrade && \
     apk add --no-cache iptables iptables-legacy

Makefile

@@ -5,7 +5,7 @@ TARGET := kube-vip
 .DEFAULT_GOAL := $(TARGET)
 
 # These will be provided to the target
-VERSION := v0.8.10
+VERSION := v1.0.0
 
 BUILD := `git rev-parse HEAD`
 
@@ -115,7 +115,7 @@ manifests:
 	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
 	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
 	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
-	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster --provider-config /etc/cloud-sa/cloud-sa.json > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
 	@-rm ./kube-vip
 
 manifest-test: 
@@ -133,10 +133,18 @@ integration-tests:
 	go test -tags=integration,e2e -v ./pkg/etcd
 
 e2e-tests:
-	E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd
+	GOMAXPROCS=4 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd
 
-e2e-tests129:
-	V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
+e2e-tests129-arp:
+	GOMAXPROCS=4 TEST_MODE=arp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
+
+e2e-tests129-rt:
+	GOMAXPROCS=4 TEST_MODE=rt V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
+
+e2e-tests129-bgp:
+	GOMAXPROCS=4 TEST_MODE=bgp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
+
+e2e-tests129: e2e-tests129-arp e2e-tests129-rt e2e-tests129-bgp
 
 service-tests:
 	E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/services -Services -simple -deployments -leaderActive -leaderFailover -localDeploy -egress -egressIPv6 -dualStack
@@ -163,3 +171,9 @@ kind-quick:
 kind-reload:
 	kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name kube-vip
 	kubectl rollout restart -n kube-system daemonset/kube-vip-ds
+
+get-gobgp:
+	mkdir -p bin
+	wget -nc --directory-prefix=bin https://github.com/osrg/gobgp/releases/download/v3.37.0/gobgp_3.37.0_linux_amd64.tar.gz
+	tar -xvzf bin/gobgp_3.37.0_linux_amd64.tar.gz -C bin
+

README.md

@@ -4,7 +4,7 @@ High Availability and Load-Balancing
 
 ![](https://github.com/kube-vip/kube-vip/raw/main/kube-vip.png)
 
-[![Build and publish main image regularly](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml/badge.svg)](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml)
+[![Build and publish main image regularly](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml/badge.svg)](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml) [![LFX Health Score](https://img.shields.io/static/v1?label=Health%20Score&message=Healthy&color=A7F3D0&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip) [![LFX Active Contributors](https://img.shields.io/static/v1?label=Active%20contributors%20(1Y)&message=212&color=0094FF&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip)
 
 ## Overview
 Kubernetes Virtual IP and Load-Balancer for both control plane and Kubernetes services

cmd/kube-vip-kubeadm.go

@@ -57,10 +57,10 @@ var kubeKubeadmInit = &cobra.Command{
 		}
 
 		// Ensure there is an address to generate the CIDR from
-		if initConfig.VIPCIDR == "" && initConfig.Address != "" {
-			initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
 			if err != nil {
-				log.Error("generating CIDR", "err", err)
+				log.Error("generating VIPSubnet", "err", err)
 				return
 			}
 		}
@@ -102,10 +102,10 @@ var kubeKubeadmJoin = &cobra.Command{
 		}
 
 		// Ensure there is an address to generate the CIDR from
-		if initConfig.VIPCIDR == "" && initConfig.Address != "" {
-			initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
 			if err != nil {
-				log.Error("generating CIDR", "err", err)
+				log.Error("generating VIPSubnet", "err", err)
 				return
 			}
 		}

cmd/kube-vip-manifests.go

@@ -7,20 +7,21 @@ import (
 
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
 	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v2"
 )
 
 // manifests will eventually deprecate the kubeadm set of subcommands
 // manifests will be used to generate:
 // - Pod spec manifest, mainly used for a static pod (kubeadm)
 // - Daemonset manifest, mainly used to run kube-vip as a deamonset within Kubernetes (k3s/rke)
+// - RBAC manifest, used to generate the RBAC permissions for kube-vip
 
-// var inCluster bool
-var taint bool
+var taint, role, rolebinding bool
 
 func init() {
 	kubeManifest.PersistentFlags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
 	kubeManifestDaemon.PersistentFlags().BoolVar(&taint, "taint", false, "Taint the manifest for only running on control planes")
+	kubeManifestRbac.PersistentFlags().BoolVar(&role, "role", false, "Generate only a Role inside the serviceNamespace access")
+	kubeManifestRbac.PersistentFlags().BoolVar(&rolebinding, "rolebinding", false, "Generate only a RoleBinding for namespaced access")
 
 	kubeManifest.AddCommand(kubeManifestPod)
 	kubeManifest.AddCommand(kubeManifestDaemon)
@@ -57,8 +58,8 @@ var kubeManifestPod = &cobra.Command{
 		}
 
 		// Ensure there is an address to generate the CIDR from
-		if initConfig.VIPCIDR == "" && initConfig.Address != "" {
-			initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
 			if err != nil {
 				log.Error("config parse", "err", err)
 				return
@@ -90,8 +91,8 @@ var kubeManifestDaemon = &cobra.Command{
 		}
 
 		// Ensure there is an address to generate the CIDR from
-		if initConfig.VIPCIDR == "" && initConfig.Address != "" {
-			initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
 			if err != nil {
 				log.Error("config parse", "err", err)
 				return
@@ -124,16 +125,26 @@ var kubeManifestRbac = &cobra.Command{
 		}
 
 		// Ensure there is an address to generate the CIDR from
-		if initConfig.VIPCIDR == "" && initConfig.Address != "" {
-			initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
 			if err != nil {
-				log.Error("generating CIDR", "err", err)
+				log.Error("generating VIPSubnet", "err", err)
 				return
 			}
 		}
+		saCfg := kubevip.GenerateSA(&initConfig)
+		roleCfg := kubevip.GenerateRole(&initConfig, role)
+		if role {
+			rolebinding = true
+		}
+		roleBindingCfg := kubevip.GenerateRoleBinding(rolebinding, saCfg, roleCfg)
 
-		cfg := kubevip.GenerateSA()
-		b, _ := yaml.Marshal(cfg)
-		fmt.Println(string(b)) // output manifest to stdout
+		// Output the YAML manifests to stdout
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(saCfg))
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(roleCfg))
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(roleBindingCfg))
 	},
 }

cmd/kube-vip.go

@@ -15,8 +15,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
 	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
 
-	"github.com/kube-vip/kube-vip/pkg/equinixmetal"
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
 	"github.com/kube-vip/kube-vip/pkg/manager"
 	"github.com/kube-vip/kube-vip/pkg/vip"
@@ -28,9 +28,6 @@ var inCluster bool
 // ConfigMap name within a Kubernetes cluster
 var configMap string
 
-// Provider Config
-var providerConfig string
-
 // Points to a kubernetes configuration file
 var kubeConfigPath string
 
@@ -56,11 +53,10 @@ func init() {
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Interface, "interface", "", "Name of the interface to bind to")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesInterface, "serviceInterface", "", "Name of the interface to bind to (for services)")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIP, "vip", "", "The Virtual IP address")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPSubnet, "vipSubnet", "", "The Virtual IP address subnet e.g. /32 /24 /8 etc..")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPSubnet, "vipSubnet", "", "The Virtual IP address subnet e.g. /32 /24 /8 etc.. (Default to 32 for IPv4 and 128 for IPv6)")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.NodeName, "nodeName", "", "Name to be used for lease holder. Must be unique for each node/instance")
 
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPCIDR, "cidr", "", "The CIDR range for the virtual IP address. Default to 32 for IPv4 and 128 for IPv6") // todo: deprecate
-
+	// VIP flags
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Address, "address", "", "an address (IP or DNS name) to use as a VIP")
 	kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.Port, "port", 6443, "Port for the VIP")
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for VIP changes")
@@ -82,13 +78,6 @@ func init() {
 	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time (in seconds) a Kubernetes leader can attempt to renew its lease")
 	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Length of time (in seconds) the LeaderElector clients should wait between tries of actions")
 
-	// Equinix Metal flags
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableMetal, "metal", false, "This will use the Equinix Metal API (requires the token ENV) to update the EIP <-> VIP")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalAPIKey, "metalKey", "", "The API token for authenticating with the Equinix Metal API")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalProject, "metalProject", "", "The name of project already created within Equinix Metal")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalProjectID, "metalProjectID", "", "The ID of project already created within Equinix Metal")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ProviderConfig, "provider-config", "", "The path to a provider configuration")
-
 	// BGP flags
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableBGP, "bgp", false, "This will enable BGP support within kube-vip")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.RouterID, "bgpRouterID", "", "The routerID for the bgp server")
@@ -104,6 +93,11 @@ func init() {
 	kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.BGPPeers, "bgppeers", []string{}, "Comma separated BGP Peer, format: address:as:password:multihop")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Annotations, "annotations", "", "Set Node annotations prefix for parsing")
 
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPConfig.Zebra.Enabled, "zebra", false, "This will enable Zebra support within kube-vip")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.URL, "zebraUrl", "unix:/var/run/frr/zserv.api", "Path to the unix domain socket for connecting to Zebra daemon")
+	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.Zebra.Version, "zebraVersion", 6, "Zebra API Version")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.SoftwareName, "zebraSoftwareName", "frr8.3", "Software Name for Zebra")
+
 	// Namespace for kube-vip
 	kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The namespace for the configmap defined within the cluster")
 
@@ -115,7 +109,7 @@ func init() {
 
 	// Routing Table flags
 	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableID, "tableID", 198, "The routing table used for all table entries")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableType, "tableType", 0, "The type of route that will be added to the routing table")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableType, "tableType", unix.RTN_UNICAST, "The type of route that will be added to the routing table")
 	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingProtocol, "routingProtocol", 248, "The routing protocol value used to create routes")
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.CleanRoutingTable, "cleanRoutingTable", false, "Clean routing table of redundant routes on start")
 
@@ -134,7 +128,7 @@ func init() {
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesLeaseName, "servicesLeaseName", "plndr-svcs-lock", "Name of the lease that is used for leader election for services (in arp mode)")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.DNSMode, "dnsMode", "first", "Name of the mode that DNS lookup will be performed (first, ipv4, ipv6, dual)")
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DisableServiceUpdates, "disableServiceUpdates", false, "If true, kube-vip will process services as usual, but will not update service's Status.LoadBalancer.Ingress slice")
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpointSlices, "enableEndpointSlices", false, "If enabled, kube-vip will only advertise services, but will use EndpointSlices instead of endpoints to get IPs of Pods")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpoints, "enableEndpoints", false, "If enabled, kube-vip will only advertise services, but will use the (deprecated since v1.33) endpoints for IP addresses")
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoInterfaceGlobalScope, "loInterfaceGlobalScope", false, "If true, kube-vip will set global scope when using the lo interface, otherwise a host scope will be used by default")
 	kubeVipCmd.PersistentFlags().IntVar(&initConfig.HealthCheckPort, "healthCheckPort", 0, "If set to non-zero (> 1024), then this is the port that the healthcheck will listen on")
 
@@ -199,6 +193,11 @@ var kubeVipService = &cobra.Command{
 			return
 		}
 
+		// Change RTN_UNSPEC to default type
+		if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
+			initConfig.RoutingProtocol = unix.RTN_UNICAST
+		}
+
 		// Set the logging level for all subsequent functions
 		log.SetLogLoggerLevel(log.Level(initConfig.Logging))
 
@@ -214,8 +213,8 @@ var kubeVipService = &cobra.Command{
 		}
 
 		// Ensure there is an address to generate the CIDR from
-		if initConfig.VIPCIDR == "" && initConfig.Address != "" {
-			initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
 			if err != nil {
 				log.Error("generating CIDR", "err", err)
 				return
@@ -249,12 +248,17 @@ var kubeVipManager = &cobra.Command{
 			return
 		}
 
+		// Change RTN_UNSPEC to default type
+		if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
+			initConfig.RoutingProtocol = unix.RTN_UNICAST
+		}
+
 		// Set the logging level for all subsequent functions
 		log.SetLogLoggerLevel(log.Level(initConfig.Logging))
 
 		// Ensure there is an address to generate the CIDR from
-		if initConfig.VIPCIDR == "" && initConfig.Address != "" {
-			initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
 			if err != nil {
 				log.Error("No interface is specified for kube-vip to bind to")
 				return
@@ -298,12 +302,12 @@ var kubeVipManager = &cobra.Command{
 			return
 		}
 
-		if !initConfig.EnableARP && strings.Contains(initConfig.VIPCIDR, kubevip.Auto) {
+		if !initConfig.EnableARP && strings.Contains(initConfig.VIPSubnet, kubevip.Auto) {
 			log.Error("auto subnet discovery cannot be used outside ARP mode")
 			return
 		}
 
-		if strings.Contains(initConfig.VIPCIDR, kubevip.Auto) && initConfig.Address != "" {
+		if strings.Contains(initConfig.VIPSubnet, kubevip.Auto) && initConfig.Address != "" {
 			log.Error("auto subnet discovery cannot be used if VIP address was provided")
 			return
 		}
@@ -372,19 +376,6 @@ var kubeVipManager = &cobra.Command{
 			configMap = envConfigMap
 		}
 
-		// If Equinix Metal is enabled and there is a provider configuration passed
-		if initConfig.EnableMetal {
-			if providerConfig != "" {
-				providerAPI, providerProject, err := equinixmetal.GetPacketConfig(providerConfig)
-				if err != nil {
-					log.Error("retrieving equinix metal config", "err", err)
-					return
-				}
-				initConfig.MetalAPIKey = providerAPI
-				initConfig.MetalProject = providerProject
-			}
-		}
-
 		// Define the new service manager
 		mgr, err := manager.New(configMap, &initConfig)
 		if err != nil {

go.mod

@@ -1,94 +1,107 @@
 module github.com/kube-vip/kube-vip
 
-go 1.23.2
+go 1.24.4
 
 require (
-	github.com/cloudflare/ipvs v0.10.3
+	github.com/cloudflare/ipvs v0.11.0
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
+	github.com/docker/docker v28.3.3+incompatible
 	github.com/florianl/go-conntrack v0.4.0
-	github.com/golang/protobuf v1.5.4
 	github.com/google/go-cmp v0.7.0
-	github.com/gookit/slog v0.5.7
+	github.com/google/nftables v0.3.0
+	github.com/gookit/slog v0.5.8
 	github.com/huin/goupnp v1.3.0
 	github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d
 	github.com/jpillora/backoff v1.0.0
 	github.com/mdlayher/ndp v1.1.0
-	github.com/onsi/ginkgo/v2 v2.22.2
-	github.com/onsi/gomega v1.36.2
-	github.com/osrg/gobgp/v3 v3.35.0
-	github.com/packethost/packngo v0.31.0
+	github.com/onsi/ginkgo/v2 v2.23.4
+	github.com/onsi/gomega v1.37.0
+	github.com/osrg/gobgp/v3 v3.37.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.21.0
+	github.com/prometheus/client_golang v1.22.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.9.1
 	github.com/stretchr/testify v1.10.0
-	github.com/vishvananda/netlink v1.3.0
-	go.etcd.io/etcd/api/v3 v3.5.18
-	go.etcd.io/etcd/client/pkg/v3 v3.5.18
-	go.etcd.io/etcd/client/v3 v3.5.18
+	github.com/vishvananda/netlink v1.3.1
+	go.etcd.io/etcd/api/v3 v3.6.1
+	go.etcd.io/etcd/client/pkg/v3 v3.6.1
+	go.etcd.io/etcd/client/v3 v3.6.1
 	go.uber.org/zap v1.27.0
 	golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
-	golang.org/x/sync v0.11.0
-	golang.org/x/sys v0.30.0
+	golang.org/x/sync v0.15.0
+	golang.org/x/sys v0.33.0
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
-	google.golang.org/protobuf v1.36.5
-	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/api v0.32.1
-	k8s.io/apimachinery v0.32.1
-	k8s.io/client-go v0.32.1
+	google.golang.org/grpc v1.73.0
+	google.golang.org/protobuf v1.36.6
+	k8s.io/api v0.33.3
+	k8s.io/apimachinery v0.33.3
+	k8s.io/client-go v0.33.3
 	k8s.io/klog/v2 v2.130.1
-	sigs.k8s.io/kind v0.27.0
+	sigs.k8s.io/kind v0.29.0
 	sigs.k8s.io/yaml v1.4.0
 )
 
 require (
 	al.essio.dev/pkg/shellescape v1.5.1 // indirect
 	github.com/BurntSushi/toml v1.4.0 // indirect
+	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect
+	github.com/distribution/reference v0.6.0 // indirect
 	github.com/dlclark/regexp2 v1.11.4 // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/eapache/channels v1.1.0 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.12.1 // indirect
 	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.8.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.9 // indirect
-	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect
-	github.com/google/safetext v0.0.0-20240722112252-5a72de7e7962 // indirect
+	github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gookit/color v1.5.4 // indirect
 	github.com/gookit/goutil v0.6.18 // indirect
 	github.com/gookit/gsr v0.1.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/k-sone/critbitgo v1.4.0 // indirect
-	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/magiconair/properties v1.8.9 // indirect
 	github.com/mailru/easyjson v0.9.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mdlayher/genetlink v1.3.2 // indirect
-	github.com/mdlayher/netlink v1.7.2 // indirect
+	github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect
 	github.com/mdlayher/packet v1.1.2 // indirect
 	github.com/mdlayher/socket v0.5.1 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/sys/atomicwriter v0.1.0 // indirect
+	github.com/moby/term v0.5.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/pborman/uuid v1.2.1 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect
@@ -113,24 +126,32 @@ require (
 	github.com/xlab/c-for-go v1.3.0 // indirect
 	github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.37.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 // indirect
+	go.opentelemetry.io/otel/metric v1.37.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.37.0 // indirect
+	go.opentelemetry.io/otel/trace v1.37.0 // indirect
+	go.uber.org/automaxprocs v1.6.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.35.0 // indirect
-	golang.org/x/mod v0.22.0 // indirect
-	golang.org/x/net v0.36.0 // indirect
-	golang.org/x/oauth2 v0.25.0 // indirect
-	golang.org/x/term v0.29.0 // indirect
-	golang.org/x/text v0.22.0 // indirect
+	golang.org/x/crypto v0.39.0 // indirect
+	golang.org/x/mod v0.25.0 // indirect
+	golang.org/x/net v0.41.0 // indirect
+	golang.org/x/oauth2 v0.30.0 // indirect
+	golang.org/x/term v0.32.0 // indirect
+	golang.org/x/text v0.26.0 // indirect
 	golang.org/x/time v0.9.0 // indirect
-	golang.org/x/tools v0.28.0 // indirect
+	golang.org/x/tools v0.33.0 // indirect
 	golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d // indirect
-	google.golang.org/grpc v1.69.2 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 // indirect
+	k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
 	k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect
 	modernc.org/cc/v4 v4.24.4 // indirect
 	modernc.org/mathutil v1.7.1 // indirect
@@ -139,5 +160,6 @@ require (
 	modernc.org/strutil v1.2.1 // indirect
 	modernc.org/token v1.1.0 // indirect
 	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.5.0 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
 )

go.sum

@@ -1,17 +1,29 @@
 al.essio.dev/pkg/shellescape v1.5.1 h1:86HrALUujYS/h+GtqoB26SBEdkWfmMI6FubjXlsXyho=
 al.essio.dev/pkg/shellescape v1.5.1/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
+github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
+github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
 github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
+github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8=
+github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
 github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
-github.com/cloudflare/ipvs v0.10.3 h1:YFhvFFRzRxV1A1+RBa28AoC0XS4cg1QSmxBr4xb2pu4=
-github.com/cloudflare/ipvs v0.10.3/go.mod h1:XsbuKcQpqb3rpjYPtsFsM8BNZlgLw+Z0iC6RhocqDd8=
+github.com/cloudflare/ipvs v0.11.0 h1:niLcbqfv8+RSYk+yI+jhiCIHobEdEhO++mcwCX0Znsw=
+github.com/cloudflare/ipvs v0.11.0/go.mod h1:XsbuKcQpqb3rpjYPtsFsM8BNZlgLw+Z0iC6RhocqDd8=
+github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
+github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
+github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
+github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
+github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
 github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
 github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
@@ -23,10 +35,16 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
 github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
+github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
 github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
-github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
-github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
+github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
+github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
+github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k=
@@ -37,6 +55,8 @@ github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtz
 github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/florianl/go-conntrack v0.4.0 h1:TlYkxytdwgVayfU0cKwkHurQA0Rd1ZSEBRckRYDUu18=
 github.com/florianl/go-conntrack v0.4.0/go.mod h1:iPDx4oIats2T7X7Jm3PFyRCJM1GfZhJaSHOWROYOrE8=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
@@ -46,8 +66,9 @@ github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/
 github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
 github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
-github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
-github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
@@ -78,15 +99,12 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
-github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg=
-github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
-github.com/google/safetext v0.0.0-20240722112252-5a72de7e7962 h1:+9C/TgFfcCmZBV7Fjb3kQCGlkpFrhtvFDgbdQHB9RaA=
-github.com/google/safetext v0.0.0-20240722112252-5a72de7e7962/go.mod h1:H3K1Iu/utuCfa10JO+GsmKUYSWi7ug57Rk6GaDRHaaQ=
+github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg=
+github.com/google/nftables v0.3.0/go.mod h1:BCp9FsrbF1Fn/Yu6CLUc9GGZFw/+hsxfluNXXmxBfRM=
+github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
+github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
-github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
@@ -95,8 +113,10 @@ github.com/gookit/goutil v0.6.18 h1:MUVj0G16flubWT8zYVicIuisUiHdgirPAkmnfD2kKgw=
 github.com/gookit/goutil v0.6.18/go.mod h1:AY/5sAwKe7Xck+mEbuxj0n/bc3qwrGNe3Oeulln7zBA=
 github.com/gookit/gsr v0.1.0 h1:0gadWaYGU4phMs0bma38t+Do5OZowRMEVlHv31p0Zig=
 github.com/gookit/gsr v0.1.0/go.mod h1:7wv4Y4WCnil8+DlDYHBjidzrEzfHhXEoFjEA0pPPWpI=
-github.com/gookit/slog v0.5.7 h1:n3Dhgmr3NP+KppkNg95+vpFcI4YD1csu9VTQwgcEYTs=
-github.com/gookit/slog v0.5.7/go.mod h1:uWCRB4YO+FmgwXEq3s8U7ob1wWP7RStOuY/2a4yC/8o=
+github.com/gookit/slog v0.5.8 h1:XZCeHLQvvOZWcSUDZcqxXITsL9+d1ESsKZoASBmK1lI=
+github.com/gookit/slog v0.5.8/go.mod h1:s0ViFOY/IgUuT4MDPF0l9x5/npcciy8pL4xwWZadnoc=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8CrSJVmaolDVOxTfS9kc36uB6H40kdbQq8=
@@ -129,8 +149,9 @@ github.com/k-sone/critbitgo v1.4.0 h1:l71cTyBGeh6X5ATh6Fibgw3+rtNT80BA0uNNWgkPrb
 github.com/k-sone/critbitgo v1.4.0/go.mod h1:7E6pyoyADnFxlUBEKcnfS49b7SUAQGMK+OAp/UQvo0s=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
-github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -164,8 +185,8 @@ github.com/mdlayher/netlink v1.3.0/go.mod h1:xK/BssKuwcRXHrtN04UBkwQ6dY9VviGGuri
 github.com/mdlayher/netlink v1.4.0/go.mod h1:dRJi5IABcZpBD2A3D0Mv/AiX8I9uDEu5oGkAVrekmf8=
 github.com/mdlayher/netlink v1.4.1/go.mod h1:e4/KuJ+s8UhfUpO9z00/fDZZmhSrs+oxyqAS9cNgn6Q=
 github.com/mdlayher/netlink v1.5.0/go.mod h1:1Kr8BBFxGyUyNmztC9WLOayqYVAd2wsgOZm18nqGuzQ=
-github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g=
-github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
+github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 h1:A1Cq6Ysb0GM0tpKMbdCXCIfBclan4oHk1Jb+Hrejirg=
+github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42/go.mod h1:BB4YCPDOzfy7FniQ/lxuYQ3dgmM2cZumHbK8RpTjN2o=
 github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY=
 github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4=
 github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00/go.mod h1:GAFlyu4/XV68LkQKYzKhIo/WW7j3Zi0YRAz/BOoanUc=
@@ -177,39 +198,51 @@ github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE9
 github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
+github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
+github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw=
+github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs=
+github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
+github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
+github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
+github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
+github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
+github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU=
-github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk=
-github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8=
-github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY=
-github.com/osrg/gobgp/v3 v3.35.0 h1:8HQyCCoByAJyXiSzTFBl77wuOyVdDXozPdXX+tjYyHQ=
-github.com/osrg/gobgp/v3 v3.35.0/go.mod h1:l2nPaHaLmIoKbFxMUzKon/h6c9BTzCp5zJI9Dhnrx5c=
-github.com/packethost/packngo v0.31.0 h1:LLH90ardhULWbagBIc3I3nl2uU75io0a7AwY6hyi0S4=
-github.com/packethost/packngo v0.31.0/go.mod h1:Io6VJqzkiqmIEQbpOjeIw9v8q9PfcTEq8TEY/tMQsfw=
+github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus=
+github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8=
+github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y=
+github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
+github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
+github.com/osrg/gobgp/v3 v3.37.0 h1:+ObuOdvj7G7nxrT0fKFta+EAupdWf/q1WzbXydr8IOY=
+github.com/osrg/gobgp/v3 v3.37.0/go.mod h1:kVHVFy1/fyZHJ8P32+ctvPeJogn9qKwa1YCeMRXXrP0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
-github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
-github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
 github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
 github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
 github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.21.0 h1:DIsaGmiaBkSangBgMtWdNfxbMNdku5IK6iNhrEqWvdA=
-github.com/prometheus/client_golang v1.21.0/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg=
+github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
+github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
+github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
+github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
 github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
 github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
 github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
@@ -218,13 +251,14 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
-github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
-github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk=
 github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0=
 github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
 github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
@@ -240,13 +274,12 @@ github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
 github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
@@ -257,9 +290,8 @@ github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 h1:pyC9PaHYZFgEKFdlp3G8
 github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk=
-github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs=
-github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
+github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
+github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
 github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
 github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
@@ -273,22 +305,34 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJu
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-go.etcd.io/etcd/api/v3 v3.5.18 h1:Q4oDAKnmwqTo5lafvB+afbgCDF7E35E4EYV2g+FNGhs=
-go.etcd.io/etcd/api/v3 v3.5.18/go.mod h1:uY03Ob2H50077J7Qq0DeehjM/A9S8PhVfbQ1mSaMopU=
-go.etcd.io/etcd/client/pkg/v3 v3.5.18 h1:mZPOYw4h8rTk7TeJ5+3udUkfVGBqc+GCjOJYd68QgNM=
-go.etcd.io/etcd/client/pkg/v3 v3.5.18/go.mod h1:BxVf2o5wXG9ZJV+/Cu7QNUiJYk4A29sAhoI5tIRsCu4=
-go.etcd.io/etcd/client/v3 v3.5.18 h1:nvvYmNHGumkDjZhTHgVU36A9pykGa2K4lAJ0yY7hcXA=
-go.etcd.io/etcd/client/v3 v3.5.18/go.mod h1:kmemwOsPU9broExyhYsBxX4spCTDX3yLgPMWtpBXG6E=
-go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=
-go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=
-go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
-go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
-go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=
-go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0=
-go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc=
-go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=
-go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
-go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
+go.etcd.io/etcd/api/v3 v3.6.1 h1:yJ9WlDih9HT457QPuHt/TH/XtsdN2tubyxyQHSHPsEo=
+go.etcd.io/etcd/api/v3 v3.6.1/go.mod h1:lnfuqoGsXMlZdTJlact3IB56o3bWp1DIlXPIGKRArto=
+go.etcd.io/etcd/client/pkg/v3 v3.6.1 h1:CxDVv8ggphmamrXM4Of8aCC8QHzDM4tGcVr9p2BSoGk=
+go.etcd.io/etcd/client/pkg/v3 v3.6.1/go.mod h1:aTkCp+6ixcVTZmrJGa7/Mc5nMNs59PEgBbq+HCmWyMc=
+go.etcd.io/etcd/client/v3 v3.6.1 h1:KelkcizJGsskUXlsxjVrSmINvMMga0VWwFF0tSPGEP0=
+go.etcd.io/etcd/client/v3 v3.6.1/go.mod h1:fCbPUdjWNLfx1A6ATo9syUmFVxqHH9bCnPLBZmnLmMY=
+go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
+go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
+go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
+go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 h1:bDMKF3RUSxshZ5OjOTi8rsHGaPKsAt76FaqgvIUySLc=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0/go.mod h1:dDT67G/IkA46Mr2l9Uj7HsQVwsjASyV9SjGofsiUZDA=
+go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
+go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
+go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
+go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
+go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
+go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
+go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
+go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
+go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
+go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
+go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
+go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
@@ -297,18 +341,17 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
-golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
+golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
+golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
 golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329 h1:9kj3STMvgqy3YA4VQXBrN7925ICMxD5wzMRcgA30588=
 golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
-golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
-golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
+golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -329,20 +372,22 @@ golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA=
-golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I=
-golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70=
-golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
+golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
+golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
+golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
+golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
-golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
+golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -369,16 +414,16 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
-golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
+golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
-golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
+golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
+golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
-golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
+golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
+golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
 golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
 golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -387,8 +432,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
-golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8=
-golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw=
+golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
+golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -397,14 +442,14 @@ golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uI
 golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA=
 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10 h1:3GDAcqdIg1ozBNLgPy4SLT84nfcBjr6rhGtXYtrkWLU=
 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10/go.mod h1:T97yPqesLiNrOYxkwmhMI0ZIlJDm+p0PMR8eRVeR5tQ=
-google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d h1:H8tOf8XM88HvKqLTxe755haY6r1fqqzLbEnfrmLXlSA=
-google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d/go.mod h1:2v7Z7gP2ZUOGsaFyxATQSRoBnKygqVq2Cwnvom7QiqY=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d h1:xJJRGY7TJcvIlpSrN3K6LAWgNFUILlO+OMAqtg9aqnw=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d/go.mod h1:3ENsm/5D1mzDyhpzeRi1NR784I0BcofWBoSc5QqqMK4=
-google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU=
-google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4=
-google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
-google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
+google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY=
+google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
+google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok=
+google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc=
+google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
+google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
@@ -414,7 +459,6 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -424,16 +468,16 @@ gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
 gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
 honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
 honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
-k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc=
-k8s.io/api v0.32.1/go.mod h1:/Yi/BqkuueW1BgpoePYBRdDYfjPF5sgTr5+YqDZra5k=
-k8s.io/apimachinery v0.32.1 h1:683ENpaCBjma4CYqsmZyhEzrGz6cjn1MY/X2jB2hkZs=
-k8s.io/apimachinery v0.32.1/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE=
-k8s.io/client-go v0.32.1 h1:otM0AxdhdBIaQh7l1Q0jQpmo7WOFIk5FFa4bg6YMdUU=
-k8s.io/client-go v0.32.1/go.mod h1:aTTKZY7MdxUaJ/KiUs8D+GssR9zJZi77ZqtzcGXIiDg=
+k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8=
+k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE=
+k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA=
+k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM=
+k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA=
+k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
-k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 h1:hcha5B1kVACrLujCKLbr8XWMxCxzQx42DY8QKYJrDLg=
-k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7/go.mod h1:GewRfANuJ70iYzvn+i4lezLDAFzvjxZYK1gn1lWcfas=
+k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4=
+k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8=
 k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0=
 k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 modernc.org/cc/v4 v4.24.4 h1:TFkx1s6dCkQpd6dKurBNmpo+G8Zl4Sq/ztJ+2+DEsh0=
@@ -454,9 +498,12 @@ pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
 pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
 sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=
 sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
-sigs.k8s.io/kind v0.27.0 h1:PQ3f0iAWNIj66LYkZ1ivhEg/+Zb6UPMbO+qVei/INZA=
-sigs.k8s.io/kind v0.27.0/go.mod h1:RZVFmy6qcwlSWwp6xeIUv7kXCPF3i8MXsEXxW/J+gJY=
-sigs.k8s.io/structured-merge-diff/v4 v4.5.0 h1:nbCitCK2hfnhyiKo6uf2HxUPTCodY6Qaf85SbDIaMBk=
-sigs.k8s.io/structured-merge-diff/v4 v4.5.0/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4=
+sigs.k8s.io/kind v0.29.0 h1:3TpCsyh908IkXXpcSnsMjWdwdWjIl7o9IMZImZCWFnI=
+sigs.k8s.io/kind v0.29.0/go.mod h1:ldWQisw2NYyM6k64o/tkZng/1qQW7OlzcN5a8geJX3o=
+sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
+sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
+sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
+sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc=
+sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
 sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

pkg/arp/arp.go

@@ -0,0 +1,149 @@
+package arp
+
+import (
+	"context"
+	log "log/slog"
+	"sync"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
+
+type Manager struct {
+	instances map[string]*Instance
+	config    *kubevip.Config
+}
+
+type Instance struct {
+	network vip.Network
+	ndp     *vip.NdpResponder
+	mu      sync.Mutex
+	counter int
+}
+
+func NewManager(config *kubevip.Config) *Manager {
+	return &Manager{
+		instances: make(map[string]*Instance),
+		config:    config,
+	}
+}
+
+func NewInstance(network vip.Network, ndp *vip.NdpResponder) *Instance {
+	return &Instance{
+		ndp:     ndp,
+		network: network,
+		counter: 1,
+	}
+}
+
+func (i *Instance) Name() string {
+	return i.network.ARPName()
+}
+
+func (m *Manager) Insert(instance *Instance) {
+	i, ok := m.instances[instance.Name()]
+	if !ok {
+		log.Info("inserting ARP/NDP instance", "name", instance.Name())
+		m.instances[instance.Name()] = instance
+	} else {
+		i.mu.Lock()
+		defer i.mu.Unlock()
+		i.counter++
+	}
+}
+
+func (m *Manager) Remove(instance *Instance) {
+	if i, ok := m.instances[instance.Name()]; ok {
+		i.mu.Lock()
+		defer i.mu.Unlock()
+		if i.counter > 1 {
+			i.counter--
+		} else {
+			log.Info("removing ARP/NDP instance", "name", instance.Name())
+			delete(m.instances, instance.Name())
+		}
+	}
+}
+
+func (m *Manager) Count(name string) int {
+	if i, ok := m.instances[name]; ok {
+		i.mu.Lock()
+		defer i.mu.Unlock()
+		return i.counter
+	}
+	return 0
+}
+
+func (m *Manager) StartAdvertisement(ctx context.Context) {
+	log.Info("Starting ARP/NDP advertisement")
+	for {
+		select {
+		case <-ctx.Done(): // if cancel() execute
+			return
+		default:
+			for _, instance := range m.instances {
+				if instance.counter > 0 {
+					ensureIPAndSendGratuitous(instance)
+				}
+			}
+		}
+		if m.config.ArpBroadcastRate < 500 {
+			log.Error("arp broadcast rate is too low", "rate (ms)", m.config.ArpBroadcastRate, "setting to (ms)", "3000")
+			m.config.ArpBroadcastRate = 3000
+		}
+		time.Sleep(time.Duration(m.config.ArpBroadcastRate) * time.Millisecond)
+	}
+}
+
+// ensureIPAndSendGratuitous - adds IP to the interface if missing, and send
+// either a gratuitous ARP or gratuitous NDP. Re-adds the interface if it is IPv6
+// and in a dadfailed state.
+func ensureIPAndSendGratuitous(instance *Instance) {
+	iface := instance.network.Interface()
+	ipString := instance.network.IP()
+
+	// Check if IP is dadfailed
+	if instance.network.IsDADFAIL() {
+		log.Warn("IP address is in dadfailed state, removing config", "ip", ipString, "interface", iface)
+		deleted, err := instance.network.DeleteIP()
+		if err != nil {
+			log.Warn(err.Error())
+		}
+		if deleted {
+			log.Info("deleted and recreating address", "IP", ipString, "interface", iface)
+			// if _, err := instance.network.AddIP(false); err != nil {
+			// 	log.Error("failed to recreate address", "IP", ipString, "interface", iface)
+			// }
+		}
+	}
+
+	// Ensure the address exists on the interface before attempting to ARP
+	// if instance.network.HasEndpoints() {
+	if added, err := instance.network.AddIP(true); err != nil {
+		log.Warn(err.Error())
+	} else if added {
+		log.Warn("Re-applied the VIP configuration", "ip", ipString, "interface", iface)
+	}
+	// }
+
+	if vip.IsIPv6(ipString) {
+		// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
+		if instance.ndp == nil {
+			log.Error("NDP responder was not created")
+		} else {
+			err := instance.ndp.SendGratuitous(ipString)
+			if err != nil {
+				log.Warn(err.Error())
+			}
+		}
+
+	} else {
+		// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
+		err := vip.ARPSendGratuitous(ipString, iface)
+		if err != nil {
+			log.Warn(err.Error())
+		}
+	}
+
+}

pkg/backend/backend.go

@@ -40,7 +40,6 @@ func (e *Entry) Check() bool {
 	switch {
 	case utils.FileExists(adminConfigPath):
 		config, err = k8s.NewRestConfig(adminConfigPath, false, k8sAddr)
-		// client, err = k8s.NewClientset(adminConfigPath, false, k8sAddr)
 		if err != nil {
 			log.Error("create k8s REST config", "path", adminConfigPath, "err", err)
 			return false

pkg/bgp/peers.go

@@ -8,7 +8,6 @@ import (
 	"strings"
 
 	//nolint
-	"github.com/golang/protobuf/ptypes/any"
 	"github.com/kube-vip/kube-vip/pkg/vip"
 	api "github.com/osrg/gobgp/v3/api"
 	"github.com/osrg/gobgp/v3/pkg/server"
@@ -142,7 +141,7 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
 				Safi: api.Family_SAFI_UNICAST,
 			},
 			Nlri:   nlri,
-			Pattrs: []*any.Any{originAttr, nhAttr},
+			Pattrs: []*anypb.Any{originAttr, nhAttr},
 		}
 	} else {
 		//nolint
@@ -160,13 +159,13 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
 		mpAttr, _ := anypb.New(&api.MpReachNLRIAttribute{
 			Family:   v6Family,
 			NextHops: []string{"::"}, // gobgp will fill this
-			Nlris:    []*any.Any{nlri},
+			Nlris:    []*anypb.Any{nlri},
 		})
 
 		path = &api.Path{
 			Family: v6Family,
 			Nlri:   nlri,
-			Pattrs: []*any.Any{originAttr, mpAttr},
+			Pattrs: []*anypb.Any{originAttr, mpAttr},
 		}
 	}
 	return

pkg/bgp/server.go

@@ -9,10 +9,11 @@ import (
 
 	api "github.com/osrg/gobgp/v3/api"
 	gobgp "github.com/osrg/gobgp/v3/pkg/server"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // NewBGPServer takes a configuration and returns a running BGP server instance
-func NewBGPServer(c *Config, peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (b *Server, err error) {
+func NewBGPServer(c *Config) (b *Server, err error) {
 	if c.AS == 0 {
 		return nil, fmt.Errorf("you need to provide AS")
 	}
@@ -28,13 +29,25 @@ func NewBGPServer(c *Config, peerStateChangeCallback func(*api.WatchEventRespons
 	b = &Server{
 		s: gobgp.NewBgpServer(),
 		c: c,
+
+		BGPSessionInfoGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "kube_vip",
+			Subsystem: "manager",
+			Name:      "bgp_session_info",
+			Help:      "Display state of session by setting metric for label value with current state to 1",
+		}, []string{"state", "peer"}),
 	}
+	return
+}
+
+// Start starts the BGP server
+func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (err error) {
 	go b.s.Serve()
 
 	if err = b.s.StartBgp(context.Background(), &api.StartBgpRequest{
 		Global: &api.Global{
-			Asn:        c.AS,
-			RouterId:   c.RouterID,
+			Asn:        b.c.AS,
+			RouterId:   b.c.RouterID,
 			ListenPort: -1,
 		},
 	}); err != nil {
@@ -52,12 +65,23 @@ func NewBGPServer(c *Config, peerStateChangeCallback func(*api.WatchEventRespons
 		return
 	}
 
-	for _, p := range c.Peers {
+	for _, p := range b.c.Peers {
 		if err = b.AddPeer(p); err != nil {
 			return
 		}
 	}
 
+	if b.c.Zebra.Enabled {
+		if err = b.s.EnableZebra(context.Background(), &api.EnableZebraRequest{
+			Url:          b.c.Zebra.URL,
+			Version:      b.c.Zebra.Version,
+			SoftwareName: b.c.Zebra.SoftwareName,
+		}); err != nil {
+			log.Error(err.Error())
+			return
+		}
+	}
+
 	return
 }
 

pkg/bgp/types.go

@@ -7,6 +7,7 @@ import (
 	"github.com/kube-vip/kube-vip/pkg/vip"
 	api "github.com/osrg/gobgp/v3/api"
 	gobgp "github.com/osrg/gobgp/v3/pkg/server"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/vishvananda/netlink"
 )
 
@@ -121,10 +122,24 @@ type Config struct {
 	KeepaliveInterval uint64
 
 	Peers []Peer
+
+	Zebra ZebraConfig
+}
+
+// Defines Zebra connection configuration. More on the topic - https://github.com/osrg/gobgp/blob/master/docs/sources/zebra.md#configuration
+type ZebraConfig struct {
+	Enabled      bool
+	URL          string
+	Version      uint32
+	SoftwareName string
 }
 
 // Server manages a server object
 type Server struct {
 	s *gobgp.BgpServer
 	c *Config
+
+	// This is a prometheus gauge indicating the state of the sessions.
+	// 1 means "ESTABLISHED", 0 means "NOT ESTABLISHED"
+	BGPSessionInfoGauge *prometheus.GaugeVec
 }

pkg/cluster/cluster.go

@@ -5,7 +5,9 @@ import (
 
 	log "log/slog"
 
+	"github.com/kube-vip/kube-vip/pkg/arp"
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/networkinterface"
 	"github.com/kube-vip/kube-vip/pkg/vip"
 )
 
@@ -15,16 +17,17 @@ type Cluster struct {
 	completed chan bool
 	once      sync.Once
 	Network   []vip.Network
+	arpMgr    *arp.Manager
 }
 
 // InitCluster - Will attempt to initialise all of the required settings for the cluster
-func InitCluster(c *kubevip.Config, disableVIP bool) (*Cluster, error) {
+func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Cluster, error) {
 	var networks []vip.Network
 	var err error
 
 	if !disableVIP {
 		// Start the Virtual IP Networking configuration
-		networks, err = startNetworking(c)
+		networks, err = startNetworking(c, intfMgr)
 		if err != nil {
 			return nil, err
 		}
@@ -32,6 +35,7 @@ func InitCluster(c *kubevip.Config, disableVIP bool) (*Cluster, error) {
 	// Initialise the Cluster structure
 	newCluster := &Cluster{
 		Network: networks,
+		arpMgr:  arpMgr,
 	}
 
 	log.Debug("service security", "enabled", c.EnableServiceSecurity)
@@ -39,7 +43,7 @@ func InitCluster(c *kubevip.Config, disableVIP bool) (*Cluster, error) {
 	return newCluster, nil
 }
 
-func startNetworking(c *kubevip.Config) ([]vip.Network, error) {
+func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vip.Network, error) {
 	address := c.VIP
 
 	if c.Address != "" {
@@ -51,7 +55,8 @@ func startNetworking(c *kubevip.Config) ([]vip.Network, error) {
 	networks := []vip.Network{}
 	for _, addr := range addresses {
 		network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.RoutingTableID,
-			c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend, c.EnableLoadBalancer)
+			c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend,
+			c.EnableLoadBalancer, c.EnableServiceSecurity, intfMgr)
 		if err != nil {
 			return nil, err
 		}

pkg/cluster/clusterLeaderElection.go

@@ -11,14 +11,11 @@ import (
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/kube-vip/kube-vip/pkg/bgp"
-	"github.com/kube-vip/kube-vip/pkg/equinixmetal"
 	"github.com/kube-vip/kube-vip/pkg/etcd"
 	"github.com/kube-vip/kube-vip/pkg/k8s"
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
 	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
 
-	"github.com/packethost/packngo"
-
 	log "log/slog"
 
 	clientv3 "go.etcd.io/etcd/client/v3"
@@ -135,10 +132,13 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
 	// (attempt to) Remove the virtual IP, in case it already exists
 
 	for i := range cluster.Network {
-		err = cluster.Network[i].DeleteIP()
+		deleted, err := cluster.Network[i].DeleteIP()
 		if err != nil {
 			log.Error("could not delete virtualIP", "err", err)
 		}
+		if deleted {
+			log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+		}
 	}
 
 	// Defer a function to check if the bgpServer has been created and if so attempt to close it
@@ -148,42 +148,16 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
 		}
 	}()
 
-	// If Equinix Metal is enabled then we can begin our preparation work
-	var packetClient *packngo.Client
-	if c.EnableMetal {
-		if c.ProviderConfig != "" {
-			key, project, err := equinixmetal.GetPacketConfig(c.ProviderConfig)
-			if err != nil {
-				log.Error("retrieve Equinix metal config", "err", err)
-			} else {
-				// Set the environment variable with the key for the project
-				os.Setenv("PACKET_AUTH_TOKEN", key)
-				// Update the configuration with the project key
-				c.MetalProjectID = project
-			}
-		}
-		packetClient, err = packngo.NewClient()
-		if err != nil {
-			log.Error("create Equinix metal client", "err", err)
-		}
-
-		// We're using Equinix Metal with BGP, populate the Peer information from the API
-		if c.EnableBGP {
-			log.Info("Looking up the BGP configuration from Equinix Metal")
-			err = equinixmetal.BGPLookup(packetClient, c)
-			if err != nil {
-				log.Error("retrieve Equinix metal BPG config", "err", err)
-			}
-		}
-	}
-
 	if c.EnableBGP && bgpServer == nil {
 		// Lets start BGP
 		log.Info("Starting the BGP server to advertise VIP routes to VGP peers")
-		bgpServer, err = bgp.NewBGPServer(&c.BGPConfig, nil)
+		bgpServer, err = bgp.NewBGPServer(&c.BGPConfig)
 		if err != nil {
 			log.Error("new BGP server", "err", err)
 		}
+		if err := bgpServer.Start(nil); err != nil {
+			log.Error("starting BGP server", "err", err)
+		}
 	}
 
 	run := &runConfig{
@@ -192,7 +166,7 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
 		sm:      sm,
 		onStartedLeading: func(ctx context.Context) { //nolint TODO: potential clean code
 			// As we're leading lets start the vip service
-			err := cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, packetClient, cancel)
+			err := cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, cancel)
 			if err != nil {
 				log.Error("starting VIP service on leader", "err", err)
 			}
@@ -215,10 +189,13 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
 			}
 
 			for i := range cluster.Network {
-				err := cluster.Network[i].DeleteIP()
+				deleted, err := cluster.Network[i].DeleteIP()
 				if err != nil {
 					log.Warn("delete VIP", "err", err)
 				}
+				if deleted {
+					log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+				}
 			}
 
 			log.Error("lost leadership, restarting kube-vip")
@@ -307,7 +284,7 @@ func (cluster *Cluster) runEtcdLeaderElectionOrDie(ctx context.Context, run *run
 	})
 }
 
-func (sm *Manager) NodeWatcher(lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
+func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
 	// Use a restartable watcher, as this should help in the event of etcd or timeout issues
 	log.Info("Kube-Vip is watching nodes for control-plane labels")
 
@@ -315,7 +292,7 @@ func (sm *Manager) NodeWatcher(lb *loadbalancer.IPVSLoadBalancer, port uint16) e
 		LabelSelector: "node-role.kubernetes.io/control-plane",
 	}
 
-	rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
+	rw, err := watchtools.NewRetryWatcherWithContext(ctxArp, "1", &cache.ListWatch{
 		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
 			return sm.RetryWatcherClient.CoreV1().Nodes().Watch(context.Background(), listOptions)
 		},

pkg/cluster/service.go

@@ -8,25 +8,25 @@ import (
 	"net"
 	"os"
 	"os/signal"
+	"sync"
 	"syscall"
-	"time"
 
 	log "log/slog"
 
+	"github.com/kube-vip/kube-vip/pkg/arp"
 	"github.com/kube-vip/kube-vip/pkg/backend"
 	"github.com/kube-vip/kube-vip/pkg/bgp"
-	"github.com/kube-vip/kube-vip/pkg/equinixmetal"
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
 	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
 	"github.com/kube-vip/kube-vip/pkg/vip"
-	"github.com/packethost/packngo"
+	"github.com/vishvananda/netlink"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 )
 
-func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Config, sm *Manager, bgpServer *bgp.Server, packetClient *packngo.Client, cancelLeaderElection context.CancelFunc) error {
+func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Config, sm *Manager, bgpServer *bgp.Server, cancelLeaderElection context.CancelFunc) error {
 	var err error
 
 	// listen for interrupts or the Linux SIGTERM signal and cancel
@@ -41,75 +41,50 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
 
 	loadbalancers := []*loadbalancer.IPVSLoadBalancer{}
 
+	var arpWG sync.WaitGroup
+
 	for i := range cluster.Network {
-		if cluster.Network[i].IsDDNS() {
+		network := cluster.Network[i]
+		if err := network.SetMask(c.VIPSubnet); err != nil {
+			log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
+			panic("")
+		}
+		if network.IsDDNS() {
 			if err := cluster.StartDDNS(ctxDNS); err != nil {
 				log.Error(err.Error())
 			}
 		}
 
 		// start the dns updater if address is dns
-		if cluster.Network[i].IsDNS() {
-			log.Info("starting the DNS updater", "address", cluster.Network[i].DNSName())
-			ipUpdater := vip.NewIPUpdater(cluster.Network[i])
+		if network.IsDNS() {
+			log.Info("starting the DNS updater", "address", network.DNSName())
+			ipUpdater := vip.NewIPUpdater(network)
 			ipUpdater.Run(ctxDNS)
 		}
 
 		if !c.EnableRoutingTable {
-			if c.EnableARP {
-				subnets := vip.Split(c.VIPCIDR)
-				subnet := ""
-				if len(subnets) > 0 {
-					subnet = subnets[0]
-				}
-				if vip.IsIPv6(cluster.Network[i].IP()) && len(subnets) > 1 {
-					subnet = subnets[1]
-				}
-				if subnet == "" {
-					log.Error("no subnet provided", "IP", cluster.Network[i].IP())
-					panic("")
-				}
-				if err = cluster.Network[i].SetMask(subnet); err != nil {
-					log.Error("failed to set mask", "subnet", subnet, "err", err)
-					panic("")
-				}
-			}
-			if err = cluster.Network[i].AddIP(false); err != nil {
+			if _, err = network.AddIP(false); err != nil {
 				log.Error(err.Error())
 			}
 		}
 
-		if c.EnableMetal {
-			// We're not using Equinix Metal with BGP
-			if !c.EnableBGP {
-				// Attempt to attach the EIP in the standard manner
-				log.Debug("Attaching the Equinix Metal EIP through the API to this host")
-				err = equinixmetal.AttachEIP(packetClient, c, c.NodeName)
-				if err != nil {
-					log.Error(err.Error())
-				}
-			}
-		}
-
 		if c.EnableBGP {
 			// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
-			cidrVip := fmt.Sprintf("%s/%s", cluster.Network[i].IP(), c.VIPCIDR)
-			log.Debug("Attempting to advertise over BGP", "address", cidrVip)
-
-			err = bgpServer.AddHost(cidrVip)
+			log.Debug("Attempting to advertise over BGP", "address", network.CIDR())
+			err = bgpServer.AddHost(network.CIDR())
 			if err != nil {
 				log.Error(err.Error())
 			}
 		}
 
 		if c.EnableLoadBalancer {
-			lb, err := loadbalancer.NewIPVSLB(cluster.Network[i].IP(), c.LoadBalancerPort, c.LoadBalancerForwardingMethod, c.BackendHealthCheckInterval, c.Interface, cancelLeaderElection, signalChan)
+			lb, err := loadbalancer.NewIPVSLB(network.IP(), c.LoadBalancerPort, c.LoadBalancerForwardingMethod, c.BackendHealthCheckInterval, c.Interface, cancelLeaderElection, signalChan)
 			if err != nil {
 				log.Error("Error creating IPVS LoadBalancer", "err", err)
 			}
 
 			go func() {
-				err = sm.NodeWatcher(lb, c.Port)
+				err = sm.NodeWatcher(ctxArp, lb, c.Port) //TODO: We're using the ctxARP as the context this will change when rkatz finishes his change
 				if err != nil {
 					log.Error("Error watching node labels", "err", err)
 				}
@@ -119,7 +94,8 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
 		}
 
 		if c.EnableARP {
-			go cluster.layer2Update(ctxArp, cluster.Network[i], c)
+			arpWG.Add(1)
+			go cluster.layer2Update(ctxArp, network, c, &arpWG)
 		}
 	}
 
@@ -190,7 +166,8 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
 
 		backend.Watch(func() {
 			for i := range cluster.Network {
-				networkIP := cluster.Network[i].IP()
+				network := cluster.Network[i]
+				networkIP := network.IP()
 				isNetworkV6, err := isV6(networkIP)
 				if err != nil {
 					log.Error("failed to check IP type", "IP", networkIP, "error", err)
@@ -204,19 +181,19 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
 
 				for entry := range *backendMap {
 					if entry.Check() {
-						err = cluster.Network[i].AddIP(true)
+						_, err = network.AddIP(true)
 						if err != nil {
 							log.Error("error adding address", "err", err)
 						}
 						if !(*backendMap)[entry] {
-							log.Info("added backend", "ip", cluster.Network[i].IP())
+							log.Info("added backend", "ip", network.IP())
 						}
 
-						err = cluster.Network[i].AddRoute(true)
+						err = network.AddRoute(true)
 						if err != nil && !errors.Is(err, fs.ErrExist) && !errors.Is(err, syscall.ESRCH) {
 							log.Warn(err.Error())
 						} else if err == nil && !(*backendMap)[entry] {
-							log.Info("added route", "route", cluster.Network[i].PrepareRoute().String())
+							log.Info("added route", "route", network.PrepareRoute().String())
 						}
 
 						(*backendMap)[entry] = true
@@ -234,24 +211,20 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
 				}
 
 				if deleteAddress {
-					err = cluster.Network[i].DeleteRoute()
+					err = network.DeleteRoute()
 					if err != nil && !errors.Is(err, fs.ErrNotExist) && !errors.Is(err, syscall.ESRCH) {
 						log.Warn("deleting route", "err", err)
 					} else if err == nil {
-						log.Info("deleted route", "route", cluster.Network[i].PrepareRoute().String())
+						log.Info("deleted route", "route", network.PrepareRoute().String())
 					}
 
-					isSet, err := cluster.Network[i].IsSet()
+					deleted, err := network.DeleteIP()
 					if err != nil {
-						log.Error("failed to check IP address", "error", err)
+						log.Error("error deleting IP", "err", err)
+						panic("")
 					}
-					if isSet {
-						err = cluster.Network[i].DeleteIP()
-						if err != nil {
-							log.Error("error deleting IP", "err", err)
-							panic("")
-						}
-						log.Info("deleted address", "ip", cluster.Network[i].IP())
+					if deleted {
+						log.Info("deleted address", "IP", network.IP(), "interface", network.Interface())
 					}
 				}
 			}
@@ -284,60 +257,48 @@ func getNodeIPs(ctx context.Context, nodename string, client *kubernetes.Clients
 }
 
 // StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
-func (cluster *Cluster) StartLoadBalancerService(c *kubevip.Config, bgp *bgp.Server) {
+func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp *bgp.Server, name string, CountRouteReferences func(*netlink.Route) int) {
 	// use a Go context so we can tell the arp loop code when we
 	// want to step down
 	//nolint
-	ctxArp, cancelArp := context.WithCancel(context.Background())
+	ctxArp, cancelArp := context.WithCancel(ctx)
 
 	cluster.stop = make(chan bool, 1)
 	cluster.completed = make(chan bool, 1)
 
+	var arpWG sync.WaitGroup
+
 	for i := range cluster.Network {
 		network := cluster.Network[i]
-
-		err := network.DeleteIP()
-		if err != nil {
-			log.Warn("Attempted to clean existing VIP", "err", err)
+		if err := network.SetMask(c.VIPSubnet); err != nil {
+			log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
+			panic("")
 		}
+		_, err := network.DeleteIP()
+		if err != nil {
+			log.Warn("attempted to clean existing VIP", "err", err)
+		}
+
 		if c.EnableRoutingTable && (c.EnableLeaderElection || c.EnableServicesElection) {
 			err = network.AddRoute(false)
 			if err != nil {
 				log.Warn(err.Error())
 			}
 		} else if !c.EnableRoutingTable {
-			if c.EnableARP {
-				subnets := vip.Split(c.VIPCIDR)
-				subnet := ""
-				if len(subnets) > 0 {
-					subnet = subnets[0]
-				}
-				if vip.IsIPv6(cluster.Network[i].IP()) && len(subnets) > 1 {
-					subnet = subnets[1]
-				}
-				if subnet == "" {
-					log.Error("no subnet provided for address", "ip", cluster.Network[i].IP())
-					panic("")
-				}
-				if err = network.SetMask(subnet); err != nil {
-					log.Error("failed to set mask", "subnet", subnet, "err", err)
-					panic("")
-				}
-			}
-			if err = network.AddIP(false); err != nil {
+			if _, err = network.AddIP(false); err != nil {
 				log.Warn(err.Error())
 			}
 		}
 
 		if c.EnableARP {
-			go cluster.layer2Update(ctxArp, cluster.Network[i], c)
+			arpWG.Add(1)
+			go cluster.layer2Update(ctxArp, network, c, &arpWG)
 		}
 
 		if c.EnableBGP && (c.EnableLeaderElection || c.EnableServicesElection) {
 			// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
-			cidrVip := fmt.Sprintf("%s/%s", network.IP(), c.VIPCIDR)
-			log.Debug("(svcs) attempting to advertise over BGP", "address", cidrVip)
-			err = bgp.AddHost(cidrVip)
+			log.Debug("(svcs) attempting to advertise over BGP", "address", network.CIDR())
+			err = bgp.AddHost(network.CIDR())
 			if err != nil {
 				log.Error(err.Error())
 			}
@@ -349,13 +310,19 @@ func (cluster *Cluster) StartLoadBalancerService(c *kubevip.Config, bgp *bgp.Ser
 		// Stop the Arp context if it is running
 		cancelArp()
 
-		log.Info("[LOADBALANCER] Stopping load balancers")
+		arpWG.Wait() // wait for all cluster ARP/NDP to be finished
+
+		log.Info("[LOADBALANCER] Stopping load balancers", "name", name)
 
 		if c.EnableRoutingTable {
 			for i := range cluster.Network {
-				log.Info("[VIP] Deleting Route for VIP [%s]", "ip", cluster.Network[i].IP())
-				if err := cluster.Network[i].DeleteRoute(); err != nil {
-					log.Warn(err.Error())
+				// chek if route is not  referenced by another service
+				r := cluster.Network[i].PrepareRoute()
+				if CountRouteReferences(r) < 1 {
+					log.Info("[VIP] Deleting Route for VIP", "IP", cluster.Network[i].IP())
+					if err := cluster.Network[i].DeleteRoute(); err != nil {
+						log.Warn(err.Error())
+					}
 				}
 			}
 
@@ -363,10 +330,17 @@ func (cluster *Cluster) StartLoadBalancerService(c *kubevip.Config, bgp *bgp.Ser
 			return
 		}
 		for i := range cluster.Network {
+			if c.EnableARP && cluster.arpMgr.Count(cluster.Network[i].ARPName()) > 0 {
+				continue
+			}
 			log.Info("[VIP] Deleting VIP", "ip", cluster.Network[i].IP())
-			if err := cluster.Network[i].DeleteIP(); err != nil {
+			deleted, err := cluster.Network[i].DeleteIP()
+			if err != nil {
 				log.Warn(err.Error())
 			}
+			if deleted {
+				log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+			}
 		}
 
 		close(cluster.completed)
@@ -374,7 +348,8 @@ func (cluster *Cluster) StartLoadBalancerService(c *kubevip.Config, bgp *bgp.Ser
 }
 
 // Layer2Update, handles the creation of the
-func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config) {
+func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config, arpWG *sync.WaitGroup) {
+	defer arpWG.Done()
 	log.Info("layer 2 broadcaster starting")
 	var ndp *vip.NdpResponder
 	var err error
@@ -397,68 +372,10 @@ func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c
 
 	log.Debug("layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
 
-	for {
-		select {
-		case <-ctx.Done(): // if cancel() execute
-			log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
-			return
-		default:
-			cluster.ensureIPAndSendGratuitous(network, ndp)
-		}
-		if c.ArpBroadcastRate < 500 {
-			log.Error("arp broadcast rate is too low", "rate (ms)", c.ArpBroadcastRate, "setting to (ms)", "3000")
-			c.ArpBroadcastRate = 3000
-		}
-		time.Sleep(time.Duration(c.ArpBroadcastRate) * time.Millisecond)
-	}
-}
-
-// ensureIPAndSendGratuitous - adds IP to the interface if missing, and send
-// either a gratuitous ARP or gratuitous NDP. Re-adds the interface if it is IPv6
-// and in a dadfailed state.
-func (cluster *Cluster) ensureIPAndSendGratuitous(network vip.Network, ndp *vip.NdpResponder) {
-	iface := network.Interface()
-	ipString := network.IP()
-
-	// Check if IP is dadfailed
-	if network.IsDADFAIL() {
-		log.Warn("IP address is in dadfailed state, removing config", "ip", ipString, "interface", iface)
-		err := network.DeleteIP()
-		if err != nil {
-			log.Warn(err.Error())
-		}
-	}
-
-	// Ensure the address exists on the interface before attempting to ARP
-	set, err := network.IsSet()
-	if err != nil {
-		log.Warn(err.Error())
-	}
-	if !set {
-		log.Warn("Re-applying the VIP configuration", "ip", ipString, "interface", iface)
-		err = network.AddIP(false)
-		if err != nil {
-			log.Warn(err.Error())
-		}
-	}
-
-	if vip.IsIPv6(ipString) {
-		// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
-		if ndp == nil {
-			log.Error("NDP responder was not created")
-		} else {
-			err := ndp.SendGratuitous(ipString)
-			if err != nil {
-				log.Warn(err.Error())
-			}
-		}
-
-	} else {
-		// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
-		err := vip.ARPSendGratuitous(ipString, iface)
-		if err != nil {
-			log.Warn(err.Error())
-		}
-	}
-
+	arpInstance := arp.NewInstance(network, ndp)
+	cluster.arpMgr.Insert(arpInstance)
+
+	<-ctx.Done() // if cancel() execute
+	log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
+	cluster.arpMgr.Remove(arpInstance)
 }

pkg/cluster/singleNode.go

@@ -5,8 +5,6 @@ import (
 
 	log "log/slog"
 
-	"github.com/packethost/packngo"
-
 	"github.com/kube-vip/kube-vip/pkg/bgp"
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
 	"github.com/kube-vip/kube-vip/pkg/vip"
@@ -26,12 +24,15 @@ func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) erro
 
 	for i := range cluster.Network {
 		if !disableVIP {
-			err := cluster.Network[i].DeleteIP()
+			deleted, err := cluster.Network[i].DeleteIP()
 			if err != nil {
 				log.Warn("Attempted to clean existing VIP", "err", err)
 			}
+			if deleted {
+				log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+			}
 
-			err = cluster.Network[i].AddIP(false)
+			_, err = cluster.Network[i].AddIP(false)
 			if err != nil {
 				log.Warn(err.Error())
 			}
@@ -53,10 +54,13 @@ func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) erro
 		if !disableVIP {
 			for i := range cluster.Network {
 				log.Info("[VIP] Releasing the VIP", "address", cluster.Network[i].IP())
-				err := cluster.Network[i].DeleteIP()
+				deleted, err := cluster.Network[i].DeleteIP()
 				if err != nil {
 					log.Warn(err.Error())
 				}
+				if deleted {
+					log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+				}
 			}
 		}
 		close(cluster.completed)
@@ -65,7 +69,7 @@ func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) erro
 	return nil
 }
 
-func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp.Server, packetClient *packngo.Client) error {
+func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp.Server) error {
 	// use a Go context so we can tell the arp loop code when we
 	// want to step down
 	ctxArp, cancelArp := context.WithCancel(context.Background())
@@ -76,5 +80,5 @@ func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp
 	ctxDNS, cancelDNS := context.WithCancel(context.Background())
 	defer cancelDNS()
 
-	return cluster.vipService(ctxArp, ctxDNS, c, sm, bgp, packetClient, nil)
+	return cluster.vipService(ctxArp, ctxDNS, c, sm, bgp, nil)
 }

pkg/egress/egress.go

@@ -0,0 +1,99 @@
+package egress
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/kube-vip/kube-vip/pkg/iptables"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/nftables"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
+
+func Teardown(podIP, vipIP, namespace, serviceUUID string, annotations map[string]string, useNftables bool) error {
+	// Look up the destination ports from the annotations on the service
+	destinationPorts := annotations[kubevip.EgressDestinationPorts]
+	deniedNetworks := annotations[kubevip.EgressDeniedNetworks]
+	allowedNetworks := annotations[kubevip.EgressAllowedNetworks]
+	internalEgress := annotations[kubevip.EgressInternal]
+
+	protocol := iptables.ProtocolIPv4
+	IPv6 := false
+	if vip.IsIPv6(podIP) {
+		protocol = iptables.ProtocolIPv6
+		IPv6 = true
+	}
+
+	// Use the internal egress implementation
+	if internalEgress != "" {
+		return nftables.DeleteSNAT(IPv6, serviceUUID)
+	}
+
+	i, err := vip.CreateIptablesClient(useNftables, namespace, protocol)
+	if err != nil {
+		return fmt.Errorf("error Creating iptables client [%s]", err)
+	}
+
+	if deniedNetworks != "" {
+		networks := strings.Split(deniedNetworks, ",")
+		for x := range networks {
+			err = i.DeleteMangleReturnForNetwork(vip.MangleChainName, networks[x])
+			if err != nil {
+				return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
+			}
+		}
+	}
+
+	if allowedNetworks != "" {
+		networks := strings.Split(allowedNetworks, ",")
+		for x := range networks {
+			err = i.DeleteMangleMarkingForNetwork(podIP, vip.MangleChainName, networks[x])
+			if err != nil {
+				return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
+			}
+		}
+	} else {
+		// Remove the marking of egress packets
+		err = i.DeleteMangleMarking(podIP, vip.MangleChainName)
+		if err != nil {
+			return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+		}
+	}
+
+	// Clear up SNAT rules
+	if destinationPorts != "" {
+		fixedPorts := strings.Split(destinationPorts, ",")
+
+		for _, fixedPort := range fixedPorts {
+			var proto, port string
+
+			data := strings.Split(fixedPort, ":")
+			if len(data) == 0 {
+				continue
+			} else if len(data) == 1 {
+				proto = "tcp"
+				port = data[0]
+			} else {
+				proto = data[0]
+				port = data[1]
+			}
+
+			err = i.DeleteSourceNatForDestinationPort(podIP, vipIP, port, proto)
+			if err != nil {
+				return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+			}
+
+		}
+	} else {
+		err = i.DeleteSourceNat(podIP, vipIP)
+		if err != nil {
+			return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+		}
+	}
+
+	err = vip.DeleteExistingSessions(podIP, false, destinationPorts, "")
+	if err != nil {
+		return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+	}
+	return nil
+}

pkg/endpoints/endpoints.go

@@ -0,0 +1,165 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	"net"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	v1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	"k8s.io/apimachinery/pkg/watch"
+)
+
+type Processor struct {
+	config    *kubevip.Config
+	provider  providers.Provider
+	bgpServer *bgp.Server
+	worker    endpointWorker
+	instances *[]*instance.Instance
+}
+
+func NewEndpointProcessor(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
+	instances *[]*instance.Instance) *Processor {
+	return &Processor{
+		config:    config,
+		provider:  provider,
+		bgpServer: bgpServer,
+		instances: instances,
+		worker:    newEndpointWorker(config, provider, bgpServer, instances),
+	}
+}
+
+func (p *Processor) AddOrModify(ctx *servicecontext.Context, event watch.Event,
+	lastKnownGoodEndpoint *string, service *v1.Service, id string, leaderElectionActive *bool,
+	serviceFunc func(context.Context, *v1.Service) error,
+	leaderCtx *context.Context, cancel *context.CancelFunc) (bool, error) {
+
+	var err error
+	if err = p.provider.LoadObject(event.Object, *cancel); err != nil {
+		return false, fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
+	}
+
+	endpoints, err := p.worker.getEndpoints(service, id)
+	if err != nil {
+		return false, err
+	}
+
+	if err := p.worker.setInstanceEndpointsStatus(service, endpoints); err != nil {
+		log.Error("updating instance", "err", err)
+	}
+
+	// Find out if we have any local endpoints
+	// if out endpoint is empty then populate it
+	// if not, go through the endpoints and see if ours still exists
+	// If we have a local endpoint then begin the leader Election, unless it's already running
+	//
+
+	// Check that we have local endpoints
+	if len(endpoints) != 0 {
+		// Ignore IPv4
+		if service.Annotations[kubevip.EgressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
+			return true, nil
+		}
+
+		p.updateLastKnownGoodEndpoint(lastKnownGoodEndpoint, endpoints, service, leaderElectionActive, *cancel)
+		// start leader election if it's enabled and not already started
+		if !*leaderElectionActive && p.config.EnableServicesElection {
+			go func() {
+				*leaderCtx, *cancel = context.WithCancel(ctx.Ctx)
+				startLeaderElection(*leaderCtx, leaderElectionActive, service, serviceFunc)
+			}()
+		}
+
+		// There are local endpoints available on the node
+		if !p.config.EnableServicesElection && !p.config.EnableLeaderElection {
+			if err := p.worker.processInstance(ctx, service, leaderElectionActive); err != nil {
+				return false, fmt.Errorf("failed to process non-empty instance: %w", err)
+			}
+		}
+	} else {
+		// There are no local endpoints
+		p.worker.clear(ctx, lastKnownGoodEndpoint, service, *cancel, leaderElectionActive)
+	}
+
+	// Set the service accordingly
+	p.updateAnnotations(service, lastKnownGoodEndpoint)
+
+	log.Debug("watcher", "provider",
+		p.provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", *lastKnownGoodEndpoint, "active leader election", *leaderElectionActive)
+
+	return false, nil
+}
+
+func (p *Processor) Delete(service *v1.Service, id string) error {
+	if err := p.worker.delete(service, id); err != nil {
+		return fmt.Errorf("[%s] error deleting service: %w", p.provider.GetLabel(), err)
+	}
+	return nil
+}
+
+func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service, leaderElectionActive *bool, cancel context.CancelFunc) {
+	// if we haven't populated one, then do so
+	if *lastKnownGoodEndpoint == "" {
+		*lastKnownGoodEndpoint = endpoints[0]
+		return
+	}
+
+	// check out previous endpoint exists
+	stillExists := false
+
+	for x := range endpoints {
+		if endpoints[x] == *lastKnownGoodEndpoint {
+			stillExists = true
+		}
+	}
+	// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
+	if !stillExists {
+		p.worker.removeEgress(service, lastKnownGoodEndpoint)
+		if *leaderElectionActive && (p.config.EnableServicesElection || p.config.EnableLeaderElection) {
+			log.Warn("existing endpoint has been removed, restarting leaderElection", "provider", p.provider.GetLabel(), "endpoint", *lastKnownGoodEndpoint)
+			// Stop the existing leaderElection
+			cancel()
+			// disable last leaderElection flag
+			*leaderElectionActive = false
+		}
+		// Set our active endpoint to an existing one
+		*lastKnownGoodEndpoint = endpoints[0]
+	}
+}
+
+func (p *Processor) updateAnnotations(service *v1.Service, lastKnownGoodEndpoint *string) {
+	// Set the service accordingly
+	if service.Annotations[kubevip.Egress] == "true" {
+		activeEndpointAnnotation := kubevip.ActiveEndpoint
+
+		if !p.config.EnableEndpoints && p.provider.GetProtocol() == string(discoveryv1.AddressTypeIPv6) {
+			activeEndpointAnnotation = kubevip.ActiveEndpointIPv6
+		}
+		service.Annotations[activeEndpointAnnotation] = *lastKnownGoodEndpoint
+	}
+}
+
+func startLeaderElection(ctx context.Context, leaderElectionActive *bool, service *v1.Service, serviceFunc func(context.Context, *v1.Service) error) {
+	// This is a blocking function, that will restart (in the event of failure)
+	for {
+		// if the context isn't cancelled restart
+		if ctx.Err() != context.Canceled {
+			*leaderElectionActive = true
+			err := serviceFunc(ctx, service)
+			if err != nil {
+				log.Error(err.Error())
+			}
+			*leaderElectionActive = false
+		} else {
+			*leaderElectionActive = false
+			break
+		}
+	}
+}

pkg/endpoints/endpoints_bgp.go

@@ -0,0 +1,125 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	v1 "k8s.io/api/core/v1"
+)
+
+type BGP struct {
+	generic
+	bgpServer *bgp.Server
+}
+
+func newBGP(generic generic, bgpServer *bgp.Server) endpointWorker {
+	return &BGP{
+		generic:   generic,
+		bgpServer: bgpServer,
+	}
+}
+
+func (b *BGP) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
+	if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
+		for _, cluster := range instance.Clusters {
+			for i := range cluster.Network {
+				if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) {
+					log.Debug("attempting to advertise BGP service", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP())
+					err := b.bgpServer.AddHost(cluster.Network[i].CIDR())
+					if err != nil {
+						log.Error("error adding BGP host", "provider", b.provider.GetLabel(), "err", err)
+					} else {
+						log.Info("added BGP host", "provider",
+							b.provider.GetLabel(), "ip", cluster.Network[i].CIDR(), "service name", service.Name, "namespace", service.Namespace)
+						ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
+						*leaderElectionActive = true
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (b *BGP) clear(ctx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
+	if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
+		// If BGP mode is enabled - routes should be deleted
+		if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
+			for _, cluster := range instance.Clusters {
+				for i := range cluster.Network {
+					err := b.bgpServer.DelHost(cluster.Network[i].CIDR())
+					if err != nil {
+						log.Error("deleting BGP host", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "err", err)
+					} else {
+						log.Info("deleted BGP host", "provider",
+							b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace)
+						ctx.ConfiguredNetworks.Delete(cluster.Network[i])
+						*leaderElectionActive = false
+					}
+				}
+			}
+
+		}
+	}
+
+	b.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
+}
+
+func (b *BGP) getEndpoints(service *v1.Service, id string) ([]string, error) {
+	return b.getAllEndpoints(service, id)
+}
+
+func (b *BGP) delete(service *v1.Service, id string) error {
+	// When no-leader-elecition mode
+	if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
+		// find all existing local endpoints
+		endpoints, err := b.getEndpoints(service, id)
+		if err != nil {
+			return fmt.Errorf("[%s] error getting endpoints: %w", b.provider.GetLabel(), err)
+		}
+
+		// If there were local endpoints deleted
+		if len(endpoints) > 0 {
+			b.deleteAction(service)
+		}
+	}
+
+	return nil
+}
+
+func (b *BGP) deleteAction(service *v1.Service) {
+	b.clearBGPHosts(service)
+}
+
+func (b *BGP) clearBGPHosts(service *v1.Service) {
+	ClearBGPHosts(service, b.instances, b.bgpServer)
+}
+
+func (b *BGP) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
+	return nil
+}
+
+func ClearBGPHosts(service *v1.Service, instances *[]*instance.Instance, bgpServer *bgp.Server) {
+	if instance := instance.FindServiceInstance(service, *instances); instance != nil {
+		ClearBGPHostsByInstance(instance, bgpServer)
+	}
+}
+
+func ClearBGPHostsByInstance(instance *instance.Instance, bgpServer *bgp.Server) {
+	for _, cluster := range instance.Clusters {
+		for i := range cluster.Network {
+			network := cluster.Network[i]
+			err := bgpServer.DelHost(network.CIDR())
+			if err != nil {
+				log.Error("[endpoint] error deleting BGP host", "err", err)
+			} else {
+				log.Debug("[endpoint] deleted BGP host", "ip",
+					network.CIDR(), "service name", instance.ServiceSnapshot.Name, "namespace", instance.ServiceSnapshot.Namespace)
+			}
+		}
+	}
+}

pkg/endpoints/endpoints_generic.go

@@ -0,0 +1,118 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/egress"
+	"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	v1 "k8s.io/api/core/v1"
+)
+
+type endpointWorker interface {
+	processInstance(svcCtx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error
+	clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool)
+	getEndpoints(service *v1.Service, id string) ([]string, error)
+	removeEgress(service *v1.Service, lastKnownGoodEndpoint *string)
+	delete(service *v1.Service, id string) error
+	setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error
+}
+
+func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server, instances *[]*instance.Instance) endpointWorker {
+	generic := newGeneric(config, provider, instances)
+
+	if config.EnableRoutingTable {
+		return newRoutingTable(generic)
+	}
+	if config.EnableBGP {
+		return newBGP(generic, bgpServer)
+	}
+
+	return &generic
+}
+
+type generic struct {
+	config    *kubevip.Config
+	provider  providers.Provider
+	instances *[]*instance.Instance
+}
+
+func newGeneric(config *kubevip.Config, provider providers.Provider, instances *[]*instance.Instance) generic {
+	return generic{
+		config:    config,
+		provider:  provider,
+		instances: instances,
+	}
+}
+
+func (g *generic) processInstance(_ *servicecontext.Context, _ *v1.Service, _ *bool) error {
+	return nil
+}
+
+func (g *generic) clear(_ *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
+	g.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
+}
+
+func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
+	if *lastKnownGoodEndpoint != "" {
+		log.Warn("existing  endpoint has been removed, no remaining endpoints for leaderElection", "provider", g.provider.GetLabel(), "endpoint", lastKnownGoodEndpoint)
+		if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP, service.Namespace, string(service.UID), service.Annotations, g.config.EgressWithNftables); err != nil {
+			log.Error("error removing redundant egress rules", "err", err)
+		}
+
+		*lastKnownGoodEndpoint = "" // reset endpoint
+		if g.config.EnableServicesElection || g.config.EnableLeaderElection {
+			cancel() // stop services watcher
+		}
+		*leaderElectionActive = false
+	}
+}
+
+func (g *generic) getEndpoints(_ *v1.Service, id string) ([]string, error) {
+	return g.getLocalEndpoints(id)
+}
+
+func (g *generic) getLocalEndpoints(id string) ([]string, error) {
+	// Build endpoints
+	var endpoints []string
+	var err error
+	if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
+		return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
+	}
+
+	return endpoints, nil
+}
+
+func (g *generic) getAllEndpoints(service *v1.Service, id string) ([]string, error) {
+	// Build endpoints
+	var err error
+	var endpoints []string
+	if !g.config.EnableLeaderElection && !g.config.EnableServicesElection &&
+		service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
+		if endpoints, err = g.provider.GetAllEndpoints(); err != nil {
+			return nil, fmt.Errorf("[%s] error getting all endpoints: %w", g.provider.GetLabel(), err)
+		}
+	} else {
+		if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
+			return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
+		}
+	}
+
+	return endpoints, nil
+}
+
+func (g *generic) removeEgress(_ *v1.Service, _ *string) {
+}
+
+func (g *generic) delete(_ *v1.Service, _ string) error {
+	return nil
+}
+
+func (g *generic) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
+	return nil
+}

pkg/endpoints/endpoints_routing_table.go

@@ -0,0 +1,178 @@
+package endpoints
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"syscall"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/egress"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	"github.com/vishvananda/netlink"
+	v1 "k8s.io/api/core/v1"
+)
+
+type RoutingTable struct {
+	generic
+}
+
+func newRoutingTable(generic generic) endpointWorker {
+	return &RoutingTable{
+		generic: generic,
+	}
+}
+
+func (rt *RoutingTable) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
+	instance := instance.FindServiceInstance(service, *rt.instances)
+	if instance != nil {
+		for _, cluster := range instance.Clusters {
+			for i := range cluster.Network {
+				if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) && cluster.Network[i].HasEndpoints() {
+					err := cluster.Network[i].AddRoute(false)
+					if err != nil {
+						if errors.Is(err, syscall.EEXIST) {
+							// If route exists, but protocol is not set (e.g. the route was created by the older version
+							// of kube-vip) try to update it if necessary
+							isUpdated, err := cluster.Network[i].UpdateRoutes()
+							if err != nil {
+								return fmt.Errorf("[%s] error updating existing routes: %w", rt.provider.GetLabel(), err)
+							}
+							if isUpdated {
+								log.Info("updated route", "provider",
+									rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
+									service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
+							} else {
+								log.Info("route already present", "provider",
+									rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
+									service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
+							}
+						} else {
+							// If other error occurs, return error
+							return fmt.Errorf("[%s] error adding route: %s", rt.provider.GetLabel(), err.Error())
+						}
+					} else {
+						log.Info("added route", "provider",
+							rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
+							service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
+						ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
+						*leaderElectionActive = true
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
+	if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
+		if errs := ClearRoutes(service, rt.instances); len(errs) == 0 {
+			svcCtx.ConfiguredNetworks.Clear()
+		} else {
+			for _, err := range errs {
+				log.Error("error while clearing routes", "err", err)
+			}
+		}
+	}
+
+	rt.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
+}
+
+func (rt *RoutingTable) getEndpoints(service *v1.Service, id string) ([]string, error) {
+	return rt.getAllEndpoints(service, id)
+}
+
+func (rt *RoutingTable) removeEgress(service *v1.Service, lastKnownGoodEndpoint *string) {
+	if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
+		service.Namespace, string(service.UID), service.Annotations, rt.config.EgressWithNftables); err != nil {
+		log.Warn("removing redundant egress rules", "err", err)
+	}
+}
+
+func (rt *RoutingTable) delete(service *v1.Service, id string) error {
+	// When no-leader-elecition mode
+	if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
+		// find all existing local endpoints
+		endpoints, err := rt.getEndpoints(service, id)
+		if err != nil {
+			return fmt.Errorf("[%s] error getting endpoints: %w", rt.provider.GetLabel(), err)
+		}
+
+		// If there were local endpoints deleted
+		if len(endpoints) > 0 {
+			rt.deleteAction(service)
+		}
+	}
+
+	return nil
+}
+
+func (rt *RoutingTable) deleteAction(service *v1.Service) {
+	ClearRoutes(service, rt.instances)
+}
+
+func (rt *RoutingTable) setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error {
+	instance := instance.FindServiceInstance(service, *rt.instances)
+	if instance == nil {
+		log.Error("failed to find the instance", "service", service.UID, "provider", rt.provider.GetLabel())
+	} else {
+		for _, c := range instance.Clusters {
+			for n := range c.Network {
+				// if there are no endpoints set HasEndpoints false just in case
+				if len(endpoints) < 1 {
+					c.Network[n].SetHasEndpoints(false)
+				}
+				// check if endpoint are available and are of same IP family as service
+				if len(endpoints) > 0 && ((net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(endpoints[0]).To4() == nil)) {
+					c.Network[n].SetHasEndpoints(true)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func ClearRoutes(service *v1.Service, instances *[]*instance.Instance) []error {
+	errs := []error{}
+	if instance := instance.FindServiceInstance(service, *instances); instance != nil {
+		for _, cluster := range instance.Clusters {
+			for i := range cluster.Network {
+				route := cluster.Network[i].PrepareRoute()
+				// check if route we are about to delete is not referenced by more than one service
+				if CountRouteReferences(route, instances) <= 1 {
+					err := cluster.Network[i].DeleteRoute()
+					if err != nil && !errors.Is(err, syscall.ESRCH) {
+						log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
+						errs = append(errs, err)
+					}
+					log.Debug("deleted route", "ip",
+						cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface())
+				}
+			}
+		}
+	}
+	return errs
+}
+
+func CountRouteReferences(route *netlink.Route, instances *[]*instance.Instance) int {
+	cnt := 0
+	for _, instance := range *instances {
+		for _, cluster := range instance.Clusters {
+			for n := range cluster.Network {
+				if cluster.Network[n].HasEndpoints() {
+					r := cluster.Network[n].PrepareRoute()
+					if r.Dst.String() == route.Dst.String() {
+						cnt++
+					}
+				}
+			}
+		}
+	}
+	return cnt
+}

pkg/endpoints/providers/endpoints.go

@@ -0,0 +1,137 @@
+package providers
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	watchtools "k8s.io/client-go/tools/watch"
+	"k8s.io/client-go/util/retry"
+
+	log "log/slog"
+)
+
+type Endpoints struct {
+	label string
+	//nolint:staticcheck // SA1019 endpoints are moving to an opt-in only
+	endpoints *v1.Endpoints
+}
+
+func NewEndpoints() Provider {
+	return &Endpoints{
+		label: "endpoints",
+	}
+}
+
+func (ep *Endpoints) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
+	service *v1.Service) (*watchtools.RetryWatcher, error) {
+	opts := metav1.ListOptions{
+		FieldSelector: fields.OneTermEqualSelector("metadata.name", service.Name).String(),
+	}
+
+	rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
+		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
+			return clientSet.CoreV1().Endpoints(service.Namespace).Watch(ctx, opts)
+		},
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("error creating endpoint watcher: %s", err.Error())
+	}
+
+	return rw, nil
+}
+
+func (ep *Endpoints) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
+	//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
+	eps, ok := endpoints.(*v1.Endpoints)
+	if !ok {
+		cancel()
+		return fmt.Errorf("[%s] unable to parse Kubernetes services from API watcher", ep.GetLabel())
+	}
+	ep.endpoints = eps
+	return nil
+}
+
+func (ep *Endpoints) GetAllEndpoints() ([]string, error) {
+	result := []string{}
+	for subset := range ep.endpoints.Subsets {
+		for address := range ep.endpoints.Subsets[subset].Addresses {
+			addr := strings.Split(ep.endpoints.Subsets[subset].Addresses[address].IP, "/")
+			result = append(result, addr[0])
+		}
+	}
+
+	return result, nil
+}
+
+func (ep *Endpoints) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
+	var localEndpoints []string
+
+	for _, subset := range ep.endpoints.Subsets {
+		for _, address := range subset.Addresses {
+			log.Debug("processing endpoint", "label", ep.label, "ip", address.IP)
+
+			// 1. Compare the Nodename
+			if address.NodeName != nil && id == *address.NodeName {
+				log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname, "nodename", *address.NodeName)
+				localEndpoints = append(localEndpoints, address.IP)
+				continue
+			}
+			// 2. Compare the Hostname (only useful if address.NodeName is not available)
+			if id == address.Hostname {
+				log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname)
+				localEndpoints = append(localEndpoints, address.IP)
+				continue
+			}
+		}
+	}
+	return localEndpoints, nil
+}
+
+func (ep *Endpoints) UpdateServiceAnnotation(endpoint string, _ string, service *v1.Service, clientSet *kubernetes.Clientset) error {
+	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		// Retrieve the latest version of Deployment before attempting update
+		// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
+		currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		currentServiceCopy := currentService.DeepCopy()
+		if currentServiceCopy.Annotations == nil {
+			currentServiceCopy.Annotations = make(map[string]string)
+		}
+
+		currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
+
+		_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
+		if err != nil {
+			log.Error("error updating Service Spec", "label", ep.GetLabel(), "name", currentServiceCopy.Name, "err", err)
+			return err
+		}
+		return nil
+	})
+
+	if retryErr != nil {
+		log.Error("failed to set Services", "label", ep.GetLabel(), "err", retryErr)
+		return retryErr
+	}
+	return nil
+}
+
+func (ep *Endpoints) GetLabel() string {
+	return ep.label
+}
+
+func (ep *Endpoints) GetProtocol() string {
+	return ""
+}

pkg/endpoints/providers/endpointslices.go

@@ -1,4 +1,4 @@
-package manager
+package providers
 
 import (
 	"context"
@@ -13,17 +13,24 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
 	watchtools "k8s.io/client-go/tools/watch"
 	"k8s.io/client-go/util/retry"
 )
 
-type endpointslicesProvider struct {
+type Endpointslices struct {
 	label     string
 	endpoints *discoveryv1.EndpointSlice
 }
 
-func (ep *endpointslicesProvider) createRetryWatcher(ctx context.Context, sm *Manager,
+func NewEndpointslices() Provider {
+	return &Endpointslices{
+		label: "endpointslices",
+	}
+}
+
+func (ep *Endpointslices) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
 	service *v1.Service) (*watchtools.RetryWatcher, error) {
 	labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/service-name": service.Name}}
 
@@ -31,9 +38,9 @@ func (ep *endpointslicesProvider) createRetryWatcher(ctx context.Context, sm *Ma
 		LabelSelector: labels.Set(labelSelector.MatchLabels).String(),
 	}
 
-	rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
+	rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
 		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
-			return sm.rwClientSet.DiscoveryV1().EndpointSlices(service.Namespace).Watch(ctx, opts)
+			return clientSet.DiscoveryV1().EndpointSlices(service.Namespace).Watch(ctx, opts)
 		},
 	})
 	if err != nil {
@@ -43,7 +50,7 @@ func (ep *endpointslicesProvider) createRetryWatcher(ctx context.Context, sm *Ma
 	return rw, nil
 }
 
-func (ep *endpointslicesProvider) loadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
+func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
 	eps, ok := endpoints.(*discoveryv1.EndpointSlice)
 	if !ok {
 		cancel()
@@ -53,7 +60,7 @@ func (ep *endpointslicesProvider) loadObject(endpoints runtime.Object, cancel co
 	return nil
 }
 
-func (ep *endpointslicesProvider) getAllEndpoints() ([]string, error) {
+func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
 	result := []string{}
 	for _, ep := range ep.endpoints.Endpoints {
 		result = append(result, ep.Addresses...)
@@ -61,7 +68,7 @@ func (ep *endpointslicesProvider) getAllEndpoints() ([]string, error) {
 	return result, nil
 }
 
-func (ep *endpointslicesProvider) getLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
+func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
 	var localEndpoints []string
 	for _, endpoint := range ep.endpoints.Endpoints {
 		if !*endpoint.Conditions.Serving {
@@ -91,11 +98,11 @@ func (ep *endpointslicesProvider) getLocalEndpoints(id string, _ *kubevip.Config
 	return localEndpoints, nil
 }
 
-func (ep *endpointslicesProvider) updateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, sm *Manager) error {
+func (ep *Endpointslices) UpdateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, clientSet *kubernetes.Clientset) error {
 	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
 		// Retrieve the latest version of Deployment before attempting update
 		// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
-		currentService, err := sm.clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
+		currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
 		if err != nil {
 			return err
 		}
@@ -105,10 +112,10 @@ func (ep *endpointslicesProvider) updateServiceAnnotation(endpoint, endpointIPv6
 			currentServiceCopy.Annotations = make(map[string]string)
 		}
 
-		currentServiceCopy.Annotations[activeEndpoint] = endpoint
-		currentServiceCopy.Annotations[activeEndpointIPv6] = endpointIPv6
+		currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
+		currentServiceCopy.Annotations[kubevip.ActiveEndpointIPv6] = endpointIPv6
 
-		_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
+		_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
 		if err != nil {
 			log.Error("error updating Service Spec", "provider", ep.label, "service name", currentServiceCopy.Name, "err", err)
 			return err
@@ -123,10 +130,10 @@ func (ep *endpointslicesProvider) updateServiceAnnotation(endpoint, endpointIPv6
 	return nil
 }
 
-func (ep *endpointslicesProvider) getLabel() string {
+func (ep *Endpointslices) GetLabel() string {
 	return ep.label
 }
 
-func (ep *endpointslicesProvider) getProtocol() string {
+func (ep *Endpointslices) GetProtocol() string {
 	return string(ep.endpoints.AddressType)
 }

pkg/endpoints/providers/interface.go

@@ -0,0 +1,22 @@
+package providers
+
+import (
+	"context"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
+	watchtools "k8s.io/client-go/tools/watch"
+)
+
+type Provider interface {
+	CreateRetryWatcher(context.Context, *kubernetes.Clientset,
+		*v1.Service) (*watchtools.RetryWatcher, error)
+	GetAllEndpoints() ([]string, error)
+	GetLocalEndpoints(string, *kubevip.Config) ([]string, error)
+	GetLabel() string
+	UpdateServiceAnnotation(string, string, *v1.Service, *kubernetes.Clientset) error
+	LoadObject(runtime.Object, context.CancelFunc) error
+	GetProtocol() string
+}

pkg/equinixmetal/bgp.go

@@ -1,64 +0,0 @@
-package equinixmetal
-
-import (
-	"fmt"
-
-	log "log/slog"
-
-	"github.com/kube-vip/kube-vip/pkg/bgp"
-	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	"github.com/packethost/packngo"
-)
-
-// BGPLookup will use the Equinix Metal API functions to populate the BGP information
-func BGPLookup(c *packngo.Client, k *kubevip.Config) error {
-	var thisDevice *packngo.Device
-	if k.MetalProjectID == "" {
-		proj := findProject(k.MetalProject, c)
-		if proj == nil {
-			return fmt.Errorf("Unable to find Project [%s]", k.MetalProject)
-		}
-		thisDevice = findSelf(c, proj.ID)
-	} else {
-		thisDevice = findSelf(c, k.MetalProjectID)
-	}
-	if thisDevice == nil {
-		return fmt.Errorf("Unable to find local/this device in Equinix Metal API")
-	}
-
-	log.Info("Querying BGP settings", "hostname", thisDevice.Hostname)
-	neighbours, _, err := c.Devices.ListBGPNeighbors(thisDevice.ID, &packngo.ListOptions{})
-	if err != nil {
-		return err
-	}
-	// Ensure neighbours exist (and it's enabled)
-	if len(neighbours) == 0 {
-		return fmt.Errorf("The server [%s]/[%s] has no BGP neighbours, ensure BGP is enabled", thisDevice.Hostname, thisDevice.ID)
-	}
-
-	// Add a warning (TODO)
-	if len(neighbours) > 1 {
-		log.Warn(fmt.Sprintf("There are [%d] neighbours, only designed to manage one", len(neighbours)))
-	}
-
-	// Ensure a peer exists
-	if len(neighbours[0].PeerIps) == 0 {
-		return fmt.Errorf("The server [%s]/[%s] has no BGP peers, ensure BGP is enabled", thisDevice.Hostname, thisDevice.ID)
-	}
-
-	k.BGPConfig.RouterID = neighbours[0].CustomerIP
-	k.BGPConfig.AS = uint32(neighbours[0].CustomerAs) //nolint (being removed)
-
-	// Add the peer(s)
-	for x := range neighbours[0].PeerIps {
-		peer := bgp.Peer{
-			Address:  neighbours[0].PeerIps[x],
-			AS:       uint32(neighbours[0].PeerAs), //nolint (being removed)
-			MultiHop: neighbours[0].Multihop,
-			Password: neighbours[0].Md5Password,
-		}
-		k.BGPConfig.Peers = append(k.BGPConfig.Peers, peer)
-	}
-
-	return nil
-}

pkg/equinixmetal/eip.go

@@ -1,66 +0,0 @@
-package equinixmetal
-
-import (
-	"fmt"
-	"path"
-
-	log "log/slog"
-
-	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	"github.com/packethost/packngo"
-)
-
-// AttachEIP will use the Equinix Metal APIs to move an EIP and attach to a host
-func AttachEIP(c *packngo.Client, k *kubevip.Config, _ string) error {
-	// Use MetalProjectID if it is defined
-	projID := k.MetalProjectID
-
-	if projID == "" {
-		// Fallback to attempting to find the project by name
-		proj := findProject(k.MetalProject, c)
-		if proj == nil {
-			return fmt.Errorf("unable to find Project [%s]", k.MetalProject)
-		}
-
-		projID = proj.ID
-	}
-
-	// Prefer Address over VIP
-	vip := k.Address
-	if vip == "" {
-		vip = k.VIP
-	}
-
-	ips, _, _ := c.ProjectIPs.List(projID, &packngo.ListOptions{})
-	for _, ip := range ips {
-		// Find the device id for our EIP
-		if ip.Address == vip {
-			log.Info("Found EIP", "address", ip.Address, "id", ip.ID)
-			// If attachments already exist then remove them
-			if len(ip.Assignments) != 0 {
-				hrefID := path.Base(ip.Assignments[0].Href)
-				_, err := c.DeviceIPs.Unassign(hrefID)
-				if err != nil {
-					return fmt.Errorf("unable to unassign deviceIP %q: %v", hrefID, err)
-				}
-			}
-		}
-	}
-
-	// Lookup this server through the Equinix Metal API
-	thisDevice := findSelf(c, projID)
-	if thisDevice == nil {
-		return fmt.Errorf("unable to find local/this device in Equinix Metal API")
-	}
-
-	// Assign the EIP to this device
-	log.Info("Assigning EIP", "hostname", thisDevice.Hostname)
-	_, _, err := c.DeviceIPs.Assign(thisDevice.ID, &packngo.AddressStruct{
-		Address: vip,
-	})
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

pkg/equinixmetal/utils.go

@@ -1,60 +0,0 @@
-package equinixmetal
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-
-	log "log/slog"
-
-	"github.com/packethost/packngo"
-)
-
-func findProject(project string, c *packngo.Client) *packngo.Project {
-	l := &packngo.ListOptions{Includes: []string{project}}
-	ps, _, err := c.Projects.List(l)
-	if err != nil {
-		log.Error(err.Error())
-	}
-	for _, p := range ps {
-
-		// Find our project
-		if p.Name == project {
-			return &p
-		}
-	}
-	return nil
-}
-
-func findSelf(c *packngo.Client, projectID string) *packngo.Device {
-	// Go through devices
-	dev, _, _ := c.Devices.List(projectID, &packngo.ListOptions{})
-	for _, d := range dev {
-		// TODO do we need to replace os.Hostname with config.NodeName here?
-		me, _ := os.Hostname()
-		if me == d.Hostname {
-			return &d
-		}
-	}
-	return nil
-}
-
-// GetPacketConfig will lookup the configuration from a file path
-func GetPacketConfig(providerConfig string) (string, string, error) {
-	var config struct {
-		AuthToken string `json:"apiKey"`
-		ProjectID string `json:"projectId"`
-	}
-	// get our token and project
-	if providerConfig != "" {
-		configBytes, err := os.ReadFile(providerConfig)
-		if err != nil {
-			return "", "", fmt.Errorf("failed to get read configuration file at path %s: %v", providerConfig, err)
-		}
-		err = json.Unmarshal(configBytes, &config)
-		if err != nil {
-			return "", "", fmt.Errorf("failed to process json of configuration file at path %s: %v", providerConfig, err)
-		}
-	}
-	return config.AuthToken, config.ProjectID, nil
-}

pkg/instance/instance.go

@@ -1,42 +1,46 @@
-package manager
+package instance
 
 import (
 	"fmt"
 	"net"
 	"strconv"
+	"strings"
 
+	"log/slog"
 	log "log/slog"
 
 	"github.com/vishvananda/netlink"
 	v1 "k8s.io/api/core/v1"
 
+	"github.com/kube-vip/kube-vip/pkg/arp"
 	"github.com/kube-vip/kube-vip/pkg/cluster"
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/networkinterface"
+	"github.com/kube-vip/kube-vip/pkg/sysctl"
 	"github.com/kube-vip/kube-vip/pkg/vip"
 )
 
 // Instance defines an instance of everything needed to manage vips
 type Instance struct {
 	// Virtual IP / Load Balancer configuration
-	vipConfigs []*kubevip.Config
+	VIPConfigs []*kubevip.Config
 
 	// cluster instances
-	clusters []*cluster.Cluster
+	Clusters []*cluster.Cluster
 
 	// Service uses DHCP
-	isDHCP              bool
-	dhcpInterface       string
-	dhcpInterfaceHwaddr string
-	dhcpInterfaceIP     string
-	dhcpHostname        string
-	dhcpClient          *vip.DHCPClient
-	HasEndpoints        bool
+	IsDHCP              bool
+	DHCPInterface       string
+	DHCPInterfaceHwaddr string
+	DHCPInterfaceIP     string
+	DHCPHostname        string
+	DHCPClient          *vip.DHCPClient
 
 	// External Gateway IP the service is forwarded from
-	upnpGatewayIPs []string
+	UPNPGatewayIPs []string
 
 	// Kubernetes service mapping
-	serviceSnapshot *v1.Service
+	ServiceSnapshot *v1.Service
 }
 
 type Port struct {
@@ -44,8 +48,8 @@ type Port struct {
 	Type string
 }
 
-func NewInstance(svc *v1.Service, config *kubevip.Config) (*Instance, error) {
-	instanceAddresses := fetchServiceAddresses(svc)
+func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Instance, error) {
+	instanceAddresses := FetchServiceAddresses(svc)
 	//instanceUID := string(svc.UID)
 
 	var newVips []*kubevip.Config
@@ -55,7 +59,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config) (*Instance, error) {
 	for _, address := range instanceAddresses {
 		// Detect if we're using a specific interface for services
 		var svcInterface string
-		svcInterface = svc.Annotations[serviceInterface] // If the service has a specific interface defined, then use it
+		svcInterface = svc.Annotations[kubevip.ServiceInterface] // If the service has a specific interface defined, then use it
 		if svcInterface == kubevip.Auto {
 			link, err = autoFindInterface(address)
 			if err != nil {
@@ -98,7 +102,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config) (*Instance, error) {
 			}
 		}
 
-		cidrs := vip.Split(config.VIPCIDR)
+		cidrs := vip.Split(config.VIPSubnet)
 
 		ipv4AutoSubnet := false
 		ipv6AutoSubnet := false
@@ -123,7 +127,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config) (*Instance, error) {
 					return nil, fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
 				}
 			} else {
-				if cidrs[0] != "" {
+				if cidrs[0] != "" && cidrs[0] != kubevip.Auto {
 					subnet = cidrs[0]
 				} else {
 					subnet = "32"
@@ -136,7 +140,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config) (*Instance, error) {
 					return nil, fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
 				}
 			} else {
-				if len(cidrs) > 1 {
+				if len(cidrs) > 1 && cidrs[1] != "" && cidrs[1] != kubevip.Auto {
 					subnet = cidrs[1]
 				} else {
 					subnet = "128"
@@ -144,8 +148,6 @@ func NewInstance(svc *v1.Service, config *kubevip.Config) (*Instance, error) {
 			}
 		}
 
-		//log.Info("new instance", "svc", *svc, "interface", svcInterface)
-
 		// Generate new Virtual IP configuration
 		newVips = append(newVips, &kubevip.Config{
 			VIP:                    address,
@@ -153,8 +155,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config) (*Instance, error) {
 			SingleNode:             true,
 			EnableARP:              config.EnableARP,
 			EnableBGP:              config.EnableBGP,
-			VIPCIDR:                subnet,
-			VIPSubnet:              config.VIPSubnet,
+			VIPSubnet:              subnet,
 			EnableRoutingTable:     config.EnableRoutingTable,
 			RoutingTableID:         config.RoutingTableID,
 			RoutingTableType:       config.RoutingTableType,
@@ -174,7 +175,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config) (*Instance, error) {
 	instance := &Instance{
 		//UID:             instanceUID,
 		//VIPs:            instanceAddresses,
-		serviceSnapshot: svc,
+		ServiceSnapshot: svc,
 	}
 	// for _, port := range svc.Spec.Ports {
 	// 	instance.ExternalPorts = append(instance.ExternalPorts, Port{
@@ -184,9 +185,9 @@ func NewInstance(svc *v1.Service, config *kubevip.Config) (*Instance, error) {
 	// }
 
 	if svc.Annotations != nil {
-		instance.dhcpInterfaceHwaddr = svc.Annotations[hwAddrKey]
-		instance.dhcpInterfaceIP = svc.Annotations[requestedIP]
-		instance.dhcpHostname = svc.Annotations[loadbalancerHostname]
+		instance.DHCPInterfaceHwaddr = svc.Annotations[kubevip.HwAddrKey]
+		instance.DHCPInterfaceIP = svc.Annotations[kubevip.RequestedIP]
+		instance.DHCPHostname = svc.Annotations[kubevip.LoadbalancerHostname]
 	}
 
 	configPorts := make([]kubevip.Port, 0)
@@ -207,7 +208,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config) (*Instance, error) {
 		vip.LoadBalancers = append(vip.LoadBalancers, newLB)
 	}
 	// Create Add configuration to the new service
-	instance.vipConfigs = newVips
+	instance.VIPConfigs = newVips
 
 	// If this was purposely created with the address 0.0.0.0,
 	// we will create a macvlan on the main interface and a DHCP client
@@ -218,18 +219,18 @@ func NewInstance(svc *v1.Service, config *kubevip.Config) (*Instance, error) {
 			return nil, err
 		}
 		select {
-		case err := <-instance.dhcpClient.ErrorChannel():
+		case err := <-instance.DHCPClient.ErrorChannel():
 			return nil, fmt.Errorf("error starting DHCP for %s/%s: error: %s",
-				instance.serviceSnapshot.Namespace, instance.serviceSnapshot.Name, err)
-		case ip := <-instance.dhcpClient.IPChannel():
-			instance.vipConfigs[0].Interface = instance.dhcpInterface
-			instance.vipConfigs[0].VIP = ip
-			instance.dhcpInterfaceIP = ip
+				instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
+		case ip := <-instance.DHCPClient.IPChannel():
+			instance.VIPConfigs[0].Interface = instance.DHCPInterface
+			instance.VIPConfigs[0].VIP = ip
+			instance.DHCPInterfaceIP = ip
 		}
 	}
 
-	for _, vipConfig := range instance.vipConfigs {
-		c, err := cluster.InitCluster(vipConfig, false)
+	for _, vipConfig := range instance.VIPConfigs {
+		c, err := cluster.InitCluster(vipConfig, false, intfMgr, arpMgr)
 		if err != nil {
 			log.Error("Failed to add Service %s/%s", svc.Namespace, svc.Name)
 			return nil, err
@@ -239,9 +240,8 @@ func NewInstance(svc *v1.Service, config *kubevip.Config) (*Instance, error) {
 			c.Network[i].SetServicePorts(svc)
 		}
 
-		instance.clusters = append(instance.clusters, c)
+		instance.Clusters = append(instance.Clusters, c)
 		log.Info("(svcs) adding VIP", "ip", vipConfig.VIP, "interface", vipConfig.Interface, "namespace", svc.Namespace, "name", svc.Name)
-
 	}
 
 	return instance, nil
@@ -305,24 +305,24 @@ func getAutoInterfaceName(link netlink.Link, defaultInterface string) string {
 }
 
 func (i *Instance) startDHCP() error {
-	if len(i.vipConfigs) != 1 {
-		return fmt.Errorf("DHCP requires exactly 1 VIP config, got: %v", len(i.vipConfigs))
+	if len(i.VIPConfigs) != 1 {
+		return fmt.Errorf("DHCP requires exactly 1 VIP config, got: %v", len(i.VIPConfigs))
 	}
-	parent, err := netlink.LinkByName(i.vipConfigs[0].Interface)
+	parent, err := netlink.LinkByName(i.VIPConfigs[0].Interface)
 	if err != nil {
 		return fmt.Errorf("error finding VIP Interface, for building DHCP Link : %v", err)
 	}
 
 	// Generate name from UID
-	interfaceName := fmt.Sprintf("vip-%s", i.serviceSnapshot.UID[0:8])
+	interfaceName := fmt.Sprintf("vip-%s", i.ServiceSnapshot.UID[0:8])
 
 	// Check if the interface doesn't exist first
 	iface, err := net.InterfaceByName(interfaceName)
 	if err != nil {
 		log.Info("creating new macvlan interface for DHCP", "interface", interfaceName)
 
-		hwaddr, err := net.ParseMAC(i.dhcpInterfaceHwaddr)
-		if i.dhcpInterfaceHwaddr != "" && err != nil {
+		hwaddr, err := net.ParseMAC(i.DHCPInterfaceHwaddr)
+		if i.DHCPInterfaceHwaddr != "" && err != nil {
 			return err
 		} else if hwaddr == nil {
 			hwaddr, err = net.ParseMAC(vip.GenerateMac())
@@ -359,28 +359,124 @@ func (i *Instance) startDHCP() error {
 		log.Info("Using existing macvlan interface for DHCP", "interface", interfaceName)
 	}
 
+	// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
+	rpfilterSetting := "0"
+
+	// Check if we need to set an override rp_filter value for the interface
+	if i.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
+		// Check the rp_filter value
+		rpFilter, err := strconv.Atoi(i.ServiceSnapshot.Annotations[kubevip.RPFilter])
+		if err != nil {
+			slog.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
+		} else {
+			if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
+				rpfilterSetting = i.ServiceSnapshot.Annotations[kubevip.RPFilter]
+			} else {
+				slog.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
+			}
+		}
+	}
+
+	err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
+	if err != nil {
+		slog.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
+	}
 	var initRebootFlag bool
-	if i.dhcpInterfaceIP != "" {
+	if i.DHCPInterfaceIP != "" {
 		initRebootFlag = true
 	}
 
-	client := vip.NewDHCPClient(iface, initRebootFlag, i.dhcpInterfaceIP)
+	client := vip.NewDHCPClient(iface, initRebootFlag, i.DHCPInterfaceIP)
 
 	// Add hostname to dhcp client if annotated
-	if i.dhcpHostname != "" {
-		log.Info("Hostname specified for dhcp lease", "interface", interfaceName, "hostname", i.dhcpHostname)
-		client.WithHostName(i.dhcpHostname)
+	if i.DHCPHostname != "" {
+		log.Info("Hostname specified for dhcp lease", "interface", interfaceName, "hostname", i.DHCPHostname)
+		client.WithHostName(i.DHCPHostname)
 	}
 
 	go client.Start()
 
 	// Set that DHCP is enabled
-	i.isDHCP = true
+	i.IsDHCP = true
 	// Set the name of the interface so that it can be removed on Service deletion
-	i.dhcpInterface = interfaceName
-	i.dhcpInterfaceHwaddr = iface.HardwareAddr.String()
+	i.DHCPInterface = interfaceName
+	i.DHCPInterfaceHwaddr = iface.HardwareAddr.String()
 	// Add the client so that we can call it to stop function
-	i.dhcpClient = client
+	i.DHCPClient = client
 
 	return nil
 }
+
+// FetchIngressAddresses tries to get the addresses from status.loadBalancerIP
+func FetchLoadBalancerIngressAddresses(s *v1.Service) []string {
+	// If the service has no status, return empty
+	lbStatusAddresses := []string{}
+
+	if len(s.Status.LoadBalancer.Ingress) == 0 {
+		return lbStatusAddresses
+	}
+
+	for _, ingress := range s.Status.LoadBalancer.Ingress {
+		if ingress.IP != "" {
+			lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
+		}
+		// TODO: Handle hostname if needed
+	}
+
+	return lbStatusAddresses
+}
+
+// FetchServiceAddresses tries to get the addresses from annotations
+// kube-vip.io/loadbalancerIPs, then from spec.loadbalancerIP
+func FetchServiceAddresses(s *v1.Service) []string {
+	annotationAvailable := false
+	if s.Annotations != nil {
+
+		if v, annotationAvailable := s.Annotations[kubevip.LoadbalancerIPAnnotation]; annotationAvailable {
+			ips := strings.Split(v, ",")
+			var trimmedIPs []string
+			for _, ip := range ips {
+				trimmedIPs = append(trimmedIPs, strings.TrimSpace(ip))
+			}
+			return trimmedIPs
+		}
+	}
+
+	lbStatusAddresses := []string{}
+	if !annotationAvailable {
+		if len(s.Status.LoadBalancer.Ingress) > 0 {
+			for _, ingress := range s.Status.LoadBalancer.Ingress {
+				lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
+			}
+		}
+	}
+
+	lbIP := net.ParseIP(s.Spec.LoadBalancerIP)
+	isLbIPv4 := vip.IsIPv4(s.Spec.LoadBalancerIP)
+
+	if len(lbStatusAddresses) > 0 {
+		for _, a := range lbStatusAddresses {
+			if lbStatusIP := net.ParseIP(a); lbStatusIP != nil && lbIP != nil && vip.IsIPv4(a) == isLbIPv4 && !lbIP.Equal(lbStatusIP) {
+				return []string{s.Spec.LoadBalancerIP}
+			}
+		}
+		return lbStatusAddresses
+	}
+
+	if s.Spec.LoadBalancerIP != "" {
+		return []string{s.Spec.LoadBalancerIP}
+	}
+
+	return []string{}
+}
+
+func FindServiceInstance(svc *v1.Service, instances []*Instance) *Instance {
+	log.Debug("finding service", "UID", svc.UID)
+	for i := range instances {
+		log.Debug("saved service", "instance", i, "UID", instances[i].ServiceSnapshot.UID)
+		if instances[i].ServiceSnapshot.UID == svc.UID {
+			return instances[i]
+		}
+	}
+	return nil
+}

pkg/kubevip/annotations.go

@@ -0,0 +1,49 @@
+package kubevip
+
+const (
+	// Hardware address of the host that has the VIP
+	HwAddrKey = "kube-vip.io/hwaddr"
+
+	// The IP address that is requested
+	RequestedIP = "kube-vip.io/requestedIP"
+
+	// The host that has the VIP
+	VipHost = "kube-vip.io/vipHost"
+
+	// Enable Egress on a service
+	Egress = "kube-vip.io/egress"
+
+	// Enable internal Egress
+	EgressInternal = "kube-vip.io/egress-internal"
+
+	// Egress should be IPv6
+	EgressIPv6 = "kube-vip.io/egress-ipv6"
+
+	// Ports that traffic is allowed to access from the egress VIP
+	EgressDestinationPorts = "kube-vip.io/egress-destination-ports"
+
+	// Allowed incoming ports to the VIP
+	EgressSourcePorts = "kube-vip.io/egress-source-ports"
+
+	// Allowed networks for the Egress to be enabled for
+	EgressAllowedNetworks = "kube-vip.io/egress-allowed-networks"
+
+	// Networks that we wont Egress for
+	EgressDeniedNetworks = "kube-vip.io/egress-denied-networks"
+
+	// The current active endpoint(pod) for the Egress VIP
+	ActiveEndpoint = "kube-vip.io/active-endpoint"
+
+	// The current active endpoint(pod) for the Egress VIP (v6)
+	ActiveEndpointIPv6 = "kube-vip.io/active-endpoint-ipv6"
+
+	// Flush the conntrack rules (remove existing sessions) once Egress is configured
+	FlushContrack = "kube-vip.io/flush-conntrack"
+
+	LoadbalancerIPAnnotation = "kube-vip.io/loadbalancerIPs"
+	LoadbalancerHostname     = "kube-vip.io/loadbalancerHostname"
+	ServiceInterface         = "kube-vip.io/serviceInterface"
+	UpnpEnabled              = "kube-vip.io/forwardUPNP"
+
+	RPFilter = "kube-vip.io/rp_filter" // Set the return path filter for a specific service interface
+)

pkg/kubevip/config_environment.go

@@ -7,6 +7,7 @@ import (
 	"math/bits"
 	"os"
 	"strconv"
+	"strings"
 
 	"github.com/kube-vip/kube-vip/pkg/bgp"
 	"github.com/kube-vip/kube-vip/pkg/detector"
@@ -49,12 +50,6 @@ func ParseEnvironment(c *Config) error {
 		c.ServicesInterface = env
 	}
 
-	// Find provider configuration
-	env = os.Getenv(providerConfig)
-	if env != "" {
-		c.ProviderConfig = env
-	}
-
 	// Find Kubernetes Leader Election configuration
 	env = os.Getenv(vipLeaderElection)
 	if env != "" {
@@ -238,12 +233,6 @@ func ParseEnvironment(c *Config) error {
 		}
 	}
 
-	// Find vip address cidr range
-	env = os.Getenv(vipCidr)
-	if env != "" {
-		c.VIPCIDR = env
-	}
-
 	// Find vip address subnet
 	env = os.Getenv(vipSubnet)
 	if env != "" {
@@ -512,28 +501,32 @@ func ParseEnvironment(c *Config) error {
 		c.BGPConfig.KeepaliveInterval = u64
 	}
 
-	// Enable the Equinix Metal API calls
-	env = os.Getenv(vipPacket)
+	env = os.Getenv(zebraEnable)
 	if env != "" {
-		b, err := strconv.ParseBool(env)
+		result, err := strconv.ParseBool(env)
 		if err != nil {
 			return err
 		}
-		c.EnableMetal = b
+		c.BGPConfig.Zebra.Enabled = result
 	}
 
-	// Find the Equinix Metal project name
-	env = os.Getenv(vipPacketProject)
+	env = os.Getenv(zebraURL)
 	if env != "" {
-		// TODO - parse address net.Host()
-		c.MetalProject = env
+		c.BGPConfig.Zebra.URL = env
 	}
 
-	// Find the Equinix Metal project ID
-	env = os.Getenv(vipPacketProjectID)
+	env = os.Getenv(zebraVersion)
 	if env != "" {
-		// TODO - parse address net.Host()
-		c.MetalProjectID = env
+		u64, err := strconv.ParseUint(env, 10, 32)
+		if err != nil {
+			return err
+		}
+		c.BGPConfig.Zebra.Version = uint32(u64)
+	}
+
+	env = os.Getenv(zebraSoftwareName)
+	if env != "" {
+		c.BGPConfig.Zebra.SoftwareName = env
 	}
 
 	// Enable the load-balancer
@@ -614,13 +607,13 @@ func ParseEnvironment(c *Config) error {
 		c.K8sConfigFile = env
 	}
 
-	env = os.Getenv(enableEndpointSlices)
+	env = os.Getenv(enableEndpoints)
 	if env != "" {
 		b, err := strconv.ParseBool(env)
 		if err != nil {
 			return err
 		}
-		c.EnableEndpointSlices = b
+		c.EnableEndpoints = b
 	}
 
 	env = os.Getenv(mirrorDestInterface)
@@ -654,5 +647,25 @@ func ParseEnvironment(c *Config) error {
 		c.HealthCheckPort = int(i)
 	}
 
+	env = os.Getenv(enableUPNP)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.EnableUPNP = b
+	}
+
+	if env = os.Getenv(egressClean); env == "" {
+		env = os.Getenv(strings.ToUpper(egressClean))
+	}
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.EgressClean = b
+	}
+
 	return nil
 }

pkg/kubevip/config_envvar.go

@@ -39,9 +39,6 @@ const (
 	// vipServicesInterface - defines the interface that the service vips should bind too
 	vipServicesInterface = "vip_servicesinterface"
 
-	// vipCidr - defines the cidr that the vip will use (for BGP)
-	vipCidr = "vip_cidr"
-
 	// vipSubnet - defines the subnet that the vip will use
 	vipSubnet = "vip_subnet"
 
@@ -86,18 +83,6 @@ const (
 	// vipStartLeader - will start this instance as the leader of the cluster
 	vipStartLeader = "vip_startleader"
 
-	// vipPacket defines that the packet API will be used for EIP
-	vipPacket = "vip_packet"
-
-	// vipPacketProject defines which project within Packet to use
-	vipPacketProject = "vip_packetproject"
-
-	// vipPacketProjectID defines which projectID within Packet to use
-	vipPacketProjectID = "vip_packetprojectid"
-
-	// providerConfig defines a path to a configuration that should be parsed
-	providerConfig = "provider_config"
-
 	// bgpEnable defines if BGP should be enabled
 	bgpEnable = "bgp_enable"
 	// bgpRouterID defines the routerID for the BGP server
@@ -125,6 +110,15 @@ const (
 	// bgpKeepaliveInterval defines bgp timers keepalive interval
 	bgpKeepaliveInterval = "bgp_keepalive_interval"
 
+	// zebraEnable defines if Zebra integraton should be enabled
+	zebraEnable = "zebra_enable"
+	// zebraUrl specifies path to the unix domain socket for connecting to Zebra daemon
+	zebraURL = "zebra_url"
+	// zebraVersion specifies Zebra API Version
+	zebraVersion = "zebra_version"
+	// zebraSoftwareName specifies Software Name for Zebra
+	zebraSoftwareName = "zebra_software_name"
+
 	// mpbgpNexthop defines MPBGP mode
 	mpbgpNexthop = "mpbgp_nexthop"
 	// mpbgpIPv4 defines fixed IPv4 to be used with MPBGP
@@ -218,8 +212,8 @@ const (
 	// disableServiceUpdates disables service updating
 	disableServiceUpdates = "disable_service_updates"
 
-	// enableEndpointSlices enables use of EndpointSlices instead of Endpoints
-	enableEndpointSlices = "enable_endpointslices"
+	// enableEndpoints enables use of Endpoints instead of EndpointSlices
+	enableEndpoints = "enable_endpoints"
 
 	// mirrorDestInterface is the network interface where all traffics that go through service interface
 	// will be mirrored to. The source interface is ServicesInterface by default, fall back to Interface if not set.
@@ -234,4 +228,10 @@ const (
 
 	// healthCheckPort, if set to non-zero will be the port the health check will listen on
 	healthCheckPort = "health_check_port"
+
+	// enableUPNP enables UPNP functions
+	enableUPNP = "enable_upnp"
+
+	// egressClean enables egress cleaning on kube-vip's start
+	egressClean = "egress_clean"
 )

pkg/kubevip/config_generator.go

@@ -2,11 +2,14 @@ package kubevip
 
 import (
 	"fmt"
+	"log"
 	"strconv"
 
 	appv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	applyCoreV1 "k8s.io/client-go/applyconfigurations/core/v1"
 	applyMetaV1 "k8s.io/client-go/applyconfigurations/meta/v1"
 	applyRbacV1 "k8s.io/client-go/applyconfigurations/rbac/v1"
@@ -14,11 +17,43 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
+// TransformApplyObjectToManifest transforms an apply object into a normal Kubernetes manifest
+func TransformApplyObjectToManifest(applyObject interface{}) string {
+	// Convert the apply object to an unstructured object
+	unstructuredObj := &unstructured.Unstructured{}
+	err := runtime.DefaultUnstructuredConverter.FromUnstructured(applyConfigToMap(applyObject), unstructuredObj)
+	if err != nil {
+		log.Fatalf("Error converting apply object to unstructured: %v", err)
+	}
+
+	// Marshal the unstructured object into YAML
+	yamlData, err := yaml.Marshal(unstructuredObj.Object)
+	if err != nil {
+		log.Fatalf("Error marshaling unstructured object to YAML: %v", err)
+	}
+
+	return string(yamlData)
+}
+
+// Helper function to convert apply configuration to a map
+func applyConfigToMap(applyConfig interface{}) map[string]interface{} {
+	data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(applyConfig)
+	if err != nil {
+		log.Fatalf("Error converting apply configuration to map: %v", err)
+	}
+	return data
+}
+
 // GenerateSA will create the service account for kube-vip
-func GenerateSA() *applyCoreV1.ServiceAccountApplyConfiguration {
+func GenerateSA(c *Config) *applyCoreV1.ServiceAccountApplyConfiguration {
 	kind := "ServiceAccount"
 	name := "kube-vip"
-	namespace := "kube-system"
+	var namespace string
+	if c.ServiceNamespace != "" {
+		namespace = c.ServiceNamespace
+	} else {
+		namespace = metav1.NamespaceSystem
+	}
 	newManifest := &applyCoreV1.ServiceAccountApplyConfiguration{
 		TypeMetaApplyConfiguration: applyMetaV1.TypeMetaApplyConfiguration{APIVersion: &corev1.SchemeGroupVersion.Version, Kind: &kind},
 		ObjectMetaApplyConfiguration: &applyMetaV1.ObjectMetaApplyConfiguration{
@@ -30,15 +65,31 @@ func GenerateSA() *applyCoreV1.ServiceAccountApplyConfiguration {
 }
 
 // GenerateCR will generate the Cluster role for kube-vip
-func GenerateCR() *applyRbacV1.ClusterRoleApplyConfiguration {
-	name := "system:kube-vip-role"
-	roleRefKind := "ClusterRole"
-	apiVersion := "rbac.authorization.k8s.io/v1"
+func GenerateRole(c *Config, role bool) *applyRbacV1.RoleApplyConfiguration {
+	var kind, name string
+	var namespace *string
+	if role {
+		kind = "Role"
+		name = "kube-vip"
+		if c.ServiceNamespace != "" {
+			namespace = &c.ServiceNamespace
+		} else {
+			// If the namespace is empty then we need to set it to the system namespace
+			copiedNamespace := metav1.NamespaceSystem
+			namespace = &copiedNamespace
+		}
 
-	newManifest := &applyRbacV1.ClusterRoleApplyConfiguration{
-		TypeMetaApplyConfiguration: applyMetaV1.TypeMetaApplyConfiguration{APIVersion: &apiVersion, Kind: &roleRefKind},
+	} else {
+		kind = "ClusterRole"
+		name = "system:kube-vip-role"
+
+	}
+	apiVersion := "rbac.authorization.k8s.io/v1"
+	newManifest := &applyRbacV1.RoleApplyConfiguration{
+		TypeMetaApplyConfiguration: applyMetaV1.TypeMetaApplyConfiguration{APIVersion: &apiVersion, Kind: &kind},
 		ObjectMetaApplyConfiguration: &applyMetaV1.ObjectMetaApplyConfiguration{
-			Name: &name,
+			Name:      &name,
+			Namespace: namespace,
 		},
 		Rules: []applyRbacV1.PolicyRuleApplyConfiguration{
 			{
@@ -49,7 +100,7 @@ func GenerateCR() *applyRbacV1.ClusterRoleApplyConfiguration {
 			{
 				APIGroups: []string{""},
 				Resources: []string{"services", "endpoints"},
-				Verbs:     []string{"list", "get", "watch", "endoints"},
+				Verbs:     []string{"list", "get", "watch", "update"},
 			},
 			{
 				APIGroups: []string{""},
@@ -61,38 +112,54 @@ func GenerateCR() *applyRbacV1.ClusterRoleApplyConfiguration {
 				Resources: []string{"leases"},
 				Verbs:     []string{"list", "get", "watch", "update", "create"},
 			},
+			{
+				APIGroups: []string{"discovery.k8s.io"},
+				Resources: []string{"endpointslices"},
+				Verbs:     []string{"list", "get", "watch", "update"},
+			},
+			{
+				APIGroups: []string{""},
+				Resources: []string{"pods"},
+				Verbs:     []string{"list"},
+			},
 		},
 	}
 	return newManifest
 }
 
-// GenerateCRB will generate the clusterRoleBinding
-func GenerateCRB() *applyRbacV1.ClusterRoleBindingApplyConfiguration {
-	kind := "ClusterRoleBinding"
+// GenerateCRB will generate the clusterRoleBinding or rolebinding
+func GenerateRoleBinding(rolebinding bool, saCfg *applyCoreV1.ServiceAccountApplyConfiguration, crCfg *applyRbacV1.RoleApplyConfiguration) *applyRbacV1.RoleBindingApplyConfiguration {
 	apiVersion := "rbac.authorization.k8s.io/v1"
-	subjectKind := "ServiceAccount"
 	apiGroup := "rbac.authorization.k8s.io"
-	roleRefKind := "ClusterRole"
-	roleRefName := "system:kube-vip-role"
-	name := "kube-vip"
-	bindName := "system:kube-vip-role-binding"
-	namespace := "kube-system"
-
-	newManifest := &applyRbacV1.ClusterRoleBindingApplyConfiguration{
+	var kind, bindName string
+	var namespace, objectNamespace *string
+	if rolebinding {
+		kind = "RoleBinding"
+		bindName = "kube-vip"
+		namespace = nil
+		objectNamespace = saCfg.Namespace
+	} else {
+		kind = "ClusterRoleBinding"
+		bindName = "system:kube-vip-binding"
+		namespace = saCfg.Namespace
+		objectNamespace = nil
+	}
+	newManifest := &applyRbacV1.RoleBindingApplyConfiguration{
 		TypeMetaApplyConfiguration: applyMetaV1.TypeMetaApplyConfiguration{APIVersion: &apiVersion, Kind: &kind},
 		ObjectMetaApplyConfiguration: &applyMetaV1.ObjectMetaApplyConfiguration{
-			Name: &bindName,
+			Name:      &bindName,
+			Namespace: objectNamespace,
 		},
 		RoleRef: &applyRbacV1.RoleRefApplyConfiguration{
 			APIGroup: &apiGroup,
-			Kind:     &roleRefKind,
-			Name:     &roleRefName,
+			Kind:     crCfg.Kind,
+			Name:     crCfg.Name,
 		},
 		Subjects: []applyRbacV1.SubjectApplyConfiguration{
 			{
-				Kind:      &subjectKind,
-				Name:      &name,
-				Namespace: &namespace,
+				Kind:      saCfg.Kind,
+				Name:      saCfg.Name,
+				Namespace: namespace,
 			},
 		},
 	}
@@ -161,18 +228,6 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
 		newEnvironment = append(newEnvironment, svcInterface...)
 	}
 
-	// If a CIDR is used add it to the manifest
-	if c.VIPCIDR != "" {
-		// build environment variables
-		cidr := []corev1.EnvVar{
-			{
-				Name:  vipCidr,
-				Value: c.VIPCIDR,
-			},
-		}
-		newEnvironment = append(newEnvironment, cidr...)
-	}
-
 	// If a subnet is required for the VIP
 	if c.VIPSubnet != "" {
 		// build environment variables
@@ -331,40 +386,6 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
 
 	}
 
-	// If we're specifying a configuration
-	if c.ProviderConfig != "" {
-		provider := []corev1.EnvVar{
-			{
-				Name:  providerConfig,
-				Value: c.ProviderConfig,
-			},
-		}
-		newEnvironment = append(newEnvironment, provider...)
-	}
-
-	// If Equinix Metal is enabled then add it to the manifest
-	if c.EnableMetal {
-		packet := []corev1.EnvVar{
-			{
-				Name:  vipPacket,
-				Value: strconv.FormatBool(c.EnableMetal),
-			},
-			{
-				Name:  vipPacketProject,
-				Value: c.MetalProject,
-			},
-			{
-				Name:  vipPacketProjectID,
-				Value: c.MetalProjectID,
-			},
-			{
-				Name:  "PACKET_AUTH_TOKEN",
-				Value: c.MetalAPIKey,
-			},
-		}
-		newEnvironment = append(newEnvironment, packet...)
-	}
-
 	// Detect and enable wireguard mode
 	if c.EnableWireguard {
 		wireguard := []corev1.EnvVar{
@@ -386,8 +407,7 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
 		}
 		newEnvironment = append(newEnvironment, routingtable...)
 	}
-
-	// If BGP, but we're not using Equinix Metal
+	// If BGP
 	if c.EnableBGP {
 		bgp := []corev1.EnvVar{
 			{
@@ -397,8 +417,9 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
 		}
 		newEnvironment = append(newEnvironment, bgp...)
 	}
-	// If BGP, but we're not using Equinix Metal
-	if c.EnableBGP && !c.EnableMetal {
+
+	// If BGP
+	if c.EnableBGP {
 		bgpConfig := []corev1.EnvVar{
 			{
 				Name:  bgpRouterID,
@@ -500,10 +521,10 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
 	}
 	newEnvironment = append(newEnvironment, prometheus...)
 
-	if c.EnableEndpointSlices {
+	if c.EnableEndpoints {
 		newEnvironment = append(newEnvironment, corev1.EnvVar{
-			Name:  enableEndpointSlices,
-			Value: strconv.FormatBool(c.EnableEndpointSlices),
+			Name:  enableEndpoints,
+			Value: strconv.FormatBool(c.EnableEndpoints),
 		})
 	}
 
@@ -612,26 +633,6 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
 		newManifest.Spec.HostAliases = append(newManifest.Spec.HostAliases, hostAlias)
 	}
 
-	if c.ProviderConfig != "" {
-		providerConfigMount := corev1.VolumeMount{
-			Name:      "cloud-sa-volume",
-			MountPath: "/etc/cloud-sa",
-			ReadOnly:  true,
-		}
-		newManifest.Spec.Containers[0].VolumeMounts = append(newManifest.Spec.Containers[0].VolumeMounts, providerConfigMount)
-
-		providerConfigVolume := corev1.Volume{
-			Name: "cloud-sa-volume",
-			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{
-					SecretName: "metal-cloud-config",
-				},
-			},
-		}
-		newManifest.Spec.Volumes = append(newManifest.Spec.Volumes, providerConfigVolume)
-
-	}
-
 	return newManifest
 }
 

pkg/kubevip/config_types.go

@@ -76,9 +76,6 @@ type Config struct {
 	// VipSubnet is the Subnet that is applied to the VIP
 	VIPSubnet string `yaml:"vipSubnet"`
 
-	// VIPCIDR is cidr range for the VIP (primarily needed for BGP)
-	VIPCIDR string `yaml:"vipCidr"`
-
 	// Address is the IP or DNS Name to use as a VirtualIP
 	Address string `yaml:"address"`
 
@@ -135,21 +132,6 @@ type Config struct {
 	BGPPeerConfig bgp.Peer
 	BGPPeers      []string
 
-	// EnableMetal, will use the metal API to update the EIP <-> VIP (if BGP is enabled then BGP will be used)
-	EnableMetal bool `yaml:"enableMetal"`
-
-	// MetalAPIKey, is the API token used to authenticate to the API
-	MetalAPIKey string
-
-	// MetalProject, is the name of a particular defined project
-	MetalProject string
-
-	// MetalProjectID, is the name of a particular defined project
-	MetalProjectID string
-
-	// ProviderConfig, is the path to a provider configuration file
-	ProviderConfig string
-
 	// LoadBalancers are the various services we can load balance over
 	LoadBalancers []LoadBalancer `yaml:"loadBalancers,omitempty"`
 
@@ -179,8 +161,8 @@ type Config struct {
 	// DisableServiceUpdates, if true, kube-vip will only advertise service, but it will not update service's Status.LoadBalancer.Ingress slice
 	DisableServiceUpdates bool `yaml:"disableServiceUpdates"`
 
-	// EnableEndpointSlices, if enabled, EndpointSlices will be used instead of Endpoints
-	EnableEndpointSlices bool `yaml:"enableEndpointSlices"`
+	// EnableEndpoints, if enabled, Endpoints will be used instead of EndpointSlices
+	EnableEndpoints bool `yaml:"enableEndpoints"`
 
 	// MirrorDestInterface is the network interface where all traffics that go through service interface
 	// will be mirrored to. If ServicesInterface is not set, fall back to Interface.
@@ -198,6 +180,12 @@ type Config struct {
 
 	// HealthCheckPort, if non-zero then will enable the healthcheck to return ok on this port
 	HealthCheckPort int `yaml:"healthCheckPort"`
+
+	// EnableUPNP, enables UPNP functions
+	EnableUPNP bool `yaml:"enableUPNP"`
+
+	// EgressClean, enables egress cleaning on Kube-vip's start
+	EgressClean bool `yaml:"egressClean"`
 }
 
 // KubernetesLeaderElection defines all of the settings for Kubernetes KubernetesLeaderElection

pkg/manager/annotations.go

@@ -1,44 +0,0 @@
-package manager
-
-const (
-	// Hardware address of the host that has the VIP
-	hwAddrKey = "kube-vip.io/hwaddr"
-
-	// The IP address that is requested
-	requestedIP = "kube-vip.io/requestedIP"
-
-	// The host that has the VIP
-	vipHost = "kube-vip.io/vipHost"
-
-	// Enable egress on a service
-	egress = "kube-vip.io/egress"
-
-	// Egress should be IPv6
-	egressIPv6 = "kube-vip.io/egress-ipv6"
-
-	// Ports that traffic is allowed to access from the egress VIP
-	egressDestinationPorts = "kube-vip.io/egress-destination-ports"
-
-	// Allowed incoming ports to the VIP
-	egressSourcePorts = "kube-vip.io/egress-source-ports"
-
-	// Allowed networks for the Egress to be enabled for
-	egressAllowedNetworks = "kube-vip.io/egress-allowed-networks"
-
-	// Networks that we wont Egress for
-	egressDeniedNetworks = "kube-vip.io/egress-denied-networks"
-
-	// The current active endpoint(pod) for the Egress VIP
-	activeEndpoint = "kube-vip.io/active-endpoint"
-
-	// The current active endpoint(pod) for the Egress VIP (v6)
-	activeEndpointIPv6 = "kube-vip.io/active-endpoint-ipv6"
-
-	// Flush the conntrack rules (remove existing sessions) once Egress is configured
-	flushContrack = "kube-vip.io/flush-conntrack"
-
-	loadbalancerIPAnnotation = "kube-vip.io/loadbalancerIPs"
-	loadbalancerHostname     = "kube-vip.io/loadbalancerHostname"
-	serviceInterface         = "kube-vip.io/serviceInterface"
-	upnpEnabled              = "kube-vip.io/forwardUPNP"
-)

pkg/manager/manager.go

@@ -7,7 +7,6 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -15,14 +14,15 @@ import (
 
 	log "log/slog"
 
+	"github.com/kube-vip/kube-vip/pkg/arp"
 	"github.com/kube-vip/kube-vip/pkg/bgp"
 	"github.com/kube-vip/kube-vip/pkg/k8s"
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	"github.com/kube-vip/kube-vip/pkg/trafficmirror"
+	"github.com/kube-vip/kube-vip/pkg/networkinterface"
+	"github.com/kube-vip/kube-vip/pkg/services"
 	"github.com/kube-vip/kube-vip/pkg/upnp"
 	"github.com/kube-vip/kube-vip/pkg/utils"
 	"github.com/prometheus/client_golang/prometheus"
-	v1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 )
@@ -39,11 +39,6 @@ type Manager struct {
 	// Manager services
 	// service bool
 
-	// Keeps track of all running instances
-	serviceInstances []*Instance
-
-	// UPNP functionality
-	upnp bool
 	// BGP Manager, this is a singleton that manages all BGP advertisements
 	bgpServer *bgp.Server
 
@@ -53,6 +48,8 @@ type Manager struct {
 	// This channel is used to signal a shutdown
 	shutdownChan chan struct{}
 
+	svcProcessor *services.Processor
+
 	// This is a prometheus counter used to count the number of events received
 	// from the service watcher
 	countServiceWatchEvent *prometheus.CounterVec
@@ -63,6 +60,12 @@ type Manager struct {
 
 	// This mutex is to protect calls from various goroutines
 	mutex sync.Mutex
+
+	// This tracks used network interfaces and guards them with mutex for concurrent changes.
+	intfMgr *networkinterface.Manager
+
+	// This tracks VIPs and performs ARP/NDP advertisement.
+	arpMgr *arp.Manager
 }
 
 // New will create a new managing object
@@ -168,6 +171,32 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
 	// 	}
 	// }
 
+	// listen for interrupts or the Linux SIGTERM signal and cancel
+	// our context, which the leader election code will observe and
+	// step down
+	signalChan := make(chan os.Signal, 1)
+	// Add Notification for Userland interrupt
+	signal.Notify(signalChan, syscall.SIGINT)
+
+	// Add Notification for SIGTERM (sent from Kubernetes)
+	signal.Notify(signalChan, syscall.SIGTERM)
+
+	// All watchers and other goroutines should have an additional goroutine that blocks on this, to shut things down
+	shutdownChan := make(chan struct{})
+
+	intfMgr := networkinterface.NewManager()
+	arpMgr := arp.NewManager(config)
+
+	var bgpServer *bgp.Server
+	if config.EnableBGP {
+		bgpServer, err = bgp.NewBGPServer(&config.BGPConfig)
+		if err != nil {
+			return nil, fmt.Errorf("creating BGP server: %w", err)
+		}
+	}
+
+	svcProcessor := services.NewServicesProcessor(config, bgpServer, clientset, rwClientSet, shutdownChan, intfMgr, arpMgr)
+
 	return &Manager{
 		clientSet:   clientset,
 		rwClientSet: rwClientSet,
@@ -185,6 +214,12 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
 			Name:      "bgp_session_info",
 			Help:      "Display state of session by setting metric for label value with current state to 1",
 		}, []string{"state", "peer"}),
+		signalChan:   signalChan,
+		shutdownChan: shutdownChan,
+		svcProcessor: svcProcessor,
+		intfMgr:      intfMgr,
+		arpMgr:       arpMgr,
+		bgpServer:    bgpServer,
 	}, nil
 }
 
@@ -237,16 +272,12 @@ func (sm *Manager) Start() error {
 	}
 
 	if sm.config.EnableARP || sm.config.EnableWireguard {
-		// Before starting the leader Election enable any additional functionality
-		upnpEnabled, _ := strconv.ParseBool(os.Getenv("enableUPNP"))
-
-		if upnpEnabled {
-			sm.upnp = true
+		if sm.config.EnableUPNP {
 			clients := upnp.GetConnectionClients(context.TODO())
 			if len(clients) == 0 {
 				log.Error("Error Enabling UPNP. No Clients found")
 				// Set the struct to false so nothing should use it in future
-				sm.upnp = false
+				sm.config.EnableUPNP = false
 			} else {
 				for _, c := range clients {
 					ip, err := c.GetExternalIPAddress()
@@ -254,12 +285,11 @@ func (sm *Manager) Start() error {
 						log.Error("unable to find IGD2 Gateway address", "err", err)
 					}
 					log.Info("Found UPNP IGD2 Gateway address", "ip", ip)
-
 				}
 			}
 		}
 		// TODO: It would be nice to run the UPNP refresh only on the leader.
-		go sm.refreshUPNPForwards()
+		go sm.svcProcessor.RefreshUPNPForwards()
 	}
 
 	// If ARP is enabled then we start a LeaderElection that will use ARP to advertise VIPs
@@ -304,89 +334,3 @@ func (sm *Manager) parseAnnotations() error {
 	}
 	return nil
 }
-
-func (sm *Manager) serviceInterface() string {
-	svcIf := sm.config.Interface
-	if sm.config.ServicesInterface != "" {
-		svcIf = sm.config.ServicesInterface
-	}
-	return svcIf
-}
-
-func (sm *Manager) startTrafficMirroringIfEnabled() error {
-	if sm.config.MirrorDestInterface != "" {
-		svcIf := sm.serviceInterface()
-		log.Info("mirroring traffic", "src", svcIf, "dest", sm.config.MirrorDestInterface)
-		if err := trafficmirror.MirrorTrafficFromNIC(svcIf, sm.config.MirrorDestInterface); err != nil {
-			return err
-		}
-	} else {
-		log.Debug("skip starting traffic mirroring since it's not enabled.")
-	}
-	return nil
-}
-
-func (sm *Manager) stopTrafficMirroringIfEnabled() error {
-	if sm.config.MirrorDestInterface != "" {
-		svcIf := sm.serviceInterface()
-		log.Info("clean up qdisc config", "interface", svcIf)
-		if err := trafficmirror.CleanupQDSICFromNIC(svcIf); err != nil {
-			return err
-		}
-	} else {
-		log.Debug("skip stopping traffic mirroring since it's not enabled.")
-	}
-	return nil
-}
-
-func (sm *Manager) findServiceInstance(svc *v1.Service) *Instance {
-	log.Debug("finding service", "UID", svc.UID)
-	for i := range sm.serviceInstances {
-		log.Debug("saved service", "instance", i, "UID", sm.serviceInstances[i].serviceSnapshot.UID)
-		if sm.serviceInstances[i].serviceSnapshot.UID == svc.UID {
-			return sm.serviceInstances[i]
-		}
-	}
-	return nil
-}
-
-func (sm *Manager) findServiceInstanceWithTimeout(ctx context.Context, svc *v1.Service) (*Instance, error) {
-	log.Debug("finding service with timeout", "UID", svc.UID)
-
-	ctxTimeout, ctxTimeoutCancel := context.WithTimeout(ctx, time.Minute)
-	defer ctxTimeoutCancel()
-
-	for {
-		for i := range sm.serviceInstances {
-			log.Debug("saved service", "instance", i, "UID", sm.serviceInstances[i].serviceSnapshot.UID)
-			if sm.serviceInstances[i].serviceSnapshot.UID == svc.UID {
-				return sm.serviceInstances[i], nil
-			}
-		}
-
-		t := time.NewTimer(time.Millisecond * 100)
-		select {
-		case <-ctxTimeout.Done():
-			t.Stop()
-			return nil, fmt.Errorf("failed to wait for the service instance: %w", ctx.Err())
-		case <-t.C:
-			log.Debug("waiting for the service to be discovered", "UID", svc.UID)
-		}
-	}
-}
-
-// Refresh UPNP Port Forwards for all Service Instances registered in the SM
-func (sm *Manager) refreshUPNPForwards() {
-	log.Info("Starting UPNP Port Refresher")
-	for {
-		time.Sleep(300 * time.Second)
-
-		log.Info("[UPNP] Refreshing Instances", "number of instances", len(sm.serviceInstances))
-		for i := range sm.serviceInstances {
-			sm.upnpMap(context.TODO(), sm.serviceInstances[i])
-			if err := sm.updateStatus(sm.serviceInstances[i]); err != nil {
-				log.Warn("[UPNP] Error updating service", "ip", sm.serviceInstances[i].serviceSnapshot.Name, "err", err)
-			}
-		}
-	}
-}

pkg/manager/manager_arp.go

@@ -2,7 +2,6 @@ package manager
 
 import (
 	"context"
-	"os"
 	"syscall"
 	"time"
 
@@ -28,6 +27,9 @@ func (sm *Manager) startARP(id string) error {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	log.Info("Start ARP/NDP advertisement")
+	go sm.arpMgr.StartAdvertisement(ctx)
+
 	// Shutdown function that will wait on this signal, unless we call it ourselves
 	go func() {
 		<-sm.signalChan
@@ -42,7 +44,7 @@ func (sm *Manager) startARP(id string) error {
 	}()
 
 	if sm.config.EnableControlPlane {
-		cpCluster, err = cluster.InitCluster(sm.config, false)
+		cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
 		if err != nil {
 			return err
 		}
@@ -80,7 +82,7 @@ func (sm *Manager) startARP(id string) error {
 	}
 
 	// This will tidy any dangling kube-vip iptables rules
-	if os.Getenv("EGRESS_CLEAN") != "" {
+	if sm.config.EgressClean {
 		vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
 	}
 
@@ -88,7 +90,7 @@ func (sm *Manager) startARP(id string) error {
 	// a lock based upon that service is created that they will all leaderElection on
 	if sm.config.EnableServicesElection {
 		log.Info("beginning watching services, leaderelection will happen for every service")
-		err = sm.startServicesWatchForLeaderElection(ctx)
+		err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
 		if err != nil {
 			return err
 		}
@@ -123,7 +125,7 @@ func (sm *Manager) startARP(id string) error {
 			RetryPeriod:     time.Duration(sm.config.RetryPeriod) * time.Second,
 			Callbacks: leaderelection.LeaderCallbacks{
 				OnStartedLeading: func(ctx context.Context) {
-					err = sm.servicesWatcher(ctx, sm.syncServices)
+					err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
 					if err != nil {
 						log.Error("service watcher", "err", err)
 						panic("") // TODO: - emulating log.fatal here
@@ -131,12 +133,10 @@ func (sm *Manager) startARP(id string) error {
 				},
 				OnStoppedLeading: func() {
 					// we can do cleanup here
+					sm.mutex.Lock()
+					defer sm.mutex.Unlock()
 					log.Info("leader lost", "new leader", id)
-					for _, instance := range sm.serviceInstances {
-						for _, cluster := range instance.clusters {
-							cluster.Stop()
-						}
-					}
+					sm.svcProcessor.Stop()
 
 					log.Error("lost leadership, restarting kube-vip")
 					panic("") // TODO: - emulating log.fatal here

pkg/manager/manager_bgp.go

@@ -3,16 +3,13 @@ package manager
 import (
 	"context"
 	"fmt"
-	"os"
 	"syscall"
 
 	log "log/slog"
 
 	"github.com/kube-vip/kube-vip/pkg/bgp"
 	"github.com/kube-vip/kube-vip/pkg/cluster"
-	"github.com/kube-vip/kube-vip/pkg/equinixmetal"
 	api "github.com/osrg/gobgp/v3/api"
-	"github.com/packethost/packngo"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -22,37 +19,15 @@ func (sm *Manager) startBGP() error {
 	// var ns string
 	var err error
 
-	// If Equinix Metal is enabled then we can begin our preparation work
-	var packetClient *packngo.Client
-	if sm.config.EnableMetal {
-		if sm.config.ProviderConfig != "" {
-			key, project, err := equinixmetal.GetPacketConfig(sm.config.ProviderConfig)
-			if err != nil {
-				return err
-			}
-			// Set the environment variable with the key for the project
-			os.Setenv("PACKET_AUTH_TOKEN", key)
-			// Update the configuration with the project key
-			sm.config.MetalProjectID = project
-
-		}
-		packetClient, err = packngo.NewClient()
+	if sm.bgpServer == nil {
+		sm.bgpServer, err = bgp.NewBGPServer(&sm.config.BGPConfig)
 		if err != nil {
-			return err
-		}
-
-		// We're using Equinix Metal with BGP, populate the Peer information from the API
-		if sm.config.EnableBGP {
-			log.Info("Looking up the BGP configuration from Equinix Metal")
-			err = equinixmetal.BGPLookup(packetClient, sm.config)
-			if err != nil {
-				return err
-			}
+			return fmt.Errorf("creating BGP server: %w", err)
 		}
 	}
 
 	log.Info("Starting the BGP server to advertise VIP routes to BGP peers")
-	sm.bgpServer, err = bgp.NewBGPServer(&sm.config.BGPConfig, func(p *api.WatchEventResponse_PeerEvent) {
+	if err := sm.bgpServer.Start(func(p *api.WatchEventResponse_PeerEvent) {
 		ipaddr := p.GetPeer().GetState().GetNeighborAddress()
 		port := uint64(179)
 		peerDescription := fmt.Sprintf("%s:%d", ipaddr, port)
@@ -68,9 +43,8 @@ func (sm *Manager) startBGP() error {
 				"peer":  peerDescription,
 			}).Set(metricValue)
 		}
-	})
-	if err != nil {
-		return err
+	}); err != nil {
+		return fmt.Errorf("starting BGP server: %w", err)
 	}
 
 	// use a Go context so we can tell the leaderelection code when we
@@ -99,7 +73,7 @@ func (sm *Manager) startBGP() error {
 	}()
 
 	if sm.config.EnableControlPlane {
-		cpCluster, err = cluster.InitCluster(sm.config, false)
+		cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
 		if err != nil {
 			return err
 		}
@@ -113,7 +87,7 @@ func (sm *Manager) startBGP() error {
 			if sm.config.EnableLeaderElection {
 				err = cpCluster.StartCluster(sm.config, clusterManager, sm.bgpServer)
 			} else {
-				err = cpCluster.StartVipService(sm.config, clusterManager, sm.bgpServer, packetClient)
+				err = cpCluster.StartVipService(sm.config, clusterManager, sm.bgpServer)
 			}
 			if err != nil {
 				log.Error("Control Plane", "err", err)
@@ -131,7 +105,7 @@ func (sm *Manager) startBGP() error {
 		}
 	}
 
-	err = sm.servicesWatcher(ctx, sm.syncServices)
+	err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
 	if err != nil {
 		return err
 	}

pkg/manager/manager_table.go

@@ -3,14 +3,13 @@ package manager
 import (
 	"context"
 	"fmt"
-	"os"
-	"strconv"
 	"syscall"
 	"time"
 
 	log "log/slog"
 
 	"github.com/kube-vip/kube-vip/pkg/cluster"
+	"github.com/kube-vip/kube-vip/pkg/endpoints"
 	"github.com/kube-vip/kube-vip/pkg/iptables"
 	"github.com/kube-vip/kube-vip/pkg/vip"
 	"github.com/vishvananda/netlink"
@@ -40,17 +39,10 @@ func (sm *Manager) startTableMode(id string) error {
 		}()
 	}
 
-	egressCleanEnv := os.Getenv("EGRESS_CLEAN")
-	if egressCleanEnv != "" {
-		egressClean, err := strconv.ParseBool(egressCleanEnv)
-		if err != nil {
-			log.Warn("failed to parse EGRESS_CLEAN env value [%s]. Egress cleaning will not be performed: %s", egressCleanEnv, err.Error())
-		}
-		if egressClean {
-			vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
-			vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv6)
-			log.Debug("IPtables rules cleaned on startup")
-		}
+	if sm.config.EgressClean {
+		vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
+		vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv6)
+		log.Debug("IPtables rules cleaned on startup")
 	}
 
 	// Shutdown function that will wait on this signal, unless we call it ourselves
@@ -66,7 +58,7 @@ func (sm *Manager) startTableMode(id string) error {
 	}()
 
 	if sm.config.EnableControlPlane {
-		cpCluster, err = cluster.InitCluster(sm.config, false)
+		cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
 		if err != nil {
 			return fmt.Errorf("cluster initialization error: %w", err)
 		}
@@ -76,7 +68,7 @@ func (sm *Manager) startTableMode(id string) error {
 			return fmt.Errorf("cluster manager initialization error: %w", err)
 		}
 
-		if err := cpCluster.StartVipService(sm.config, clusterManager, nil, nil); err != nil {
+		if err := cpCluster.StartVipService(sm.config, clusterManager, nil); err != nil {
 			log.Error("Control Plane", "err", err)
 			// Trigger the shutdown of this manager instance
 			sm.signalChan <- syscall.SIGINT
@@ -92,7 +84,7 @@ func (sm *Manager) startTableMode(id string) error {
 		// a lock based upon that service is created that they will all leaderElection on
 		if sm.config.EnableServicesElection {
 			log.Info("beginning watching services, leaderelection will happen for every service")
-			err = sm.startServicesWatchForLeaderElection(ctx)
+			err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
 			if err != nil {
 				return err
 			}
@@ -126,7 +118,7 @@ func (sm *Manager) startTableMode(id string) error {
 				RetryPeriod:     time.Duration(sm.config.RetryPeriod) * time.Second,
 				Callbacks: leaderelection.LeaderCallbacks{
 					OnStartedLeading: func(ctx context.Context) {
-						err = sm.servicesWatcher(ctx, sm.syncServices)
+						err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
 						if err != nil {
 							log.Error(err.Error())
 							panic("")
@@ -134,12 +126,10 @@ func (sm *Manager) startTableMode(id string) error {
 					},
 					OnStoppedLeading: func() {
 						// we can do cleanup here
+						sm.mutex.Lock()
+						defer sm.mutex.Unlock()
 						log.Info("leader lost", "id", id)
-						for _, instance := range sm.serviceInstances {
-							for _, cluster := range instance.clusters {
-								cluster.Stop()
-							}
-						}
+						sm.svcProcessor.Stop()
 
 						log.Error("lost leadership, restarting kube-vip")
 						panic("")
@@ -156,7 +146,7 @@ func (sm *Manager) startTableMode(id string) error {
 			})
 		} else {
 			log.Info("beginning watching services without leader election")
-			err = sm.servicesWatcher(ctx, sm.syncServices)
+			err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
 			if err != nil {
 				log.Error("Cannot watch services", "err", err)
 			}
@@ -167,6 +157,8 @@ func (sm *Manager) startTableMode(id string) error {
 }
 
 func (sm *Manager) cleanRoutes() error {
+	sm.mutex.Lock()
+	defer sm.mutex.Unlock()
 	routes, err := vip.ListRoutes(sm.config.RoutingTableID, sm.config.RoutingProtocol)
 	if err != nil {
 		return fmt.Errorf("error getting routes: %w", err)
@@ -177,7 +169,7 @@ func (sm *Manager) cleanRoutes() error {
 		if sm.config.EnableControlPlane {
 			found = (routes[i].Dst.IP.String() == sm.config.Address)
 		} else {
-			found = sm.countRouteReferences(&routes[i]) > 0
+			found = endpoints.CountRouteReferences(&routes[i], &sm.svcProcessor.ServiceInstances) > 0
 		}
 
 		if !found {
@@ -191,20 +183,3 @@ func (sm *Manager) cleanRoutes() error {
 	}
 	return nil
 }
-
-func (sm *Manager) countRouteReferences(route *netlink.Route) int {
-	cnt := 0
-	for _, instance := range sm.serviceInstances {
-		if instance.HasEndpoints {
-			for _, cluster := range instance.clusters {
-				for n := range cluster.Network {
-					r := cluster.Network[n].PrepareRoute()
-					if r.Dst.String() == route.Dst.String() {
-						cnt++
-					}
-				}
-			}
-		}
-	}
-	return cnt
-}

pkg/manager/manager_wireguard.go

@@ -56,7 +56,7 @@ func (sm *Manager) startWireguard(id string) error {
 	// a lock based upon that service is created that they will all leaderElection on
 	if sm.config.EnableServicesElection {
 		log.Info("beginning watching services, leaderelection will happen for every service")
-		err = sm.startServicesWatchForLeaderElection(ctx)
+		err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
 		if err != nil {
 			return err
 		}
@@ -91,7 +91,7 @@ func (sm *Manager) startWireguard(id string) error {
 			RetryPeriod:     time.Duration(sm.config.RetryPeriod) * time.Second,
 			Callbacks: leaderelection.LeaderCallbacks{
 				OnStartedLeading: func(ctx context.Context) {
-					err = sm.servicesWatcher(ctx, sm.syncServices)
+					err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
 					if err != nil {
 						log.Error(err.Error())
 						panic("")
@@ -99,12 +99,10 @@ func (sm *Manager) startWireguard(id string) error {
 				},
 				OnStoppedLeading: func() {
 					// we can do cleanup here
+					sm.mutex.Lock()
+					defer sm.mutex.Unlock()
 					log.Info("leader lost", "id", id)
-					for _, instance := range sm.serviceInstances {
-						for _, cluster := range instance.clusters {
-							cluster.Stop()
-						}
-					}
+					sm.svcProcessor.Stop()
 
 					log.Error("lost leadership, restarting kube-vip")
 					panic("")

pkg/manager/prom.go

@@ -4,5 +4,12 @@ import "github.com/prometheus/client_golang/prometheus"
 
 // PrometheusCollector defines a service watch event counter.
 func (sm *Manager) PrometheusCollector() []prometheus.Collector {
-	return []prometheus.Collector{sm.countServiceWatchEvent, sm.bgpSessionInfoGauge}
+	collectors := []prometheus.Collector{}
+	if sm.svcProcessor != nil {
+		collectors = append(collectors, sm.svcProcessor.CountServiceWatchEvent)
+	}
+	if sm.bgpServer != nil {
+		collectors = append(collectors, sm.bgpServer.BGPSessionInfoGauge)
+	}
+	return collectors
 }

pkg/manager/services.go

@@ -1,497 +0,0 @@
-package manager
-
-import (
-	"context"
-	"fmt"
-	"net"
-	"slices"
-	"strings"
-	"time"
-
-	log "log/slog"
-
-	"github.com/google/go-cmp/cmp"
-	"github.com/vishvananda/netlink"
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/client-go/util/retry"
-
-	"github.com/kube-vip/kube-vip/pkg/upnp"
-	"github.com/kube-vip/kube-vip/pkg/vip"
-)
-
-func (sm *Manager) syncServices(ctx context.Context, svc *v1.Service) error {
-	log.Debug("[STARTING] Service Sync")
-
-	// Iterate through the synchronising services
-	foundInstance := false
-	newServiceAddresses := fetchServiceAddresses(svc)
-	newServiceUID := svc.UID
-
-	ingressIPs := []string{}
-
-	for _, ingress := range svc.Status.LoadBalancer.Ingress {
-		ingressIPs = append(ingressIPs, ingress.IP)
-	}
-
-	shouldBreake := false
-
-	for x := range sm.serviceInstances {
-		if shouldBreake {
-			break
-		}
-		for _, newServiceAddress := range newServiceAddresses {
-			log.Debug("service", "isDHCP", sm.serviceInstances[x].isDHCP, "newServiceAddress", newServiceAddress)
-			if sm.serviceInstances[x].serviceSnapshot.UID == newServiceUID {
-				// If the found instance's DHCP configuration doesn't match the new service, delete it.
-				if (sm.serviceInstances[x].isDHCP && newServiceAddress != "0.0.0.0") ||
-					(!sm.serviceInstances[x].isDHCP && newServiceAddress == "0.0.0.0") ||
-					(!sm.serviceInstances[x].isDHCP && len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, newServiceAddress)) ||
-					(len(svc.Status.LoadBalancer.Ingress) > 0 && !comparePortsAndPortStatuses(svc)) ||
-					(sm.serviceInstances[x].isDHCP && len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, sm.serviceInstances[x].dhcpInterfaceIP)) {
-					if err := sm.deleteService(newServiceUID); err != nil {
-						return err
-					}
-					shouldBreake = true
-					break
-				}
-				foundInstance = true
-			}
-		}
-	}
-
-	// This instance wasn't found, we need to add it to the manager
-	if !foundInstance && len(newServiceAddresses) > 0 {
-		if err := sm.addService(ctx, svc); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func comparePortsAndPortStatuses(svc *v1.Service) bool {
-	portsStatus := svc.Status.LoadBalancer.Ingress[0].Ports
-	if len(portsStatus) != len(svc.Spec.Ports) {
-		return false
-	}
-	for i, portSpec := range svc.Spec.Ports {
-		if portsStatus[i].Port != portSpec.Port || portsStatus[i].Protocol != portSpec.Protocol {
-			return false
-		}
-	}
-	return true
-}
-
-func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
-	startTime := time.Now()
-
-	newService, err := NewInstance(svc, sm.config)
-	if err != nil {
-		return err
-	}
-
-	for x := range newService.vipConfigs {
-		newService.clusters[x].StartLoadBalancerService(newService.vipConfigs[x], sm.bgpServer)
-	}
-
-	sm.upnpMap(ctx, newService)
-
-	if newService.isDHCP && len(newService.vipConfigs) == 1 {
-		go func() {
-			for ip := range newService.dhcpClient.IPChannel() {
-				log.Debug("IP changed", "ip", ip)
-				newService.vipConfigs[0].VIP = ip
-				newService.dhcpInterfaceIP = ip
-				if !sm.config.DisableServiceUpdates {
-					if err := sm.updateStatus(newService); err != nil {
-						log.Warn("updating svc", "err", err)
-					}
-				}
-			}
-			log.Debug("IP update channel closed, stopping")
-		}()
-	}
-
-	sm.serviceInstances = append(sm.serviceInstances, newService)
-
-	if !sm.config.DisableServiceUpdates {
-		log.Debug("service update", "namespace", newService.serviceSnapshot.Namespace, "name", newService.serviceSnapshot.Name)
-		if err := sm.updateStatus(newService); err != nil {
-			log.Error("updating status", "namespace", newService.serviceSnapshot.Namespace, "name", newService.serviceSnapshot.Name, "err", err)
-		}
-	}
-
-	serviceIPs := fetchServiceAddresses(svc)
-	// Check if we need to flush any conntrack connections (due to some dangling conntrack connections)
-	if svc.Annotations[flushContrack] == "true" {
-
-		log.Debug("Flushing conntrack rules", "service", svc.Name)
-		for _, serviceIP := range serviceIPs {
-			err = vip.DeleteExistingSessions(serviceIP, false, svc.Annotations[egressDestinationPorts], svc.Annotations[egressSourcePorts])
-			if err != nil {
-				log.Error("flushing any remaining egress connections", "err", err)
-			}
-			err = vip.DeleteExistingSessions(serviceIP, true, svc.Annotations[egressDestinationPorts], svc.Annotations[egressSourcePorts])
-			if err != nil {
-				log.Error("flushing any remaining ingress connections", "err", err)
-			}
-		}
-	}
-
-	// Check if egress is enabled on the service, if so we'll need to configure some rules
-	if svc.Annotations[egress] == "true" && len(serviceIPs) > 0 {
-		log.Debug("enabling egress", "service", svc.Name)
-		// We will need to modify the iptables rules
-		err = sm.iptablesCheck()
-		if err != nil {
-			log.Error("configuring egress", "service", svc.Name, "err", err)
-		}
-		var podIP string
-		errList := []error{}
-
-		// Should egress be IPv6
-		if svc.Annotations[egressIPv6] == "true" {
-			// Does the service have an active IPv6 endpoint
-			if svc.Annotations[activeEndpointIPv6] != "" {
-				for _, serviceIP := range serviceIPs {
-					if sm.config.EnableEndpointSlices && vip.IsIPv6(serviceIP) {
-
-						podIP = svc.Annotations[activeEndpointIPv6]
-
-						err = sm.configureEgress(serviceIP, podIP, svc.Namespace, svc.Annotations)
-						if err != nil {
-							errList = append(errList, err)
-							log.Error("configuring egress", "service", svc.Name, "err", err)
-						}
-					}
-				}
-			}
-		} else if svc.Annotations[activeEndpoint] != "" { // Not expected to be IPv6, so should be an IPv4 address
-			for _, serviceIP := range serviceIPs {
-				podIPs := svc.Annotations[activeEndpoint]
-				if sm.config.EnableEndpointSlices && vip.IsIPv6(serviceIP) {
-					podIPs = svc.Annotations[activeEndpointIPv6]
-				}
-				err = sm.configureEgress(serviceIP, podIPs, svc.Namespace, svc.Annotations)
-				if err != nil {
-					errList = append(errList, err)
-					log.Error("configuring egress", "service", svc.Name, "err", err)
-				}
-			}
-		}
-		if len(errList) == 0 {
-			var provider epProvider
-			if !sm.config.EnableEndpointSlices {
-				provider = &endpointsProvider{label: "endpoints"}
-			} else {
-				provider = &endpointslicesProvider{label: "endpointslices"}
-			}
-			err = provider.updateServiceAnnotation(svc.Annotations[activeEndpoint], svc.Annotations[activeEndpointIPv6], svc, sm)
-			if err != nil {
-				log.Error("configuring egress", "service", svc.Name, "err", err)
-			}
-		}
-	}
-
-	finishTime := time.Since(startTime)
-	log.Info("[service] synchronised", "in", fmt.Sprintf("%dms", finishTime.Milliseconds()))
-
-	return nil
-}
-
-func (sm *Manager) deleteService(uid types.UID) error {
-	// protect multiple calls
-	sm.mutex.Lock()
-	defer sm.mutex.Unlock()
-
-	var updatedInstances []*Instance
-	var serviceInstance *Instance
-	found := false
-	for x := range sm.serviceInstances {
-		log.Debug("service lookup", "target UID", uid, "found UID ", sm.serviceInstances[x].serviceSnapshot.UID)
-		// Add the running services to the new array
-		if sm.serviceInstances[x].serviceSnapshot.UID != uid {
-			updatedInstances = append(updatedInstances, sm.serviceInstances[x])
-		} else {
-			// Flip the found when we match
-			found = true
-			serviceInstance = sm.serviceInstances[x]
-		}
-	}
-	// If we've been through all services and not found the correct one then error
-	if !found {
-		// TODO: - fix UX
-		// return fmt.Errorf("unable to find/stop service [%s]", uid)
-		return nil
-	}
-
-	// Determine if this this VIP is shared with other loadbalancers
-	shared := false
-	vipSet := make(map[string]interface{})
-	for x := range updatedInstances {
-		for _, vip := range fetchServiceAddresses(updatedInstances[x].serviceSnapshot) { //updatedInstances[x].serviceSnapshot.Spec.LoadBalancerIP {
-			vipSet[vip] = nil
-		}
-	}
-	for _, vip := range fetchServiceAddresses(serviceInstance.serviceSnapshot) {
-		if _, found := vipSet[vip]; found {
-			shared = true
-		}
-	}
-	if !shared {
-		for x := range serviceInstance.clusters {
-			serviceInstance.clusters[x].Stop()
-		}
-		if serviceInstance.isDHCP {
-			serviceInstance.dhcpClient.Stop()
-			macvlan, err := netlink.LinkByName(serviceInstance.dhcpInterface)
-			if err != nil {
-				return fmt.Errorf("error finding VIP Interface: %v", err)
-			}
-
-			err = netlink.LinkDel(macvlan)
-			if err != nil {
-				return fmt.Errorf("error deleting DHCP Link : %v", err)
-			}
-		}
-		// TODO: Implement dual-stack loadbalancer support if BGP is enabled
-		for i := range serviceInstance.vipConfigs {
-			if serviceInstance.vipConfigs[i].EnableBGP {
-				cidrVip := fmt.Sprintf("%s/%s", serviceInstance.vipConfigs[i].VIP, serviceInstance.vipConfigs[i].VIPCIDR)
-				err := sm.bgpServer.DelHost(cidrVip)
-				if err != nil {
-					return fmt.Errorf("[BGP] error deleting BGP host: %v", err)
-				}
-				log.Debug("[BGP] delete", "host", cidrVip)
-			}
-		}
-
-		// We will need to tear down the egress
-		if serviceInstance.serviceSnapshot.Annotations[egress] == "true" {
-			if serviceInstance.serviceSnapshot.Annotations[activeEndpoint] != "" {
-				log.Info("egress re-write enabled", "service", serviceInstance.serviceSnapshot.Name)
-				err := sm.TeardownEgress(serviceInstance.serviceSnapshot.Annotations[activeEndpoint], serviceInstance.serviceSnapshot.Spec.LoadBalancerIP, serviceInstance.serviceSnapshot.Namespace, serviceInstance.serviceSnapshot.Annotations)
-				if err != nil {
-					log.Error("egress teardown", "err", err)
-				}
-			}
-		}
-	}
-
-	// Update the service array
-	sm.serviceInstances = updatedInstances
-
-	log.Info("Removed instance from manager", "uid", uid, "remaining advertised services", len(sm.serviceInstances))
-
-	return nil
-}
-
-// Set up UPNP forwards for a service
-// We first try to use the more modern Pinhole API introduced in UPNPv2 and fall back to UPNPv2 Port Forwarding if no forward was successful
-func (sm *Manager) upnpMap(ctx context.Context, s *Instance) {
-	if !isUPNPEnabled(s.serviceSnapshot) {
-		// Skip services missing the annotation
-		return
-	}
-	if !sm.upnp {
-		log.Warn("[UPNP] Found kube-vip.io/forwardUPNP on service while UPNP forwarding is disabled in the kube-vip config. Not forwarding", "service", s.serviceSnapshot.Name)
-	}
-	// If upnp is enabled then update the gateway/router with the address
-	// TODO - check if this implementation for dualstack is correct
-
-	gateways := upnp.GetGatewayClients(ctx)
-
-	// Reset Gateway IPs to remove stale addresses
-	s.upnpGatewayIPs = make([]string, 0)
-
-	for _, vip := range fetchServiceAddresses(s.serviceSnapshot) {
-		for _, port := range s.serviceSnapshot.Spec.Ports {
-			for _, gw := range gateways {
-				log.Info("[UPNP] Adding map", "vip", vip, "port", port.Port, "service", s.serviceSnapshot.Name, "gateway", gw.WANIPv6FirewallControlClient.Location)
-
-				forwardSucessful := false
-				if gw.WANIPv6FirewallControlClient != nil {
-					pinholeID, pinholeErr := gw.WANIPv6FirewallControlClient.AddPinholeCtx(ctx, "0.0.0.0", uint16(port.Port), vip, uint16(port.Port), upnp.MapProtocolToIANA(string(port.Protocol)), 3600) //nolint  TODO
-					if pinholeErr == nil {
-						forwardSucessful = true
-						log.Info("[UPNP] Service should be accessible externally", "port", port.Port, "pinhold ID", pinholeID)
-					} else {
-						//TODO: Cleanup
-						log.Error("[UPNP] Unable to map port to gateway using Pinhole API", "err", pinholeErr.Error())
-					}
-				}
-				// Fallback to PortForward
-				if !forwardSucessful {
-					portMappingErr := gw.ConnectionClient.AddPortMapping("0.0.0.0", uint16(port.Port), strings.ToUpper(string(port.Protocol)), uint16(port.Port), vip, true, s.serviceSnapshot.Name, 3600) //nolint  TODO
-					if portMappingErr == nil {
-						log.Info("[UPNP] Service should be accessible externally", "port", port.Port)
-						forwardSucessful = true
-					} else {
-						//TODO: Cleanup
-						log.Error("[UPNP] Unable to map port to gateway using PortForward API", "err", portMappingErr.Error())
-					}
-				}
-
-				if forwardSucessful {
-					ip, err := gw.ConnectionClient.GetExternalIPAddress()
-					if err == nil {
-						s.upnpGatewayIPs = append(s.upnpGatewayIPs, ip)
-					}
-				}
-			}
-		}
-	}
-
-	// Remove duplicate IPs
-	slices.Sort(s.upnpGatewayIPs)
-	s.upnpGatewayIPs = slices.Compact(s.upnpGatewayIPs)
-}
-
-func (sm *Manager) updateStatus(i *Instance) error {
-	// let's retry status update every 10ms for 30s
-	retryConfig := wait.Backoff{
-		Steps:    3000,
-		Duration: 10 * time.Millisecond,
-		Factor:   0,
-		Jitter:   0.1,
-	}
-	// will retry for every error encountered, TODO: should a list of errors that will trigger retry be specified?
-	err := retry.OnError(retryConfig, func(error) bool { return true }, func() error {
-		// Retrieve the latest version of Deployment before attempting update
-		// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
-		currentService, err := sm.clientSet.CoreV1().Services(i.serviceSnapshot.Namespace).Get(context.TODO(), i.serviceSnapshot.Name, metav1.GetOptions{})
-		if err != nil {
-			return err
-		}
-
-		currentServiceCopy := currentService.DeepCopy()
-		if currentServiceCopy.Annotations == nil {
-			currentServiceCopy.Annotations = make(map[string]string)
-		}
-
-		// If we're using ARP then we can only broadcast the VIP from one place, add an annotation to the service
-		if sm.config.EnableARP {
-			// Add the current host
-			currentServiceCopy.Annotations[vipHost] = sm.config.NodeName
-		}
-		if i.dhcpInterfaceHwaddr != "" || i.dhcpInterfaceIP != "" {
-			currentServiceCopy.Annotations[hwAddrKey] = i.dhcpInterfaceHwaddr
-			currentServiceCopy.Annotations[requestedIP] = i.dhcpInterfaceIP
-		}
-
-		if currentService.Annotations["development.kube-vip.io/synthetic-api-server-error-on-update"] == "true" {
-			log.Error("(Synthetic error ) updating Spec", "service", i.serviceSnapshot.Name, "err", err)
-			return fmt.Errorf("(Synthetic) simulating api server errors")
-		}
-
-		if !cmp.Equal(currentService, currentServiceCopy) {
-			currentService, err = sm.clientSet.CoreV1().Services(currentServiceCopy.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
-			if err != nil {
-				log.Error("updating Spec", "service", i.serviceSnapshot.Name, "err", err)
-				return err
-			}
-		}
-
-		ports := make([]v1.PortStatus, 0, len(i.serviceSnapshot.Spec.Ports))
-		for _, port := range i.serviceSnapshot.Spec.Ports {
-			ports = append(ports, v1.PortStatus{
-				Port:     port.Port,
-				Protocol: port.Protocol,
-			})
-		}
-
-		ingresses := []v1.LoadBalancerIngress{}
-
-		for _, c := range i.vipConfigs {
-			if !vip.IsIP(c.VIP) {
-				ips, err := vip.LookupHost(c.VIP, sm.config.DNSMode)
-				if err != nil {
-					return err
-				}
-				for _, ip := range ips {
-					i := v1.LoadBalancerIngress{
-						IP:    ip,
-						Ports: ports,
-					}
-					ingresses = append(ingresses, i)
-				}
-			} else {
-				i := v1.LoadBalancerIngress{
-					IP:    c.VIP,
-					Ports: ports,
-				}
-				ingresses = append(ingresses, i)
-			}
-			if isUPNPEnabled(currentService) {
-				for _, ip := range i.upnpGatewayIPs {
-					i := v1.LoadBalancerIngress{
-						IP:    ip,
-						Ports: ports,
-					}
-					ingresses = append(ingresses, i)
-				}
-			}
-		}
-		if !cmp.Equal(currentService.Status.LoadBalancer.Ingress, ingresses) {
-			currentService.Status.LoadBalancer.Ingress = ingresses
-			_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).UpdateStatus(context.TODO(), currentService, metav1.UpdateOptions{})
-			if err != nil {
-				log.Error("updating Service", "namespace", i.serviceSnapshot.Namespace, "name", i.serviceSnapshot.Name, "err", err)
-				return err
-			}
-		}
-		return nil
-	})
-
-	return err
-}
-
-// fetchServiceAddresses tries to get the addresses from annotations
-// kube-vip.io/loadbalancerIPs, then from spec.loadbalancerIP
-func fetchServiceAddresses(s *v1.Service) []string {
-	annotationAvailable := false
-	if s.Annotations != nil {
-		if v, annotationAvailable := s.Annotations[loadbalancerIPAnnotation]; annotationAvailable {
-			ips := strings.Split(v, ",")
-			var trimmedIPs []string
-			for _, ip := range ips {
-				trimmedIPs = append(trimmedIPs, strings.TrimSpace(ip))
-			}
-			return trimmedIPs
-		}
-	}
-
-	lbStatusAddresses := []string{}
-	if !annotationAvailable {
-		if len(s.Status.LoadBalancer.Ingress) > 0 {
-			for _, ingress := range s.Status.LoadBalancer.Ingress {
-				lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
-			}
-		}
-	}
-
-	lbIP := net.ParseIP(s.Spec.LoadBalancerIP)
-	isLbIPv4 := vip.IsIPv4(s.Spec.LoadBalancerIP)
-
-	if len(lbStatusAddresses) > 0 {
-		for _, a := range lbStatusAddresses {
-			if lbStatusIP := net.ParseIP(a); lbStatusIP != nil && lbIP != nil && vip.IsIPv4(a) == isLbIPv4 && !lbIP.Equal(lbStatusIP) {
-				return []string{s.Spec.LoadBalancerIP}
-			}
-		}
-		return lbStatusAddresses
-	}
-
-	if s.Spec.LoadBalancerIP != "" {
-		return []string{s.Spec.LoadBalancerIP}
-	}
-
-	return []string{}
-}
-
-func isUPNPEnabled(s *v1.Service) bool {
-	return metav1.HasAnnotation(s.ObjectMeta, upnpEnabled) && s.Annotations[upnpEnabled] == "true"
-}

pkg/manager/watch_annotations.go

@@ -57,7 +57,8 @@ func (sm *Manager) annotationsWatcher() error {
 	// they're as needed
 	log.Warn(err.Error())
 
-	rw, err := watchtools.NewRetryWatcher(node.ResourceVersion, &cache.ListWatch{
+	// TODO, will need refactoring as part of rikatz work
+	rw, err := watchtools.NewRetryWatcherWithContext(context.TODO(), node.ResourceVersion, &cache.ListWatch{
 		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
 			return sm.rwClientSet.CoreV1().Nodes().Watch(context.Background(), listOptions)
 		},

pkg/manager/watch_endpoints.go

@@ -1,501 +0,0 @@
-package manager
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"net"
-	"strings"
-	"syscall"
-
-	log "log/slog"
-
-	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	v1 "k8s.io/api/core/v1"
-	discoveryv1 "k8s.io/api/discovery/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/fields"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/watch"
-	"k8s.io/client-go/tools/cache"
-	watchtools "k8s.io/client-go/tools/watch"
-	"k8s.io/client-go/util/retry"
-)
-
-type epProvider interface {
-	createRetryWatcher(context.Context, *Manager,
-		*v1.Service) (*watchtools.RetryWatcher, error)
-	getAllEndpoints() ([]string, error)
-	getLocalEndpoints(string, *kubevip.Config) ([]string, error)
-	getLabel() string
-	updateServiceAnnotation(string, string, *v1.Service, *Manager) error
-	loadObject(runtime.Object, context.CancelFunc) error
-	getProtocol() string
-}
-
-type endpointsProvider struct {
-	label     string
-	endpoints *v1.Endpoints
-}
-
-func (ep *endpointsProvider) createRetryWatcher(ctx context.Context, sm *Manager,
-	service *v1.Service) (*watchtools.RetryWatcher, error) {
-	opts := metav1.ListOptions{
-		FieldSelector: fields.OneTermEqualSelector("metadata.name", service.Name).String(),
-	}
-
-	rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
-		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
-			return sm.rwClientSet.CoreV1().Endpoints(service.Namespace).Watch(ctx, opts)
-		},
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("error creating endpoint watcher: %s", err.Error())
-	}
-
-	return rw, nil
-}
-
-func (ep *endpointsProvider) loadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
-	eps, ok := endpoints.(*v1.Endpoints)
-	if !ok {
-		cancel()
-		return fmt.Errorf("[%s] unable to parse Kubernetes services from API watcher", ep.getLabel())
-	}
-	ep.endpoints = eps
-	return nil
-}
-
-func (ep *endpointsProvider) getAllEndpoints() ([]string, error) {
-	result := []string{}
-	for subset := range ep.endpoints.Subsets {
-		for address := range ep.endpoints.Subsets[subset].Addresses {
-			addr := strings.Split(ep.endpoints.Subsets[subset].Addresses[address].IP, "/")
-			result = append(result, addr[0])
-		}
-	}
-
-	return result, nil
-}
-
-func (ep *endpointsProvider) getLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
-	var localEndpoints []string
-
-	for _, subset := range ep.endpoints.Subsets {
-		for _, address := range subset.Addresses {
-			log.Debug("processing endpoint", "label", ep.label, "ip", address.IP)
-
-			// 1. Compare the Nodename
-			if address.NodeName != nil && id == *address.NodeName {
-				log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname, "nodename", *address.NodeName)
-				localEndpoints = append(localEndpoints, address.IP)
-				continue
-			}
-			// 2. Compare the Hostname (only useful if address.NodeName is not available)
-			if id == address.Hostname {
-				log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname)
-				localEndpoints = append(localEndpoints, address.IP)
-				continue
-			}
-		}
-	}
-	return localEndpoints, nil
-}
-
-func (ep *endpointsProvider) updateServiceAnnotation(endpoint string, _ string, service *v1.Service, sm *Manager) error {
-	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
-		// Retrieve the latest version of Deployment before attempting update
-		// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
-		currentService, err := sm.clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
-		if err != nil {
-			return err
-		}
-
-		currentServiceCopy := currentService.DeepCopy()
-		if currentServiceCopy.Annotations == nil {
-			currentServiceCopy.Annotations = make(map[string]string)
-		}
-
-		currentServiceCopy.Annotations[activeEndpoint] = endpoint
-
-		_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
-		if err != nil {
-			log.Error("error updating Service Spec", "label", ep.getLabel(), "name", currentServiceCopy.Name, "err", err)
-			return err
-		}
-		return nil
-	})
-
-	if retryErr != nil {
-		log.Error("failed to set Services", "label", ep.getLabel(), "err", retryErr)
-		return retryErr
-	}
-	return nil
-}
-
-func (ep *endpointsProvider) getLabel() string {
-	return ep.label
-}
-
-func (ep *endpointsProvider) getProtocol() string {
-	return ""
-}
-
-func (sm *Manager) watchEndpoint(ctx context.Context, id string, service *v1.Service, provider epProvider) error {
-	log.Info("watching", "provider", provider.getLabel(), "service_name", service.Name, "namespace", service.Namespace)
-	// Use a restartable watcher, as this should help in the event of etcd or timeout issues
-	leaderContext, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	var leaderElectionActive bool
-
-	rw, err := provider.createRetryWatcher(leaderContext, sm, service)
-	if err != nil {
-		cancel()
-		return fmt.Errorf("[%s] error watching endpoints: %w", provider.getLabel(), err)
-	}
-
-	exitFunction := make(chan struct{})
-	go func() {
-		select {
-		case <-ctx.Done():
-			log.Debug("context cancelled", "provider", provider.getLabel())
-			// Stop the retry watcher
-			rw.Stop()
-			// Cancel the context, which will in turn cancel the leadership
-			cancel()
-			return
-		case <-sm.shutdownChan:
-			log.Debug("shutdown called", "provider", provider.getLabel())
-			// Stop the retry watcher
-			rw.Stop()
-			// Cancel the context, which will in turn cancel the leadership
-			cancel()
-			return
-		case <-exitFunction:
-			log.Debug("function ending", "provider", provider.getLabel())
-			// Stop the retry watcher
-			rw.Stop()
-			// Cancel the context, which will in turn cancel the leadership
-			cancel()
-			return
-		}
-	}()
-
-	ch := rw.ResultChan()
-
-	var lastKnownGoodEndpoint string
-	for event := range ch {
-		activeEndpointAnnotation := activeEndpoint
-		// We need to inspect the event and get ResourceVersion out of it
-		switch event.Type {
-
-		case watch.Added, watch.Modified:
-
-			if err = provider.loadObject(event.Object, cancel); err != nil {
-				return fmt.Errorf("[%s] error loading k8s object: %w", provider.getLabel(), err)
-			}
-
-			if sm.config.EnableEndpointSlices && provider.getProtocol() == string(discoveryv1.AddressTypeIPv6) {
-				activeEndpointAnnotation = activeEndpointIPv6
-			}
-
-			// Build endpoints
-			var endpoints []string
-			if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
-				service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
-				if endpoints, err = provider.getAllEndpoints(); err != nil {
-					return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
-				}
-			} else {
-				if endpoints, err = provider.getLocalEndpoints(id, sm.config); err != nil {
-					return fmt.Errorf("[%s] error getting local endpoints: %w", provider.getLabel(), err)
-				}
-			}
-
-			if sm.config.EnableRoutingTable {
-				instance := sm.findServiceInstance(service)
-				if instance == nil {
-					log.Error("error finding instance", "service", service.UID, "provider", provider.getLabel(), "err", err)
-				} else {
-					instance.HasEndpoints = len(endpoints) > 0
-				}
-			}
-
-			// Find out if we have any local endpoints
-			// if out endpoint is empty then populate it
-			// if not, go through the endpoints and see if ours still exists
-			// If we have a local endpoint then begin the leader Election, unless it's already running
-			//
-
-			// Check that we have local endpoints
-			if len(endpoints) != 0 {
-				// Ignore IPv4
-				if service.Annotations[egressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
-					continue
-				}
-
-				// if we haven't populated one, then do so
-				if lastKnownGoodEndpoint != "" {
-
-					// check out previous endpoint exists
-					stillExists := false
-
-					for x := range endpoints {
-						if endpoints[x] == lastKnownGoodEndpoint {
-							stillExists = true
-						}
-					}
-					// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
-					if !stillExists {
-						if sm.config.EnableRoutingTable {
-							if err := sm.TeardownEgress(lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
-								service.Namespace, service.Annotations); err != nil {
-								log.Warn("removing redundant egress rules", "err", err)
-							}
-						}
-						if leaderElectionActive && (sm.config.EnableServicesElection || sm.config.EnableLeaderElection) {
-							log.Warn(" existing endpoint has been removed, restarting leaderElection", "provider", provider.getLabel(), "endpoint", lastKnownGoodEndpoint)
-							// Stop the existing leaderElection
-							cancel()
-							// disable last leaderElection flag
-							leaderElectionActive = false
-						}
-						// Set our active endpoint to an existing one
-						lastKnownGoodEndpoint = endpoints[0]
-					}
-				} else {
-					lastKnownGoodEndpoint = endpoints[0]
-				}
-
-				if !leaderElectionActive && sm.config.EnableServicesElection {
-					go func() {
-						leaderContext, cancel = context.WithCancel(ctx)
-
-						// This is a blocking function, that will restart (in the event of failure)
-						for {
-							// if the context isn't cancelled restart
-							if leaderContext.Err() != context.Canceled {
-								leaderElectionActive = true
-								err := sm.StartServicesLeaderElection(leaderContext, service)
-								if err != nil {
-									log.Error(err.Error())
-								}
-								leaderElectionActive = false
-							} else {
-								leaderElectionActive = false
-								break
-							}
-						}
-					}()
-				}
-
-				isRouteConfigured, err := isRouteConfigured(service.UID)
-				if err != nil {
-					return fmt.Errorf("[%s] error while checking if route is configured: %w", provider.getLabel(), err)
-				}
-				// There are local endpoints available on the node
-				if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection && !isRouteConfigured {
-					// If routing table mode is enabled - routes should be added per node
-					if sm.config.EnableRoutingTable {
-						instance, err := sm.findServiceInstanceWithTimeout(ctx, service)
-						if err != nil {
-							log.Error("error finding instance", "service", service.UID, "provider", provider.getLabel(), "err", err)
-						}
-						if instance != nil {
-							for _, cluster := range instance.clusters {
-								for i := range cluster.Network {
-									err := cluster.Network[i].AddRoute(false)
-									if err != nil {
-										if errors.Is(err, syscall.EEXIST) {
-											// If route exists try to update it if necessary
-											isUpdated, err := cluster.Network[i].UpdateRoutes()
-											if err != nil {
-												return fmt.Errorf("[%s] error updating existing routes: %w", provider.getLabel(), err)
-											}
-											if isUpdated {
-												log.Debug("updated route", "provider", provider.getLabel(), "ip", cluster.Network[i].IP())
-											}
-										} else {
-											// If other error occurs, return error
-											return fmt.Errorf("[%s] error adding route: %s", provider.getLabel(), err.Error())
-										}
-									} else {
-										log.Info("added route", "provider",
-											provider.getLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", sm.config.RoutingTableID)
-										configuredLocalRoutes.Store(string(service.UID), true)
-										leaderElectionActive = true
-									}
-								}
-							}
-						}
-					}
-
-					// If BGP mode is enabled - hosts should be added per node
-					if sm.config.EnableBGP {
-						instance, err := sm.findServiceInstanceWithTimeout(ctx, service)
-						if err != nil {
-							log.Error("error finding instance", "service", service.UID, "provider", provider.getLabel(), "err", err)
-						}
-						if instance != nil {
-							for _, cluster := range instance.clusters {
-								for i := range cluster.Network {
-									address := fmt.Sprintf("%s/%s", cluster.Network[i].IP(), sm.config.VIPCIDR)
-									log.Debug("attempting to advertise BGP service", "provider", provider.getLabel(), "ip", address)
-									err := sm.bgpServer.AddHost(address)
-									if err != nil {
-										log.Error("error adding BGP host", "provider", provider.getLabel(), "err", err)
-									} else {
-										log.Info("added BGP host", "provider",
-											provider.getLabel(), "ip", address, "service name", service.Name, "namespace", service.Namespace)
-										configuredLocalRoutes.Store(string(service.UID), true)
-										leaderElectionActive = true
-									}
-								}
-							}
-						}
-					}
-				}
-			} else {
-				// There are no local endpoints
-				if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection {
-					// If routing table mode is enabled - routes should be deleted
-					if sm.config.EnableRoutingTable {
-						if errs := sm.clearRoutes(service); len(errs) == 0 {
-							configuredLocalRoutes.Store(string(service.UID), false)
-						} else {
-							for _, err := range errs {
-								log.Error("error while clearing routes", "err", err)
-							}
-						}
-					}
-
-					// If BGP mode is enabled - routes should be deleted
-					if sm.config.EnableBGP {
-						if instance := sm.findServiceInstance(service); instance != nil {
-							for _, cluster := range instance.clusters {
-								for i := range cluster.Network {
-									address := fmt.Sprintf("%s/%s", cluster.Network[i].IP(), sm.config.VIPCIDR)
-									err := sm.bgpServer.DelHost(address)
-									if err != nil {
-										log.Error("deleting BGP host", "provider", provider.getLabel(), "ip", address, "err", err)
-									} else {
-										log.Info("deleted BGP host", "provider",
-											provider.getLabel(), "ip", address, "service name", service.Name, "namespace", service.Namespace)
-										configuredLocalRoutes.Store(string(service.UID), false)
-										leaderElectionActive = false
-									}
-								}
-							}
-						}
-					}
-				}
-
-				// If there are no local endpoints, and we had one then remove it and stop the leaderElection
-				if lastKnownGoodEndpoint != "" {
-					log.Warn("existing endpoint has been removed, no remaining endpoints for leaderElection", "provider", provider.getLabel(), "endpoint", lastKnownGoodEndpoint)
-					if err := sm.TeardownEgress(lastKnownGoodEndpoint, service.Spec.LoadBalancerIP, service.Namespace, service.Annotations); err != nil {
-						log.Error("error removing redundant egress rules", "err", err)
-					}
-
-					lastKnownGoodEndpoint = "" // reset endpoint
-					if sm.config.EnableServicesElection || sm.config.EnableLeaderElection {
-						cancel() // stop services watcher
-					}
-					leaderElectionActive = false
-				}
-			}
-			// Set the service accordingly
-			if service.Annotations[egress] == "true" {
-				service.Annotations[activeEndpointAnnotation] = lastKnownGoodEndpoint
-			}
-
-			log.Debug("watcher", "provider",
-				provider.getLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", lastKnownGoodEndpoint, "active leader election", leaderElectionActive)
-
-		case watch.Deleted:
-			// When no-leader-elecition mode
-			if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection {
-				// find all existing local endpoints
-				var endpoints []string
-				if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
-					service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
-					if endpoints, err = provider.getAllEndpoints(); err != nil {
-						return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
-					}
-				} else {
-					if endpoints, err = provider.getLocalEndpoints(id, sm.config); err != nil {
-						return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
-					}
-				}
-
-				// If there were local endpoints deleted
-				if len(endpoints) > 0 {
-					// Delete all routes in routing table mode
-					if sm.config.EnableRoutingTable {
-						sm.clearRoutes(service)
-					}
-
-					// Delete all hosts in BGP mode
-					if sm.config.EnableBGP {
-						sm.clearBGPHosts(service)
-					}
-				}
-			}
-
-			// Close the goroutine that will end the retry watcher, then exit the endpoint watcher function
-			close(exitFunction)
-			log.Info("stopping watching", "provider", provider.getLabel(), "service name", service.Name, "namespace", service.Namespace)
-
-			return nil
-		case watch.Error:
-			errObject := apierrors.FromObject(event.Object)
-			statusErr, _ := errObject.(*apierrors.StatusError)
-			log.Error("watch error", "provider", provider.getLabel(), "err", statusErr)
-		}
-	}
-	close(exitFunction)
-	log.Info("stopping watching", "provider", provider.getLabel(), "service name", service.Name, "namespace", service.Namespace)
-	return nil //nolint:govet
-}
-
-func (sm *Manager) clearRoutes(service *v1.Service) []error {
-	errs := []error{}
-	if instance := sm.findServiceInstance(service); instance != nil {
-		for _, cluster := range instance.clusters {
-			for i := range cluster.Network {
-				route := cluster.Network[i].PrepareRoute()
-				// check if route we are about to delete is not referenced by more than one service
-				if sm.countRouteReferences(route) <= 1 {
-					err := cluster.Network[i].DeleteRoute()
-					if err != nil && !errors.Is(err, syscall.ESRCH) {
-						log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
-						errs = append(errs, err)
-					}
-					log.Debug("deleted route", "ip",
-						cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", sm.config.RoutingTableID)
-				}
-			}
-		}
-	}
-	return errs
-}
-
-func (sm *Manager) clearBGPHosts(service *v1.Service) {
-	if instance := sm.findServiceInstance(service); instance != nil {
-		for _, cluster := range instance.clusters {
-			for i := range cluster.Network {
-				address := fmt.Sprintf("%s/%s", cluster.Network[i].IP(), sm.config.VIPCIDR)
-				err := sm.bgpServer.DelHost(address)
-				if err != nil {
-					log.Error("[endpoint] error deleting BGP host", "err", err)
-				} else {
-					log.Debug("[endpoint] deleted BGP host", "ip",
-						address, "service name", service.Name, "namespace", service.Namespace)
-				}
-			}
-		}
-	}
-}

pkg/manager/watch_services.go

@@ -1,411 +0,0 @@
-package manager
-
-import (
-	"context"
-	"fmt"
-	"reflect"
-	"sync"
-
-	log "log/slog"
-
-	"github.com/davecgh/go-spew/spew"
-	"github.com/kube-vip/kube-vip/pkg/vip"
-	"github.com/prometheus/client_golang/prometheus"
-	v1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/watch"
-	"k8s.io/client-go/tools/cache"
-	watchtools "k8s.io/client-go/tools/watch"
-)
-
-// TODO: Fix the naming of these contexts
-
-// activeServiceLoadBalancer keeps track of services that already have a leaderElection in place
-var activeServiceLoadBalancer map[string]context.Context
-
-// activeServiceLoadBalancer keeps track of services that already have a leaderElection in place
-var activeServiceLoadBalancerCancel map[string]func()
-
-// activeService keeps track of services that already have a leaderElection in place
-var activeService map[string]bool
-
-// watchedService keeps track of services that are already being watched
-var watchedService map[string]bool
-
-// watchedService keeps track of routes that has been configured on the node
-var configuredLocalRoutes sync.Map
-
-func init() {
-	// Set up the caches for monitoring existing active or watched services
-	activeServiceLoadBalancerCancel = make(map[string]func())
-	activeServiceLoadBalancer = make(map[string]context.Context)
-	activeService = make(map[string]bool)
-	watchedService = make(map[string]bool)
-}
-
-// This function handles the watching of a services endpoints and updates a load balancers endpoint configurations accordingly
-func (sm *Manager) servicesWatcher(ctx context.Context, serviceFunc func(context.Context, *v1.Service) error) error {
-	// first start port mirroring if enabled
-	if err := sm.startTrafficMirroringIfEnabled(); err != nil {
-		return err
-	}
-	defer func() {
-		// clean up traffic mirror related config
-		err := sm.stopTrafficMirroringIfEnabled()
-		if err != nil {
-			log.Error("Stopping traffic mirroring", "err", err)
-		}
-	}()
-
-	if sm.config.ServiceNamespace == "" {
-		// v1.NamespaceAll is actually "", but we'll stay with the const in case things change upstream
-		sm.config.ServiceNamespace = v1.NamespaceAll
-		log.Info("(svcs) starting services watcher for all namespaces")
-	} else {
-		log.Info("(svcs) starting services watcher", "namespace", sm.config.ServiceNamespace)
-	}
-
-	// Use a restartable watcher, as this should help in the event of etcd or timeout issues
-	rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
-		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
-			return sm.rwClientSet.CoreV1().Services(sm.config.ServiceNamespace).Watch(ctx, metav1.ListOptions{})
-		},
-	})
-	if err != nil {
-		return fmt.Errorf("error creating services watcher: %s", err.Error())
-	}
-	exitFunction := make(chan struct{})
-	go func() {
-		select {
-		case <-sm.shutdownChan:
-			log.Debug("(svcs) shutdown called")
-			// Stop the retry watcher
-			rw.Stop()
-			return
-		case <-exitFunction:
-			log.Debug("(svcs) function ending")
-			// Stop the retry watcher
-			rw.Stop()
-			return
-		}
-	}()
-	ch := rw.ResultChan()
-
-	// Used for tracking an active endpoint / pod
-	for event := range ch {
-		sm.countServiceWatchEvent.With(prometheus.Labels{"type": string(event.Type)}).Add(1)
-
-		// We need to inspect the event and get ResourceVersion out of it
-		switch event.Type {
-		case watch.Added, watch.Modified:
-			// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
-			svc, ok := event.Object.(*v1.Service)
-			if !ok {
-				return fmt.Errorf("unable to parse Kubernetes services from API watcher")
-			}
-
-			// We only care about LoadBalancer services
-			if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
-				break
-			}
-
-			// Check if we ignore this service
-			if svc.Annotations["kube-vip.io/ignore"] == "true" {
-				log.Info("ignore annotation for kube-vip", "service name", svc.Name)
-				break
-			}
-
-			// Select loadbalancer class filtering function
-			lbClassFilterFunc := sm.lbClassFilter
-			if sm.config.LoadBalancerClassLegacyHandling {
-				lbClassFilterFunc = sm.lbClassFilterLegacy
-			}
-
-			// Check the loadBalancer class
-			if lbClassFilterFunc(svc) {
-				break
-			}
-
-			svcAddresses := fetchServiceAddresses(svc)
-
-			// We only care about LoadBalancer services that have been allocated an address
-			if len(svcAddresses) <= 0 {
-				break
-			}
-
-			// The modified event should only be triggered if the service has been modified (i.e. moved somewhere else)
-			if event.Type == watch.Modified {
-				for _, addr := range svcAddresses {
-					// log.Debugf("(svcs) Retreiving local addresses, to ensure that this modified address doesn't exist: %s", addr)
-					f, err := vip.GarbageCollect(sm.config.Interface, addr)
-					if err != nil {
-						log.Error("(svcs) cleaning existing address error", "err", err)
-					}
-					if f {
-						log.Warn("(svcs) already found existing config", "address", addr, "adapter", sm.config.Interface)
-					}
-				}
-				// This service has been modified, but it was also active..
-				if activeService[string(svc.UID)] {
-
-					i := sm.findServiceInstance(svc)
-					if i != nil {
-						originalService := fetchServiceAddresses(i.serviceSnapshot)
-						newService := fetchServiceAddresses(svc)
-						if !reflect.DeepEqual(originalService, newService) {
-
-							// Calls the cancel function of the context
-							if activeServiceLoadBalancerCancel[string(svc.UID)] != nil {
-								log.Warn("(svcs) The load balancer has changed, cancelling original load balancer")
-								activeServiceLoadBalancerCancel[string(svc.UID)]()
-								<-activeServiceLoadBalancer[string(svc.UID)].Done()
-								log.Warn("(svcs) waiting for load balancer to finish")
-							}
-							err = sm.deleteService(svc.UID)
-							if err != nil {
-								log.Error("(svc) unable to remove", "service", svc.UID)
-							}
-							activeService[string(svc.UID)] = false
-							watchedService[string(svc.UID)] = false
-							delete(activeServiceLoadBalancer, string(svc.UID))
-							configuredLocalRoutes.Store(string(svc.UID), false)
-						}
-						// in theory this should never fail
-
-					}
-
-				}
-			}
-
-			// Architecture walkthrough: (Had to do this as this code path is making my head hurt)
-
-			// Is the service active (bool), if not then process this new service
-			// Does this service use an election per service?
-			//
-
-			if !activeService[string(svc.UID)] {
-				log.Debug("(svcs) has been added/modified with addresses", "service name", svc.Name, "ip", fetchServiceAddresses(svc))
-
-				activeServiceLoadBalancer[string(svc.UID)], activeServiceLoadBalancerCancel[string(svc.UID)] = context.WithCancel(ctx)
-
-				if sm.config.EnableServicesElection || // Service Election
-					((sm.config.EnableRoutingTable || sm.config.EnableBGP) && // Routing table mode or BGP
-						(!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection)) { // No leaderelection or services election
-
-					// If this load balancer Traffic Policy is "local"
-					if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
-
-						// Start an endpoint watcher if we're not watching it already
-						if !watchedService[string(svc.UID)] {
-							// background the endpoint watcher
-							go func() {
-								if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
-									// Add Endpoint or EndpointSlices watcher
-									var provider epProvider
-									if !sm.config.EnableEndpointSlices {
-										provider = &endpointsProvider{label: "endpoints"}
-									} else {
-										provider = &endpointslicesProvider{label: "endpointslices"}
-									}
-									if err = sm.watchEndpoint(activeServiceLoadBalancer[string(svc.UID)], sm.config.NodeName, svc, provider); err != nil {
-										log.Error(err.Error())
-									}
-								}
-							}()
-
-							if (sm.config.EnableRoutingTable || sm.config.EnableBGP) && (!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection) {
-								go func() {
-									err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc)
-									if err != nil {
-										log.Error(err.Error())
-									}
-								}()
-							}
-							// We're now watching this service
-							watchedService[string(svc.UID)] = true
-						}
-					} else if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && (!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection) {
-						go func() {
-							if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
-								// Add Endpoint watcher
-								var provider epProvider
-								if !sm.config.EnableEndpointSlices {
-									provider = &endpointsProvider{label: "endpoints"}
-								} else {
-									provider = &endpointslicesProvider{label: "endpointslices"}
-								}
-								if err = sm.watchEndpoint(activeServiceLoadBalancer[string(svc.UID)], sm.config.NodeName, svc, provider); err != nil {
-									log.Error(err.Error())
-								}
-							}
-						}()
-
-						go func() {
-							err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc)
-							if err != nil {
-								log.Error(err.Error())
-							}
-						}()
-					} else {
-
-						go func() {
-							for {
-								select {
-								case <-activeServiceLoadBalancer[string(svc.UID)].Done():
-									log.Warn("(svcs) restartable service watcher ending", "uid", svc.UID)
-									return
-								default:
-									log.Info("(svcs) restartable service watcher starting", "uid", svc.UID)
-									err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc)
-
-									if err != nil {
-										log.Error(err.Error())
-									}
-								}
-							}
-
-						}()
-					}
-				} else {
-					// Increment the waitGroup before the service Func is called (Done is completed in there)
-					err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc)
-					if err != nil {
-						log.Error(err.Error())
-					}
-				}
-				activeService[string(svc.UID)] = true
-			}
-		case watch.Deleted:
-			svc, ok := event.Object.(*v1.Service)
-			if !ok {
-				return fmt.Errorf("unable to parse Kubernetes services from API watcher")
-			}
-			if activeService[string(svc.UID)] {
-
-				// We only care about LoadBalancer services
-				if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
-					break
-				}
-
-				// We can ignore this service
-				if svc.Annotations["kube-vip.io/ignore"] == "true" {
-					log.Info("(svcs)ignore annotation for kube-vip", "service name", svc.Name)
-					break
-				}
-
-				isRouteConfigured, err := isRouteConfigured(svc.UID)
-				if err != nil {
-					return fmt.Errorf("error while checkig if route is configured: %w", err)
-				}
-				// If no leader election is enabled, delete routes here
-				if !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
-					sm.config.EnableRoutingTable && isRouteConfigured {
-					if errs := sm.clearRoutes(svc); len(errs) == 0 {
-						configuredLocalRoutes.Store(string(svc.UID), false)
-					}
-				}
-
-				// If this is an active service then and additional leaderElection will handle stopping
-				err = sm.deleteService(svc.UID)
-				if err != nil {
-					log.Error(err.Error())
-				}
-
-				// Calls the cancel function of the context
-				if activeServiceLoadBalancerCancel[string(svc.UID)] != nil {
-					activeServiceLoadBalancerCancel[string(svc.UID)]()
-				}
-				activeService[string(svc.UID)] = false
-				watchedService[string(svc.UID)] = false
-			}
-
-			if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && sm.config.EnableLeaderElection && !sm.config.EnableServicesElection {
-				if sm.config.EnableBGP {
-					instance := sm.findServiceInstance(svc)
-					for _, vip := range instance.vipConfigs {
-						vipCidr := fmt.Sprintf("%s/%s", vip.VIP, vip.VIPCIDR)
-						err = sm.bgpServer.DelHost(vipCidr)
-						if err != nil {
-							log.Error("error deleting host", "ip", vipCidr, "err", err)
-						}
-					}
-				} else {
-					sm.clearRoutes(svc)
-				}
-			}
-
-			log.Info("(svcs) deleted", "service name", svc.Name, "namespace", svc.Namespace)
-		case watch.Bookmark:
-			// Un-used
-		case watch.Error:
-			log.Error("Error attempting to watch Kubernetes services")
-
-			// This round trip allows us to handle unstructured status
-			errObject := apierrors.FromObject(event.Object)
-			statusErr, ok := errObject.(*apierrors.StatusError)
-			if !ok {
-				log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
-			}
-
-			status := statusErr.ErrStatus
-			log.Error("services", "err", status)
-		default:
-		}
-	}
-	close(exitFunction)
-	log.Warn("Stopping watching services for type: LoadBalancer in all namespaces")
-	return nil
-}
-
-func (sm *Manager) lbClassFilterLegacy(svc *v1.Service) bool {
-	if svc == nil {
-		log.Info("(svcs) service is nil, ignoring")
-		return true
-	}
-	if svc.Spec.LoadBalancerClass != nil {
-		// if this isn't nil then it has been configured, check if it the kube-vip loadBalancer class
-		if *svc.Spec.LoadBalancerClass != sm.config.LoadBalancerClassName {
-			log.Info("(svcs) specified the wrong loadBalancer class", "service name", svc.Name, "lbClass", *svc.Spec.LoadBalancerClass)
-			return true
-		}
-	} else if sm.config.LoadBalancerClassOnly {
-		// if kube-vip is configured to only recognize services with kube-vip's lb class, then ignore the services without any lb class
-		log.Info("(svcs) kube-vip configured to only recognize services with kube-vip's lb class but the service didn't specify any loadBalancer class, ignoring", "service name", svc.Name)
-		return true
-	}
-	return false
-}
-
-func (sm *Manager) lbClassFilter(svc *v1.Service) bool {
-	if svc == nil {
-		log.Info("(svcs) service is nil, ignoring")
-		return true
-	}
-	if svc.Spec.LoadBalancerClass == nil && sm.config.LoadBalancerClassName != "" {
-		log.Info("(svcs) no loadBalancer class, ignoring", "service name", svc.Name, "expected lbClass", sm.config.LoadBalancerClassName)
-		return true
-	}
-	if svc.Spec.LoadBalancerClass == nil && sm.config.LoadBalancerClassName == "" {
-		return false
-	}
-	if *svc.Spec.LoadBalancerClass != sm.config.LoadBalancerClassName {
-		log.Info("(svcs) specified wrong loadBalancer class, ignoring", "service name", svc.Name, "wrong lbClass", *svc.Spec.LoadBalancerClass, "expected lbClass", sm.config.LoadBalancerClassName)
-		return true
-	}
-	return false
-}
-
-func isRouteConfigured(serviceUID types.UID) (bool, error) {
-	isConfigured := false
-	value, ok := configuredLocalRoutes.Load(string(serviceUID))
-	if ok {
-		isConfigured, ok = value.(bool)
-		if !ok {
-			return false, fmt.Errorf("error converting configuredLocalRoute item to boolean value")
-		}
-	}
-
-	return isConfigured, nil
-}

pkg/networkinterface/networkinterface.go

@@ -0,0 +1,34 @@
+package networkinterface
+
+import (
+	"sync"
+
+	"github.com/vishvananda/netlink"
+)
+
+type Manager struct {
+	interfaces map[string]*Link
+}
+
+type Link struct {
+	Lock sync.Mutex
+	Intf netlink.Link
+}
+
+func NewManager() *Manager {
+	return &Manager{
+		interfaces: make(map[string]*Link),
+	}
+}
+
+func (m *Manager) Get(intf netlink.Link) *Link {
+	if l, ok := m.interfaces[intf.Attrs().Name]; ok {
+		return l
+	}
+	result := &Link{
+		Intf: intf,
+	}
+
+	m.interfaces[intf.Attrs().Name] = result
+	return result
+}

pkg/nftables/nftables.go

@@ -0,0 +1,458 @@
+package nftables
+
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+	"net"
+	"strconv"
+	"strings"
+
+	"github.com/google/nftables"
+	"github.com/google/nftables/binaryutil"
+	"github.com/google/nftables/expr"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	NatTable  = "kube_vip_%s"
+	SNatChain = "kube_vip_snat_%s"
+)
+
+func ApplySNAT(podIP, vipIP, service, destinationPorts string, ignoreCIDR []string, IPv6 bool) error {
+
+	conn, err := nftables.New()
+	if err != nil {
+		return err
+	}
+
+	var tableName string
+	if IPv6 {
+		tableName = fmt.Sprintf(NatTable, "v6")
+	} else {
+		tableName = fmt.Sprintf(NatTable, "v4")
+	}
+	// Look up the table
+	if t, err := FilterTable(conn, tableName, IPv6); err != nil {
+		if t == nil {
+			// If it doesn't exist then create it
+			slog.Debug("[egress]", "Creating Table", tableName)
+			conn.AddTable(GetTable(IPv6))
+		}
+	}
+	slog.Debug("[egress]", "Creating Chain for service", service, "IPv6", IPv6)
+	// These don't return errors, so not 100% sure how to guarantee things were created
+	conn.AddChain(GetSNatChain(IPv6, service))
+	conn.Flush()
+	// Create our nftables rule
+	rule, err := CreateRule(podIP, vipIP, service, destinationPorts, ignoreCIDR, conn, IPv6)
+	if err != nil {
+		return err
+	}
+	slog.Debug("[egress]", "table", rule.Table.Name, "chain", rule.Chain.Name, "expr", rule.Exprs)
+	conn.AddRule(rule) // Add the rule
+
+	err = conn.Flush() // Commit the rule to nftables
+	if err != nil {
+		return err
+	}
+
+	return conn.CloseLasting() // Close out any remaining netlink communication
+}
+
+func DeleteSNAT(IPv6 bool, service string) error {
+	conn, err := nftables.New()
+	if err != nil {
+		return err
+	}
+
+	var chainName = fmt.Sprintf(SNatChain, service)
+	slog.Info("[egress]", "Looking for", chainName)
+
+	chain, err := conn.ListChain(GetTable(IPv6), chainName)
+	if err != nil {
+		return err
+	}
+	if chain != nil {
+		slog.Info("[egress]", "Deleting chain", chainName)
+		conn.DelChain(chain)
+		return conn.Flush()
+
+	}
+
+	return fmt.Errorf("unable to find chain [%s]", chainName)
+}
+
+func GetTable(IPv6 bool) *nftables.Table {
+	var tableName string
+	if IPv6 {
+		tableName = fmt.Sprintf(NatTable, "v6")
+	} else {
+		tableName = fmt.Sprintf(NatTable, "v4")
+	}
+	// Default to IPv4
+	table := &nftables.Table{
+		Family: nftables.TableFamilyIPv4,
+		Name:   tableName,
+	}
+
+	// Move to IPv6 if needed
+	if IPv6 {
+		table.Family = nftables.TableFamilyIPv6
+	}
+	return table
+}
+
+func GetSNatChain(IPv6 bool, service string) *nftables.Chain {
+	var chainName = fmt.Sprintf(SNatChain, service)
+	policy := nftables.ChainPolicyAccept
+	return &nftables.Chain{
+		Name:     chainName,
+		Table:    GetTable(IPv6),
+		Type:     nftables.ChainTypeNAT,
+		Hooknum:  nftables.ChainHookPostrouting,
+		Priority: nftables.ChainPriorityNATSource,
+		Policy:   &policy,
+	}
+}
+
+func FilterTable(conn *nftables.Conn, tableName string, IPv6 bool) (*nftables.Table, error) {
+	if IPv6 {
+		return conn.ListTableOfFamily(tableName, nftables.TableFamilyIPv6)
+	}
+	return conn.ListTableOfFamily(tableName, nftables.TableFamilyIPv4)
+}
+
+// ClearTable will remove the original tables and create new empty ones
+func ClearTable(conn *nftables.Conn) error {
+	tableName := fmt.Sprintf(NatTable, "v6")
+	if t, err := FilterTable(conn, tableName, false); err != nil {
+		return err
+	} else if t != nil {
+		conn.DelTable(t)
+	}
+
+	// These don't return errors, so not 100% sure how to guarantee things were created
+	conn.AddTable(GetTable(true))
+	tableName = fmt.Sprintf(NatTable, "v4")
+	if t, err := FilterTable(conn, tableName, true); err != nil {
+		return err
+	} else if t != nil {
+		conn.DelTable(t)
+	}
+
+	// These don't return errors, so not 100% sure how to guarantee things were created
+	conn.AddTable(GetTable(false))
+	return nil
+}
+
+// Create our nftables rule
+func CreateRule(podIP, vipIP, service, destinationPorts string, ignoreCIDR []string, conn *nftables.Conn, IPv6 bool) (*nftables.Rule, error) {
+
+	// Validate pod IP
+	if net.ParseIP(podIP) == nil {
+		return nil, errors.New("ip is invalid")
+	}
+
+	// Validate vip IP
+	if net.ParseIP(vipIP) == nil {
+		return nil, errors.New("output_ip is not a valid ip")
+	}
+
+	// Get the kube-vip table
+	table := GetTable(IPv6)
+
+	// Create our rule
+	rule := &nftables.Rule{
+		Table: table,
+		Exprs: []expr.Any{},
+	}
+	// Set the correct chain
+	rule.Chain = GetSNatChain(IPv6, service)
+
+	// Create a set for our original/source address
+	set := &nftables.Set{
+		Table:     table,
+		Anonymous: true,
+		Constant:  true,
+		KeyType:   nftables.TypeIPAddr,
+		Interval:  false,
+	}
+	if IPv6 {
+		set.KeyType = nftables.TypeIP6Addr
+	} else {
+		set.KeyType = nftables.TypeIPAddr
+	}
+
+	// Create an element using our pod IP
+	elements := []nftables.SetElement{}
+	if IPv6 {
+		elements = append(elements, nftables.SetElement{Key: net.ParseIP(podIP).To16()})
+	} else {
+		elements = append(elements, nftables.SetElement{Key: net.ParseIP(podIP).To4()})
+	}
+
+	// Add the elements to the set
+	err := conn.AddSet(set, elements)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the expression using the set
+	expression := []expr.Any{}
+
+	payload := &expr.Payload{
+		OperationType:  expr.PayloadLoad,
+		Base:           expr.PayloadBaseNetworkHeader,
+		DestRegister:   1,
+		SourceRegister: 0,
+	}
+
+	// Set the length of the data based upon the type of IP version being used
+	if IPv6 {
+		payload.Offset = 8
+		payload.Len = 16
+	} else {
+		payload.Offset = 12
+		payload.Len = 4
+	}
+	lookup := &expr.Lookup{
+		SourceRegister: 1,
+		DestRegister:   0,
+		SetID:          set.ID,
+	}
+
+	// Add expressions
+	expression = append(expression, payload)
+	expression = append(expression, lookup)
+
+	// Add expression to the rule
+	rule.Exprs = append(rule.Exprs, expression...)
+
+	// If we filter on ports protocols then parse them
+	if destinationPorts != "" {
+		fixedPorts := strings.Split(destinationPorts, ",")
+
+		// Create an element using our pod IP
+		tcpElements := []nftables.SetElement{}
+		udpElements := []nftables.SetElement{}
+		sctpElements := []nftables.SetElement{}
+
+		tcpSet := &nftables.Set{
+			Anonymous: true,
+			Constant:  true,
+			Table:     table,
+			KeyType:   nftables.TypeInetService,
+		}
+		udpSet := &nftables.Set{
+			Anonymous: true,
+			Constant:  true,
+			Table:     table,
+			KeyType:   nftables.TypeInetService,
+		}
+		sctpSet := &nftables.Set{
+			Anonymous: true,
+			Constant:  true,
+			Table:     table,
+			KeyType:   nftables.TypeInetService,
+		}
+		for _, fixedPort := range fixedPorts {
+			data := strings.Split(fixedPort, ":")
+			if len(data) == 0 {
+				continue
+			} else if len(data) == 2 { // Ensure we have two elements { proto:port }
+				// parse the port to a number
+				port, err := strconv.Atoi(data[1])
+				if err != nil {
+					slog.Error("[egress]", "unable to process port", data[1])
+					continue
+				}
+				// Ensure the port is within the valid range for uint16
+				if port < 0 || port > 65535 {
+					slog.Error("[egress]", "port out of range for uint16", data[1])
+					continue
+				}
+
+				switch data[0] {
+				case "tcp":
+					//nolint:gosec
+					tcpElements = append(tcpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
+				case "udp":
+					//nolint:gosec
+					udpElements = append(udpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
+				case "sctp":
+					//nolint:gosec
+					sctpElements = append(sctpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
+				default:
+					slog.Error("[egress]", "unknown protocol", data[0])
+				}
+			}
+		}
+
+		// Add TCP Ports
+		if len(tcpElements) != 0 {
+			err = conn.AddSet(tcpSet, tcpElements)
+			if err != nil {
+				return nil, err
+			}
+			expression := []expr.Any{
+				&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
+				// [ cmp eq reg 1 0x00000006 ]
+				&expr.Cmp{
+					Op:       expr.CmpOpEq,
+					Register: 1,
+					Data:     []byte{unix.IPPROTO_TCP},
+				},
+
+				// [ payload load 2b @ transport header + 2 => reg 1 ]
+				&expr.Payload{
+					DestRegister: 1,
+					Base:         expr.PayloadBaseTransportHeader,
+					Offset:       2,
+					Len:          2,
+				},
+				// [ lookup reg 1 set __set%d ]
+				&expr.Lookup{
+					SourceRegister: 1,
+					SetName:        tcpSet.Name,
+					SetID:          tcpSet.ID,
+				},
+			}
+			rule.Exprs = append(rule.Exprs, expression...)
+		}
+
+		// Add UDP ports
+		if len(udpElements) != 0 {
+			err = conn.AddSet(udpSet, udpElements)
+			if err != nil {
+				return nil, err
+			}
+			expression := []expr.Any{
+				&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
+				// [ cmp eq reg 1 0x00000006 ]
+				&expr.Cmp{
+					Op:       expr.CmpOpEq,
+					Register: 1,
+					Data:     []byte{unix.IPPROTO_UDP},
+				},
+
+				// [ payload load 2b @ transport header + 2 => reg 1 ]
+				&expr.Payload{
+					DestRegister: 1,
+					Base:         expr.PayloadBaseTransportHeader,
+					Offset:       2,
+					Len:          2,
+				},
+				// [ lookup reg 1 set __set%d ]
+				&expr.Lookup{
+					SourceRegister: 1,
+					SetName:        udpSet.Name,
+					SetID:          udpSet.ID,
+				},
+			}
+			rule.Exprs = append(rule.Exprs, expression...)
+		}
+
+		// Add SCTP Ports
+		if len(sctpElements) != 0 {
+			err = conn.AddSet(sctpSet, sctpElements)
+			if err != nil {
+				return nil, err
+			}
+			expression := []expr.Any{
+				&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
+				// [ cmp eq reg 1 0x00000006 ]
+				&expr.Cmp{
+					Op:       expr.CmpOpEq,
+					Register: 1,
+					Data:     []byte{unix.IPPROTO_SCTP},
+				},
+
+				// [ payload load 2b @ transport header + 2 => reg 1 ]
+				&expr.Payload{
+					DestRegister: 1,
+					Base:         expr.PayloadBaseTransportHeader,
+					Offset:       2,
+					Len:          2,
+				},
+				// [ lookup reg 1 set __set%d ]
+				&expr.Lookup{
+					SourceRegister: 1,
+					SetName:        sctpSet.Name,
+					SetID:          sctpSet.ID,
+				},
+			}
+			rule.Exprs = append(rule.Exprs, expression...)
+		}
+	}
+
+	// Parse which CIDRs we will not SNAT for
+	for _, cidr := range ignoreCIDR {
+		start, end, err := nftables.NetFirstAndLastIP(cidr)
+		if err != nil {
+			return nil, err
+		}
+		expression = []expr.Any{}
+
+		payload := &expr.Payload{
+			DestRegister: 1,
+			Base:         expr.PayloadBaseNetworkHeader,
+		}
+		notEqualRange := &expr.Range{
+			Op:       expr.CmpOpNeq,
+			Register: 1,
+		}
+
+		if IPv6 {
+			payload.Len = 16
+			payload.Offset = 24
+			notEqualRange.FromData = start.To16()
+			notEqualRange.ToData = end.To16()
+		} else {
+			payload.Offset = 16
+			payload.Len = 4
+			notEqualRange.FromData = start.To4()
+			notEqualRange.ToData = end.To4()
+		}
+		// Add expressions
+		expression = append(expression, payload)
+		expression = append(expression, notEqualRange)
+
+		// // Add expression to the rule
+		rule.Exprs = append(rule.Exprs, expression...)
+	}
+
+	// Final expression to the rule is the SNAT to the VIP address
+	expression = []expr.Any{}
+
+	immediate := &expr.Immediate{
+		Register: 1,
+	}
+
+	nat := &expr.NAT{
+		Type:        expr.NATTypeSourceNAT,
+		RegAddrMin:  1,
+		RegAddrMax:  1,
+		RegProtoMin: 0,
+		RegProtoMax: 0,
+		Random:      false,
+		FullyRandom: false,
+		Persistent:  false,
+		Prefix:      false,
+	}
+
+	if IPv6 {
+		immediate.Data = net.ParseIP(vipIP).To16()
+		nat.Family = unix.NFPROTO_IPV6
+	} else {
+		immediate.Data = net.ParseIP(vipIP).To4()
+		nat.Family = unix.NFPROTO_IPV4
+	}
+	// https://github.com/google/nftables/blob/main/nftables_test.go#L5375
+	// Add expressions
+	expression = append(expression, immediate)
+	expression = append(expression, nat)
+	rule.Exprs = append(rule.Exprs, expression...)
+
+	return rule, nil
+}

pkg/servicecontext/servicecontext.go

@@ -0,0 +1,36 @@
+package servicecontext
+
+import (
+	"context"
+	"sync"
+)
+
+type Context struct {
+	Ctx                context.Context
+	Cancel             context.CancelFunc
+	IsActive           bool
+	IsWatched          bool
+	ConfiguredNetworks sync.Map
+}
+
+func New(ctx context.Context) *Context {
+	svcCtx, svcCancel := context.WithCancel(ctx)
+	return &Context{
+		Ctx:    svcCtx,
+		Cancel: svcCancel,
+	}
+}
+
+func (ctx *Context) HasConfiguredNetworks() bool {
+	cnt := 0
+	ctx.ConfiguredNetworks.Range(func(_ any, _ any) bool {
+		cnt++
+		return cnt < 1
+	})
+	return cnt > 0
+}
+
+func (ctx *Context) IsNetworkConfigured(ip string) bool {
+	_, exists := ctx.ConfiguredNetworks.Load(ip)
+	return exists
+}

pkg/services/context.go

@@ -0,0 +1,36 @@
+package services
+
+import (
+	"context"
+	"sync"
+)
+
+type Context struct {
+	Ctx                context.Context
+	Cancel             context.CancelFunc
+	IsActive           bool
+	IsWatched          bool
+	ConfiguredNetworks sync.Map
+}
+
+func NewContext(ctx context.Context) *Context {
+	svcCtx, svcCancel := context.WithCancel(ctx)
+	return &Context{
+		Ctx:    svcCtx,
+		Cancel: svcCancel,
+	}
+}
+
+func (ctx *Context) HasConfiguredNetworks() bool {
+	cnt := 0
+	ctx.ConfiguredNetworks.Range(func(_ any, _ any) bool {
+		cnt++
+		return cnt < 1
+	})
+	return cnt > 0
+}
+
+func (ctx *Context) IsNetworkConfigured(ip string) bool {
+	_, exists := ctx.ConfiguredNetworks.Load(ip)
+	return exists
+}

pkg/services/egress.go

@@ -1,4 +1,4 @@
-package manager
+package services
 
 import (
 	"bufio"
@@ -11,6 +11,8 @@ import (
 	log "log/slog"
 
 	"github.com/kube-vip/kube-vip/pkg/iptables"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/nftables"
 	"github.com/kube-vip/kube-vip/pkg/vip"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -21,7 +23,7 @@ const (
 	defaultServiceCIDR = "10.96.0.0/12"
 )
 
-func (sm *Manager) iptablesCheck() error {
+func (p *Processor) iptablesCheck() error {
 	file, err := os.Open("/proc/modules")
 	if err != nil {
 		return err
@@ -48,6 +50,31 @@ func (sm *Manager) iptablesCheck() error {
 	return nil
 }
 
+func (p *Processor) nftablesCheck() error {
+	file, err := os.Open("/proc/modules")
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	scanner := bufio.NewScanner(file)
+	scanner.Split(bufio.ScanLines)
+	var queue, ct bool
+	for scanner.Scan() {
+		line := strings.Fields(scanner.Text())
+		switch line[0] {
+		case "nft_queue":
+			queue = true
+		case "nft_ct":
+			ct = true
+		}
+	}
+
+	if !queue || !ct {
+		return fmt.Errorf("missing nftables modules -> nft_ct [%t] -> ntf_queue [%t]", ct, queue)
+	}
+	return nil
+}
+
 func getSameFamilyCidr(sourceCidrs, ip string) string { //Todo: not sure how this ever worked
 	cidrs := strings.Split(sourceCidrs, ",")
 	isV6 := vip.IsIPv6(ip)
@@ -107,26 +134,27 @@ func checkCIDR(ip, cidr string) (string, error) {
 	return "", nil
 }
 
-func (sm *Manager) configureEgress(vipIP, podIP, namespace string, annotations map[string]string) error {
+func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string, annotations map[string]string) error {
 	var podCidr, serviceCidr string
 	var autoServiceCIDR, autoPodCIDR string
 	var discoverErr error
 
 	// Look up the destination ports from the annotations on the service
-	destinationPorts := annotations[egressDestinationPorts]
-	deniedNetworks := annotations[egressDeniedNetworks]
-	allowedNetworks := annotations[egressAllowedNetworks]
+	destinationPorts := annotations[kubevip.EgressDestinationPorts]
+	deniedNetworks := annotations[kubevip.EgressDeniedNetworks]
+	allowedNetworks := annotations[kubevip.EgressAllowedNetworks]
+	internalEgress := annotations[kubevip.EgressInternal]
 
-	if sm.config.EgressPodCidr == "" || sm.config.EgressServiceCidr == "" {
-		autoServiceCIDR, autoPodCIDR, discoverErr = sm.AutoDiscoverCIDRs()
+	if p.config.EgressPodCidr == "" || p.config.EgressServiceCidr == "" {
+		autoServiceCIDR, autoPodCIDR, discoverErr = p.AutoDiscoverCIDRs()
 	}
 
 	if discoverErr != nil {
 		log.Warn("autodiscover CIDR", "err", discoverErr)
 	}
 
-	if sm.config.EgressPodCidr != "" {
-		podCidr = getSameFamilyCidr(sm.config.EgressPodCidr, podIP)
+	if p.config.EgressPodCidr != "" {
+		podCidr = getSameFamilyCidr(p.config.EgressPodCidr, podIP)
 	} else {
 		if discoverErr == nil {
 			podCidr = getSameFamilyCidr(autoPodCIDR, podIP)
@@ -141,8 +169,8 @@ func (sm *Manager) configureEgress(vipIP, podIP, namespace string, annotations m
 		podCidr = defaultPodCIDR
 	}
 
-	if sm.config.EgressServiceCidr != "" {
-		serviceCidr = getSameFamilyCidr(sm.config.EgressServiceCidr, vipIP)
+	if p.config.EgressServiceCidr != "" {
+		serviceCidr = getSameFamilyCidr(p.config.EgressServiceCidr, vipIP)
 	} else {
 		if discoverErr == nil {
 			serviceCidr = getSameFamilyCidr(autoServiceCIDR, vipIP)
@@ -180,7 +208,30 @@ func (sm *Manager) configureEgress(vipIP, podIP, namespace string, annotations m
 		protocol = iptables.ProtocolIPv6
 	}
 
-	i, err := vip.CreateIptablesClient(sm.config.EgressWithNftables, namespace, protocol)
+	// Use the internal egress implementation
+	if internalEgress != "" {
+		// Create an array of CIDRs that we wont SNAT to.
+		ignoreCIDRs := []string{
+			podCidr,
+			serviceCidr,
+		}
+
+		// Add any specifically denied networks
+		if deniedNetworks != "" {
+			networks := strings.Split(strings.TrimSpace(deniedNetworks), ",") //Remove whitespace characters and then create an array from the CIDRs
+			ignoreCIDRs = append(ignoreCIDRs, networks...)
+
+		}
+
+		// Apply the SNAT rules
+		err := nftables.ApplySNAT(podIP, vipIP, serviceUUID, destinationPorts, ignoreCIDRs, vip.IsIPv6(vipIP))
+		if err != nil {
+			return fmt.Errorf("error performing netlink nftables [%s]", err)
+		}
+		return nil
+	}
+
+	i, err := vip.CreateIptablesClient(p.config.EgressWithNftables, namespace, protocol)
 	if err != nil {
 		return fmt.Errorf("error Creating iptables client [%s]", err)
 	}
@@ -279,12 +330,12 @@ func (sm *Manager) configureEgress(vipIP, podIP, namespace string, annotations m
 	return nil
 }
 
-func (sm *Manager) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error) {
+func (p *Processor) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error) {
 	log.Debug("Trying to automatically discover Service and Pod CIDRs")
 	options := v1.ListOptions{
 		LabelSelector: "component=kube-controller-manager",
 	}
-	podList, err := sm.clientSet.CoreV1().Pods("kube-system").List(context.TODO(), options)
+	podList, err := p.clientSet.CoreV1().Pods("kube-system").List(context.TODO(), options)
 	if err != nil {
 		return "", "", fmt.Errorf("[Egress] Unable to get kube-controller-manager pod: %w", err)
 	}
@@ -307,83 +358,3 @@ func (sm *Manager) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error)
 
 	return
 }
-
-func (sm *Manager) TeardownEgress(podIP, vipIP, namespace string, annotations map[string]string) error {
-	// Look up the destination ports from the annotations on the service
-	destinationPorts := annotations[egressDestinationPorts]
-	deniedNetworks := annotations[egressDeniedNetworks]
-	allowedNetworks := annotations[egressAllowedNetworks]
-
-	protocol := iptables.ProtocolIPv4
-	if vip.IsIPv6(podIP) {
-		protocol = iptables.ProtocolIPv6
-	}
-
-	i, err := vip.CreateIptablesClient(sm.config.EgressWithNftables, namespace, protocol)
-	if err != nil {
-		return fmt.Errorf("error Creating iptables client [%s]", err)
-	}
-
-	if deniedNetworks != "" {
-		networks := strings.Split(deniedNetworks, ",")
-		for x := range networks {
-			err = i.DeleteMangleReturnForNetwork(vip.MangleChainName, networks[x])
-			if err != nil {
-				return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
-			}
-		}
-	}
-
-	if allowedNetworks != "" {
-		networks := strings.Split(allowedNetworks, ",")
-		for x := range networks {
-			err = i.DeleteMangleMarkingForNetwork(podIP, vip.MangleChainName, networks[x])
-			if err != nil {
-				return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
-			}
-		}
-	} else {
-		// Remove the marking of egress packets
-		err = i.DeleteMangleMarking(podIP, vip.MangleChainName)
-		if err != nil {
-			return fmt.Errorf("error changing iptables rules for egress [%s]", err)
-		}
-	}
-
-	// Clear up SNAT rules
-	if destinationPorts != "" {
-		fixedPorts := strings.Split(destinationPorts, ",")
-
-		for _, fixedPort := range fixedPorts {
-			var proto, port string
-
-			data := strings.Split(fixedPort, ":")
-			if len(data) == 0 {
-				continue
-			} else if len(data) == 1 {
-				proto = "tcp"
-				port = data[0]
-			} else {
-				proto = data[0]
-				port = data[1]
-			}
-
-			err = i.DeleteSourceNatForDestinationPort(podIP, vipIP, port, proto)
-			if err != nil {
-				return fmt.Errorf("error changing iptables rules for egress [%s]", err)
-			}
-
-		}
-	} else {
-		err = i.DeleteSourceNat(podIP, vipIP)
-		if err != nil {
-			return fmt.Errorf("error changing iptables rules for egress [%s]", err)
-		}
-	}
-
-	err = vip.DeleteExistingSessions(podIP, false, destinationPorts, "")
-	if err != nil {
-		return fmt.Errorf("error changing iptables rules for egress [%s]", err)
-	}
-	return nil
-}

pkg/services/egress_test.go

@@ -1,4 +1,4 @@
-package manager
+package services
 
 import "testing"
 

pkg/services/leader.go

@@ -1,4 +1,4 @@
-package manager
+package services
 
 import (
 	"context"
@@ -22,15 +22,15 @@ func init() {
 	svcLocks = make(map[string]*sync.Mutex)
 }
 
-// The startServicesWatchForLeaderElection function will start a services watcher, the
-func (sm *Manager) startServicesWatchForLeaderElection(ctx context.Context) error {
-	err := sm.servicesWatcher(ctx, sm.StartServicesLeaderElection)
+// The StartServicesWatchForLeaderElection function will start a services watcher, the
+func (p *Processor) StartServicesWatchForLeaderElection(ctx context.Context) error {
+	err := p.ServicesWatcher(ctx, p.StartServicesLeaderElection)
 	if err != nil {
 		return err
 	}
 
-	for _, instance := range sm.serviceInstances {
-		for _, cluster := range instance.clusters {
+	for _, instance := range p.ServiceInstances {
+		for _, cluster := range instance.Clusters {
 			for i := range cluster.Network {
 				_ = cluster.Network[i].DeleteRoute()
 			}
@@ -44,9 +44,9 @@ func (sm *Manager) startServicesWatchForLeaderElection(ctx context.Context) erro
 }
 
 // The startServicesWatchForLeaderElection function will start a services watcher, the
-func (sm *Manager) StartServicesLeaderElection(ctx context.Context, service *v1.Service) error {
+func (p *Processor) StartServicesLeaderElection(ctx context.Context, service *v1.Service) error {
 	serviceLease := fmt.Sprintf("kubevip-%s", service.Name)
-	log.Info("new leader election", "service", service.Name, "namespace", service.Namespace, "lock_name", serviceLease, "host_id", sm.config.NodeName)
+	log.Info("new leader election", "service", service.Name, "namespace", service.Namespace, "lock_name", serviceLease, "host_id", p.config.NodeName)
 	// we use the Lease lock type since edits to Leases are less common
 	// and fewer objects in the cluster watch "all Leases".
 	lock := &resourcelock.LeaseLock{
@@ -54,9 +54,9 @@ func (sm *Manager) StartServicesLeaderElection(ctx context.Context, service *v1.
 			Name:      serviceLease,
 			Namespace: service.Namespace,
 		},
-		Client: sm.clientSet.CoordinationV1(),
+		Client: p.clientSet.CoordinationV1(),
 		LockConfig: resourcelock.ResourceLockConfig{
-			Identity: sm.config.NodeName,
+			Identity: p.config.NodeName,
 		},
 	}
 	childCtx, childCancel := context.WithCancel(ctx)
@@ -69,7 +69,16 @@ func (sm *Manager) StartServicesLeaderElection(ctx context.Context, service *v1.
 	svcLocks[serviceLease].Lock()
 	defer svcLocks[serviceLease].Unlock()
 
-	activeService[string(service.UID)] = true
+	svcCtx, err := p.getServiceContext(service.UID)
+	if err != nil {
+		return fmt.Errorf("failed to get context for service %q with UID %q: %w", service.Name, service.UID, err)
+	}
+	if svcCtx == nil {
+		return fmt.Errorf("failed to get context for service %q with UID %q: nil context", service.Name, service.UID)
+	}
+
+	svcCtx.IsActive = true
+
 	// start the leader election code loop
 	leaderelection.RunOrDie(childCtx, leaderelection.LeaderElectionConfig{
 		Lock: lock,
@@ -80,32 +89,32 @@ func (sm *Manager) StartServicesLeaderElection(ctx context.Context, service *v1.
 		// get elected before your background loop finished, violating
 		// the stated goal of the lease.
 		ReleaseOnCancel: true,
-		LeaseDuration:   time.Duration(sm.config.LeaseDuration) * time.Second,
-		RenewDeadline:   time.Duration(sm.config.RenewDeadline) * time.Second,
-		RetryPeriod:     time.Duration(sm.config.RetryPeriod) * time.Second,
+		LeaseDuration:   time.Duration(p.config.LeaseDuration) * time.Second,
+		RenewDeadline:   time.Duration(p.config.RenewDeadline) * time.Second,
+		RetryPeriod:     time.Duration(p.config.RetryPeriod) * time.Second,
 		Callbacks: leaderelection.LeaderCallbacks{
 			OnStartedLeading: func(ctx context.Context) {
 				// Mark this service as active (as we've started leading)
 				// we run this in background as it's blocking
-				if err := sm.syncServices(ctx, service); err != nil {
+				if err := p.SyncServices(ctx, service); err != nil {
 					log.Error("service sync", "err", err)
 					childCancel()
 				}
 			},
 			OnStoppedLeading: func() {
 				// we can do cleanup here
-				log.Info("leadership lost", "service", service.Name, "leader", sm.config.NodeName)
-				if activeService[string(service.UID)] {
-					if err := sm.deleteService(service.UID); err != nil {
+				log.Info("leadership lost", "service", service.Name, "leader", p.config.NodeName)
+				if svcCtx.IsActive {
+					if err := p.deleteService(service.UID); err != nil {
 						log.Error("service deletion", "err", err)
 					}
 				}
 				// Mark this service is inactive
-				activeService[string(service.UID)] = false
+				svcCtx.IsActive = false
 			},
 			OnNewLeader: func(identity string) {
 				// we're notified when new leader elected
-				if identity == sm.config.NodeName {
+				if identity == p.config.NodeName {
 					// I just got the lock
 					return
 				}

pkg/services/processor.go

@@ -0,0 +1,344 @@
+package services
+
+import (
+	"context"
+	"fmt"
+	log "log/slog"
+	"reflect"
+	"sync"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/endpoints"
+	"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/networkinterface"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/vishvananda/netlink"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+)
+
+type Processor struct {
+	config        *kubevip.Config
+	lbClassFilter func(svc *v1.Service, config *kubevip.Config) bool
+	svcMap        sync.Map
+
+	// Keeps track of all running instances
+	ServiceInstances []*instance.Instance
+
+	mutex     sync.Mutex
+	bgpServer *bgp.Server
+
+	clientSet   *kubernetes.Clientset
+	rwClientSet *kubernetes.Clientset
+
+	shutdownChan chan struct{}
+
+	// This is a prometheus counter used to count the number of events received
+	// from the service watcher
+	CountServiceWatchEvent *prometheus.CounterVec
+
+	intfMgr *networkinterface.Manager
+	arpMgr  *arp.Manager
+}
+
+func NewServicesProcessor(config *kubevip.Config, bgpServer *bgp.Server,
+	clientSet *kubernetes.Clientset, rwClientSet *kubernetes.Clientset, shutdownChan chan struct{},
+	intfMgr *networkinterface.Manager, arpMgr *arp.Manager) *Processor {
+	lbClassFilterFunc := lbClassFilter
+	if config.LoadBalancerClassLegacyHandling {
+		lbClassFilterFunc = lbClassFilterLegacy
+	}
+
+	return &Processor{
+		config:           config,
+		lbClassFilter:    lbClassFilterFunc,
+		ServiceInstances: []*instance.Instance{},
+		bgpServer:        bgpServer,
+		clientSet:        clientSet,
+		rwClientSet:      rwClientSet,
+		shutdownChan:     shutdownChan,
+		CountServiceWatchEvent: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Namespace: "kube_vip",
+			Subsystem: "manager",
+			Name:      "all_services_events",
+			Help:      "Count all events fired by the service watcher categorised by event type",
+		}, []string{"type"}),
+
+		intfMgr: intfMgr,
+		arpMgr:  arpMgr,
+	}
+}
+
+func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceFunc func(context.Context, *v1.Service) error) (bool, error) {
+	// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
+	svc, ok := event.Object.(*v1.Service)
+	if !ok {
+		return false, fmt.Errorf("unable to parse Kubernetes services from API watcher")
+	}
+
+	// We only care about LoadBalancer services
+	if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
+		return true, nil
+	}
+
+	// Check if we ignore this service
+	if svc.Annotations["kube-vip.io/ignore"] == "true" {
+		log.Info("ignore annotation for kube-vip", "service name", svc.Name)
+		return true, nil
+	}
+
+	// Check the loadBalancer class
+	if p.lbClassFilter(svc, p.config) {
+		return true, nil
+	}
+
+	svcAddresses := instance.FetchServiceAddresses(svc)
+
+	// We only care about LoadBalancer services that have been allocated an address
+	if len(svcAddresses) <= 0 {
+		return true, nil
+	}
+
+	svcCtx, err := p.getServiceContext(svc.UID)
+	if err != nil {
+		return false, fmt.Errorf("failed to get service context: %w", err)
+	}
+
+	// The modified event should only be triggered if the service has been modified (i.e. moved somewhere else)
+	if event.Type == watch.Modified {
+		i := instance.FindServiceInstance(svc, p.ServiceInstances)
+		var originalService []string
+		shouldGarbageCollect := true
+		if i != nil {
+			originalService = instance.FetchServiceAddresses(i.ServiceSnapshot)
+			shouldGarbageCollect = !reflect.DeepEqual(originalService, svcAddresses)
+		}
+		if shouldGarbageCollect {
+			for _, addr := range svcAddresses {
+				// log.Debugf("(svcs) Retreiving local addresses, to ensure that this modified address doesn't exist: %s", addr)
+				f, err := vip.GarbageCollect(p.config.Interface, addr, p.intfMgr)
+				if err != nil {
+					log.Error("(svcs) cleaning existing address error", "err", err)
+				}
+				if f {
+					log.Warn("(svcs) already found existing config", "address", addr, "adapter", p.config.Interface)
+				}
+			}
+		}
+		// This service has been modified, but it was also active.
+		if svcCtx != nil && svcCtx.IsActive {
+			if i != nil {
+				originalService := instance.FetchServiceAddresses(i.ServiceSnapshot)
+				newService := instance.FetchServiceAddresses(svc)
+				if !reflect.DeepEqual(originalService, newService) {
+
+					// Calls the cancel function of the context
+					if svcCtx != nil {
+						log.Warn("(svcs) The load balancer has changed, cancelling original load balancer")
+						svcCtx.Cancel()
+						log.Warn("(svcs) waiting for load balancer to finish")
+						<-svcCtx.Ctx.Done()
+					}
+
+					err = p.deleteService(svc.UID)
+					if err != nil {
+						log.Error("(svc) unable to remove", "service", svc.UID)
+					}
+
+					p.svcMap.Delete(svc.UID)
+				}
+				// in theory this should never fail
+			}
+		}
+	}
+
+	// Architecture walkthrough: (Had to do this as this code path is making my head hurt)
+
+	// Is the service active (bool), if not then process this new service
+	// Does this service use an election per service?
+	//
+
+	if svcCtx == nil || svcCtx != nil && !svcCtx.IsActive {
+		log.Debug("(svcs) has been added/modified with addresses", "service name", svc.Name, "ip", instance.FetchServiceAddresses(svc))
+
+		if svcCtx == nil {
+			svcCtx = servicecontext.New(ctx)
+			p.svcMap.Store(svc.UID, svcCtx)
+		}
+
+		if p.config.EnableServicesElection || // Service Election
+			((p.config.EnableRoutingTable || p.config.EnableBGP) && // Routing table mode or BGP
+				(!p.config.EnableLeaderElection && !p.config.EnableServicesElection)) { // No leaderelection or services election
+
+			// If this load balancer Traffic Policy is "local"
+			if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
+
+				// Start an endpoint watcher if we're not watching it already
+				if !svcCtx.IsWatched {
+					// background the endpoint watcher
+					if (p.config.EnableRoutingTable || p.config.EnableBGP) && (!p.config.EnableLeaderElection && !p.config.EnableServicesElection) {
+						err = serviceFunc(svcCtx.Ctx, svc)
+						if err != nil {
+							log.Error(err.Error())
+						}
+					}
+
+					go func() {
+						if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
+							// Add Endpoint or EndpointSlices watcher
+							var provider providers.Provider
+							if p.config.EnableEndpoints {
+								provider = providers.NewEndpoints()
+							} else {
+								provider = providers.NewEndpointslices()
+							}
+							if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
+								log.Error(err.Error())
+							}
+						}
+					}()
+
+					// We're now watching this service
+					svcCtx.IsWatched = true
+				}
+			} else if (p.config.EnableBGP || p.config.EnableRoutingTable) && (!p.config.EnableLeaderElection && !p.config.EnableServicesElection) {
+				err = serviceFunc(svcCtx.Ctx, svc)
+				if err != nil {
+					log.Error(err.Error())
+				}
+
+				go func() {
+					if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
+						// Add Endpoint watcher
+						var provider providers.Provider
+						if p.config.EnableEndpoints {
+							provider = providers.NewEndpoints()
+						} else {
+							provider = providers.NewEndpointslices()
+						}
+						if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
+							log.Error(err.Error())
+						}
+					}
+				}()
+				// We're now watching this service
+				svcCtx.IsWatched = true
+			} else {
+
+				go func() {
+					for {
+						select {
+						case <-svcCtx.Ctx.Done():
+							log.Warn("(svcs) restartable service watcher ending", "uid", svc.UID)
+							return
+						default:
+							log.Info("(svcs) restartable service watcher starting", "uid", svc.UID)
+							err = serviceFunc(svcCtx.Ctx, svc)
+
+							if err != nil {
+								log.Error(err.Error())
+							}
+						}
+					}
+
+				}()
+			}
+		} else {
+			// Increment the waitGroup before the service Func is called (Done is completed in there)
+			err = serviceFunc(svcCtx.Ctx, svc)
+			if err != nil {
+				log.Error(err.Error())
+			}
+		}
+		svcCtx.IsActive = true
+	}
+
+	return false, nil
+}
+
+func (p *Processor) Delete(event watch.Event) (bool, error) {
+	svc, ok := event.Object.(*v1.Service)
+	if !ok {
+		return false, fmt.Errorf("unable to parse Kubernetes services from API watcher")
+	}
+	svcCtx, err := p.getServiceContext(svc.UID)
+	if err != nil {
+		return false, fmt.Errorf("(svcs) unable to get context: %w", err)
+	}
+	if svcCtx != nil && svcCtx.IsActive {
+		// We only care about LoadBalancer services
+		if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
+			return true, nil
+		}
+
+		// We can ignore this service
+		if svc.Annotations["kube-vip.io/ignore"] == "true" {
+			log.Info("(svcs)ignore annotation for kube-vip", "service name", svc.Name)
+			return true, nil
+		}
+
+		// If no leader election is enabled, delete routes here
+		if !p.config.EnableLeaderElection && !p.config.EnableServicesElection &&
+			p.config.EnableRoutingTable && svcCtx.HasConfiguredNetworks() {
+			if errs := endpoints.ClearRoutes(svc, &p.ServiceInstances); len(errs) == 0 {
+				svcCtx.ConfiguredNetworks.Clear()
+			}
+		}
+
+		// If this is an active service then and additional leaderElection will handle stopping
+		err = p.deleteService(svc.UID)
+		if err != nil {
+			log.Error(err.Error())
+		}
+
+		// Calls the cancel function of the context
+		log.Warn("(svcs) The load balancer was deleted, cancelling context")
+		svcCtx.Cancel()
+		log.Warn("(svcs) waiting for load balancer to finish")
+		<-svcCtx.Ctx.Done()
+		p.svcMap.Delete(svc.UID)
+	}
+
+	if p.config.EnableLeaderElection && !p.config.EnableServicesElection {
+		if p.config.EnableBGP {
+			endpoints.ClearBGPHosts(svc, &p.ServiceInstances, p.bgpServer)
+		} else if p.config.EnableRoutingTable {
+			endpoints.ClearRoutes(svc, &p.ServiceInstances)
+		}
+	}
+
+	log.Info("(svcs) deleted", "service name", svc.Name, "namespace", svc.Namespace)
+
+	return true, nil
+}
+
+func (p *Processor) Stop() {
+	for _, instance := range p.ServiceInstances {
+		for _, cluster := range instance.Clusters {
+			cluster.Stop()
+		}
+	}
+}
+
+func (p *Processor) getServiceContext(uid types.UID) (*servicecontext.Context, error) {
+	svcCtx, ok := p.svcMap.Load(uid)
+	if !ok {
+		return nil, nil
+	}
+	ctx, ok := svcCtx.(*servicecontext.Context)
+	if !ok {
+		return nil, fmt.Errorf("failed to cast service context pointer - UID: %s", uid)
+	}
+	return ctx, nil
+}
+
+func (p *Processor) CountRouteReferences(route *netlink.Route) int {
+	return endpoints.CountRouteReferences(route, &p.ServiceInstances)
+}

pkg/services/services.go

@@ -0,0 +1,516 @@
+package services
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"strings"
+	"time"
+
+	log "log/slog"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/vishvananda/netlink"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/retry"
+
+	"github.com/kube-vip/kube-vip/pkg/egress"
+	"github.com/kube-vip/kube-vip/pkg/endpoints"
+	"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/upnp"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
+
+type ServiceInstanceAction string
+
+const (
+	ActionDelete ServiceInstanceAction = "delete"
+	ActionAdd    ServiceInstanceAction = "add"
+	ActionNone   ServiceInstanceAction = "none"
+)
+
+func (p *Processor) SyncServices(ctx context.Context, svc *v1.Service) error {
+	log.Debug("[STARTING] Service Sync", "namespace", svc.Namespace, "name", svc.Name)
+
+	// Iterate through the synchronising services
+
+	action := p.getServiceInstanceAction(svc)
+	switch action {
+	case ActionDelete:
+		log.Debug("[service] delete", "namespace", svc.Namespace, "name", svc.Name)
+		if err := p.deleteService(svc.UID); err != nil {
+			return fmt.Errorf("error deleting service %s/%s: %w", svc.Namespace, svc.Name, err)
+		}
+	case ActionAdd:
+		log.Debug("[service] add", "namespace", svc.Namespace, "name", svc.Name)
+		if err := p.addService(ctx, svc); err != nil {
+			return fmt.Errorf("error adding service %s/%s: %w", svc.Namespace, svc.Name, err)
+		}
+	case ActionNone:
+		log.Debug("[service] no action", "namespace", svc.Namespace, "name", svc.Name)
+	}
+	log.Debug("[FINISHED] Service Sync", "namespace", svc.Namespace, "name", svc.Name)
+	return nil
+}
+
+func (p *Processor) getServiceInstanceAction(svc *v1.Service) ServiceInstanceAction {
+	// protect against multiple calls
+	addresses := instance.FetchServiceAddresses(svc)
+	ingressIPs := instance.FetchLoadBalancerIngressAddresses(svc)
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
+
+	for _, instance := range p.ServiceInstances {
+		if instance != nil && instance.ServiceSnapshot.UID == svc.UID {
+			for _, address := range addresses {
+				// handle the case where the service instance needs to be deleted
+				if instance.IsDHCP {
+					if address != "0.0.0.0" {
+						return ActionDelete
+					}
+					if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, instance.DHCPInterfaceIP) {
+						return ActionDelete
+					}
+				}
+				if !instance.IsDHCP {
+					if address == "0.0.0.0" {
+						return ActionDelete
+					}
+					if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, address) {
+						return ActionDelete
+					}
+				}
+				if !comparePortsAndPortStatuses(svc) {
+					return ActionDelete
+				}
+			}
+			// If we reach here, it means the service instance matches the service UID and is not a DHCP service, so we can return "no action"
+			return ActionNone
+		}
+	}
+	if len(addresses) > 0 {
+		log.Debug("No matching service instance found", "service", svc.Name, "namespace", svc.Namespace, "addresses", addresses)
+		return ActionAdd // If no matching instance is found, we need to add a new service instance
+	}
+	return ActionNone
+}
+
+func comparePortsAndPortStatuses(svc *v1.Service) bool {
+	if len(svc.Status.LoadBalancer.Ingress) == 0 {
+		return false
+	}
+	portsStatus := svc.Status.LoadBalancer.Ingress[0].Ports
+	if len(portsStatus) != len(svc.Spec.Ports) {
+		return false
+	}
+	for i, portSpec := range svc.Spec.Ports {
+		if portsStatus[i].Port != portSpec.Port || portsStatus[i].Protocol != portSpec.Protocol {
+			return false
+		}
+	}
+	return true
+}
+
+func (p *Processor) addService(ctx context.Context, svc *v1.Service) error {
+	// protect against addService while reading
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
+
+	startTime := time.Now()
+
+	newService, err := instance.NewInstance(svc, p.config, p.intfMgr, p.arpMgr)
+	if err != nil {
+		return err
+	}
+
+	for x := range newService.VIPConfigs {
+		log.Debug("starting loadbalancer for service", "name", svc.Name, "namespace", svc.Namespace)
+		newService.Clusters[x].StartLoadBalancerService(ctx, newService.VIPConfigs[x], p.bgpServer, svc.Name, p.CountRouteReferences)
+	}
+
+	p.upnpMap(ctx, newService)
+
+	if newService.IsDHCP && len(newService.VIPConfigs) == 1 {
+		go func() {
+			for ip := range newService.DHCPClient.IPChannel() {
+				log.Debug("IP changed", "ip", ip)
+				newService.VIPConfigs[0].VIP = ip
+				newService.DHCPInterfaceIP = ip
+				if !p.config.DisableServiceUpdates {
+					if err := p.updateStatus(newService); err != nil {
+						log.Warn("updating svc", "err", err)
+					}
+				}
+			}
+			log.Debug("IP update channel closed, stopping")
+		}()
+	}
+
+	p.ServiceInstances = append(p.ServiceInstances, newService)
+
+	if !p.config.DisableServiceUpdates {
+		log.Debug("[service] update", "namespace", newService.ServiceSnapshot.Namespace, "name", newService.ServiceSnapshot.Name)
+		if err := p.updateStatus(newService); err != nil {
+			log.Error("[service] updating status", "namespace", newService.ServiceSnapshot.Namespace, "name", newService.ServiceSnapshot.Name, "err", err)
+		}
+	}
+
+	serviceIPs := instance.FetchServiceAddresses(svc)
+	// Check if we need to flush any conntrack connections (due to some dangling conntrack connections)
+	if svc.Annotations[kubevip.FlushContrack] == "true" {
+
+		log.Debug("[service] Flushing conntrack rules", "service", svc.Name, "namespace", svc.Namespace)
+		for _, serviceIP := range serviceIPs {
+			err = vip.DeleteExistingSessions(serviceIP, false, svc.Annotations[kubevip.EgressDestinationPorts], svc.Annotations[kubevip.EgressSourcePorts])
+			if err != nil {
+				log.Error("[service] flushing any remaining egress connections", "service", svc.Name, "namespace", svc.Namespace, "err", err)
+			}
+			err = vip.DeleteExistingSessions(serviceIP, true, svc.Annotations[kubevip.EgressDestinationPorts], svc.Annotations[kubevip.EgressSourcePorts])
+			if err != nil {
+				log.Error("[service] flushing any remaining ingress connections", "service", svc.Name, "namespace", svc.Namespace, "err", err)
+			}
+		}
+	}
+
+	// Check if egress is enabled on the service, if so we'll need to configure some rules
+	if svc.Annotations[kubevip.Egress] == "true" && len(serviceIPs) > 0 {
+		log.Debug("[service] enabling egress", "service", svc.Name, "namespace", svc.Namespace)
+		// If we'er not using NFtables, then ensure that the correct iptables modules are loaded
+		if p.config.EgressWithNftables {
+			// Ensure that kernel modules are loaded and report back missing modules.
+			err = p.nftablesCheck()
+			if err != nil {
+				log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
+			}
+		} else {
+			// Ensure that kernel modules are loaded and report back missing modules.
+			err = p.iptablesCheck()
+			if err != nil {
+				log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
+			}
+		}
+		var podIP string
+		errList := []error{}
+
+		// Should egress be IPv6
+		if svc.Annotations[kubevip.EgressIPv6] == "true" {
+			// Does the service have an active IPv6 endpoint
+			if svc.Annotations[kubevip.ActiveEndpointIPv6] != "" {
+				for _, serviceIP := range serviceIPs {
+					if !p.config.EnableEndpoints && vip.IsIPv6(serviceIP) {
+
+						podIP = svc.Annotations[kubevip.ActiveEndpointIPv6]
+
+						err = p.configureEgress(serviceIP, podIP, svc.Namespace, string(svc.UID), svc.Annotations)
+						if err != nil {
+							errList = append(errList, err)
+							log.Error("[service] configuring egress IPv6", "service", svc.Name, "namespace", svc.Namespace, "err", err)
+						}
+					}
+				}
+			}
+		} else if svc.Annotations[kubevip.ActiveEndpoint] != "" { // Not expected to be IPv6, so should be an IPv4 address
+			for _, serviceIP := range serviceIPs {
+				podIPs := svc.Annotations[kubevip.ActiveEndpoint]
+				if !p.config.EnableEndpoints && vip.IsIPv6(serviceIP) {
+					podIPs = svc.Annotations[kubevip.ActiveEndpointIPv6]
+				}
+				err = p.configureEgress(serviceIP, podIPs, svc.Namespace, string(svc.UID), svc.Annotations)
+				if err != nil {
+					errList = append(errList, err)
+					log.Error("[service] configuring egress IPv4", "service", svc.Name, "namespace", svc.Namespace, "err", err)
+				}
+			}
+		}
+		if len(errList) == 0 {
+			var provider providers.Provider
+			if p.config.EnableEndpoints {
+				provider = providers.NewEndpoints()
+			} else {
+				provider = providers.NewEndpointslices()
+			}
+			err = provider.UpdateServiceAnnotation(svc.Annotations[kubevip.ActiveEndpoint], svc.Annotations[kubevip.ActiveEndpointIPv6], svc, p.clientSet)
+			if err != nil {
+				log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
+			}
+		}
+	}
+
+	finishTime := time.Since(startTime)
+	log.Info("[service]", "service", svc.Name, "namespace", svc.Namespace, "synchronised in", fmt.Sprintf("%dms", finishTime.Milliseconds()))
+
+	return nil
+}
+
+func (p *Processor) deleteService(uid types.UID) error {
+	// protect multiple calls
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
+
+	var updatedInstances []*instance.Instance
+	var serviceInstance *instance.Instance
+	found := false
+	for x := range p.ServiceInstances {
+		log.Debug("[service] lookup", "target UID", uid, "found UID ", p.ServiceInstances[x].ServiceSnapshot.UID, "name", p.ServiceInstances[x].ServiceSnapshot.Name, "namespace", p.ServiceInstances[x].ServiceSnapshot.Namespace)
+		// Add the running services to the new array
+		if p.ServiceInstances[x].ServiceSnapshot.UID != uid {
+			updatedInstances = append(updatedInstances, p.ServiceInstances[x])
+		} else {
+			// Flip the found when we match
+			found = true
+			serviceInstance = p.ServiceInstances[x]
+		}
+	}
+	// If we've been through all services and not found the correct one then error
+	if !found {
+		// TODO: - fix UX
+		// return fmt.Errorf("unable to find/stop service [%s]", uid)
+		return nil
+	}
+
+	for _, c := range serviceInstance.Clusters {
+		for n := range c.Network {
+			c.Network[n].SetHasEndpoints(false)
+		}
+	}
+
+	// Determine if this this VIP is shared with other loadbalancers
+	shared := false
+	vipSet := make(map[string]interface{})
+	for x := range updatedInstances {
+		for _, vip := range instance.FetchServiceAddresses(updatedInstances[x].ServiceSnapshot) { //updatedInstances[x].ServiceSnapshot.Spec.LoadBalancerIP {
+			vipSet[vip] = nil
+		}
+	}
+	for _, vip := range instance.FetchServiceAddresses(serviceInstance.ServiceSnapshot) {
+		if _, found := vipSet[vip]; found {
+			shared = true
+		}
+	}
+	if !shared {
+		for x := range serviceInstance.Clusters {
+			serviceInstance.Clusters[x].Stop()
+		}
+		if serviceInstance.IsDHCP {
+			serviceInstance.DHCPClient.Stop()
+			macvlan, err := netlink.LinkByName(serviceInstance.DHCPInterface)
+			if err != nil {
+				return fmt.Errorf("[service] error finding VIP Interface: %v", err)
+			}
+
+			err = netlink.LinkDel(macvlan)
+			if err != nil {
+				return fmt.Errorf("[service] error deleting DHCP Link : %v", err)
+			}
+		}
+		for i := range serviceInstance.VIPConfigs {
+			if serviceInstance.VIPConfigs[i].EnableBGP {
+				endpoints.ClearBGPHostsByInstance(serviceInstance, p.bgpServer)
+			}
+		}
+
+		// We will need to tear down the egress
+		if serviceInstance.ServiceSnapshot.Annotations[kubevip.Egress] == "true" {
+			if serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint] != "" {
+				log.Info("[service] egress re-write enabled", "service", serviceInstance.ServiceSnapshot.Name)
+				err := egress.Teardown(serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint], serviceInstance.ServiceSnapshot.Spec.LoadBalancerIP, serviceInstance.ServiceSnapshot.Namespace, string(serviceInstance.ServiceSnapshot.UID), serviceInstance.ServiceSnapshot.Annotations, p.config.EgressWithNftables)
+				if err != nil {
+					log.Error("[service] egress teardown", "err", err)
+				}
+			}
+		}
+	}
+
+	// Update the service array
+	p.ServiceInstances = updatedInstances
+
+	log.Info("Removed instance from manager", "uid", uid, "remaining advertised services", len(p.ServiceInstances))
+
+	return nil
+}
+
+// Set up UPNP forwards for a service
+// We first try to use the more modern Pinhole API introduced in UPNPv2 and fall back to UPNPv2 Port Forwarding if no forward was successful
+func (p *Processor) upnpMap(ctx context.Context, s *instance.Instance) {
+	if !isUPNPEnabled(s.ServiceSnapshot) {
+		// Skip services missing the annotation
+		return
+	}
+	if !p.config.EnableUPNP {
+		log.Warn("[UPNP] Found kube-vip.io/forwardUPNP on service while UPNP forwarding is disabled in the kube-vip config. Not forwarding", "service", s.ServiceSnapshot.Name)
+		return
+	}
+	// If upnp is enabled then update the gateway/router with the address
+	// TODO - check if this implementation for dualstack is correct
+
+	gateways := upnp.GetGatewayClients(ctx)
+
+	// Reset Gateway IPs to remove stale addresses
+	s.UPNPGatewayIPs = make([]string, 0)
+
+	for _, vip := range instance.FetchServiceAddresses(s.ServiceSnapshot) {
+		for _, port := range s.ServiceSnapshot.Spec.Ports {
+			for _, gw := range gateways {
+				log.Info("[UPNP] Adding map", "vip", vip, "port", port.Port, "service", s.ServiceSnapshot.Name, "gateway", gw.WANIPv6FirewallControlClient.Location)
+
+				forwardSucessful := false
+				if gw.WANIPv6FirewallControlClient != nil {
+					pinholeID, pinholeErr := gw.WANIPv6FirewallControlClient.AddPinholeCtx(ctx, "0.0.0.0", uint16(port.Port), vip, uint16(port.Port), upnp.MapProtocolToIANA(string(port.Protocol)), 3600) //nolint  TODO
+					if pinholeErr == nil {
+						forwardSucessful = true
+						log.Info("[UPNP] Service should be accessible externally", "port", port.Port, "pinhold ID", pinholeID)
+					} else {
+						//TODO: Cleanup
+						log.Error("[UPNP] Unable to map port to gateway using Pinhole API", "err", pinholeErr.Error())
+					}
+				}
+				// Fallback to PortForward
+				if !forwardSucessful {
+					portMappingErr := gw.ConnectionClient.AddPortMapping("0.0.0.0", uint16(port.Port), strings.ToUpper(string(port.Protocol)), uint16(port.Port), vip, true, s.ServiceSnapshot.Name, 3600) //nolint  TODO
+					if portMappingErr == nil {
+						log.Info("[UPNP] Service should be accessible externally", "port", port.Port)
+						forwardSucessful = true
+					} else {
+						//TODO: Cleanup
+						log.Error("[UPNP] Unable to map port to gateway using PortForward API", "err", portMappingErr.Error())
+					}
+				}
+
+				if forwardSucessful {
+					ip, err := gw.ConnectionClient.GetExternalIPAddress()
+					if err == nil {
+						s.UPNPGatewayIPs = append(s.UPNPGatewayIPs, ip)
+					}
+				}
+			}
+		}
+	}
+
+	// Remove duplicate IPs
+	slices.Sort(s.UPNPGatewayIPs)
+	s.UPNPGatewayIPs = slices.Compact(s.UPNPGatewayIPs)
+}
+
+func (p *Processor) updateStatus(i *instance.Instance) error {
+	// let's retry status update every 10ms for 30s
+	retryConfig := wait.Backoff{
+		Steps:    3000,
+		Duration: 10 * time.Millisecond,
+		Factor:   0,
+		Jitter:   0.1,
+	}
+	// will retry for every error encountered, TODO: should a list of errors that will trigger retry be specified?
+	err := retry.OnError(retryConfig, func(error) bool { return true }, func() error {
+		// Retrieve the latest version of Deployment before attempting update
+		// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
+		currentService, err := p.clientSet.CoreV1().Services(i.ServiceSnapshot.Namespace).Get(context.TODO(), i.ServiceSnapshot.Name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		currentServiceCopy := currentService.DeepCopy()
+		if currentServiceCopy.Annotations == nil {
+			currentServiceCopy.Annotations = make(map[string]string)
+		}
+
+		// If we're using ARP then we can only broadcast the VIP from one place, add an annotation to the service
+		if p.config.EnableARP {
+			// Add the current host
+			currentServiceCopy.Annotations[kubevip.VipHost] = p.config.NodeName
+		}
+		if i.DHCPInterfaceHwaddr != "" || i.DHCPInterfaceIP != "" {
+			currentServiceCopy.Annotations[kubevip.HwAddrKey] = i.DHCPInterfaceHwaddr
+			currentServiceCopy.Annotations[kubevip.RequestedIP] = i.DHCPInterfaceIP
+		}
+
+		if currentService.Annotations["development.kube-vip.io/synthetic-api-server-error-on-update"] == "true" {
+			log.Error("(Synthetic error ) updating Spec", "service", i.ServiceSnapshot.Name, "err", err)
+			return fmt.Errorf("(Synthetic) simulating api server errors")
+		}
+
+		if !cmp.Equal(currentService, currentServiceCopy) {
+			currentService, err = p.clientSet.CoreV1().Services(currentServiceCopy.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
+			if err != nil {
+				log.Error("updating Spec", "service", i.ServiceSnapshot.Name, "err", err)
+				return err
+			}
+		}
+
+		ports := make([]v1.PortStatus, 0, len(i.ServiceSnapshot.Spec.Ports))
+		for _, port := range i.ServiceSnapshot.Spec.Ports {
+			ports = append(ports, v1.PortStatus{
+				Port:     port.Port,
+				Protocol: port.Protocol,
+			})
+		}
+
+		ingresses := []v1.LoadBalancerIngress{}
+
+		for _, c := range i.VIPConfigs {
+			if !vip.IsIP(c.VIP) {
+				ips, err := vip.LookupHost(c.VIP, p.config.DNSMode)
+				if err != nil {
+					return err
+				}
+				for _, ip := range ips {
+					i := v1.LoadBalancerIngress{
+						IP:    ip,
+						Ports: ports,
+					}
+					ingresses = append(ingresses, i)
+				}
+			} else {
+				i := v1.LoadBalancerIngress{
+					IP:    c.VIP,
+					Ports: ports,
+				}
+				ingresses = append(ingresses, i)
+			}
+			if isUPNPEnabled(currentService) {
+				for _, ip := range i.UPNPGatewayIPs {
+					i := v1.LoadBalancerIngress{
+						IP:    ip,
+						Ports: ports,
+					}
+					ingresses = append(ingresses, i)
+				}
+			}
+		}
+		if !cmp.Equal(currentService.Status.LoadBalancer.Ingress, ingresses) {
+			currentService.Status.LoadBalancer.Ingress = ingresses
+			_, err = p.clientSet.CoreV1().Services(currentService.Namespace).UpdateStatus(context.TODO(), currentService, metav1.UpdateOptions{})
+			if err != nil {
+				log.Error("updating Service", "namespace", i.ServiceSnapshot.Namespace, "name", i.ServiceSnapshot.Name, "err", err)
+				return err
+			}
+		}
+		return nil
+	})
+
+	return err
+}
+
+func isUPNPEnabled(s *v1.Service) bool {
+	return metav1.HasAnnotation(s.ObjectMeta, kubevip.UpnpEnabled) && s.Annotations[kubevip.UpnpEnabled] == "true"
+}
+
+// Refresh UPNP Port Forwards for all Service Instances registered in the processor
+func (p *Processor) RefreshUPNPForwards() {
+	log.Info("Starting UPNP Port Refresher")
+	for {
+		time.Sleep(300 * time.Second)
+
+		log.Info("[UPNP] Refreshing Instances", "number of instances", len(p.ServiceInstances))
+		for i := range p.ServiceInstances {
+			p.upnpMap(context.TODO(), p.ServiceInstances[i])
+			if err := p.updateStatus(p.ServiceInstances[i]); err != nil {
+				log.Warn("[UPNP] Error updating service", "ip", p.ServiceInstances[i].ServiceSnapshot.Name, "err", err)
+			}
+		}
+	}
+}

pkg/services/watch_endpoints.go

@@ -0,0 +1,94 @@
+package services
+
+import (
+	"context"
+	"fmt"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/endpoints"
+	"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/watch"
+)
+
+func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, service *v1.Service, provider providers.Provider) error {
+	log.Info("watching", "provider", provider.GetLabel(), "service_name", service.Name, "namespace", service.Namespace)
+	// Use a restartable watcher, as this should help in the event of etcd or timeout issues
+
+	leaderCtx, cancel := context.WithCancel(svcCtx.Ctx)
+	defer cancel()
+
+	var leaderElectionActive bool
+
+	rw, err := provider.CreateRetryWatcher(leaderCtx, p.rwClientSet, service)
+	if err != nil {
+		return fmt.Errorf("[%s] error watching endpoints: %w", provider.GetLabel(), err)
+	}
+
+	exitFunction := make(chan struct{})
+	go func() {
+		select {
+		case <-svcCtx.Ctx.Done():
+			log.Debug("context cancelled", "provider", provider.GetLabel())
+			// Stop the retry watcher
+			rw.Stop()
+			// Cancel the context, which will in turn cancel the leadership
+			cancel()
+			return
+		case <-p.shutdownChan:
+			log.Debug("shutdown called", "provider", provider.GetLabel())
+			// Stop the retry watcher
+			rw.Stop()
+			// Cancel the context, which will in turn cancel the leadership
+			cancel()
+			return
+		case <-exitFunction:
+			log.Debug("function ending", "provider", provider.GetLabel())
+			// Stop the retry watcher
+			rw.Stop()
+			// Cancel the context, which will in turn cancel the leadership
+			cancel()
+			return
+		}
+	}()
+
+	ch := rw.ResultChan()
+
+	epProcessor := endpoints.NewEndpointProcessor(p.config, provider, p.bgpServer, &p.ServiceInstances)
+
+	var lastKnownGoodEndpoint string
+	for event := range ch {
+		// We need to inspect the event and get ResourceVersion out of it
+		switch event.Type {
+
+		case watch.Added, watch.Modified:
+			restart, err := epProcessor.AddOrModify(svcCtx, event, &lastKnownGoodEndpoint, service, id, &leaderElectionActive, p.StartServicesLeaderElection, &leaderCtx, &cancel)
+			if restart {
+				continue
+			} else if err != nil {
+				return fmt.Errorf("[%s] error while processing add/modify event: %w", provider.GetLabel(), err)
+			}
+
+		case watch.Deleted:
+			if err := epProcessor.Delete(service, id); err != nil {
+				return fmt.Errorf("[%s] error while processing delete event: %w", provider.GetLabel(), err)
+			}
+
+			// Close the goroutine that will end the retry watcher, then exit the endpoint watcher function
+			close(exitFunction)
+			log.Info("stopping watching", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
+
+			return nil
+		case watch.Error:
+			errObject := apierrors.FromObject(event.Object)
+			statusErr, _ := errObject.(*apierrors.StatusError)
+			log.Error("watch error", "provider", provider.GetLabel(), "err", statusErr)
+		}
+	}
+	close(exitFunction)
+	log.Info("stopping watching", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
+	return nil //nolint:govet
+}

pkg/services/watch_services.go

@@ -0,0 +1,183 @@
+package services
+
+import (
+	"context"
+	"fmt"
+
+	log "log/slog"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/trafficmirror"
+	"github.com/prometheus/client_golang/prometheus"
+	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/tools/cache"
+	watchtools "k8s.io/client-go/tools/watch"
+)
+
+// This function handles the watching of a services endpoints and updates a load balancers endpoint configurations accordingly
+func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc func(context.Context, *v1.Service) error) error {
+	// first start port mirroring if enabled
+	if err := p.startTrafficMirroringIfEnabled(); err != nil {
+		return err
+	}
+	defer func() {
+		// clean up traffic mirror related config
+		err := p.stopTrafficMirroringIfEnabled()
+		if err != nil {
+			log.Error("Stopping traffic mirroring", "err", err)
+		}
+	}()
+
+	if p.config.ServiceNamespace == "" {
+		// v1.NamespaceAll is actually "", but we'll stay with the const in case things change upstream
+		p.config.ServiceNamespace = v1.NamespaceAll
+		log.Info("(svcs) starting services watcher for all namespaces")
+	} else {
+		log.Info("(svcs) starting services watcher", "namespace", p.config.ServiceNamespace)
+	}
+
+	// Use a restartable watcher, as this should help in the event of etcd or timeout issues
+	rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
+		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
+			return p.rwClientSet.CoreV1().Services(p.config.ServiceNamespace).Watch(ctx, metav1.ListOptions{})
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("error creating services watcher: %s", err.Error())
+	}
+	exitFunction := make(chan struct{})
+	go func() {
+		select {
+		case <-p.shutdownChan:
+			log.Debug("(svcs) shutdown called")
+			// Stop the retry watcher
+			rw.Stop()
+			return
+		case <-exitFunction:
+			log.Debug("(svcs) function ending")
+			// Stop the retry watcher
+			rw.Stop()
+			return
+		}
+	}()
+	ch := rw.ResultChan()
+
+	// Used for tracking an active endpoint / pod
+	for event := range ch {
+		p.CountServiceWatchEvent.With(prometheus.Labels{"type": string(event.Type)}).Add(1)
+
+		// We need to inspect the event and get ResourceVersion out of it
+		switch event.Type {
+		case watch.Added, watch.Modified:
+			restart, err := p.AddOrModify(ctx, event, serviceFunc)
+			if restart {
+				break
+			}
+			if err != nil {
+				return fmt.Errorf("add/modify service error: %w", err)
+			}
+		case watch.Deleted:
+			restart, err := p.Delete(event)
+			if restart {
+				break
+			}
+			if err != nil {
+				return fmt.Errorf("delete service error: %w", err)
+			}
+		case watch.Bookmark:
+			// Un-used
+		case watch.Error:
+			log.Error("Error attempting to watch Kubernetes services")
+
+			// This round trip allows us to handle unstructured status
+			errObject := apierrors.FromObject(event.Object)
+			statusErr, ok := errObject.(*apierrors.StatusError)
+			if !ok {
+				log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
+			}
+
+			status := statusErr.ErrStatus
+			log.Error("services", "err", status)
+		default:
+		}
+	}
+	close(exitFunction)
+	log.Warn("Stopping watching services for type: LoadBalancer in all namespaces")
+	return nil
+}
+
+func lbClassFilterLegacy(svc *v1.Service, config *kubevip.Config) bool {
+	if svc == nil {
+		log.Info("(svcs) service is nil, ignoring")
+		return true
+	}
+	if svc.Spec.LoadBalancerClass != nil {
+		// if this isn't nil then it has been configured, check if it the kube-vip loadBalancer class
+		if *svc.Spec.LoadBalancerClass != config.LoadBalancerClassName {
+			log.Info("(svcs) specified the wrong loadBalancer class", "service name", svc.Name, "lbClass", *svc.Spec.LoadBalancerClass)
+			return true
+		}
+	} else if config.LoadBalancerClassOnly {
+		// if kube-vip is configured to only recognize services with kube-vip's lb class, then ignore the services without any lb class
+		log.Info("(svcs) kube-vip configured to only recognize services with kube-vip's lb class but the service didn't specify any loadBalancer class, ignoring", "service name", svc.Name)
+		return true
+	}
+	return false
+}
+
+func lbClassFilter(svc *v1.Service, config *kubevip.Config) bool {
+	if svc == nil {
+		log.Info("(svcs) service is nil, ignoring")
+		return true
+	}
+	if svc.Spec.LoadBalancerClass == nil && config.LoadBalancerClassName != "" {
+		log.Info("(svcs) no loadBalancer class, ignoring", "service name", svc.Name, "expected lbClass", config.LoadBalancerClassName)
+		return true
+	}
+	if svc.Spec.LoadBalancerClass == nil && config.LoadBalancerClassName == "" {
+		return false
+	}
+	if *svc.Spec.LoadBalancerClass != config.LoadBalancerClassName {
+		log.Info("(svcs) specified wrong loadBalancer class, ignoring", "service name", svc.Name, "wrong lbClass", *svc.Spec.LoadBalancerClass, "expected lbClass", config.LoadBalancerClassName)
+		return true
+	}
+	return false
+}
+
+func (p *Processor) serviceInterface() string {
+	svcIf := p.config.Interface
+	if p.config.ServicesInterface != "" {
+		svcIf = p.config.ServicesInterface
+	}
+	return svcIf
+}
+
+func (p *Processor) startTrafficMirroringIfEnabled() error {
+	if p.config.MirrorDestInterface != "" {
+		svcIf := p.serviceInterface()
+		log.Info("mirroring traffic", "src", svcIf, "dest", p.config.MirrorDestInterface)
+		if err := trafficmirror.MirrorTrafficFromNIC(svcIf, p.config.MirrorDestInterface); err != nil {
+			return err
+		}
+	} else {
+		log.Debug("skip starting traffic mirroring since it's not enabled.")
+	}
+	return nil
+}
+
+func (p *Processor) stopTrafficMirroringIfEnabled() error {
+	if p.config.MirrorDestInterface != "" {
+		svcIf := p.serviceInterface()
+		log.Info("clean up qdisc config", "interface", svcIf)
+		if err := trafficmirror.CleanupQDSICFromNIC(svcIf); err != nil {
+			return err
+		}
+	} else {
+		log.Debug("skip stopping traffic mirroring since it's not enabled.")
+	}
+	return nil
+}

pkg/utils/utils.go

@@ -1,6 +1,10 @@
 package utils
 
-import "os"
+import (
+	"fmt"
+	"net"
+	"os"
+)
 
 func FileExists(filename string) bool {
 	info, err := os.Stat(filename)
@@ -9,3 +13,15 @@ func FileExists(filename string) bool {
 	}
 	return !info.IsDir()
 }
+
+// FormatIPWithSubnetMask takes a raw IP address and a subnet mask, and returns a formatted string in CIDR notation.
+func FormatIPWithSubnetMask(rawIP string, subnetMask string) (string, error) {
+
+	addr := fmt.Sprintf("%s/%s", rawIP, subnetMask)
+	// Check if the input is valid
+	_, _, err := net.ParseCIDR(addr)
+	if err != nil {
+		return "", fmt.Errorf("invalid CIDR: %q, %w", addr, err)
+	}
+	return addr, nil
+}

pkg/vip/address.go

@@ -2,8 +2,9 @@ package vip
 
 import (
 	"fmt"
+	"math"
 	"net"
-	"os"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -17,6 +18,8 @@ import (
 	v1 "k8s.io/api/core/v1"
 
 	"github.com/kube-vip/kube-vip/pkg/iptables"
+	"github.com/kube-vip/kube-vip/pkg/networkinterface"
+	"github.com/kube-vip/kube-vip/pkg/utils"
 )
 
 const (
@@ -28,13 +31,14 @@ const (
 
 // Network is an interface that enable managing operations for a given IP
 type Network interface {
-	AddIP(precheck bool) error
+	AddIP(precheck bool) (bool, error)
 	AddRoute(precheck bool) error
-	DeleteIP() error
+	DeleteIP() (bool, error)
 	DeleteRoute() error
 	UpdateRoutes() (bool, error)
 	IsSet() (bool, error)
 	IP() string
+	CIDR() string
 	IPisLinkLocal() bool
 	PrepareRoute() *netlink.Route
 	SetIP(ip string) error
@@ -46,6 +50,9 @@ type Network interface {
 	DDNSHostName() string
 	DNSName() string
 	SetMask(mask string) error
+	SetHasEndpoints(value bool)
+	HasEndpoints() bool
+	ARPName() string
 }
 
 // network - This allows network configuration
@@ -53,9 +60,10 @@ type network struct {
 	mu sync.Mutex
 
 	address        *netlink.Addr
-	link           netlink.Link
+	link           *networkinterface.Link
 	ports          []v1.ServicePort
 	serviceName    string
+	enableSecurity bool
 	ignoreSecurity bool
 
 	dnsName string
@@ -69,18 +77,14 @@ type network struct {
 	routingProtocol  int
 
 	ipvsEnabled bool
-}
 
-func netlinkParse(addr string) (*netlink.Addr, error) {
-	mask, err := GetFullMask(addr)
-	if err != nil {
-		return nil, err
-	}
-	return netlink.ParseAddr(addr + mask)
+	hasEndpoints bool
 }
 
 // NewConfig will attempt to provide an interface to the kernel network configuration
-func NewConfig(address string, iface string, loGlobalScope bool, subnet string, isDDNS bool, tableID int, tableType int, routingProtocol int, dnsMode, forwardMethod, iptablesBackend string, ipvsEnabled bool) ([]Network, error) {
+func NewConfig(address string, iface string, loGlobalScope bool, subnet string, isDDNS bool, tableID int, tableType int,
+	routingProtocol int, dnsMode, forwardMethod, iptablesBackend string,
+	ipvsEnabled, enableSecurity bool, intfMgr *networkinterface.Manager) ([]Network, error) {
 	networks := []Network{}
 
 	link, err := netlink.LinkByName(iface)
@@ -88,9 +92,11 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
 		return networks, errors.Wrapf(err, "could not get link for interface '%s'", iface)
 	}
 
+	networkLink := intfMgr.Get(link)
+
 	if IsIP(address) {
 		result := &network{
-			link:             link,
+			link:             networkLink,
 			routeTable:       tableID,
 			routingTableType: tableType,
 			routingProtocol:  routingProtocol,
@@ -99,19 +105,27 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
 			ipvsEnabled:      ipvsEnabled,
 		}
 
-		// Check if the subnet needs overriding
-		if subnet != "" {
-			result.address, err = netlink.ParseAddr(address + subnet)
-			if err != nil {
-				return networks, errors.Wrapf(err, "could not parse address '%s'", address)
-			}
-		} else {
-			result.address, err = netlinkParse(address)
-			if err != nil {
-				return networks, errors.Wrapf(err, "could not parse address '%s'", address)
-			}
+		subnet, err = SelectSubnet(address, subnet)
+		if err != nil {
+			return networks, fmt.Errorf("unable to select subnet for IP %q from %q: %w", address, subnet, err)
 		}
 
+		// Check if the subnet needs overriding
+		cidr, err := utils.FormatIPWithSubnetMask(address, subnet)
+		if err != nil {
+			return networks, errors.Wrapf(err, "could not format address '%s' with subnetMask '%s'", address, subnet)
+		}
+		result.address, err = netlink.ParseAddr(cidr)
+		if err != nil {
+			return networks, errors.Wrapf(err, "could not parse address '%s'", address)
+		}
+
+		// set address as deprecated so it isn't used as source address according to RFC 3484
+		result.address.PreferedLft = 0
+
+		// Also set ValidLft so the netlink library actually sets them
+		result.address.ValidLft = math.MaxInt
+
 		if iface == "lo" && !loGlobalScope {
 			// set host scope on loopback, otherwise global scope will be used by default
 			result.address.Scope = unix.RT_SCOPE_HOST
@@ -126,7 +140,7 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
 			// when leader starts, should do get IP from DHCP for the domain
 			if isDDNS {
 				result := &network{
-					link:             link,
+					link:             networkLink,
 					routeTable:       tableID,
 					routingTableType: tableType,
 					routingProtocol:  routingProtocol,
@@ -134,6 +148,8 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
 					iptablesBackend:  iptablesBackend,
 					isDDNS:           isDDNS,
 					dnsName:          address,
+					ipvsEnabled:      ipvsEnabled,
+					enableSecurity:   enableSecurity,
 				}
 
 				networks = append(networks, result)
@@ -144,7 +160,7 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
 
 		for _, ip := range ips {
 			result := &network{
-				link:             link,
+				link:             networkLink,
 				routeTable:       tableID,
 				routingTableType: tableType,
 				routingProtocol:  routingProtocol,
@@ -152,15 +168,21 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
 				iptablesBackend:  iptablesBackend,
 				isDDNS:           isDDNS,
 				dnsName:          address,
+				ipvsEnabled:      ipvsEnabled,
+				enableSecurity:   enableSecurity,
 			}
 
 			// we're able to resolve store this as the initial IP
-			if result.address, err = netlinkParse(ip); err != nil {
+
+			if result.address, err = netlink.ParseAddr(fmt.Sprintf("%s/%s", ip, subnet)); err != nil {
 				return networks, err
 			}
 			// set ValidLft so that the VIP expires if the DNS entry is updated, otherwise it'll be refreshed by the DNS prober
 			result.address.ValidLft = defaultValidLft
 
+			// set address as deprecated so it isn't used as source address according to RFC 3484
+			result.address.PreferedLft = 0
+
 			networks = append(networks, result)
 		}
 
@@ -203,7 +225,7 @@ func (configurator *network) PrepareRoute() *netlink.Route {
 	route := &netlink.Route{
 		Scope:     routeScope,
 		Dst:       configurator.address.IPNet,
-		LinkIndex: configurator.link.Attrs().Index,
+		LinkIndex: configurator.link.Intf.Attrs().Index,
 		Table:     configurator.routeTable,
 		Type:      configurator.routingTableType,
 		Protocol:  netlink.RouteProtocol(configurator.routingProtocol),
@@ -213,6 +235,8 @@ func (configurator *network) PrepareRoute() *netlink.Route {
 
 // AddRoute - Add an IP address to a route table
 func (configurator *network) AddRoute(precheck bool) error {
+	configurator.link.Lock.Lock()
+	defer configurator.link.Lock.Unlock()
 	route := configurator.PrepareRoute()
 
 	exists := false
@@ -234,7 +258,7 @@ func (configurator *network) AddRoute(precheck bool) error {
 }
 
 func (configurator *network) routeExists(route *netlink.Route) (bool, error) {
-	routes, err := netlink.RouteList(configurator.link, netlink.FAMILY_ALL)
+	routes, err := netlink.RouteList(configurator.link.Intf, netlink.FAMILY_ALL)
 	if err != nil {
 		return false, errors.Wrap(err, "failed to list routes")
 	}
@@ -284,30 +308,34 @@ func (configurator *network) UpdateRoutes() (bool, error) {
 }
 
 // AddIP - Add an IP address to the interface
-func (configurator *network) AddIP(precheck bool) error {
+func (configurator *network) AddIP(precheck bool) (bool, error) {
+	configurator.link.Lock.Lock()
+	defer configurator.link.Lock.Unlock()
 	exists := false
 	var err error
 	if precheck {
-		if exists, err = configurator.addressExists(); err != nil {
-			return errors.Wrap(err, "could not check if address exists")
+		if exists, err = configurator.IsSet(); err != nil {
+			return false, errors.Wrap(err, "could not check if address exists")
 		}
 	}
 
-	if !exists {
-		if err := netlink.AddrReplace(configurator.link, configurator.address); err != nil {
-			return errors.Wrap(err, "could not add ip")
-		}
-
-		if err := configurator.configureIPTables(); err != nil {
-			return errors.Wrap(err, "could not configure IPTables")
-		}
+	if exists {
+		return false, nil
 	}
 
-	return nil
+	if err := netlink.AddrReplace(configurator.link.Intf, configurator.address); err != nil {
+		return false, errors.Wrap(err, "could not add ip")
+	}
+
+	if err := configurator.configureIPTables(); err != nil {
+		return true, errors.Wrap(err, "could not configure IPTables")
+	}
+
+	return true, nil
 }
 
 func (configurator *network) configureIPTables() error {
-	if os.Getenv("enable_service_security") == "true" && !configurator.ignoreSecurity {
+	if configurator.enableSecurity && !configurator.ignoreSecurity {
 		if err := configurator.addIptablesRulesToLimitTrafficPorts(); err != nil {
 			return errors.Wrap(err, "could not add iptables rules to limit traffic ports")
 		}
@@ -323,21 +351,6 @@ func (configurator *network) configureIPTables() error {
 	return nil
 }
 
-func (configurator *network) addressExists() (bool, error) {
-	addrs, err := netlink.AddrList(configurator.link, netlink.FAMILY_ALL)
-	if err != nil {
-		return false, errors.Wrap(err, "could not list addresses")
-	}
-
-	for _, addr := range addrs {
-		if addr.Equal(*configurator.address) {
-			return true, nil
-		}
-	}
-
-	return false, nil
-}
-
 func (configurator *network) addIptablesRulesToLimitTrafficPorts() error {
 	ipt, err := iptables.New()
 	if err != nil {
@@ -467,34 +480,37 @@ func (configurator *network) removeIptablesRuleToLimitTrafficPorts() error {
 }
 
 // DeleteIP - Remove an IP address from the interface
-func (configurator *network) DeleteIP() error {
+func (configurator *network) DeleteIP() (bool, error) {
+	configurator.link.Lock.Lock()
+	defer configurator.link.Lock.Unlock()
+
 	result, err := configurator.IsSet()
 	if err != nil {
-		return errors.Wrap(err, "ip check in DeleteIP failed")
+		return false, errors.Wrap(err, "ip check in DeleteIP failed")
 	}
 
 	// Nothing to delete
 	if !result {
-		return nil
+		return false, nil
 	}
 
-	if err = netlink.AddrDel(configurator.link, configurator.address); err != nil {
-		return errors.Wrap(err, "could not delete ip")
+	if err = netlink.AddrDel(configurator.link.Intf, configurator.address); err != nil {
+		return false, errors.Wrap(err, "could not delete ip")
 	}
 
-	if os.Getenv("enable_service_security") == "true" && !configurator.ignoreSecurity {
+	if configurator.enableSecurity && !configurator.ignoreSecurity {
 		if err := configurator.removeIptablesRuleToLimitTrafficPorts(); err != nil {
-			return errors.Wrap(err, "could not remove iptables rules to limit traffic ports")
+			return true, errors.Wrap(err, "could not remove iptables rules to limit traffic ports")
 		}
 	}
 
 	if configurator.ipvsEnabled && configurator.forwardMethod == "masquerade" && configurator.address.IP.To4() != nil {
 		if err := configurator.removeIptablesRulesForMasquerade(); err != nil {
-			return errors.Wrap(err, "could not remove iptables masquerade rules ")
+			return true, errors.Wrap(err, "could not remove iptables masquerade rules ")
 		}
 	}
 
-	return nil
+	return true, nil
 }
 
 func (configurator *network) addIptablesRulesForMasquerade() error {
@@ -561,12 +577,15 @@ func delMasqueradeRuleForVIP(ipt *iptables.IPTables, vip, comment string) error
 
 // IsDADFAIL - Returns true if the address is IPv6 and has DADFAILED flag
 func (configurator *network) IsDADFAIL() bool {
+	configurator.link.Lock.Lock()
+	defer configurator.link.Lock.Unlock()
+
 	if configurator.address == nil || !IsIPv6(configurator.address.IP.String()) {
 		return false
 	}
 
 	// Get all the address
-	addresses, err := netlink.AddrList(configurator.link, netlink.FAMILY_V6)
+	addresses, err := netlink.AddrList(configurator.link.Intf, netlink.FAMILY_V6)
 	if err != nil {
 		return false
 	}
@@ -585,7 +604,7 @@ func addressHasDADFAILEDFlag(address netlink.Addr) bool {
 	return address.Flags&unix.IFA_F_DADFAILED != 0
 }
 
-// IsSet - Check to see if VIP is set
+// isSet - Check to see if VIP is set
 func (configurator *network) IsSet() (result bool, err error) {
 	var addresses []netlink.Addr
 
@@ -593,11 +612,11 @@ func (configurator *network) IsSet() (result bool, err error) {
 		return false, nil
 	}
 
-	addresses, err = netlink.AddrList(configurator.link, 0)
+	addresses, err = netlink.AddrList(configurator.link.Intf, 0)
 	if err != nil {
 		err = errors.Wrap(err, "could not list addresses")
 
-		return
+		return false, err
 	}
 
 	for _, address := range addresses {
@@ -614,13 +633,30 @@ func (configurator *network) SetIP(ip string) error {
 	configurator.mu.Lock()
 	defer configurator.mu.Unlock()
 
-	addr, err := netlinkParse(ip)
+	configurator.link.Lock.Lock()
+	defer configurator.link.Lock.Unlock()
+
+	if strings.Contains("/", ip) {
+		return fmt.Errorf("ip should not contain CIDR notation got: %s", ip)
+	}
+	ones, _ := configurator.address.Mask.Size()
+	cidr, err := utils.FormatIPWithSubnetMask(ip, strconv.Itoa(ones))
+	if err != nil {
+		return fmt.Errorf("could not format address '%s' with subnetMask '%s'", ip, strconv.Itoa(ones))
+	}
+	addr, err := netlink.ParseAddr(cidr)
 	if err != nil {
 		return err
 	}
 	if configurator.address != nil && configurator.IsDNS() {
 		addr.ValidLft = defaultValidLft
+	} else {
+		addr.ValidLft = math.MaxInt
 	}
+
+	// set address as deprecated so it isn't used as source address according to RFC 3484
+	addr.PreferedLft = 0
+
 	configurator.address = addr
 	return nil
 }
@@ -644,6 +680,13 @@ func (configurator *network) IP() string {
 	return configurator.address.IP.String()
 }
 
+func (configurator *network) CIDR() string {
+	configurator.mu.Lock()
+	defer configurator.mu.Unlock()
+
+	return configurator.address.IPNet.String()
+}
+
 // IP - return the IP Address
 func (configurator *network) IPisLinkLocal() bool {
 	configurator.mu.Lock()
@@ -677,19 +720,23 @@ func (configurator *network) DDNSHostName() string {
 
 // Interface - return the Interface name
 func (configurator *network) Interface() string {
-	return configurator.link.Attrs().Name
+	return configurator.link.Intf.Attrs().Name
 }
 
-func GarbageCollect(adapter, address string) (found bool, err error) {
-
+func GarbageCollect(adapter, address string, intfMgr *networkinterface.Manager) (found bool, err error) {
 	// Get adapter
 	link, err := netlink.LinkByName(adapter)
 	if err != nil {
 		return true, errors.Wrapf(err, "could not get link for interface '%s'", adapter)
 	}
 
+	l := intfMgr.Get(link)
+
+	l.Lock.Lock()
+	defer l.Lock.Unlock()
+
 	// Get addresses on adapter
-	addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL)
+	addrs, err := netlink.AddrList(l.Intf, netlink.FAMILY_ALL)
 	if err != nil {
 		return false, err
 	}
@@ -701,7 +748,7 @@ func GarbageCollect(adapter, address string) (found bool, err error) {
 			found = true
 			// linting issue
 			existing := existing
-			if err = netlink.AddrDel(link, &existing); err != nil {
+			if err = netlink.AddrDel(l.Intf, &existing); err != nil {
 				return true, errors.Wrap(err, "could not delete ip")
 			}
 		}
@@ -710,7 +757,12 @@ func GarbageCollect(adapter, address string) (found bool, err error) {
 }
 
 func (configurator *network) SetMask(mask string) error {
-	m, err := strconv.Atoi(mask)
+	selectedMask, err := SelectSubnet(configurator.IP(), mask)
+	if err != nil {
+		return fmt.Errorf("failed to select mask %q: %w", mask, err)
+	}
+
+	m, err := strconv.Atoi(selectedMask)
 	if err != nil {
 		return err
 	}
@@ -738,3 +790,50 @@ func (configurator *network) SetMask(mask string) error {
 	configurator.address.Mask = toSet
 	return nil
 }
+
+func (configurator *network) SetHasEndpoints(value bool) {
+	log.Debug("setting HasEndpoints", "ip", configurator.IP(), "value", value)
+	configurator.hasEndpoints = value
+}
+
+func (configurator *network) HasEndpoints() bool {
+	log.Debug("getting HasEndpoints", "ip", configurator.IP(), "value", configurator.hasEndpoints)
+	return configurator.hasEndpoints
+}
+
+func (configurator *network) ARPName() string {
+	return fmt.Sprintf("%s-%s", configurator.CIDR(), configurator.Interface())
+}
+
+// SelectSubnet formats an IP address with the appropriate CIDR based on the input.
+// The input SubnetMasks can be "32,128" (dual-stack), "32", "128" (SingleStack).
+func SelectSubnet(rawIP string, subnetMasks string) (string, error) {
+	// Split the SubnetMasks input into DualStack or SingleStack
+	// If the input is "32,128", it will be split into ["32", "128"]
+	subnetMasksParts := strings.Split(subnetMasks, ",")
+	if len(subnetMasksParts) == 0 {
+		return "", fmt.Errorf("no subnetMasks provided got: %q", subnetMasks)
+	} else if len(subnetMasksParts) > 2 {
+		return "", fmt.Errorf("invalid subnetMasks provided got: %q", subnetMasks)
+	}
+	if slices.Contains(subnetMasksParts, "auto") {
+		return "", fmt.Errorf("auto subnet discovery only works for services: %q", subnetMasks)
+	}
+
+	// Parse the raw IP address
+	ip := net.ParseIP(rawIP)
+	if ip == nil {
+		return "", fmt.Errorf("invalid IP address: %s", rawIP)
+	}
+	if ip.To4() != nil {
+		return subnetMasksParts[0], nil
+	}
+	if ip.To16() != nil {
+		subnetMask := subnetMasksParts[0]
+		if len(subnetMasksParts) == 2 {
+			subnetMask = subnetMasksParts[1]
+		}
+		return subnetMask, nil
+	}
+	return "", fmt.Errorf("unable to select subnet mask for: IP %q and masks %q", rawIP, subnetMasks)
+}

pkg/vip/arp.go

@@ -135,9 +135,7 @@ func sendARP(iface *net.Interface, m *arpMessage) error {
 		Halen:    m.hardwareAddressLength,
 	}
 	target := ethernetBroadcast
-	for i := 0; i < len(target); i++ { //nolint
-		ll.Addr[i] = target[i]
-	}
+	copy(ll.Addr[:], target)
 
 	b, err := m.bytes()
 	if err != nil {

pkg/vip/dns.go

@@ -49,7 +49,7 @@ func (d *ipUpdater) Run(ctx context.Context) {
 					log.Error("setting IP", "address", ip, "err", err)
 				}
 
-				if err := d.vip.AddIP(false); err != nil {
+				if _, err := d.vip.AddIP(false); err != nil {
 					log.Error("error adding virtual IP", "err", err)
 				}
 

pkg/vip/util.go

@@ -121,17 +121,6 @@ func IsIPv6CIDR(cidr string) bool {
 	return ip.To4() == nil
 }
 
-// GetFullMask returns /32 for an IPv4 address and /128 for an IPv6 address
-func GetFullMask(address string) (string, error) {
-	if IsIPv4(address) {
-		return "/32", nil
-	}
-	if IsIPv6(address) {
-		return "/128", nil
-	}
-	return "", fmt.Errorf("failed to parse %s as either IPv4 or IPv6", address)
-}
-
 // GetDefaultGatewayInterface return default gateway interface link
 func GetDefaultGatewayInterface() (*net.Interface, error) {
 	routes, err := netlink.RouteList(nil, syscall.AF_INET)

test.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/bash
+
+rm -f logs.txt
+
+for i in $(seq 1 100);
+do
+  echo "RUN $i"
+  GOMAXPROCS=4 make e2e-tests129
+  if [[ "$?" -ne 0 ]]; then 
+    echo "FAILED AT RUN $i"
+    break
+  fi
+done

testing/e2e/bgp/config.toml.tmpl

@@ -0,0 +1,3 @@
+[global.config]
+  as = {{ .AS }}
+  router-id = "1.1.1.1"

testing/e2e/e2e_bgp_test.go

@@ -0,0 +1,683 @@
+//go:build e2e
+// +build e2e
+
+package e2e_test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"text/template"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+	kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
+	"sigs.k8s.io/kind/pkg/log"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	"github.com/kube-vip/kube-vip/testing/e2e"
+	"github.com/kube-vip/kube-vip/testing/services/pkg/deployment"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/client"
+	api "github.com/osrg/gobgp/v3/api"
+)
+
+const (
+	goBGPAS   uint32 = 65500
+	kubevipAS uint32 = 65501
+	goBGPPort uint32 = 50051
+
+	defaultFixedNexthopv6 = "fc00:1000:1000:1000::100"
+	defaultFixedNexthopv4 = "172.18.0.100"
+)
+
+var _ = Describe("kube-vip BGP mode", Ordered, func() {
+	if Mode == ModeBGP {
+		var (
+			logger                     log.Logger
+			imagePath                  string
+			k8sImagePath               string
+			configPath                 string
+			kubeVIPBGPManifestTemplate *template.Template
+			goBGPConfigTemplate        *template.Template
+			tempDirPath                string
+			v129                       bool
+			localIPv4                  string
+			localIPv6                  string
+			curDir                     string
+			networkInterface           string
+
+			bgpKill chan any
+		)
+
+		BeforeAll(func() {
+			klog.SetOutput(GinkgoWriter)
+			logger = e2e.TestLogger{}
+
+			imagePath = os.Getenv("E2E_IMAGE_PATH")    // Path to kube-vip image
+			configPath = os.Getenv("CONFIG_PATH")      // path to the api server config
+			k8sImagePath = os.Getenv("K8S_IMAGE_PATH") // path to the kubernetes image (version for kind)
+			if configPath == "" {
+				configPath = "/etc/kubernetes/admin.conf"
+			}
+			if networkInterface = os.Getenv("NETWORK_INTERFACE"); networkInterface == "" {
+				networkInterface = "br-"
+			}
+
+			_, v129 = os.LookupEnv("V129")
+			var err error
+			curDir, err = os.Getwd()
+			Expect(err).NotTo(HaveOccurred())
+
+			templateBGPPath := filepath.Join(curDir, "kube-vip-bgp.yaml.tmpl")
+			kubeVIPBGPManifestTemplate, err = template.New("kube-vip-bgp.yaml.tmpl").ParseFiles(templateBGPPath)
+			Expect(err).NotTo(HaveOccurred())
+
+			tempDirPath, err = os.MkdirTemp("", "kube-vip-test")
+			Expect(err).NotTo(HaveOccurred())
+			v4addr, _, err := deployment.GetLocalIPv4(networkInterface)
+			Expect(err).ToNot(HaveOccurred())
+			localIPv4 = v4addr.String()
+
+			v6addr, _, err := deployment.GetLocalIPv6(networkInterface)
+			Expect(err).ToNot(HaveOccurred())
+			localIPv6 = v6addr.String()
+
+			goBGPConfig := &e2e.BGPPeerValues{
+				AS: goBGPAS,
+			}
+
+			bgpKill = make(chan any)
+
+			goBGPConfigPath := filepath.Join(filepath.Join(curDir, "bgp"), "config.toml.tmpl")
+			goBGPConfigTemplate, err = template.New("config.toml.tmpl").ParseFiles(goBGPConfigPath)
+			Expect(err).ToNot(HaveOccurred())
+
+			goBGPConfigPath = filepath.Join(tempDirPath, "config.toml")
+
+			f, err := os.OpenFile(goBGPConfigPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
+			Expect(err).ToNot(HaveOccurred())
+			defer f.Close()
+
+			err = goBGPConfigTemplate.Execute(f, goBGPConfig)
+			Expect(err).ToNot(HaveOccurred())
+
+			go startGoBGP(goBGPConfigPath, bgpKill)
+		})
+
+		AfterAll(func() {
+			close(bgpKill)
+		})
+
+		Describe("kube-vip IPv4 services BGP mode functionality", Ordered, func() {
+			var (
+				cpVIP          string
+				clusterName    string
+				client         kubernetes.Interface
+				manifestValues *e2e.KubevipManifestValues
+				gobgpClient    api.GobgpApiClient
+				gobgpPeers     []*e2e.BGPPeerValues
+
+				nodesNumber = 1
+			)
+
+			BeforeAll(func() {
+				setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath,
+					k8sImagePath, e2e.IPv4Family, e2e.IPv4Family, []string{e2e.IPv4Family}, &client, &gobgpPeers, v129,
+					kubeVIPBGPManifestTemplate, &gobgpClient, logger, nodesNumber, "", "bgp-ipv4")
+			})
+
+			AfterAll(func() {
+				for _, p := range gobgpPeers {
+					_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
+						Address: p.IP,
+					})
+					Expect(err).ToNot(HaveOccurred())
+				}
+				cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
+			})
+
+			DescribeTable("advertise IPv4 routes for services",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, "")
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+
+			DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, "")
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+		})
+
+		Describe("kube-vip IPv6 services BGP mode functionality", Ordered, func() {
+			var (
+				cpVIP          string
+				clusterName    string
+				client         kubernetes.Interface
+				manifestValues *e2e.KubevipManifestValues
+				gobgpClient    api.GobgpApiClient
+				gobgpPeers     []*e2e.BGPPeerValues
+
+				nodesNumber = 1
+			)
+
+			BeforeAll(func() {
+				setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath,
+					k8sImagePath, e2e.IPv6Family, e2e.IPv6Family, []string{e2e.IPv6Family}, &client, &gobgpPeers, v129,
+					kubeVIPBGPManifestTemplate, &gobgpClient, logger, nodesNumber, "", "bgp-ipv4")
+			})
+
+			AfterAll(func() {
+				for _, p := range gobgpPeers {
+					_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
+						Address: p.IP,
+					})
+					Expect(err).ToNot(HaveOccurred())
+				}
+				cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
+			})
+
+			DescribeTable("advertise IPv6 routes for services",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, "")
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+
+			DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, "")
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+		})
+
+		Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv6 over IPv4 - fixed nexthop", Ordered, func() {
+			var (
+				cpVIP          string
+				clusterName    string
+				client         kubernetes.Interface
+				manifestValues *e2e.KubevipManifestValues
+				gobgpClient    api.GobgpApiClient
+				gobgpPeers     []*e2e.BGPPeerValues
+
+				nodesNumber = 1
+			)
+
+			BeforeAll(func() {
+				setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
+					e2e.DualstackFamily, e2e.IPv4Family, []string{e2e.IPv4Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
+					logger, nodesNumber, "fixed", "mpbgp-ipv4")
+			})
+
+			AfterAll(func() {
+				for _, p := range gobgpPeers {
+					_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
+						Address: p.IP,
+					})
+					Expect(err).ToNot(HaveOccurred())
+				}
+				cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
+			})
+
+			DescribeTable("advertise IPv6 routes over IPv4 session",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, defaultFixedNexthopv6)
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+
+			DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, defaultFixedNexthopv6)
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+		})
+
+		Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv4 over IPv6 - fixed nexthop", Ordered, func() {
+			var (
+				cpVIP          string
+				clusterName    string
+				client         kubernetes.Interface
+				manifestValues *e2e.KubevipManifestValues
+				gobgpClient    api.GobgpApiClient
+				gobgpPeers     []*e2e.BGPPeerValues
+
+				nodesNumber = 1
+			)
+
+			BeforeAll(func() {
+				setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
+					e2e.DualstackFamilyIPv6, e2e.IPv6Family, []string{e2e.IPv6Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
+					logger, nodesNumber, "fixed", "mpbgp-ipv6")
+			})
+
+			AfterAll(func() {
+				for _, n := range gobgpPeers {
+					_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
+						Address: n.IP,
+					})
+					Expect(err).ToNot(HaveOccurred())
+				}
+				cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
+			})
+
+			DescribeTable("advertise IPv4 routes over IPv6 session",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, defaultFixedNexthopv4)
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+
+			DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, defaultFixedNexthopv4)
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+		})
+
+		Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv6 over IPv4 - auto_sourceif nexthop", Ordered, func() {
+			var (
+				cpVIP          string
+				clusterName    string
+				client         kubernetes.Interface
+				manifestValues *e2e.KubevipManifestValues
+				gobgpClient    api.GobgpApiClient
+				gobgpPeers     []*e2e.BGPPeerValues
+				containerIP    string
+
+				nodesNumber = 1
+			)
+
+			BeforeAll(func() {
+				_, containerIP = setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
+					e2e.DualstackFamily, e2e.IPv4Family, []string{e2e.IPv4Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
+					logger, nodesNumber, "auto_sourceif", "mpbgp-if-ipv4")
+			})
+
+			AfterAll(func() {
+				for _, p := range gobgpPeers {
+					_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
+						Address: p.IP,
+					})
+					Expect(err).ToNot(HaveOccurred())
+				}
+				cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
+			})
+
+			DescribeTable("advertise IPv6 routes over IPv4 session",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, containerIP)
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+
+			DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, containerIP)
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+		})
+
+		Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv4 over IPv6 - fixed nexthop", Ordered, func() {
+			var (
+				cpVIP          string
+				clusterName    string
+				client         kubernetes.Interface
+				manifestValues *e2e.KubevipManifestValues
+				gobgpClient    api.GobgpApiClient
+				gobgpPeers     []*e2e.BGPPeerValues
+				containerIP    string
+
+				nodesNumber = 1
+			)
+
+			BeforeAll(func() {
+				containerIP, _ = setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
+					e2e.DualstackFamilyIPv6, e2e.IPv6Family, []string{e2e.IPv6Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
+					logger, nodesNumber, "auto_sourceif", "mpbgp-if-ipv6")
+			})
+
+			AfterAll(func() {
+				for _, n := range gobgpPeers {
+					_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
+						Address: n.IP,
+					})
+					Expect(err).ToNot(HaveOccurred())
+				}
+				cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
+			})
+
+			DescribeTable("advertise IPv4 routes over IPv6 session",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, containerIP)
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+
+			DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, containerIP)
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+		})
+	}
+})
+
+func testBGP(offset uint, lbFamily string, afiFamily api.Family_Afi, svcFamily []corev1.IPFamily, svcName string,
+	trafficPolicy corev1.ServiceExternalTrafficPolicy, client kubernetes.Interface, numberOfServices int, gobgpClient api.GobgpApiClient, expectedNexthop string) {
+	lbAddress := e2e.GenerateVIP(lbFamily, offset)
+	routeCheckFamily := &api.Family{
+		Afi:  afiFamily,
+		Safi: api.Family_SAFI_UNICAST,
+	}
+	testServiceBGP(svcName, lbAddress, trafficPolicy, client, svcFamily, numberOfServices, gobgpClient, routeCheckFamily, expectedNexthop)
+}
+
+func setupEnv(tempDirPath, cpVIP, clusterName *string, manifestValues *e2e.KubevipManifestValues,
+	localIPv4, localIPv6, imagePath, configPath, k8sImagePath, clusterAddrFamily, bgpClientAddrFamily string, peerAddrFamily []string, client *kubernetes.Interface,
+	gobgpPeers *[]*e2e.BGPPeerValues, v129 bool, kubeVIPBGPManifestTemplate *template.Template, gobgpClient *api.GobgpApiClient,
+	logger log.Logger, nodesNumber int, mpbgpnexthop, clusterNameSuffix string) (string, string) {
+	var err error
+	*tempDirPath, err = os.MkdirTemp("", "kube-vip-test")
+	Expect(err).ToNot(HaveOccurred())
+
+	*cpVIP = e2e.GenerateVIP(clusterAddrFamily, SOffset.Get())
+
+	var clusterIPFamily kindconfigv1alpha4.ClusterIPFamily
+	var podSubnet, serviceSubnet string
+	switch clusterAddrFamily {
+	case e2e.IPv6Family:
+		clusterIPFamily = kindconfigv1alpha4.IPv6Family
+	case e2e.DualstackFamily:
+		clusterIPFamily = kindconfigv1alpha4.DualStackFamily
+	case e2e.DualstackFamilyIPv6:
+		clusterIPFamily = kindconfigv1alpha4.DualStackFamily
+		podSubnet = "fd00:10:244::/56,10.244.0.0/16"
+		serviceSubnet = "fd00:10:96::/112,10.96.0.0/16"
+	default:
+		clusterIPFamily = kindconfigv1alpha4.IPv4Family
+	}
+
+	networking := &kindconfigv1alpha4.Networking{
+		IPFamily: clusterIPFamily,
+	}
+
+	if podSubnet != "" && serviceSubnet != "" {
+		networking.PodSubnet = podSubnet
+		networking.ServiceSubnet = serviceSubnet
+	}
+
+	kvPeers := []*e2e.BGPPeerValues{}
+	if slices.Contains(peerAddrFamily, e2e.IPv4Family) {
+		kvPeers = append(kvPeers, &e2e.BGPPeerValues{
+			IP:       localIPv4,
+			AS:       goBGPAS,
+			IPFamily: e2e.IPv4Family,
+		})
+	}
+
+	if slices.Contains(peerAddrFamily, e2e.IPv6Family) {
+		kvPeers = append(kvPeers, &e2e.BGPPeerValues{
+			IP:       localIPv6,
+			AS:       goBGPAS,
+			IPFamily: e2e.IPv6Family,
+		})
+	}
+
+	kvPeersStr := []string{}
+	for _, p := range kvPeers {
+		kvPeersStr = append(kvPeersStr, p.String())
+	}
+
+	manifestValues = &e2e.KubevipManifestValues{
+		ControlPlaneVIP:    *cpVIP,
+		ImagePath:          imagePath,
+		ConfigPath:         configPath,
+		ControlPlaneEnable: "false",
+		SvcEnable:          "true",
+		SvcElectionEnable:  "false",
+		BGPAS:              kubevipAS,
+		BGPPeers:           strings.Join(kvPeersStr, ","),
+		MPBGPNexthop:       mpbgpnexthop,
+		MPBGPNexthopIPv4:   defaultFixedNexthopv4,
+		MPBGPNexthopIPv6:   defaultFixedNexthopv6,
+	}
+
+	By(manifestValues.BGPPeers)
+
+	*clusterName, *client = prepareCluster(*tempDirPath, clusterNameSuffix, k8sImagePath, v129, kubeVIPBGPManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
+
+	container := fmt.Sprintf("%s-control-plane", *clusterName)
+
+	containerIPv4, containerIPv6, err := GetContainerIPs(container)
+	Expect(err).ToNot(HaveOccurred())
+
+	if slices.Contains(peerAddrFamily, e2e.IPv4Family) {
+		*gobgpPeers = append(*gobgpPeers, &e2e.BGPPeerValues{
+			IP:       containerIPv4,
+			AS:       kubevipAS,
+			IPFamily: e2e.IPv4Family,
+		})
+	}
+
+	if slices.Contains(peerAddrFamily, e2e.IPv6Family) {
+		*gobgpPeers = append(*gobgpPeers, &e2e.BGPPeerValues{
+			IP:       containerIPv6,
+			AS:       kubevipAS,
+			IPFamily: e2e.IPv6Family,
+		})
+	}
+
+	if bgpClientAddrFamily == e2e.IPv6Family {
+		*gobgpClient, err = newGoBGPClient(localIPv6, goBGPPort)
+	} else {
+		*gobgpClient, err = newGoBGPClient(localIPv4, goBGPPort)
+	}
+	Expect(err).ToNot(HaveOccurred())
+
+	for _, p := range *gobgpPeers {
+		if slices.Contains(peerAddrFamily, p.IPFamily) {
+			peerCtx := context.TODO()
+			Eventually(peerCtx, func() error {
+				_, err = (*gobgpClient).AddPeer(context.TODO(), &api.AddPeerRequest{
+					Peer: &api.Peer{
+						Conf: &api.PeerConf{
+							NeighborAddress: p.IP,
+							PeerAsn:         uint32(p.AS),
+						},
+						AfiSafis: []*api.AfiSafi{
+							{
+								Config: &api.AfiSafiConfig{
+									Enabled: true,
+									Family: &api.Family{
+										Afi:  api.Family_AFI_IP6,
+										Safi: api.Family_SAFI_UNICAST,
+									},
+								},
+							},
+							{
+								Config: &api.AfiSafiConfig{
+									Enabled: true,
+									Family: &api.Family{
+										Afi:  api.Family_AFI_IP,
+										Safi: api.Family_SAFI_UNICAST,
+									},
+								},
+							},
+						},
+					},
+				})
+				return err
+			}, "120s", "100ms").Should(Succeed())
+		}
+	}
+
+	return containerIPv4, containerIPv6
+}
+
+func testServiceBGP(svcName, lbAddress string, trafficPolicy corev1.ServiceExternalTrafficPolicy,
+	client kubernetes.Interface, serviceAddrFamily []corev1.IPFamily, numberOfServices int,
+	gobgpClient api.GobgpApiClient, gobgpFamily *api.Family, expectedNexthop string) {
+	lbAddresses := vip.Split(lbAddress)
+
+	services := []string{}
+	for i := range numberOfServices {
+		services = append(services, fmt.Sprintf("%s-%d", svcName, i))
+	}
+
+	for _, svc := range services {
+		createTestService(svc, dsNamespace, dsName, lbAddress,
+			client, corev1.IPFamilyPolicyPreferDualStack, serviceAddrFamily, trafficPolicy)
+	}
+
+	for _, addr := range lbAddresses {
+		paths := checkGoBGPPaths(context.Background(), gobgpClient, gobgpFamily, []*api.TableLookupPrefix{{Prefix: addr}}, 1)
+		Expect(strings.Contains(paths[0].Prefix, lbAddress)).To(BeTrue())
+		if expectedNexthop != "" {
+			Expect(strings.Contains(paths[0].String(), fmt.Sprintf("next_hop:\"%s\"", expectedNexthop)) || strings.Contains(paths[0].String(), fmt.Sprintf("next_hops:\"%s\"", expectedNexthop))).To(BeTrue())
+		}
+	}
+
+	for i := range numberOfServices {
+		err := client.CoreV1().Services(dsNamespace).Delete(context.TODO(), services[i], metav1.DeleteOptions{})
+		Expect(err).ToNot(HaveOccurred())
+		if i < numberOfServices-1 {
+			for _, addr := range lbAddresses {
+				paths := checkGoBGPPaths(context.Background(), gobgpClient, gobgpFamily, []*api.TableLookupPrefix{{Prefix: addr}}, 1)
+				Expect(strings.Contains(paths[0].Prefix, lbAddress)).To(BeTrue())
+				if expectedNexthop != "" {
+					Expect(strings.Contains(paths[0].String(), fmt.Sprintf("next_hop:\"%s\"", expectedNexthop)) || strings.Contains(paths[0].String(), fmt.Sprintf("next_hops:\"%s\"", expectedNexthop))).To(BeTrue())
+				}
+			}
+		}
+	}
+
+	for _, addr := range lbAddresses {
+		checkGoBGPPaths(context.Background(), gobgpClient, gobgpFamily, []*api.TableLookupPrefix{{Prefix: addr}}, 0)
+	}
+}
+
+func GetContainerIPs(containerName string) (string, string, error) {
+	cli, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		panic(err)
+	}
+	containers, err := cli.ContainerList(context.Background(), container.ListOptions{})
+	if err != nil {
+		return "", "", fmt.Errorf("failed to list containers: %w", err)
+	}
+
+	for _, c := range containers {
+		for _, n := range c.Names {
+			if n[1:] == containerName {
+				fmt.Println(n)
+				for _, n := range c.NetworkSettings.Networks {
+					return n.IPAddress, n.GlobalIPv6Address, nil
+				}
+			}
+		}
+	}
+
+	return "", "", nil
+}
+
+func newGoBGPClient(address string, port uint32) (api.GobgpApiClient, error) {
+	grpcOpts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
+	target := net.JoinHostPort(address, strconv.Itoa(int(port)))
+	conn, err := grpc.NewClient(target, grpcOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to GoBGP server %q: %w", target, err)
+	}
+
+	return api.NewGobgpApiClient(conn), nil
+}
+
+func checkGoBGPPaths(ctx context.Context, client api.GobgpApiClient, family *api.Family, prefixes []*api.TableLookupPrefix, expectedPaths int) []*api.Destination {
+	var paths []*api.Destination
+	Eventually(func() error {
+		var err error
+		paths, err = getGoBGPPaths(ctx, client, family, prefixes)
+		if err != nil {
+			return err
+		}
+		if len(paths) != expectedPaths {
+			return fmt.Errorf("expected %d paths, but found %d", expectedPaths, len(paths))
+		}
+		return nil
+	}, "120s").ShouldNot(HaveOccurred())
+	return paths
+}
+
+func getGoBGPPaths(ctx context.Context, client api.GobgpApiClient, family *api.Family, prefixes []*api.TableLookupPrefix) ([]*api.Destination, error) {
+	pathCtx, cancel := context.WithTimeout(ctx, time.Second*5)
+	defer cancel()
+	stream, err := client.ListPath(pathCtx, &api.ListPathRequest{
+		TableType: api.TableType_GLOBAL,
+		Family:    family,
+		Name:      "",
+		Prefixes:  prefixes,
+		SortType:  api.ListPathRequest_PREFIX,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	rib := make([]*api.Destination, 0)
+	for {
+		r, err := stream.Recv()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+		rib = append(rib, r.Destination)
+	}
+
+	return rib, nil
+}
+
+func startGoBGP(config string, kill chan any) {
+	By("starting GoBGP server")
+	cmd := exec.Command("../../bin/gobgpd", "-f", config)
+	go cmd.Run()
+	<-kill
+	By("stopping GoBGP server")
+	err := cmd.Process.Kill()
+	Expect(err).ToNot(HaveOccurred())
+}

testing/e2e/e2e_rt_test.go

@@ -0,0 +1,583 @@
+//go:build e2e
+// +build e2e
+
+package e2e_test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strconv"
+	"text/template"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+	kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
+	"sigs.k8s.io/kind/pkg/log"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	"github.com/kube-vip/kube-vip/testing/e2e"
+	"github.com/kube-vip/kube-vip/testing/services/pkg/deployment"
+)
+
+var _ = Describe("kube-vip routing table mode", func() {
+	if Mode == ModeRT {
+		var (
+			logger                              log.Logger
+			imagePath                           string
+			k8sImagePath                        string
+			configPath                          string
+			kubeVIPRoutingTableManifestTemplate *template.Template
+			tempDirPath                         string
+			v129                                bool
+		)
+
+		BeforeEach(func() {
+			klog.SetOutput(GinkgoWriter)
+			logger = e2e.TestLogger{}
+
+			imagePath = os.Getenv("E2E_IMAGE_PATH")    // Path to kube-vip image
+			configPath = os.Getenv("CONFIG_PATH")      // path to the api server config
+			k8sImagePath = os.Getenv("K8S_IMAGE_PATH") // path to the kubernetes image (version for kind)
+			if configPath == "" {
+				configPath = "/etc/kubernetes/admin.conf"
+			}
+			_, v129 = os.LookupEnv("V129")
+			curDir, err := os.Getwd()
+			Expect(err).NotTo(HaveOccurred())
+
+			templateRoutingTablePath := filepath.Join(curDir, "kube-vip-routing-table.yaml.tmpl")
+			kubeVIPRoutingTableManifestTemplate, err = template.New("kube-vip-routing-table.yaml.tmpl").ParseFiles(templateRoutingTablePath)
+			Expect(err).NotTo(HaveOccurred())
+
+			tempDirPath, err = os.MkdirTemp("", "kube-vip-test")
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		Describe("kube-vip IPv4 control-plane routing table mode functionality", Ordered, func() {
+			var (
+				cpVIP       string
+				clusterName string
+
+				nodesNumber = 3
+			)
+
+			BeforeAll(func() {
+				cpVIP = e2e.GenerateVIP(e2e.IPv4Family, SOffset.Get())
+
+				networking := &kindconfigv1alpha4.Networking{
+					IPFamily: kindconfigv1alpha4.IPv4Family,
+				}
+
+				manifestValues := &e2e.KubevipManifestValues{
+					ControlPlaneVIP:    cpVIP,
+					ImagePath:          imagePath,
+					ConfigPath:         configPath,
+					ControlPlaneEnable: "true",
+					SvcEnable:          "false",
+					SvcElectionEnable:  "false",
+				}
+
+				clusterName, _ = prepareCluster(tempDirPath, "rt-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
+			})
+
+			AfterAll(func() {
+				cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
+			})
+
+			It("setups IPv4 address and route on control-plane node", func() {
+				By(withTimestamp("sitting for a few seconds to hopefully allow kube-vip to start"))
+				time.Sleep(30 * time.Second)
+
+				for i := 1; i <= nodesNumber; i++ {
+					var container string
+					if i > 1 {
+						container = fmt.Sprintf("%s-control-plane%d", clusterName, i)
+					} else {
+						container = fmt.Sprintf("%s-control-plane", clusterName)
+					}
+
+					checkIPAddress(cpVIP, container, true)
+					e2e.CheckRoutePresence(cpVIP, container, true)
+				}
+			})
+		})
+
+		Describe("kube-vip IPv6 control-plane routing table mode functionality", Ordered, func() {
+			var (
+				cpVIP       string
+				clusterName string
+
+				nodesNumber = 3
+			)
+
+			BeforeAll(func() {
+
+				cpVIP = e2e.GenerateVIP(e2e.IPv6Family, SOffset.Get())
+
+				networking := &kindconfigv1alpha4.Networking{
+					IPFamily: kindconfigv1alpha4.IPv6Family,
+				}
+
+				manifestValues := &e2e.KubevipManifestValues{
+					ControlPlaneVIP:    cpVIP,
+					ImagePath:          imagePath,
+					ConfigPath:         configPath,
+					ControlPlaneEnable: "true",
+					SvcEnable:          "false",
+					SvcElectionEnable:  "false",
+				}
+
+				clusterName, _ = prepareCluster(tempDirPath, "rt-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
+			})
+
+			AfterAll(func() {
+				cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
+			})
+
+			It("setups IPv6 address and route on control-plane node", func() {
+				By(withTimestamp("sitting for a few seconds to hopefully allow kube-vip to start"))
+				time.Sleep(30 * time.Second)
+
+				for i := 1; i <= nodesNumber; i++ {
+					var container string
+					if i > 1 {
+						container = fmt.Sprintf("%s-control-plane%d", clusterName, i)
+					} else {
+						container = fmt.Sprintf("%s-control-plane", clusterName)
+					}
+
+					checkIPAddress(cpVIP, container, true)
+					e2e.CheckRoutePresence(cpVIP, container, true)
+				}
+			})
+		})
+
+		Describe("kube-vip DualStack control-plane routing table mode functionality - IPv4 primary", Ordered, func() {
+			var (
+				cpVIP       string
+				clusterName string
+
+				nodesNumber = 3
+			)
+
+			BeforeAll(func() {
+				cpVIP = e2e.GenerateDualStackVIP(SOffset.Get())
+
+				networking := &kindconfigv1alpha4.Networking{
+					IPFamily: kindconfigv1alpha4.DualStackFamily,
+				}
+
+				manifestValues := &e2e.KubevipManifestValues{
+					ControlPlaneVIP:    cpVIP,
+					ImagePath:          imagePath,
+					ConfigPath:         configPath,
+					ControlPlaneEnable: "true",
+					SvcEnable:          "false",
+					SvcElectionEnable:  "false",
+				}
+
+				networkInterface := ""
+				if networkInterface = os.Getenv("NETWORK_INTERFACE"); networkInterface == "" {
+					networkInterface = "br-"
+				}
+
+				localIPv6, localIPv6Net, err := deployment.GetLocalIPv6(networkInterface)
+				Expect(err).ToNot(HaveOccurred())
+
+				addSAN := &san{
+					ip:    localIPv6,
+					ipnet: localIPv6Net,
+				}
+
+				clusterName, _ = prepareCluster(tempDirPath, "rt-ds-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, addSAN)
+			})
+
+			AfterAll(func() {
+				cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
+			})
+
+			It("setups DualStack addresses and routes on control-plane nodes", func() {
+				By(withTimestamp("sitting for a few seconds to hopefully allow kube-vip to start"))
+				time.Sleep(30 * time.Second)
+
+				for i := 1; i <= nodesNumber; i++ {
+					var container string
+					if i > 1 {
+						container = fmt.Sprintf("%s-control-plane%d", clusterName, i)
+					} else {
+						container = fmt.Sprintf("%s-control-plane", clusterName)
+					}
+
+					addresses := vip.Split(cpVIP)
+
+					for _, addr := range addresses {
+						checkIPAddress(addr, container, true)
+						e2e.CheckRoutePresence(addr, container, true)
+					}
+				}
+			})
+		})
+
+		Describe("kube-vip DualStack control-plane routing table mode functionality - IPv6 primary", Ordered, func() {
+			var (
+				cpVIP       string
+				clusterName string
+
+				nodesNumber = 3
+			)
+
+			BeforeAll(func() {
+				cpVIP = e2e.GenerateDualStackVIP(SOffset.Get())
+
+				networking := &kindconfigv1alpha4.Networking{
+					IPFamily:      kindconfigv1alpha4.DualStackFamily,
+					PodSubnet:     "fd00:10:244::/56,10.244.0.0/16",
+					ServiceSubnet: "fd00:10:96::/112,10.96.0.0/16",
+				}
+
+				manifestValues := &e2e.KubevipManifestValues{
+					ControlPlaneVIP:    cpVIP,
+					ImagePath:          imagePath,
+					ConfigPath:         configPath,
+					ControlPlaneEnable: "true",
+					SvcEnable:          "false",
+					SvcElectionEnable:  "false",
+				}
+
+				networkInterface := ""
+				if networkInterface = os.Getenv("NETWORK_INTERFACE"); networkInterface == "" {
+					networkInterface = "br-"
+				}
+
+				localIPv4, localIPv4Net, err := deployment.GetLocalIPv4(networkInterface)
+				Expect(err).ToNot(HaveOccurred())
+
+				addSAN := &san{
+					ip:    localIPv4,
+					ipnet: localIPv4Net,
+				}
+
+				clusterName, _ = prepareCluster(tempDirPath, "rt-ds-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, addSAN)
+			})
+
+			AfterAll(func() {
+				cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
+			})
+
+			It("setups DualStack addresses and routes on control-plane nodes", func() {
+				By(withTimestamp("sitting for a few seconds to hopefully allow kube-vip to start"))
+				time.Sleep(30 * time.Second)
+
+				for i := 1; i <= nodesNumber; i++ {
+					var container string
+					if i > 1 {
+						container = fmt.Sprintf("%s-control-plane%d", clusterName, i)
+					} else {
+						container = fmt.Sprintf("%s-control-plane", clusterName)
+					}
+
+					addresses := vip.Split(cpVIP)
+
+					for _, addr := range addresses {
+						checkIPAddress(addr, container, true)
+						e2e.CheckRoutePresence(addr, container, true)
+					}
+				}
+			})
+		})
+
+		Describe("kube-vip IPv4 services routing table mode functionality", Ordered, func() {
+			var (
+				cpVIP          string
+				clusterName    string
+				client         kubernetes.Interface
+				manifestValues *e2e.KubevipManifestValues
+				svcElection    bool
+				ipFamily       []corev1.IPFamily
+
+				nodesNumber = 1
+			)
+
+			BeforeAll(func() {
+				cpVIP = e2e.GenerateVIP(e2e.IPv4Family, SOffset.Get())
+
+				networking := &kindconfigv1alpha4.Networking{
+					IPFamily: kindconfigv1alpha4.IPv4Family,
+				}
+
+				manifestValues = &e2e.KubevipManifestValues{
+					ControlPlaneVIP:    cpVIP,
+					ImagePath:          imagePath,
+					ConfigPath:         configPath,
+					ControlPlaneEnable: "false",
+					SvcEnable:          "true",
+					SvcElectionEnable:  "false",
+				}
+
+				var err error
+				svcElection, err = strconv.ParseBool(manifestValues.SvcElectionEnable)
+				Expect(err).ToNot(HaveOccurred())
+
+				ipFamily = []corev1.IPFamily{corev1.IPv4Protocol}
+
+				clusterName, client = prepareCluster(tempDirPath, "rt-svc-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
+			})
+
+			AfterAll(func() {
+				cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
+			})
+
+			DescribeTable("configures an IPv4 routes for services",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					lbAddress := e2e.GenerateVIP(e2e.IPv4Family, offset)
+					testServiceRT(svcName, lbAddress, fmt.Sprintf("kubevip-%s", svcName), dsNamespace, clusterName, trafficPolicy, client, svcElection, ipFamily, 1)
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+
+			DescribeTable("only removes route if it was referenced by multiple services and all of them were deleted",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					lbAddress := e2e.GenerateVIP(e2e.IPv4Family, offset)
+					testServiceRT(svcName, lbAddress, "plndr-svcs-lock", "kube-system", clusterName, trafficPolicy, client, svcElection, ipFamily, 2)
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+		})
+
+		Describe("kube-vip IPv6 services routing table mode functionality", Ordered, func() {
+			var (
+				cpVIP          string
+				clusterName    string
+				client         kubernetes.Interface
+				manifestValues *e2e.KubevipManifestValues
+				svcElection    bool
+				ipFamily       []corev1.IPFamily
+
+				nodesNumber = 1
+			)
+
+			BeforeAll(func() {
+				cpVIP = e2e.GenerateVIP(e2e.IPv6Family, SOffset.Get())
+
+				networking := &kindconfigv1alpha4.Networking{
+					IPFamily: kindconfigv1alpha4.IPv6Family,
+				}
+
+				manifestValues = &e2e.KubevipManifestValues{
+					ControlPlaneVIP:    cpVIP,
+					ImagePath:          imagePath,
+					ConfigPath:         configPath,
+					ControlPlaneEnable: "false",
+					SvcEnable:          "true",
+					SvcElectionEnable:  "false",
+				}
+
+				var err error
+				svcElection, err = strconv.ParseBool(manifestValues.SvcElectionEnable)
+				Expect(err).ToNot(HaveOccurred())
+
+				ipFamily = []corev1.IPFamily{corev1.IPv6Protocol}
+
+				clusterName, client = prepareCluster(tempDirPath, "rt-svc-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
+			})
+
+			AfterAll(func() {
+				cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
+			})
+
+			DescribeTable("configures an IPv6 routes for services",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					lbAddress := e2e.GenerateVIP(e2e.IPv6Family, offset)
+					testServiceRT(svcName, lbAddress, fmt.Sprintf("kubevip-%s", svcName), dsNamespace, clusterName, trafficPolicy, client, svcElection, ipFamily, 1)
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+
+			DescribeTable("only removes route if it was referenced by multiple services and all of them were deleted",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					lbAddress := e2e.GenerateVIP(e2e.IPv6Family, offset)
+					testServiceRT(svcName, lbAddress, "plndr-svcs-lock", "kube-system", clusterName, trafficPolicy, client, svcElection, ipFamily, 2)
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+		})
+
+		Describe("kube-vip DualStack services routing table mode functionality - IPv4 primary", Ordered, func() {
+			var (
+				cpVIP          string
+				clusterName    string
+				client         kubernetes.Interface
+				manifestValues *e2e.KubevipManifestValues
+				svcElection    bool
+				ipFamily       []corev1.IPFamily
+
+				nodesNumber = 1
+			)
+
+			BeforeAll(func() {
+				cpVIP = e2e.GenerateDualStackVIP(SOffset.Get())
+
+				networking := &kindconfigv1alpha4.Networking{
+					IPFamily: kindconfigv1alpha4.DualStackFamily,
+				}
+
+				manifestValues = &e2e.KubevipManifestValues{
+					ControlPlaneVIP:      cpVIP,
+					ImagePath:            imagePath,
+					ConfigPath:           configPath,
+					ControlPlaneEnable:   "false",
+					SvcEnable:            "true",
+					SvcElectionEnable:    "false",
+					EnableEndpointslices: "true",
+				}
+
+				var err error
+				svcElection, err = strconv.ParseBool(manifestValues.SvcElectionEnable)
+				Expect(err).ToNot(HaveOccurred())
+
+				ipFamily = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}
+
+				clusterName, client = prepareCluster(tempDirPath, "rt-ds-svc-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
+			})
+
+			AfterAll(func() {
+				cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
+			})
+
+			DescribeTable("configures an IPv4 and IPv6 routes for services",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					lbAddress := e2e.GenerateDualStackVIP(offset)
+					testServiceRT(svcName, lbAddress, fmt.Sprintf("kubevip-%s", svcName), dsNamespace, clusterName, trafficPolicy, client, svcElection, ipFamily, 1)
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+
+			DescribeTable("only removes route if it was referenced by multiple services and all of them were deleted",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					lbAddress := e2e.GenerateDualStackVIP(offset)
+					testServiceRT(svcName, lbAddress, "plndr-svcs-lock", "kube-system", clusterName, trafficPolicy, client, svcElection, ipFamily, 2)
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+		})
+
+		Describe("kube-vip DualStack services routing table mode functionality - IPv6 primary", Ordered, func() {
+			var (
+				cpVIP          string
+				clusterName    string
+				client         kubernetes.Interface
+				manifestValues *e2e.KubevipManifestValues
+				svcElection    bool
+				ipFamily       []corev1.IPFamily
+
+				nodesNumber = 1
+			)
+
+			BeforeAll(func() {
+				cpVIP = e2e.GenerateDualStackVIP(SOffset.Get())
+
+				networking := &kindconfigv1alpha4.Networking{
+					IPFamily:      kindconfigv1alpha4.DualStackFamily,
+					PodSubnet:     "fd00:10:244::/56,10.244.0.0/16",
+					ServiceSubnet: "fd00:10:96::/112,10.96.0.0/16",
+				}
+
+				manifestValues = &e2e.KubevipManifestValues{
+					ControlPlaneVIP:      cpVIP,
+					ImagePath:            imagePath,
+					ConfigPath:           configPath,
+					ControlPlaneEnable:   "false",
+					SvcEnable:            "true",
+					SvcElectionEnable:    "false",
+					EnableEndpointslices: "true",
+				}
+
+				var err error
+				svcElection, err = strconv.ParseBool(manifestValues.SvcElectionEnable)
+				Expect(err).ToNot(HaveOccurred())
+
+				ipFamily = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}
+
+				clusterName, client = prepareCluster(tempDirPath, "rt-ds-svc-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
+			})
+
+			AfterAll(func() {
+				cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
+			})
+
+			DescribeTable("configures an IPv4 and IPv6 routes for services",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					lbAddress := e2e.GenerateDualStackVIP(offset)
+					testServiceRT(svcName, lbAddress, fmt.Sprintf("kubevip-%s", svcName), dsNamespace, clusterName, trafficPolicy, client, svcElection, ipFamily, 1)
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+
+			DescribeTable("only removes route if it was referenced by multiple services and all of them were deleted",
+				func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
+					lbAddress := e2e.GenerateDualStackVIP(offset)
+					testServiceRT(svcName, lbAddress, "plndr-svcs-lock", "kube-system", clusterName, trafficPolicy, client, svcElection, ipFamily, 2)
+				},
+				Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
+				Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
+			)
+		})
+	}
+})
+
+func testServiceRT(svcName, lbAddress, leaseName, leaseNamespace, clusterName string, trafficPolicy corev1.ServiceExternalTrafficPolicy,
+	client kubernetes.Interface, serviceElection bool, ipFamily []corev1.IPFamily, numberOfServices int) {
+	lbAddresses := vip.Split(lbAddress)
+
+	services := []string{}
+	for i := range numberOfServices {
+		services = append(services, fmt.Sprintf("%s-%d", svcName, i))
+	}
+
+	for _, svc := range services {
+		createTestService(svc, dsNamespace, dsName, lbAddress,
+			client, corev1.IPFamilyPolicyPreferDualStack, ipFamily, trafficPolicy)
+	}
+
+	var container string
+	if serviceElection {
+		container = e2e.GetLeaseHolder(leaseName, leaseNamespace, client)
+	} else {
+		container = fmt.Sprintf("%s-control-plane", clusterName)
+	}
+
+	for _, addr := range lbAddresses {
+		e2e.CheckRoutePresence(addr, container, true)
+	}
+
+	for i := range numberOfServices {
+		expected := i < numberOfServices-1
+		err := client.CoreV1().Services(dsNamespace).Delete(context.TODO(), services[i], metav1.DeleteOptions{})
+		Expect(err).ToNot(HaveOccurred())
+
+		for _, addr := range lbAddresses {
+			e2e.CheckRoutePresence(addr, container, expected)
+		}
+	}
+}
+
+type san struct {
+	ip    *net.IP
+	ipnet *net.IPNet
+}

testing/e2e/e2e_suite_test.go

@@ -4,15 +4,41 @@
 package e2e_test
 
 import (
+	"log"
+	"os"
+	"sync"
 	"testing"
 
+	"github.com/kube-vip/kube-vip/testing/e2e"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+)
 
-	"github.com/kube-vip/kube-vip/testing/e2e"
+const (
+	ModeEnv = "TEST_MODE"
+	ModeARP = "arp"
+	ModeRT  = "rt"
+	ModeBGP = "bgp"
+)
+
+var (
+	SOffset   *e2e.SecureOffset
+	ConfigMtx *sync.Mutex
+	Mode      string
 )
 
 func TestE2E(t *testing.T) {
+	mode := os.Getenv(ModeEnv)
+	if mode == "" {
+		Mode = ModeARP
+	} else if mode != ModeARP && mode != ModeRT && mode != ModeBGP {
+		log.Fatal("invalid", "mode", mode)
+		os.Exit(1)
+	} else {
+		Mode = mode
+	}
+	SOffset = e2e.NewOffset(5)
+	ConfigMtx = &sync.Mutex{}
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "E2E Suite")
 }

testing/e2e/etcd/cluster.go

@@ -268,15 +268,6 @@ func (c *Cluster) Delete() {
 	Expect(c.provider.Delete(c.Name, "")).To(Succeed())
 }
 
-func startKubeletForEtcd(node nodes.Node) {
-	e2e.RunInNode(node,
-		"kubeadm", "init", "phase", "kubeconfig", "admin", "--config", "/kind/kubeadm.conf",
-	)
-	e2e.RunInNode(node,
-		"kubeadm", "init", "phase", "kubelet-start", "--config", "/kind/kubeadm.conf",
-	)
-}
-
 func createCerts(node nodes.Node) {
 	e2e.RunInNode(node,
 		"kubeadm",

testing/e2e/etcd/election_test.go

@@ -60,7 +60,7 @@ var _ = Describe("kube-vip with etcd leader election", func() {
 
 			test.kubeVipImage = os.Getenv("E2E_IMAGE_PATH")
 
-			test.vip = e2e.GenerateIPv4VIP()
+			test.vip = e2e.GenerateVIP(e2e.IPv4Family, 5)
 			test.logger.Printf("Selected VIP %s", test.vip)
 
 			test.currentDir, err = os.Getwd()

testing/e2e/ip.go

@@ -6,22 +6,34 @@ package e2e
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"encoding/binary"
 	"fmt"
 	"io"
 	"net"
 	"os/exec"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/kube-vip/kube-vip/pkg/vip"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/onsi/gomega/gexec"
+	coordinationv1 "k8s.io/api/coordination/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 	kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
 	"sigs.k8s.io/kind/pkg/cluster"
 )
 
+const (
+	IPv4Family          = "IPv4"
+	IPv6Family          = "IPv6"
+	DualstackFamily     = "DualStack"
+	DualstackFamilyIPv6 = "DualStackIPv6"
+)
+
 func EnsureKindNetwork() {
 	By("checking if the Docker \"kind\" network exists")
 	cmd := exec.Command("docker", "inspect", "kind")
@@ -58,53 +70,47 @@ func EnsureKindNetwork() {
 	Eventually(session).Should(gexec.Exit(0))
 }
 
-func GenerateIPv6VIP() string {
+func GenerateVIP(family string, offset uint) string {
+	if family == DualstackFamily || family == DualstackFamilyIPv6 {
+		return fmt.Sprintf("%s,%s", GenerateVIP(IPv4Family, offset), GenerateVIP(IPv6Family, offset))
+	}
+
 	cidrs := getKindNetworkSubnetCIDRs()
 
 	for _, cidr := range cidrs {
-		ip, ipNet, parseErr := net.ParseCIDR(cidr)
-		Expect(parseErr).NotTo(HaveOccurred())
+		if cidr != "" {
+			ip, ipNet, parseErr := net.ParseCIDR(cidr)
+			Expect(parseErr).NotTo(HaveOccurred())
 
-		if ip.To4() == nil {
-			lowerMask := binary.BigEndian.Uint64(ipNet.Mask[8:])
-			lowerStart := binary.BigEndian.Uint64(ipNet.IP[8:])
-			lowerEnd := (lowerStart & lowerMask) | (^lowerMask)
+			if ip.To4() == nil && family == IPv6Family {
+				lowerMask := binary.BigEndian.Uint64(ipNet.Mask[8:])
+				lowerStart := binary.BigEndian.Uint64(ipNet.IP[8:])
+				lowerEnd := (lowerStart & lowerMask) | (^lowerMask)
 
-			chosenVIP := make([]byte, 16)
-			// Copy upper half into chosenVIP
-			copy(chosenVIP, ipNet.IP[0:8])
-			// Copy lower half into chosenVIP
-			binary.BigEndian.PutUint64(chosenVIP[8:], lowerEnd-5)
-			return net.IP(chosenVIP).String()
+				chosenVIP := make([]byte, 16)
+				// Copy upper half into chosenVIP
+				copy(chosenVIP, ipNet.IP[0:8])
+				// Copy lower half into chosenVIP
+				binary.BigEndian.PutUint64(chosenVIP[8:], lowerEnd-uint64(offset))
+				return net.IP(chosenVIP).String()
+			} else if ip.To4() != nil && family == IPv4Family {
+				mask := binary.BigEndian.Uint32(ipNet.Mask)
+				start := binary.BigEndian.Uint32(ipNet.IP)
+				end := (start & mask) | (^mask)
+
+				chosenVIP := make([]byte, 4)
+				binary.BigEndian.PutUint32(chosenVIP, end-uint32(offset))
+				return net.IP(chosenVIP).String()
+			}
 		}
 	}
-	Fail("Could not find any IPv6 CIDRs in the Docker \"kind\" network")
+
+	Fail("Could not find any " + family + " CIDRs in the Docker \"kind\" network")
 	return ""
 }
 
-func GenerateIPv4VIP() string {
-	cidrs := getKindNetworkSubnetCIDRs()
-
-	for _, cidr := range cidrs {
-		ip, ipNet, parseErr := net.ParseCIDR(cidr)
-		Expect(parseErr).NotTo(HaveOccurred())
-
-		if ip.To4() != nil {
-			mask := binary.BigEndian.Uint32(ipNet.Mask)
-			start := binary.BigEndian.Uint32(ipNet.IP)
-			end := (start & mask) | (^mask)
-
-			chosenVIP := make([]byte, 4)
-			binary.BigEndian.PutUint32(chosenVIP, end-5)
-			return net.IP(chosenVIP).String()
-		}
-	}
-	Fail("Could not find any IPv4 CIDRs in the Docker \"kind\" network")
-	return ""
-}
-
-func GenerateDualStackVIP() string {
-	return GenerateIPv4VIP() + "," + GenerateIPv6VIP()
+func GenerateDualStackVIP(offset uint) string {
+	return GenerateVIP(IPv4Family, offset) + "," + GenerateVIP(IPv6Family, offset)
 }
 
 func getKindNetworkSubnetCIDRs() []string {
@@ -133,30 +139,88 @@ func getKindNetworkSubnetCIDRs() []string {
 	return cidrs
 }
 
-func CheckIPAddressPresence(ip string, container string) bool {
-	cmdOut := new(bytes.Buffer)
+func CheckIPAddressPresence(ip string, container string, expected bool) bool {
 	family := "-4"
 	if vip.IsIPv6(ip) {
 		family = "-6"
 	}
+
 	cmd := exec.Command(
 		"docker", "exec", container, "ip", family, "addr", "show", "dev", "eth0",
 	)
+
+	cmdOut := new(bytes.Buffer)
+	cmdErr := new(bytes.Buffer)
 	cmd.Stdout = cmdOut
-	Eventually(cmd.Run(), "20s").Should(Succeed())
-	return strings.Contains(cmdOut.String(), ip)
+	cmd.Stderr = cmdErr
+	if err := cmd.Run(); err != nil {
+		return false
+	}
+
+	return strings.Contains(cmdOut.String(), ip) == expected
 }
 
-func CheckRoutePresence(ip string, container string) bool {
-	cmdOut := new(bytes.Buffer)
+func CheckIPAddressPresenceByLease(name, namespace, ip string, client kubernetes.Interface, expected bool) bool {
+	container := GetLeaseHolder(name, namespace, client)
+	if container == "" {
+		return false
+	}
+	return CheckIPAddressPresence(ip, container, expected)
+}
+
+func CheckRoutePresence(ip string, container string, expected bool) bool {
 	family := "-4"
 	if vip.IsIPv6(ip) {
 		family = "-6"
 	}
-	cmd := exec.Command(
-		"docker", "exec", container, "ip", family, "route", "show", "table", "198",
-	)
-	cmd.Stdout = cmdOut
-	Eventually(cmd.Run(), "20s").Should(Succeed())
-	return strings.Contains(cmdOut.String(), ip)
+
+	result := false
+	Eventually(func() bool {
+		cmdOut := new(bytes.Buffer)
+		cmdErr := new(bytes.Buffer)
+
+		cmd := exec.Command(
+			"docker", "exec", container, "ip", family, "route", "show", "table", "198",
+		)
+
+		cmd.Stdout = cmdOut
+		cmd.Stderr = cmdErr
+		cmd.Run()
+
+		result = strings.Contains(cmdOut.String(), ip) == expected
+
+		return result
+	}, "60s", "1s").Should(BeTrue())
+
+	return result
+}
+
+func GetLeaseHolder(name, namespace string, client kubernetes.Interface) string {
+	var lease *coordinationv1.Lease
+	Eventually(func() error {
+		var err error
+		lease, err = client.CoordinationV1().Leases(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+		return err
+	}, "300s").ShouldNot(HaveOccurred())
+
+	Expect(lease).ToNot(BeNil())
+	return *lease.Spec.HolderIdentity
+}
+
+type SecureOffset struct {
+	value int
+	mu    sync.Mutex
+}
+
+func NewOffset(value int) *SecureOffset {
+	return &SecureOffset{
+		value: value,
+	}
+}
+
+func (so *SecureOffset) Get() uint {
+	so.mu.Lock()
+	defer so.mu.Unlock()
+	so.value++
+	return uint(so.value - 1)
 }

testing/e2e/kube-vip-bgp.yaml.tmpl

@@ -0,0 +1,82 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-vip
+  namespace: kube-system
+spec:
+  containers:
+  - name: kube-vip
+    args:
+    - manager
+    - --prometheusHTTPServer
+    - ""
+    env:
+    - name: vip_loglevel
+      value: "-4"
+    - name: lb_class_legacy_handling
+      value: "false"
+    - name: lb_class_name
+    - name: prometheus_server
+      value: :2112
+    - name: disable_service_updates
+      value: "false"
+    - name: vip_arp
+      value: "false"
+    - name: port
+      value: "6443"
+    - name: vip_cidr
+      value: "32"
+    - name: vip_leaderelection
+      value: "false"
+    - name: vip_interface
+      value: lo
+    - name: bgp_enable
+      value: "true"
+    - name: bgp_hold_time
+      value: "3"
+    - name: bgp_keepalive_interval
+      value: "1"
+    - name: bgp_sourceif
+      value: "eth0"
+    - name: bgp_routerid
+      value: "2.2.2.2"
+    - name: bgp_as
+      value: "{{ .BGPAS }}"
+    - name: bgp_peers
+      value: "{{ .BGPPeers }}"
+    - name: mpbgp_nexthop
+      value: "{{ .MPBGPNexthop }}"
+    - name: mpbgp_ipv4
+      value: "{{ .MPBGPNexthopIPv4 }}"
+    - name: mpbgp_ipv6
+      value: "{{ .MPBGPNexthopIPv6 }}"
+    - name: address
+      value: "{{ .ControlPlaneVIP }}"
+    - name: cp_enable
+      value: "{{ .ControlPlaneEnable }}"
+    - name: svc_enable
+      value: "{{ .SvcEnable }}"
+    - name: svc_election
+      value: "{{ .SvcElectionEnable }}"
+    - name: enable_endpointslices
+      value: "{{ .EnableEndpointslices }}"
+    image: "{{ .ImagePath }}"
+    imagePullPolicy: Never
+    securityContext:
+      capabilities:
+        add:
+        - NET_ADMIN
+        - NET_RAW
+    volumeMounts:
+    - mountPath: /etc/kubernetes/admin.conf
+      name: kubeconfig
+  hostAliases:
+    - hostnames:
+      - kubernetes
+      ip: 127.0.0.1
+  hostNetwork: true
+  volumes:
+  - hostPath:
+      path: "{{ .ConfigPath }}"
+    name: kubeconfig

testing/e2e/kube-vip-routing-table.yaml.tmpl

@@ -12,6 +12,8 @@ spec:
     - --prometheusHTTPServer
     - ""
     env:
+    - name: vip_loglevel
+      value: "-4"
     - name: vip_arp
       value: "false"
     - name: vip_interface
@@ -27,19 +29,23 @@ spec:
     - name: vip_retryperiod
       value: "1"
     - name: cp_enable
-      value: "true"
+      value: "{{ .ControlPlaneEnable }}"
     - name: vip_nodename
       valueFrom:
         fieldRef:
           fieldPath: spec.nodeName
-    - name: vip_cidr
-      value: '32'
+    - name: vip_subnet
+      value: '32,128'
     - name: cp_namespace
       value: kube-system
-    - name: svc_enable
-      value: 'false'
     - name: vip_routingtable
       value: 'true'
+    - name: svc_enable
+      value: "{{ .SvcEnable }}"
+    - name: svc_election
+      value: "{{ .SvcElectionEnable }}"
+    - name: enable_endpointslices
+      value: "{{ .EnableEndpointslices }}"
     image: "{{ .ImagePath }}"
     imagePullPolicy: Never
     securityContext:

testing/e2e/kube-vip.yaml.tmpl

@@ -12,6 +12,8 @@ spec:
     - --prometheusHTTPServer
     - ""
     env:
+    - name: vip_loglevel
+      value: "-4"
     - name: vip_arp
       value: "true"
     - name: vip_interface
@@ -32,6 +34,12 @@ spec:
       valueFrom:
         fieldRef:
           fieldPath: spec.nodeName
+    - name: svc_enable
+      value: "{{ .SvcEnable }}"
+    - name: svc_election
+      value: "{{ .SvcElectionEnable }}"
+    - name: enable_endpointslices
+      value: "{{ .EnableEndpointslices }}"
     image: "{{ .ImagePath }}"
     imagePullPolicy: Never
     securityContext:

testing/e2e/template.go

@@ -3,8 +3,47 @@
 
 package e2e
 
+import (
+	"fmt"
+	"net"
+)
+
 type KubevipManifestValues struct {
-	ControlPlaneVIP string
-	ImagePath       string
-	ConfigPath      string
+	ControlPlaneVIP      string
+	ImagePath            string
+	ConfigPath           string
+	SvcEnable            string
+	SvcElectionEnable    string
+	EnableEndpointslices string
+	ControlPlaneEnable   string
+	BGPAS                uint32
+	BGPPeers             string
+	MPBGPNexthop         string
+	MPBGPNexthopIPv4     string
+	MPBGPNexthopIPv6     string
+}
+
+type BGPPeerValues struct {
+	IP       string
+	AS       uint32
+	MPBGP    string
+	IPFamily string
+}
+
+func (pv *BGPPeerValues) String() string {
+	tmpIP := pv.IP
+	ip := net.ParseIP(tmpIP)
+	if ip == nil {
+		return ""
+	}
+
+	if ip.To4() == nil {
+		tmpIP = fmt.Sprintf("[%s]", tmpIP)
+	}
+
+	if pv.MPBGP != "" {
+		return fmt.Sprintf("%s:%d::false/mpbgp_nexthop=%s", tmpIP, pv.AS, pv.MPBGP)
+	}
+
+	return fmt.Sprintf("%s:%d::false", tmpIP, pv.AS)
 }

testing/services/main.go

@@ -38,11 +38,14 @@ func main() {
 	flag.BoolVar(&t.LeaderActive, "leaderActive", false, "Perform a test on the active leader")
 	flag.BoolVar(&t.LocalDeploy, "localDeploy", false, "Perform a test on the active leader")
 	flag.BoolVar(&t.Egress, "egress", false, "Perform an egress test")
+	flag.BoolVar(&t.EgressInternal, "egressInternal", false, "Perform an egress test, using the internal functionality")
 	flag.BoolVar(&t.EgressIPv6, "egressIPv6", false, "Perform an egress test")
 	flag.BoolVar(&t.DualStack, "dualStack", false, "Perform an dual stack test")
 	flag.BoolVar(&t.RetainCluster, "retain", false, "Retain the cluster")
 	flag.StringVar(&t.KindVersionImage, "kindImage", "", "The image to use for the kind nodes e.g. (kindest/node:v1.21.14)")
 	flag.BoolVar(&existing, "existing", false, "Use an existing cluster")
+	flag.BoolVar(&t.IPv6, "ipv6", false, "Perform an IPv6-only test")
+	flag.StringVar(&t.DockerNIC, "networkInterface", "br-", "Selects networking interface to use")
 
 	flag.BoolVar(&t.Cilium, "cilium", false, "Use cilium as a CNI")
 
@@ -50,9 +53,6 @@ func main() {
 
 	slog.Infof("🔬 beginning e2e tests, image: [%s] DualStack [%t]", t.ImagePath, t.DualStack)
 
-	// if !t.ignoreDualStack {
-	// 	t.Dualstack = true
-	// }
 	t.Affinity = os.Getenv("NODE_TOLERATE")
 
 	t.DeploymentName = "kube-vip-deploy"
@@ -134,7 +134,7 @@ func main() {
 			}
 		}
 
-		t.StartServiceTest(ctx, clientset)
+		errs := t.StartServiceTest(ctx, clientset)
+		os.Exit(len(errs))
 	}
-
 }

testing/services/pkg/deployment/kind.go

@@ -3,11 +3,13 @@ package deployment
 import (
 	"bytes"
 	"fmt"
+	"net"
 	"os/exec"
 	"strings"
 	"time"
 
 	"github.com/gookit/slog"
+	"github.com/vishvananda/netlink"
 	kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
 	"sigs.k8s.io/kind/pkg/cluster"
 	"sigs.k8s.io/kind/pkg/cmd"
@@ -16,6 +18,11 @@ import (
 
 var provider *cluster.Provider
 
+const (
+	defaultIPv4Pool = "172.18.100.10-172.18.100.30"
+	defaultIPv6Pool = "fd34:70db:8529:1e3d:0000:0000:0000:0010-fd34:70db:8529:1e3d:0000:0000:0000:0030"
+)
+
 type kubevipManifestValues struct {
 	ControlPlaneVIP string
 	ImagePath       string
@@ -39,13 +46,12 @@ func (config *TestConfig) CreateKind() error {
 		},
 	}
 
-	if config.IPv6 {
-		// Change Networking Family
-		clusterConfig.Networking.IPFamily = kindconfigv1alpha4.IPv6Family
-	}
 	if config.DualStack {
-		// Change Networking Family
 		clusterConfig.Networking.IPFamily = kindconfigv1alpha4.DualStackFamily
+	} else {
+		if config.IPv6 || !config.Egress && config.EgressIPv6 {
+			clusterConfig.Networking.IPFamily = kindconfigv1alpha4.IPv6Family
+		}
 	}
 
 	if config.Cilium {
@@ -140,11 +146,52 @@ func (config *TestConfig) CreateKind() error {
 			}
 		}
 
-		globalRange := "172.18.100.10-172.18.100.30"
-		if config.IPv6 {
-			globalRange = "fd34:70db:8529:1e3d:0000:0000:0000:0010-fd34:70db:8529:1e3d:0000:0000:0000:0030"
+		ipv4Pool := defaultIPv4Pool
+		ipv6Pool := defaultIPv6Pool
+
+		var ipv4Net *net.IPNet
+		var ipv6Net *net.IPNet
+
+		links, err := netlink.LinkList()
+		if err != nil {
+			return fmt.Errorf("netlink: failed to list links: %w", err)
 		}
 
+		for _, link := range links {
+			if strings.Contains(link.Attrs().Name, config.DockerNIC) {
+				_, ipv4Net, err = getNetwork(link, netlink.FAMILY_V4)
+				if err != nil {
+					return fmt.Errorf("failed to get IPv4 network: %w", err)
+				}
+				if ipv4Net == nil {
+					return fmt.Errorf("failed to find IPv4 network on the docker interface")
+				}
+
+				ipv4Pool = generateRange(ipv4Net)
+
+				_, ipv6Net, err = getNetwork(link, netlink.FAMILY_V6)
+				if err != nil {
+					return fmt.Errorf("failed to get IPv6 network: %w", err)
+				}
+				if ipv6Net == nil {
+					return fmt.Errorf("failed to find IPv6 network on the docker interface")
+				}
+
+				ipv6Pool = generateRange(ipv6Net)
+			}
+		}
+
+		globalRange := ipv4Pool
+		if config.IPv6 {
+			globalRange = ipv6Pool
+		}
+
+		if config.DualStack {
+			globalRange = fmt.Sprintf("%s,%s", ipv4Pool, ipv6Pool)
+		}
+
+		slog.Infof("Cloud provider's range: %s", globalRange)
+
 		cmd := exec.Command("kubectl", "create", "configmap", "--namespace", "kube-system", "kubevip", "--from-literal", "range-global="+globalRange)
 		if _, err := cmd.CombinedOutput(); err != nil {
 			return err
@@ -163,6 +210,43 @@ func (config *TestConfig) CreateKind() error {
 	return nil
 }
 
+func getNetwork(link netlink.Link, family int) (*net.IP, *net.IPNet, error) {
+	addrs, err := netlink.AddrList(link, family)
+	if err != nil {
+		return nil, nil, fmt.Errorf("netlink: failed to get addresses for link %q: %w", link.Attrs().Name, err)
+	}
+	if len(addrs) > 0 {
+		for _, a := range addrs {
+			if a.Scope == int(netlink.SCOPE_UNIVERSE) {
+				ip, cidr, err := net.ParseCIDR(a.IPNet.String())
+				if err != nil {
+					return nil, nil, fmt.Errorf("failed to parse CIDR: %w", err)
+				}
+				return &ip, cidr, nil
+			}
+		}
+	}
+
+	return nil, nil, nil
+}
+
+func generateRange(network *net.IPNet) string {
+	if network.IP.To4() != nil {
+		parts := strings.Split(network.IP.String(), ".")
+		p1 := fmt.Sprintf("%s.%s.100.10", parts[0], parts[1])
+		p2 := fmt.Sprintf("%s.%s.100.100", parts[0], parts[1])
+		return fmt.Sprintf("%s-%s", p1, p2)
+	}
+
+	if network.IP.To16() != nil {
+		parts := strings.Split(network.IP.String(), ":")
+		p1 := fmt.Sprintf("%s:%s:%s:%s:0000:0000:0100:0010", parts[0], parts[1], parts[2], parts[3])
+		p2 := fmt.Sprintf("%s:%s:%s:%s:0000:0000:0100:0100", parts[0], parts[1], parts[2], parts[3])
+		return fmt.Sprintf("%s-%s", p1, p2)
+	}
+	return ""
+}
+
 func DeleteKind() error {
 	slog.Info("🧽 deleting Kind cluster")
 	return provider.Delete("services", "")

testing/services/pkg/deployment/kubernetes.go

@@ -18,13 +18,14 @@ import (
 
 // service defines the settings for a new service
 type Service struct {
-	name          string
-	egress        bool // enable egress
-	egressIPv6    bool // egress should be IPv6
-	policyLocal   bool // set the policy to local pods
-	testHTTP      bool
-	testDualstack bool // test dualstack loadbalancer services
-	timeout       int  // how long to wait for the service to be created
+	name           string
+	egress         bool // enable egress
+	egressInternal bool // enable Internal egress
+	egressIPv6     bool // egress should be IPv6
+	policyLocal    bool // set the policy to local pods
+	testHTTP       bool
+	testDualstack  bool // test dualstack loadbalancer services
+	timeout        int  // how long to wait for the service to be created
 }
 
 type Deployment struct {
@@ -71,7 +72,7 @@ func (d *Deployment) CreateKVDs(ctx context.Context, clientset *kubernetes.Clien
 									Value: "true",
 								},
 								{
-									Name:  "vip_cidr",
+									Name:  "vip_subnet",
 									Value: "32",
 								},
 								{
@@ -155,7 +156,7 @@ func (d *Deployment) CreateDeployment(ctx context.Context, clientset *kubernetes
 					Containers: []v1.Container{
 						{
 							Name:  "kube-vip-web",
-							Image: "plndr/e2e:0.0.1",
+							Image: "docker.io/plndr/e2e:0.0.1",
 							Ports: []v1.ContainerPort{
 								{
 									Name:          "http",
@@ -238,6 +239,11 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
 			"kube-vip.io/egress": "true",
 		}
 	}
+
+	if s.egressInternal {
+		svc.Annotations["kube-vip.io/egress-internal"] = "true"
+	}
+
 	if s.egressIPv6 {
 		svc.Annotations["kube-vip.io/egress-ipv6"] = "true"
 	}
@@ -276,7 +282,7 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
 		slog.Fatal(err)
 	}
 	// Use a restartable watcher, as this should help in the event of etcd or timeout issues
-	rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
+	rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
 		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
 			return clientset.CoreV1().Services(v1.NamespaceDefault).Watch(ctx, metav1.ListOptions{})
 		},
@@ -297,7 +303,6 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
 		// We need to inspect the event and get ResourceVersion out of it
 		switch event.Type {
 		case watch.Added, watch.Modified:
-			// slog.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
 			svc, ok := event.Object.(*v1.Service)
 			if !ok {
 				slog.Fatalf("unable to parse Kubernetes services from API watcher")

testing/services/pkg/deployment/services.go

@@ -7,9 +7,11 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/gookit/slog"
+	"github.com/vishvananda/netlink"
 
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
@@ -24,12 +26,14 @@ import (
 
 // 1. Create a deployment
 // 2. Expose the deployment
-func (config *TestConfig) StartServiceTest(ctx context.Context, clientset *kubernetes.Clientset) {
+func (config *TestConfig) StartServiceTest(ctx context.Context, clientset *kubernetes.Clientset) []error {
 	var err error
+	var errs []error
 	if config.Simple {
 		err = config.SimpleDeployment(ctx, clientset)
 		if err != nil {
 			slog.Error(err)
+			errs = append(errs, err)
 		}
 	}
 
@@ -37,6 +41,7 @@ func (config *TestConfig) StartServiceTest(ctx context.Context, clientset *kuber
 		err = config.MultipleDeployments(ctx, clientset)
 		if err != nil {
 			slog.Error(err)
+			errs = append(errs, err)
 		}
 	}
 
@@ -45,6 +50,7 @@ func (config *TestConfig) StartServiceTest(ctx context.Context, clientset *kuber
 		err = config.Failover(ctx, clientset)
 		if err != nil {
 			slog.Error(err)
+			errs = append(errs, err)
 		}
 	}
 
@@ -53,41 +59,72 @@ func (config *TestConfig) StartServiceTest(ctx context.Context, clientset *kuber
 		err = config.Failover(ctx, clientset)
 		if err != nil {
 			slog.Error(err)
+			errs = append(errs, err)
 		}
 	}
+
 	if config.LocalDeploy {
 		// Failover tests
 		err = config.LocalDeployment(ctx, clientset)
 		if err != nil {
 			slog.Error(err)
+			errs = append(errs, err)
 		}
 	}
 
 	if config.Egress {
-		// Failover tests
-		err = config.EgressDeployment(ctx, clientset)
+		// Egress test
+		err = config.EgressDeployment(ctx, clientset, false)
 		if err != nil {
 			slog.Error(err)
+			errs = append(errs, err)
+		}
+	}
+
+	if config.Egress && config.EgressInternal {
+		// Egress test
+		err = config.EgressDeployment(ctx, clientset, true)
+		if err != nil {
+			slog.Error(err)
+			errs = append(errs, err)
 		}
 	}
 
 	if config.EgressIPv6 {
-		// Failover tests
-		err = config.Egressv6Deployment(ctx, clientset)
+		// Egress v6 tests
+		err = config.Egressv6Deployment(ctx, clientset, false)
 		if err != nil {
 			slog.Error(err)
+			errs = append(errs, err)
+		}
+	}
+
+	if config.EgressIPv6 && config.EgressInternal {
+		// Egress v6 tests
+		err = config.Egressv6Deployment(ctx, clientset, true)
+		if err != nil {
+			slog.Error(err)
+			errs = append(errs, err)
 		}
 	}
 
 	if config.DualStack {
-		// Failover tests
+		// Dualstack tests
 		err = config.DualStackDeployment(ctx, clientset)
 		if err != nil {
 			slog.Error(err)
+			errs = append(errs, err)
 		}
 	}
 
-	slog.Infof("🏆 Testing Complete [%d] passed", config.SuccessCounter)
+	const testComplete = "🏆 Testing Complete [%d] passed / [%d] failed"
+	if len(errs) > 0 {
+		slog.Errorf(testComplete, config.SuccessCounter, len(errs))
+		return errs
+	}
+	slog.Infof(testComplete, config.SuccessCounter, len(errs))
+
+	return errs
 }
 
 func httpTest(address string) error {
@@ -147,7 +184,7 @@ func leaderFailover(ctx context.Context, name, leaderNode *string, clientset *ku
 	slog.Infof("👀 service [%s] for updates", *name)
 
 	// Use a restartable watcher, as this should help in the event of etcd or timeout issues
-	rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
+	rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
 		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
 			return clientset.CoreV1().Services(v1.NamespaceDefault).Watch(ctx, metav1.ListOptions{})
 		},
@@ -230,7 +267,7 @@ func podFailover(ctx context.Context, name, leaderNode *string, clientset *kuber
 	slog.Infof("👀 service [%s] for updates", *name)
 
 	// Use a restartable watcher, as this should help in the event of etcd or timeout issues
-	rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
+	rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
 		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
 			return clientset.CoreV1().Services(v1.NamespaceDefault).Watch(ctx, metav1.ListOptions{})
 		},
@@ -282,43 +319,62 @@ func podFailover(ctx context.Context, name, leaderNode *string, clientset *kuber
 	return nil
 }
 
-func tcpServer(IPv6 bool, egressAddress *string, timeout int) bool {
+func tcpServer(egressAddress *string, timeout int, network string) bool {
 	var listen net.Listener
 	var err error
-	if !IPv6 {
-		listen, err = net.Listen("tcp", ":12345") //nolint
-	} else {
-		listen, err = net.Listen("tcp6", ":12346") //nolint
 
+	port := ":12345"
+	if network == "tcp6" {
+		port = ":12346"
 	}
+
+	listen, err = net.Listen(network, port) //nolint
 	if err != nil {
 		slog.Error(err)
 	}
-	// close listener
-	go func() {
-		time.Sleep(time.Second * time.Duration(timeout))
-		listen.Close()
-	}()
-	for {
-		conn, err := listen.Accept()
-		if err != nil {
-			return false
-			// slog.Fatal(err)
-		}
-		remoteAddress, _, _ := net.SplitHostPort(conn.RemoteAddr().String())
-		// If it is IPv6 expand the address
-		// if IPv6 {
-		// 	addr, _ := netip.ParseAddr(remoteAddress)
-		// 	remoteAddress = addr.StringExpanded()
-		// }
 
-		if remoteAddress == *egressAddress {
-			slog.Infof("📞 👍 incoming from egress Address [%s]", remoteAddress)
-			return true
+	srvChan := make(chan any)
+	finishChan := make(chan any)
+
+	go func() {
+		defer func() {
+			err = listen.Close()
+			if err != nil {
+				slog.Error(err)
+			}
+			close(finishChan)
+		}()
+		select {
+		case <-srvChan:
+			return
+		case <-time.After(time.Second * time.Duration(timeout)):
+			return
 		}
-		slog.Infof("📞 👎 incoming from pod address [%s]", remoteAddress)
-		go handleRequest(conn)
-	}
+	}()
+
+	result := false
+	go func() {
+		defer close(srvChan)
+		for {
+			conn, err := listen.Accept()
+			if err != nil {
+				result = false
+				return
+			}
+
+			remoteAddress, _, _ := net.SplitHostPort(conn.RemoteAddr().String())
+			if remoteAddress == *egressAddress {
+				slog.Infof("📞 👍 incoming from egress Address [%s]", remoteAddress)
+				result = true
+				return
+			}
+			slog.Infof("📞 👎 incoming from pod address [%s]", remoteAddress)
+			go handleRequest(conn)
+		}
+	}()
+
+	<-finishChan
+	return result
 }
 
 func handleRequest(conn net.Conn) {
@@ -339,34 +395,32 @@ func handleRequest(conn net.Conn) {
 	conn.Close()
 }
 
-func GetLocalIPv4() string {
-	addrs, err := net.InterfaceAddrs()
+func GetLocalIP(ifName string, family int) (*net.IP, *net.IPNet, error) {
+	links, err := netlink.LinkList()
 	if err != nil {
-		return ""
+		return nil, nil, fmt.Errorf("netlink: failed to list links: %w", err)
 	}
-	for _, address := range addrs {
-		// check the address type and if it is not a loopback the display it
-		if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
-			if ipnet.IP.To4() != nil {
-				return ipnet.IP.String()
+
+	for _, link := range links {
+		if strings.Contains(link.Attrs().Name, ifName) {
+			ip, ipnet, err := getNetwork(link, family)
+			if err != nil {
+				return nil, nil, fmt.Errorf("failed to get IPv4 address: %w", err)
 			}
+			if ip == nil {
+				return nil, nil, fmt.Errorf("failed to find IPv4 address on the interface %q", ifName)
+			}
+			return ip, ipnet, nil
 		}
 	}
-	return ""
+
+	return nil, nil, nil
 }
 
-func GetLocalIPv6() string {
-	addrs, err := net.InterfaceAddrs()
-	if err != nil {
-		return ""
-	}
-	for _, address := range addrs {
-		// check the address type and if it is not a loopback the display it
-		if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() && ipnet.IP.To4() == nil && !ipnet.IP.IsLinkLocalUnicast() {
-			if ipnet.IP.To16() != nil {
-				return ipnet.IP.String()
-			}
-		}
-	}
-	return ""
+func GetLocalIPv4(ifName string) (*net.IP, *net.IPNet, error) {
+	return GetLocalIP(ifName, netlink.FAMILY_V4)
+}
+
+func GetLocalIPv6(ifName string) (*net.IP, *net.IPNet, error) {
+	return GetLocalIP(ifName, netlink.FAMILY_V6)
 }

testing/services/pkg/deployment/tests.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"net"
-	"time"
 
 	"github.com/gookit/slog"
 	v1 "k8s.io/api/core/v1"
@@ -33,6 +32,7 @@ type TestConfig struct {
 	LocalDeploy        bool
 	DualStack          bool
 	Egress             bool
+	EgressInternal     bool
 	EgressIPv6         bool
 	RetainCluster      bool
 	SkipHostnameChange bool
@@ -45,6 +45,9 @@ type TestConfig struct {
 
 	// Cilium config
 	Cilium bool
+
+	// Docker config
+	DockerNIC string
 }
 
 func (config *TestConfig) SimpleDeployment(ctx context.Context, clientset *kubernetes.Clientset) error {
@@ -305,8 +308,8 @@ func (config *TestConfig) LocalDeployment(ctx context.Context, clientset *kubern
 	return nil
 }
 
-func (config *TestConfig) EgressDeployment(ctx context.Context, clientset *kubernetes.Clientset) error {
-	// pod Failover tests
+func (config *TestConfig) EgressDeployment(ctx context.Context, clientset *kubernetes.Clientset, internal bool) error {
+	// egress test
 
 	var err error
 	defer func() error {
@@ -322,14 +325,10 @@ func (config *TestConfig) EgressDeployment(ctx context.Context, clientset *kuber
 		return nil
 	}() //nolint
 
-	slog.Infof("🧪 ---> egress IP re-write (local policy) <---")
+	slog.Infof("🧪 ---> egress IP re-write (local policy, internal: %t) <---", internal)
 	var egress string
 	var found bool
 	timeout := 30
-	// Set up a local listener
-	go func() {
-		found = tcpServer(false, &egress, timeout)
-	}()
 
 	deploy := Deployment{
 		name:         config.DeploymentName,
@@ -339,7 +338,11 @@ func (config *TestConfig) EgressDeployment(ctx context.Context, clientset *kuber
 	}
 
 	// Find this machines IP address
-	deploy.address = GetLocalIPv4()
+	addr, _, err := GetLocalIPv4(config.DockerNIC)
+	if err != nil {
+		return fmt.Errorf("unable to detect local IP address: %w", err)
+	}
+	deploy.address = addr.String()
 	if deploy.address == "" {
 		return fmt.Errorf("unable to detect local IP address")
 	}
@@ -358,30 +361,34 @@ func (config *TestConfig) EgressDeployment(ctx context.Context, clientset *kuber
 		timeout:     30,
 	}
 
+	if internal {
+		svc.egressInternal = true
+	}
+
 	_, lbAddresses, err := svc.CreateService(ctx, clientset)
 	if err != nil {
 		return err
 	}
-	egress = lbAddresses[0]
-
-	for i := 1; i < 15; i++ {
-		if found {
-			slog.Infof("🕵️  egress has correct IP address")
-			config.SuccessCounter++
-			break
-		}
-		time.Sleep(time.Second * 1)
+	if len(lbAddresses) < 1 {
+		return fmt.Errorf("no loadbalancer address found")
 	}
 
-	if !found {
+	egress = lbAddresses[0]
+
+	found = tcpServer(&egress, timeout, "tcp4")
+
+	if found {
+		slog.Infof("🕵️  egress has correct IP address")
+		config.SuccessCounter++
+	} else {
 		return fmt.Errorf("😱 No traffic found from loadbalancer address ")
 	}
 
-	return err
+	return nil
 }
 
-func (config *TestConfig) Egressv6Deployment(ctx context.Context, clientset *kubernetes.Clientset) error {
-	// pod Failover tests
+func (config *TestConfig) Egressv6Deployment(ctx context.Context, clientset *kubernetes.Clientset, internal bool) error {
+	// egress v6 test
 
 	var err error
 	defer func() error {
@@ -397,14 +404,10 @@ func (config *TestConfig) Egressv6Deployment(ctx context.Context, clientset *kub
 		return nil
 	}() //nolint
 
-	slog.Infof("🧪 ---> egress IP re-write IPv6 (local policy) <---")
+	slog.Infof("🧪 ---> egress IP re-write IPv6 (local policy, internal: %t) <---", internal)
 	var egress string
 	var found bool
-	timeout := 60
-	// Set up a local listener
-	go func() {
-		found = tcpServer(true, &egress, timeout)
-	}()
+	timeout := 30
 
 	deploy := Deployment{
 		name:         config.DeploymentName,
@@ -414,9 +417,13 @@ func (config *TestConfig) Egressv6Deployment(ctx context.Context, clientset *kub
 	}
 
 	// Find this machines IP address
-	deploy.address = GetLocalIPv6()
+	addr, _, err := GetLocalIPv6(config.DockerNIC)
+	if err != nil {
+		return fmt.Errorf("unable to detect local IP address: %w", err)
+	}
+	deploy.address = addr.String()
 	if deploy.address == "" {
-		slog.Fatalf("Unable to detect local IP address")
+		return fmt.Errorf("unable to detect local IP address")
 	}
 	slog.Infof("📠 found local address [%s]", deploy.address)
 	// Create a deployment that connects back to this machines IP address
@@ -430,20 +437,23 @@ func (config *TestConfig) Egressv6Deployment(ctx context.Context, clientset *kub
 		name:          config.ServiceName,
 		egress:        true,
 		egressIPv6:    true,
-		testHTTP:      false,
 		timeout:       timeout,
 		testDualstack: true,
 	}
 
+	if internal {
+		svc.egressInternal = true
+	}
+
 	_, lbAddresses, err := svc.CreateService(ctx, clientset)
 	if err != nil {
 		return err
 	}
 	for x := range lbAddresses {
 		ip := net.ParseIP(lbAddresses[x])
-		// if ip == nil {
-		// 	return errors.New("invalid address")
-		// }
+		if ip == nil {
+			return fmt.Errorf("invalid address")
+		}
 		if ip.To4() == nil {
 			// use brackets for IPv6 address
 			egress = lbAddresses[x]
@@ -451,20 +461,20 @@ func (config *TestConfig) Egressv6Deployment(ctx context.Context, clientset *kub
 		}
 	}
 
-	for i := 1; i < timeout; i++ {
-		if found {
-			slog.Infof("🕵️  egress has correct IP address")
-			config.SuccessCounter++
-			break
-		}
-		time.Sleep(time.Second * 1)
+	if egress == "" {
+		return fmt.Errorf("no loadbalancer address found")
 	}
 
-	if !found {
-		slog.Error("😱 No traffic found from loadbalancer address ")
+	found = tcpServer(&egress, timeout, "tcp6")
+
+	if found {
+		slog.Infof("🕵️  egress has correct IP address")
+		config.SuccessCounter++
+	} else {
+		return fmt.Errorf("😱 No traffic found from loadbalancer address ")
 	}
 
-	return err
+	return nil
 }
 
 func (config *TestConfig) DualStackDeployment(ctx context.Context, clientset *kubernetes.Clientset) error {