fix: regression on tests as we moved to context

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# Enable GitHub funding
+
+github: [kube-vip]
+

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,35 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Environment (please complete the following information):**
+ - OS/Distro: [e.g. Ubuntu 1804]
+ - Kubernetes Version: [e.g. v.1.18]
+ - Kube-vip Version: [e.g. 0.2.3]
+
+**`Kube-vip.yaml`:**
+If Possible add in your kube-vip manifest (please remove anything that is confidential)
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: gomod
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: /
+    schedule:
+      interval: weekly

.github/suggestion-comment.md

@@ -0,0 +1,13 @@
+I'll help you add a suggestion. Unfortunately, I can't directly add a suggestion to an existing comment through the API. However, here's what I recommend:
+
+**Option 1: Reply with a suggestion**
+Create a new comment with a suggested fix:
+
+```suggestion
+failed to get an IPv6 address after %d attempt(s), giving up, error: %s
+```
+
+**Option 2: Edit your existing comment**
+Update your comment to include the suggestion details pointing out that line 284 in the error message says "IPv4" but should say "IPv6" since this is the DHCPv6Client.
+
+Would you like me to create a new reply comment with the suggestion instead?

.github/workflows/anchore-syft.yml

@@ -0,0 +1,31 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# This workflow checks out code, builds an image, performs a container image
+# scan with Anchore's Syft tool, and uploads the results to the GitHub Dependency
+# submission API.
+
+# For more information on the Anchore sbom-action usage
+# and parameters, see https://github.com/anchore/sbom-action. For more
+# information about the Anchore SBOM tool, Syft, see
+# https://github.com/anchore/syft
+name: Anchore Syft SBOM scan
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  sbom:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+        with:
+          ref: ${{ github.ref_name }}
+      - name: Anchore SBOM Action
+        uses: anchore/sbom-action@v0.24.2
+        with:
+          format: cyclonedx-json

.github/workflows/ci-pull-request.yaml

@@ -0,0 +1,88 @@
+name: For each PR
+on:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  e2e-tests:
+    runs-on: ubuntu-latest
+    name: E2E tests
+    timeout-minutes: 120
+    env:
+      GINKGO_PROCS: ${{ matrix.ginkgo-procs }}
+    strategy:
+      fail-fast: false
+      max-parallel: 3
+      matrix:
+        include:
+          - mode: arp
+            ginkgo-procs: 4
+          - mode: rt
+            ginkgo-procs: 4
+          - mode: bgp
+            ginkgo-procs: 4
+    steps:
+      - name: Get current date
+        id: date
+        run: echo "date=$(date +'%Y-%m-%d-%H-%M')" >> "$GITHUB_OUTPUT"
+      - name: Ensure fs wont cause issues
+        run: sudo sysctl fs.inotify.max_user_instances=8192 &&  sudo sysctl fs.inotify.max_user_watches=524288
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Install Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+      - name: Build image locally
+        run: make dockerx86Local
+      - name: Run Manifest generation tests
+        run: make manifest-test
+        if: matrix.mode == 'arp'
+      - name: Run ARP mode tests
+        run: DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true GINKGO_ARGS="--output-dir=/tmp --json-report=kube-vip-test-report-arp.json" make e2e-tests-arp
+        if: matrix.mode == 'arp'
+      - name: Run RT mode tests
+        run: DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true GINKGO_ARGS="--output-dir=/tmp --json-report=kube-vip-test-report-rt.json" make e2e-tests-rt
+        if: matrix.mode == 'rt'
+      - name: Get GoBGP binaries
+        run: make get-gobgp
+        if: matrix.mode == 'bgp'
+      - name: Run BGP mode tests
+        run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true GINKGO_ARGS="--output-dir=/tmp --json-report=kube-vip-test-report-bgp.json" make e2e-tests-bgp
+        if: matrix.mode == 'bgp'
+      - name: Change log directory permissions
+        run: sudo chmod -R 755 /tmp/kube-vip-test*
+        if: matrix.mode == 'bgp' && always()
+      - name: Save logs
+        uses: actions/upload-artifact@v7
+        with:
+          name: e2e-test-logs-${{ matrix.mode }}-${{ steps.date.outputs.date }}
+          path: /tmp/kube-vip-test*
+        if: always()
+  service-e2e-tests:
+    runs-on: ubuntu-latest
+    name: E2E service tests
+    timeout-minutes: 60
+    steps:
+      - name: Get current date
+        id: date
+        run: echo "date=$(date +'%Y-%m-%d-%H-%M')" >> "$GITHUB_OUTPUT"
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Install Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+      - name: Build image with iptables
+        run: make dockerx86ActionIPTables
+      - name: Run tests
+        run: DOCKER_API_VERSION=1.48 DOCKERTAG=action E2E_KEEP_LOGS=true make service-tests
+      - name: Save logs
+        uses: actions/upload-artifact@v7
+        with:
+          name: services-test-logs-${{ steps.date.outputs.date }}
+          path: /tmp/kube-vip-services*
+        if: always()

.github/workflows/ci.yaml

@@ -0,0 +1,88 @@
+name: For each commit
+on:
+  push:
+    branches: [main]
+    tags: ['v*']
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  validation:
+    runs-on: ubuntu-latest
+    name: Checks and linters
+    timeout-minutes: 30
+    steps:
+      - name: Init
+        run: sudo apt-get update && sudo apt-get install -y build-essential &&  sudo sysctl fs.inotify.max_user_instances=8192 &&  sudo sysctl fs.inotify.max_user_watches=524288
+      - name: Install golangci-lint
+        uses: golangci/golangci-lint-action@v9
+        with:
+          version: v2.12
+          install-only: true
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Install Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+      - name: Verify gofmt
+        run: |
+          unformatted=$(gofmt -l .)
+          if [ -n "$unformatted" ]; then
+            echo "The following files are not gofmt-formatted:"
+            echo "$unformatted"
+            exit 1
+          fi
+      - name: All checks
+        run: make check
+  unit-tests:
+    runs-on: ubuntu-latest
+    name: Unit tests
+    timeout-minutes: 30
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Install Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+      - name: Run tests
+        run: make unit-tests
+      - name: Run privileged network tests
+        run: |
+          sudo -E env PATH="$PATH" KUBE_VIP_REQUIRE_NETNS=1 go test -race ./pkg/services ./pkg/vip ./pkg/instance \
+            -run 'TestRecover|TestServiceAddressRetained|TestRetainControlPlaneVIPs|TestAddressProtocol|TestKubeVIPAddressProtocol|TestCleanupKubeVIPAddresses|TestMonitorDefaultInterfaceReturnsErrorWhenTestLinkIsSetDown|TestCleanupLinkAttachmentsOnlyDeletesOwnedVLAN'
+  integration-tests:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Install Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+      - name: Run tests
+        run: make integration-tests
+  image-vul-check:
+    runs-on: ubuntu-latest
+    name: Image vulnerability scan
+    timeout-minutes: 30
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Build image
+        run: make dockerx86Action
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: "plndr/kube-vip:action"
+          format: "table"
+          exit-code: "1"
+          ignore-unfixed: true
+          vuln-type: "os,library"
+          severity: "CRITICAL,HIGH"

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,75 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '17 10 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v7
+      
+    - name: Install Go
+      uses: actions/setup-go@v7
+      with:
+        go-version-file: go.mod
+        
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v4
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v4
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v4

.github/workflows/main.yaml

@@ -0,0 +1,52 @@
+name: Build and publish main image regularly
+
+on:
+  schedule:
+    - cron: '25 0 * * *'
+  workflow_dispatch:
+
+
+jobs:
+  nightly_build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+      - name: Login to DockerHub
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to Github Packages
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}   
+      - name: Build standard version
+        id: docker_build
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: >-
+            plndr/kube-vip:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip:${{ github.ref_name }}
+      - name: Build iptables version
+        id: docker_build_iptables
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: Dockerfile_iptables
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: >-
+            plndr/kube-vip-iptables:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip-iptables:${{ github.ref_name }}
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/nightly-e2e.yaml

@@ -0,0 +1,111 @@
+name: Nightly e2e
+
+on:
+  schedule:
+    - cron: '30 2 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  unit-coverage:
+    runs-on: ubuntu-latest
+    name: Unit tests with coverage
+    timeout-minutes: 30
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Install Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+      - name: Run tests
+        run: make unit-tests
+      - name: Summarize coverage
+        if: always()
+        run: |
+          if test -f coverage.out; then
+            echo "### Unit coverage" >> "$GITHUB_STEP_SUMMARY"
+            go tool cover -func=coverage.out | tail -1 >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "### Unit coverage: report missing" >> "$GITHUB_STEP_SUMMARY"
+          fi
+      - name: Upload coverage
+        uses: actions/upload-artifact@v7
+        with:
+          name: unit-coverage
+          path: coverage.out
+          if-no-files-found: error
+        if: always()
+  etcd-e2e:
+    runs-on: ubuntu-latest
+    name: Etcd E2E tests
+    timeout-minutes: 60
+    steps:
+      - name: Ensure fs wont cause issues
+        run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Install Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+      - name: Build image locally
+        run: make dockerx86Local
+      - name: Prepare Etcd artifacts
+        if: always()
+        run: |
+          mkdir -p /tmp/kube-vip-etcd-artifacts
+          : > /tmp/kube-vip-etcd-artifacts/suite.log
+          printf '[]\n' > /tmp/kube-vip-etcd-artifacts/report.json
+      - name: Run Etcd tests
+        id: etcd
+        # Scheduled failures are tolerated only during the initial stabilization window.
+        # The enforcement step below makes manual runs and later schedules blocking.
+        continue-on-error: true
+        shell: bash
+        run: |
+          set +e
+          set -o pipefail
+          DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true \
+            GINKGO_ARGS="--json-report=report.json --output-dir=/tmp/kube-vip-etcd-artifacts" \
+            make e2e-tests-etcd 2>&1 | tee /tmp/kube-vip-etcd-artifacts/suite.log
+          exit_code=${PIPESTATUS[0]}
+          echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT"
+          exit "$exit_code"
+      - name: Summarize Etcd suite
+        if: always()
+        env:
+          OUTCOME: ${{ steps.etcd.outcome }}
+          EXIT_CODE: ${{ steps.etcd.outputs.exit_code }}
+        run: |
+          echo "### Etcd E2E result: ${OUTCOME}" >> "$GITHUB_STEP_SUMMARY"
+          printf '{"outcome":"%s","exit_code":%s,"event":"%s","cutoff":"2026-10-01"}\n' \
+            "${OUTCOME:-skipped}" "${EXIT_CODE:-null}" "$GITHUB_EVENT_NAME" \
+            > /tmp/kube-vip-etcd-artifacts/result.json
+      - name: Save logs
+        uses: actions/upload-artifact@v7
+        continue-on-error: true
+        with:
+          name: etcd-e2e-logs
+          path: |
+            /tmp/kube-vip-etcd-artifacts
+            /tmp/kube-vip-test*
+          if-no-files-found: warn
+        if: always()
+      - name: Enforce Etcd result
+        if: always()
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          OUTCOME: ${{ steps.etcd.outcome }}
+        run: |
+          if test "$OUTCOME" = success; then
+            exit 0
+          fi
+          if test "$EVENT_NAME" = schedule && test "$(date -u +%Y-%m-%d)" \< 2026-10-01; then
+            echo "::warning::etcd e2e suite outcome was ${OUTCOME:-skipped} during stabilization through 2026-09-30"
+            exit 0
+          fi
+          echo "::error::etcd e2e suite outcome was ${OUTCOME:-skipped}; see the etcd-e2e-logs artifact"
+          exit 1

.github/workflows/release.yaml

@@ -0,0 +1,64 @@
+name: Publish Releases to Docker Hub and GitHub Container Registry
+
+on:
+  push:
+    tags:
+      - '*'
+  workflow_dispatch:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+      - name: Login to DockerHub
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to Github Packages
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Generate Metadata
+        uses: docker/metadata-action@v6.2.0
+        id: metadata
+        with:
+          labels: |
+            org.opencontainers.image.documentation=https://kube-vip.io/docs/
+      - name: Build and push main branch
+        id: docker_build
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          labels: ${{ steps.metadata.outputs.labels }}
+          tags: >-
+            plndr/kube-vip:${{ github.ref_name }},
+            plndr/kube-vip:latest,
+            ghcr.io/kube-vip/kube-vip:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip:latest
+      - name: Build iptables version and push main branch
+        id: docker_build_iptables
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: Dockerfile_iptables
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          labels: ${{ steps.metadata.outputs.labels }}
+          tags: >-
+            plndr/kube-vip-iptables:${{ github.ref_name }},
+            plndr/kube-vip-iptables:latest,
+            ghcr.io/kube-vip/kube-vip-iptables:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip-iptables:latest
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

.gitignore

@@ -0,0 +1,9 @@
+.idea
+kube-vip
+.vscode
+bin
+testing/e2e/etcd/certs
+coverage.out
+pkg/etcd/etcd.pid
+pkg/etcd/etcd-data
+testing/e2e/e2e.test

.golangci.yml

@@ -0,0 +1,38 @@
+version: "2"
+linters:
+  enable:
+    - bodyclose
+    - gosec
+    - misspell
+    - unconvert
+    - unparam
+  settings:
+    misspell:
+      ignore-rules:
+        - creater
+    staticcheck:
+      checks:
+        - all
+        # Disable QF1008 to retain embedded fields for better readability.
+        - "-QF1008"
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  enable:
+    - gofmt
+    - goimports
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

CHANGELOG.md

@@ -0,0 +1,62 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Fixed
+- Propagate `bgp_attach_ip_to_interface` into per-service config so it attaches BGP-mode Service VIPs to the interface as configured.
+- Add a configurable kube-vip instance name and use it to isolate internal nftables egress tables, persist table ownership on Services, and migrate per-Service chains without affecting other deployments. Fixes #1634.
+- Retry on 403 Forbidden and 401 Unauthorized in `ServicesWatcher` at startup with exponential backoff. Fixes #1464.
+- Reintroduce BGP config via node annotations. Fixes #1488.
+- Fail fast in runtime `manager` and `service` paths when legacy `vip_address` is used without `vip_subnet` in control-plane ARP, BGP, or Routing Table mode.
+- Cancel the mode context on init or configuration failure before waiting on goroutines during shutdown.
+
+
+### Added
+- Configurable control-plane health check for BGP mode without leader election
+  - Polls a configurable HTTP(S) endpoint (e.g. `https://localhost:6443/livez`) to verify the exposed service is healthy (usually the local kube-apiserver)
+  - Withdraws the BGP route after a configurable number of consecutive failures, removing the unhealthy node from the ECMP set
+  - Re-announces the route automatically once the endpoint recovers
+  - Gracefully withdraws the route on shutdown (SIGTERM)
+  - Supports custom CA certificates for TLS verification
+  - Configuration via environment variables or CLI flags:
+    - `control_plane_health_check_address` / `--controlPlaneHealthCheckAddress`: URL to poll
+    - `control_plane_health_check_period_seconds` / `--controlPlaneHealthCheckPeriodSeconds`: interval between checks (default: 5)
+    - `control_plane_health_check_timeout_seconds` / `--controlPlaneHealthCheckTimeoutSeconds`: per-request timeout (default: 3)
+    - `control_plane_health_check_failure_threshold` / `--controlPlaneHealthCheckFailureThreshold`: consecutive failures before withdrawal (default: 3)
+    - `control_plane_health_check_ca_path` / `--controlPlaneHealthCheckCAPath`: CA cert for HTTPS verification
+- SIGUSR1 signal handler for runtime configuration dumps (#1301)
+  - Send SIGUSR1 to kube-vip process to dump current configuration to stdout
+  - Configuration dump includes:
+    - Basic configuration (VIP, interface, port, namespace settings)
+    - BGP configuration (enabled status, AS number, router ID, peers)
+    - ARP/NDP configuration (enabled status, broadcast rate)
+    - Services configuration (enabled status, load balancer settings)
+    - Network interfaces status
+    - Leader election configuration (type, lease details)
+    - Runtime statistics (load balancer, Prometheus, health check settings)
+  - Output format: Human-readable plaintext via fmt.Printf()
+  - Thread-safe implementation using mutex protection
+  - Non-disruptive: Process continues running after configuration dump
+  - Added comprehensive unit tests for all dump methods
+  - Added E2E tests for signal handling
+- Opt-in support for endpointless `LoadBalancer` services with `externalTrafficPolicy: Cluster`
+  - Annotation: `kube-vip.io/allow-reconcile-without-endpoints: "true"`
+  - Starts service handling path for opted-in endpointless Cluster services while preserving default endpoint-gated behavior for non-opt-in services and `Local` policy
+  - Added endpoint behavior tests and README usage documentation
+  - Added support in ipoib interfaces in ARP mode. Fixes #694
+
+### Changed
+- BGP mode now honours `enable_leader_election` for services: a single global services leader advertises the service VIPs instead of every node advertising them. Deployments that enabled `enable_leader_election` for the control plane and relied on ECMP/multipath for services must unset it (or switch to `enable_service_election`) to keep the previous datapath. kube-vip logs a warning on startup when this path is taken.
+- Updated signal handlers in manager_arp.go, manager_bgp.go, manager_wireguard.go, and manager_table.go to use switch statement pattern for handling multiple signals (SIGUSR1, SIGINT, SIGTERM)
+- wireguard.go now manages a complete wireguard interface on the current network namespace
+- manager_wireguard.go uses the new wireguard.go implementation
+
+## [v1.0.1] - Previous Release
+
+### Previous changes
+- See git history for changes prior to CHANGELOG.md introduction

CODEOWNERS

@@ -0,0 +1,10 @@
+
+# For more information on the syntax of the CODEOWNERS file, see:
+# https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners
+
+# The Kube-Vip maintainers team
+*   @thebsdbox @yastij
+
+# Emeritus Maintainers
+#
+# N/A

CODE_OF_CONDUCT.md

@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+conduct@kube-vip.io.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

CONTRIBUTING.md

@@ -0,0 +1,289 @@
+# Developer Guide
+
+Thank you for taking the time out to contribute to the **kube-vip** project!
+
+This guide will walk you through the process of making your first commit and how
+to effectively get it merged upstream.
+
+<!-- toc -->
+- [Getting Started](#getting-started)
+- [Contribute](#contribute)
+  - [GitHub Workflow](#github-workflow)
+  - [Getting reviewers](#getting-reviewers)
+  - [Inclusive Naming](#inclusive-naming)
+  - [Building and testing your change](#building-and-testing-your-change)
+  - [Reverting a commit](#reverting-a-commit)
+  - [Sign-off Your Work](#sign-off-your-work)
+- [Issue and PR Management](#issue-and-pr-management)
+  - [Filing An Issue](#filing-an-issue)
+  - [Issue Triage](#issue-triage)
+<!-- /toc -->
+
+## Getting Started
+
+To get started, let's ensure you have completed the following prerequisites for
+contributing to project **kube-vip**:
+
+1. Read and observe the [code of conduct](CODE_OF_CONDUCT.md).
+2. Check out the [Architecture documentation](https://kube-vip.io) for the **kube-vip**
+   architecture and design.
+3. Set up your [development environment](docs/contributors/manual-installation.md)
+
+Now that you're setup, skip ahead to learn how to [contribute](#contribute).
+
+**Also**, A GitHub account will be required in order to submit code changes and
+interact with the project. For committing any changes in  **Github** it is required for
+you to have a [github account](https://github.com/join).
+
+## Contribute
+
+There are multiple ways in which you can contribute, either by contributing
+code in the form of new features or bug-fixes or non-code contributions like
+helping with code reviews, triaging of bugs, documentation updates, filing
+[new issues](#filing-an-issue) or writing blogs/manuals etc.
+
+### GitHub Workflow
+
+Developers work in their own forked copy of the repository and when ready,
+submit pull requests to have their changes considered and merged into the
+project's repository.
+
+1. Fork your own copy of the repository to your GitHub account by clicking on
+   `Fork` button on [kube-vip's GitHub repository](https://github.com/kube-vip/kube-vip).
+2. Clone the forked repository on your local setup.
+
+    ```bash
+    git clone https://github.com/$user/kube-vip
+    ```
+
+    Add a remote upstream to track upstream kube-vip repository.
+
+    ```bash
+    git remote add upstream https://github.com/kube-vip/kube-vip
+    ```
+
+    Never push to upstream remote
+
+    ```bash
+    git remote set-url --push upstream no_push
+    ```
+
+3. Create a topic branch.
+
+    ```bash
+    git checkout -b branchName
+    ```
+
+4. Make changes and commit it locally. Make sure that your commit is
+   [signed](#sign-off-your-work).
+
+    ```bash
+    git add <modifiedFile>
+    git commit -s
+    ```
+
+5. Update the "Unreleased" section of the [CHANGELOG](CHANGELOG.md) for any
+   significant change that impacts users.
+6. Keeping branch in sync with upstream.
+
+    ```bash
+    git checkout branchName
+    git fetch upstream
+    git rebase upstream/main
+    ```
+
+7. Push local branch to your forked repository.
+
+    ```bash
+    git push -f $remoteBranchName branchName
+    ```
+
+8. Create a Pull request on GitHub.
+   Visit your fork at `https://github.com/kube-vip/kube-vip` and click
+   `Compare & Pull Request` button next to your `remoteBranchName` branch.
+
+### Getting reviewers
+
+Once you have opened a Pull Request (PR), reviewers will be assigned to your
+PR and they may provide review comments which you need to address.
+Commit changes made in response to review comments to the same branch on your
+fork. Once a PR is ready to merge, squash any *fix review feedback, typo*
+and *merged* sorts of commits.
+
+To make it easier for reviewers to review your PR, consider the following:
+
+1. Follow the golang [coding conventions](https://github.com/golang/go/wiki/CodeReviewComments).
+2. Format your code with `make simplify` to automatically fix formatting issues.
+2. Lint your code with `make check`; if the linters flag an issue that cannot be fixed automatically, an error message will be displayed so you can address the issue.
+3. Follow [git commit](https://chris.beams.io/posts/git-commit/) guidelines.
+4. Follow [logging](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md) guidelines.
+
+If your PR fixes a bug or implements a new feature, add the appropriate test
+cases to our [automated test suite](ci/README.md) to guarantee enough
+coverage. A PR that makes significant code changes without contributing new test
+cases will be flagged by reviewers and will not be accepted.
+
+### Inclusive Naming
+
+For symbol names and documentation, do not introduce new usage of harmful
+language such as 'master / slave' (or 'slave' independent of 'master') and
+'blacklist / whitelist'. For more information about what constitutes harmful
+language and for a reference word replacement list, please refer to the
+[Inclusive Naming Initiative](https://inclusivenaming.org/).
+
+We are committed to removing all harmful language from the project. If you
+detect existing usage of harmful language in code or documentation, please
+report the issue to us or open a Pull Request to address it directly. Thanks!
+
+### Building and testing your change
+
+To build the **kube-vip** Docker image together with all **kube-vip** bits, you can simply
+do:
+
+1. Checkout your feature branch and `cd` into it.
+2. Run `make dockerx86`
+
+The second step will compile the **kube-vip** code in a `golang` container, and build
+a `Ubuntu 20.04` Docker image that includes all the generated binaries. [`Docker`](https://docs.docker.com/install)
+must be installed on your local machine in advance.
+
+Alternatively, you can build the **kube-vip** code in your local Go environment. The
+**kube-vip** project uses the [Go modules support](https://github.com/golang/go/wiki/Modules) which was introduced in Go 1.11. It
+facilitates dependency tracking and no longer requires projects to live inside
+the `$GOPATH`.
+
+To develop locally, you can follow these steps:
+
+ 1. [Install Go 1.19](https://golang.org/doc/install)
+ 2. Checkout your feature branch and `cd` into it.
+ 3. To build all Go files and install them under `bin`, run `make bin`
+ 4. To run all Go unit tests, run `make test-unit`
+ 5. To build the **kube-vip** Ubuntu Docker image separately with the binaries generated in step 2, run `make ubuntu`
+
+### CI testing
+
+For more information about the tests we run as part of CI, please refer to
+[ci/README.md](ci/README.md).
+
+### Reverting a commit
+
+1. Create a branch in your forked repo
+
+    ```bash
+    git checkout -b revertName
+    ```
+
+2. Sync the branch with upstream
+
+    ```bash
+    git fetch upstream
+    git rebase upstream/main
+    ```
+
+3. Create a revert based on the SHA of the commit. The commit needs to be
+   [signed](#sign-off-your-work).
+
+    ```bash
+    git revert -s SHA
+    ```
+
+4. Push this new commit.
+
+    ```bash
+    git push $remoteRevertName revertName
+    ```
+
+5. Create a Pull Request on GitHub.
+   Visit your fork at `https://github.com/kube-vip/kube-vip` and click
+   `Compare & Pull Request` button next to your `remoteRevertName` branch.
+
+### Sign-off Your Work
+
+It is recommended to sign your work when contributing to the **kube-vip**
+repository. 
+
+Git provides the `-s` command-line option to append the required line
+automatically to the commit message:
+
+```bash
+git commit -s -m 'This is my commit message'
+```
+
+For an existing commit, you can also use this option with `--amend`:
+
+```bash
+git commit -s --amend
+```
+
+If more than one person works on something it's possible for more than one
+person to sign-off on it. For example:
+
+```bash
+Signed-off-by: Some Developer somedev@example.com
+Signed-off-by: Another Developer anotherdev@example.com
+```
+
+We use the [DCO Github App](https://github.com/apps/dco) to enforce that all
+commits in a Pull Request include the required `Signed-off-by` line. If this is
+not the case, the app will report a failed status for the Pull Request and it
+will be blocked from being merged.
+
+Compared to our earlier CLA, DCO tends to make the experience simpler for new
+contributors. If you are contributing as an employee, there is no need for your
+employer to sign anything; the DCO assumes you are authorized to submit
+contributions (it's your responsibility to check with your employer).
+
+## Issue and PR Management
+
+We use labels and workflows (some manual, some automated with GitHub Actions) to
+help us manage triage, prioritize, and track issue progress. For a detailed
+discussion, see [docs/issue-management.md](docs/contributors/issue-management.md).
+
+### Filing An Issue
+
+Help is always appreciated. If you find something that needs fixing, please file
+an issue [here](https://github.com/kube-vip/kube-vip/issues). Please ensure
+that the issue is self explanatory and has enough information for an assignee to
+get started.
+
+Before picking up a task, go through the existing
+[issues](https://github.com/kube-vip/kube-vip/issues) and make sure that your
+change is not already being worked on. If it does not exist, please create a new
+issue and discuss it with other members.
+
+For simple contributions to **kube-vip**, please ensure that this minimum set of
+labels are included on your issue:
+
+* **kind** -- common ones are `kind/feature`, `kind/support`, `kind/bug`,
+  `kind/documentation`, or `kind/design`. For an overview of the different types
+  of issues that can be submitted, see [Issue and PR
+  Kinds](#issue-and-pr-kinds).
+  The kind of issue will determine the issue workflow.
+* **area** (optional) -- if you know the area the issue belongs in, you can assign it.
+  Otherwise, another community member will label the issue during triage. The
+  area label will identify the area of interest an issue or PR belongs in and
+  will ensure the appropriate reviewers shepherd the issue or PR through to its
+  closure. For an overview of areas, see the
+  [`docs/github-labels.md`](docs/contributors/github-labels.md).
+* **size** (optional) -- if you have an idea of the size (lines of code, complexity,
+  effort) of the issue, you can label it using a [size label](#size). The size
+  can be updated during backlog grooming by contributors. This estimate is used
+  to guide the number of features selected for a milestone.
+
+All other labels will be assigned during issue triage.
+
+### Issue Triage
+
+Once an issue has been submitted, the CI (GitHub actions) or a human will
+automatically review the submitted issue or PR to ensure that it has all relevant
+information. If information is lacking or there is another problem with the
+submitted issue, an appropriate `triage/<?>` label will be applied.
+
+After an issue has been triaged, the maintainers can prioritize the issue with
+an appropriate `priority/<?>` label.
+
+Once an issue has been submitted, categorized, triaged, and prioritized it
+is marked as `ready-to-work`. A ready-to-work issue should have labels
+indicating assigned areas, prioritization, and should not have any remaining
+triage labels.
+

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:experimental
 
-FROM golang:1.15-alpine as dev
+FROM golang:1.27.1-alpine3.23 as dev
 RUN apk add --no-cache git ca-certificates make
 RUN adduser -D appuser
 COPY . /src/
@@ -12,5 +12,8 @@ RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
     CGO_ENABLED=0 GOOS=linux make build
 
 FROM scratch
+# Add Certificates into the image, for anything that does API calls
+COPY --from=dev /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+# Add kube-vip binary
 COPY --from=dev /src/kube-vip /
-ENTRYPOINT ["/kube-vip"]
+ENTRYPOINT ["/kube-vip"]

Dockerfile_iptables

@@ -0,0 +1,21 @@
+# syntax=docker/dockerfile:experimental
+
+FROM golang:1.27.1-alpine3.23 as dev
+RUN apk add --no-cache git make
+RUN adduser -D appuser
+COPY . /src/
+WORKDIR /src
+
+ENV GO111MODULE=on
+RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
+    --mount=type=cache,sharing=locked,id=goroot,target=/root/.cache/go-build \
+    CGO_ENABLED=0 GOOS=linux make build
+
+FROM alpine:3.24.1
+# Update pkgs and add iptables
+RUN apk upgrade && \
+    apk add --no-cache iptables iptables-legacy
+
+# Add kube-vip binary
+COPY --from=dev /src/kube-vip /
+ENTRYPOINT ["/kube-vip"]

Makefile

@@ -1,12 +1,12 @@
-
 SHELL := /bin/sh
 
 # The name of the executable (default is current directory name)
 TARGET := kube-vip
-.DEFAULT_GOAL: $(TARGET)
+.DEFAULT_GOAL := $(TARGET)
 
 # These will be provided to the target
-VERSION := 0.1.9
+VERSION := v1.2.3
+
 BUILD := `git rev-parse HEAD`
 
 # Operating System Default (LINUX)
@@ -14,18 +14,20 @@ TARGETOS=linux
 
 # Use linker flags to provide version/build settings to the target
 LDFLAGS=-ldflags "-s -w -X=main.Version=$(VERSION) -X=main.Build=$(BUILD) -extldflags -static"
-
-# go source files, ignore vendor directory
-SRC = $(shell find . -type f -name '*.go' -not -path "./vendor/*")
-
 DOCKERTAG ?= $(VERSION)
-REPOSITORY = plndr
+REPOSITORY ?= docker.io/plndr
+GO_VERSION := $(word 2,$(shell grep '^go ' go.mod))
+K8S_VERSION ?= v1.35.0
+GINKGO_ARGS ?=
+GINKGO_PROCS ?=
+GINKGO_PARALLEL := $(if $(GINKGO_PROCS),--procs=$(GINKGO_PROCS),-p)
+BUILDX_CACHE_FLAGS ?=
 
-.PHONY: all build clean install uninstall fmt simplify check run
+.PHONY: all build clean install uninstall simplify check run e2e-tests unit-tests integration-tests unit-tests-docker integration-tests-docker e2e-tests-etcd
 
 all: check install
 
-$(TARGET): $(SRC)
+$(TARGET):
 	@go build $(LDFLAGS) -o $(TARGET)
 
 build: $(TARGET)
@@ -41,44 +43,149 @@ install:
 uninstall: clean
 	@rm -f $$(which ${TARGET})
 
-fmt:
-	@gofmt -l -w $(SRC)
-
 demo:
 	@cd demo
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 	@cd ..
 
 ## Remote (push of images)
-
 # This build a local docker image (x86 only) for quick testing
+
+dockerx86Dev:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 --push -t $(REPOSITORY)/$(TARGET):dev .
+	@echo New single x86 Architecture Docker image created
+
+dockerx86Iptables:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 -f ./Dockerfile_iptables --push -t $(REPOSITORY)/$(TARGET):dev .
+	@echo New single x86 Architecture Docker image created
+
+dockerx86IptablesLocal:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 -f ./Dockerfile_iptables -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@echo New single x86 Architecture Docker image created
+
 dockerx86:
+	@-rm ./kube-vip
 	@docker buildx build  --platform linux/amd64 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New single x86 Architecture Docker image created
 
 docker:
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 ## Local (docker load of images)
-
 # This will build a local docker image (x86 only), use make dockerLocal for all architectures
 dockerx86Local:
-	@docker buildx build  --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@-rm ./kube-vip
+	@docker buildx build --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) $(BUILDX_CACHE_FLAGS) .
+	@echo New Multi Architecture Docker image created
+
+dockerx86Action:
+	@-rm ./kube-vip
+	@docker buildx build --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):action $(BUILDX_CACHE_FLAGS) .
+	@echo New Multi Architecture Docker image created
+
+dockerx86ActionIPTables:
+	@-rm ./kube-vip
+	@docker buildx build --platform linux/amd64 -f ./Dockerfile_iptables --load -t $(REPOSITORY)/$(TARGET):action $(BUILDX_CACHE_FLAGS) .
 	@echo New Multi Architecture Docker image created
 
 dockerLocal:
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7 --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 simplify:
-	@gofmt -s -l -w $(SRC)
+	@gofmt -s -l -w *.go pkg cmd
 
 check:
-	@test -z $(shell gofmt -l main.go | tee /dev/stderr) || echo "[WARN] Fix formatting issues with 'make fmt'"
-	@for d in $$(go list ./... | grep -v /vendor/); do golint $${d}; done
-	@go tool vet ${SRC}
+	go mod tidy
+	test -z "$(git status --porcelain)"
+	test -z $(shell gofmt -l *.go pkg cmd) || echo "[WARN] Fix formatting issues with 'make simplify'"
+	golangci-lint run
+	go vet ./...
 
 run: install
 	@$(TARGET)
+
+manifests:
+	@make build
+	@mkdir -p ./docs/manifests/$(VERSION)/
+	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-arp.yaml
+	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-lb.yaml
+	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-bgp.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
+	@-rm ./kube-vip
+
+manifest-test: 
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --enableLoadBalancer
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --controlplane --services
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster
+
+unit-tests:
+	go test -race -coverprofile=coverage.out -covermode=atomic ./...
+
+unit-tests-docker:
+	docker run --rm -w /kube-vip -v $$(pwd):/kube-vip -v kube-vip-gomod-cache:/go/pkg/mod -v kube-vip-gobuild-cache:/root/.cache/go-build golang:$(GO_VERSION) sh -c "make unit-tests; status=$$?; chmod 666 coverage.out 2>/dev/null || true; exit $$status"
+
+integration-tests:
+	go test -tags=integration,e2e -v ./pkg/etcd
+
+e2e-tests-arp: get-whoami
+	GOMAXPROCS=4 TEST_MODE=arp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e
+
+e2e-tests-rt: get-whoami
+	GOMAXPROCS=4 TEST_MODE=rt K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e
+
+e2e-tests-bgp: get-whoami get-gobgp
+	GOMAXPROCS=4 TEST_MODE=bgp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e
+
+e2e-tests-etcd: get-whoami
+	GOMAXPROCS=4 K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e/etcd
+
+e2e-tests: e2e-tests-arp e2e-tests-rt e2e-tests-bgp
+
+service-tests:
+	$(MAKE) -C testing/e2e/e2e dockerLocal
+	E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/services -Services -simple -deployments -leaderActive -leaderFailover -localDeploy -electionFaults -egress -egressIPv6 -dualStack -egressInternal
+
+trivy: dockerx86ActionIPTables
+	docker run -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy:0.47.0 \
+		image  \
+		--format table \
+		--exit-code  1 \
+		--ignore-unfixed \
+		--vuln-type  'os,library' \
+		--severity  'CRITICAL,HIGH'  \
+		$(REPOSITORY)/$(TARGET):action
+
+kind-quick:
+	echo "Standing up your cluster"
+	kind create cluster --config ./testing/kind/kind.yaml --name kube-vip
+	kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
+	kubectl create configmap --namespace kube-system kubevip --from-literal range-global=172.18.100.10-172.18.100.30
+	kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
+	kind load docker-image --name kube-vip $(REPOSITORY)/$(TARGET):$(DOCKERTAG)
+	docker run --network host --rm $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --services --inCluster --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --servicesElection --interface  eth0 | kubectl apply -f -
+
+kind-reload:
+	kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name services
+	kubectl rollout restart -n kube-system daemonset/kube-vip-ds
+
+get-gobgp:
+	mkdir -p bin
+	wget -nc --directory-prefix=bin https://github.com/osrg/gobgp/releases/download/v4.6.0/gobgp_4.6.0_linux_amd64.tar.gz
+	tar -xvzf bin/gobgp_4.6.0_linux_amd64.tar.gz -C bin
+
+get-whoami:
+	docker pull ghcr.io/traefik/whoami:v1.11

README.md

@@ -1,25 +1,41 @@
 # kube-vip
 
+High Availability and Load-Balancing
 
-**NOTE** All documentation of both usage and architecture are now available at [https://kube-vip.io](https://kube-vip.io)
+![](https://github.com/kube-vip/kube-vip/raw/main/kube-vip.png)
 
+[![Build and publish main image regularly](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml/badge.svg)](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml) [![LFX Health Score](https://img.shields.io/static/v1?label=Health%20Score&message=Healthy&color=A7F3D0&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip) [![LFX Active Contributors](https://img.shields.io/static/v1?label=Active%20contributors%20(1Y)&message=212&color=0094FF&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip)
 
 ## Overview
-Kubernetes Virtual IP and Load-Balancer for both control pane and Kubernetes services
+Kubernetes Virtual IP and Load-Balancer for both control plane and Kubernetes services
 
 The idea behind `kube-vip` is a small self-contained Highly-Available option for all environments, especially:
 
 - Bare-Metal
 - Edge (arm / Raspberry PI)
+- Virtualisation
 - Pretty much anywhere else :)
 
-![](/overview.png)
+**NOTE:** All documentation of both usage and architecture are now available at [https://kube-vip.io](https://kube-vip.io).
 
-The `kube-vip` application builds a multi-node or multi-pod cluster to provide High-Availability. When a leader is elected, this node will inherit the Virtual IP and become the leader of the load-balancing within the cluster. 
+For upgrading an existing install in place (static Pod or DaemonSet), see the [upgrade guide](https://kube-vip.io/docs/upgrade/).
 
-When running **out of cluster** it will use [raft](https://en.wikipedia.org/wiki/Raft_(computer_science)) clustering technology 
+## Features
 
-When running **in cluster** it will use [leader election](https://godoc.org/k8s.io/client-go/tools/leaderelection)
+Kube-Vip was originally created to provide a HA solution for the Kubernetes control plane, over time it has evolved to incorporate that same functionality into Kubernetes service type [load-balancers](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer).
+
+- VIP addresses can be both IPv4 or IPv6
+- Control Plane with ARP (Layer 2) or BGP (Layer 3)
+- Control Plane using either [leader election](https://godoc.org/k8s.io/client-go/tools/leaderelection) or [raft](https://en.wikipedia.org/wiki/Raft_(computer_science))
+- Control Plane HA with kubeadm (static Pods)
+- Control Plane HA with K3s/and others (daemonsets)
+- Service LoadBalancer using [leader election](https://godoc.org/k8s.io/client-go/tools/leaderelection) for ARP (Layer 2)
+- Service LoadBalancer using multiple nodes with BGP
+- Service LoadBalancer address pools per namespace or global
+- Service LoadBalancer address via (existing network DHCP)
+- Service LoadBalancer address exposure to gateway via UPNP
+- Egress! Kube-vip will utilise a service loadbalancer as both the ingress and **egress** for a pod. 
+- ... manifest generation, vendor API integrations and many more...
 
 ## Why?
 
@@ -31,53 +47,94 @@ The purpose of `kube-vip` is to simplify the building of HA Kubernetes clusters,
 
 **VIP**:
 - [Keepalived](https://www.keepalived.org/)
-- [Ucarp](https://ucarp.wordpress.com/)
+- [UCARP](https://ucarp.wordpress.com/)
 - Hardware Load-balancer (functionality differs per vendor)
 
 
 **LoadBalancing**:
 - [HAProxy](http://www.haproxy.org/)
 - [Nginx](http://nginx.com)
-- Hardware Load-balancer(functionality differs per vendor)
+- Hardware Load-balancer (functionality differs per vendor)
 
 All of these would require a separate level of configuration and in some infrastructures multiple teams in order to implement. Also when considering the software components, they may require packaging into containers or if they’re pre-packaged then security and transparency may be an issue. Finally, in edge environments we may have limited room for hardware (no HW load-balancer) or packages solutions in the correct architectures might not exist (e.g. ARM). Luckily with `kube-vip` being written in GO, it’s small(ish) and easy to build for multiple architectures, with the added security benefit of being the only thing needed in the container.
 
+## Troubleshooting and Feedback
 
-## Standalone Usage
+### SELinux and IPVS kernel modules
 
-The usage of `kube-vip` can either be directly by taking the binary / building yourself (`make build`), or alternatively through a pre-built docker container which can be found in the plunder Docker Hub repository [https://hub.docker.com/r/plndr/kube-vip](https://hub.docker.com/r/plndr/kube-vip). For further 
+When using IPVS load balancing on nodes with SELinux enforcing, kube-vip may be
+blocked from requesting kernel modules from inside the container. Symptoms can
+include the kube-vip pod entering `Error` or `CrashLoopBackOff`, logs that show
+`ensure IPVS kernel modules are loaded`, or audit denials for `module_request`
+from `container_t`.
 
-### Configuration
+Load the required IPVS modules on every node that can run kube-vip before
+deploying it:
 
-To generate the basic `yaml` configuration:
-
-```
-kube-vip sample config > config.yaml
+```shell
+sudo modprobe ip_vs
+sudo modprobe ip_vs_rr
 ```
 
-Modify the `localPeer` section to match this particular instance (local IP address/port etc..) and ensure that the `remotePeers` section is correct for the current instance and all other instances in the cluster. Also ensure that the `interface` is the correct interface that the `vip` will bind to.
+To persist this across reboots, add the modules to a file such as
+`/etc/modules-load.d/kube-vip-ipvs.conf`:
 
-
-## Starting a simple cluster
-
-To start `kube-vip` ensure the configuration for the `localPeers` and `remotePeers` is correct for each instance and the cluster as a whole and start:
-
-```
-kube-vip start -c /config.yaml
-INFO[0000] Reading configuration from [config.yaml]       
-INFO[0000] 2020-02-01T15:41:04.287Z [INFO]  raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server1 Address:192.168.0.70:10000} {Suffrage:Voter ID:server2 Address:192.168.0.71:10000} {Suffrage:Voter ID:server3 Address:192.168.0.72:10000}]" 
-INFO[0000] 2020-02-01T15:41:04.287Z [INFO]  raft: entering follower state: follower="Node at 192.168.0.70:10000 [Follower]" leader= 
-INFO[0000] Started                                      
-INFO[0000] The Node [] is leading                       
-INFO[0001] The Node [] is leading                       
-INFO[0001] 2020-02-01T15:41:05.522Z [WARN]  raft: heartbeat timeout reached, starting election: last-leader= 
-INFO[0001] 2020-02-01T15:41:05.522Z [INFO]  raft: entering candidate state: node="Node at 192.168.0.70:10000 [Candidate]" term=2 
-INFO[0001] 2020-02-01T15:41:05.522Z [DEBUG] raft: votes: needed=2 
-INFO[0001] 2020-02-01T15:41:05.522Z [DEBUG] raft: vote granted: from=server1 term=2 tally=1 
-INFO[0001] 2020-02-01T15:41:05.523Z [DEBUG] raft: newer term discovered, fallback to follower 
-INFO[0001] 2020-02-01T15:41:05.523Z [INFO]  raft: entering follower state: follower="Node at 192.168.0.70:10000 [Follower]" leader= 
-INFO[0001] 2020-02-01T15:41:05.838Z [WARN]  raft: failed to get previous log: previous-index=2 last-index=1 error="log not found" 
-INFO[0002] The Node [192.168.0.72:10000] is leading    
+```text
+ip_vs
+ip_vs_rr
 ```
 
-After a few seconds with additional nodes started a leader election will take place and the leader will assume the **vip**.
+Preloading only the required modules is preferred to enabling the SELinux
+`domain_kernel_load_modules` boolean for containers.
+
+### Gateway API `LoadBalancer` services with no endpoints
+
+Some Gateway API controllers create `LoadBalancer` services that intentionally have no Endpoints/EndpointSlices backends.
+
+If you want kube-vip to reconcile such a service, opt in with:
+
+```yaml
+metadata:
+  annotations:
+    kube-vip.io/allow-reconcile-without-endpoints: "true"
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Cluster
+```
+
+Scope:
+- Works only with `externalTrafficPolicy: Cluster`
+- No effect for `Local`
+- Default endpoint-gated behavior remains unchanged for services without this annotation
+
+Please raise issues on the GitHub repository and as mentioned check the documentation at [https://kube-vip.io](https://kube-vip.io/).
+
+## Community Tools
+
+- **[KubeStellar Console — Guided kube-vip Install](https://console.kubestellar.io/missions/install-kube-vip)** — A step-by-step guided installation experience for kube-vip with pre-flight checks, validation, troubleshooting, and rollback support.
+
+## Contributing
+
+Thanks for taking the time to join our community and start contributing! We welcome pull requests. Feel free to dig through the [issues](https://github.com/kube-vip/kube-vip/issues) and jump in.
+
+:warning: This project has issue compiling on MacOS, please compile it on linux distribution
+
+Additionally it is now relatively easy and quick to develop with [skaffold](https://skaffold.dev/), and the `skaffold.yaml` exists within the root folder of the gir repository.
+
+### Set up a kind development environment
+
+1. `kind create cluster --config ./testing/kind.yaml`
+2. `kubectl apply -f https://kube-vip.io/manifests/rbac.yaml`
+3. Create a load balancer range `configMap` from the kind cluster
+4. Apply the CCM manifest
+5. Start skaffold with `skaffold dev`
+6. Start developing and see changes applied in real-time.
+
+## Star History
+
+[![Star History Chart](https://star-history.dera.page/svg?repos=kube-vip/kube-vip&type=Date)](https://star-history.dera.page/#kube-vip/kube-vip&type=date)
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fkube-vip%2Fkube-vip.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fkube-vip%2Fkube-vip?ref=badge_shield)
+
+
+## License
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fkube-vip%2Fkube-vip.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fkube-vip%2Fkube-vip?ref=badge_large)

ROADMAP.md

@@ -0,0 +1,35 @@
+# Kube-Vip Roadmap
+
+This document outlines the roadmap for the **kube-vip** project and only covers the technologies within this particular project, other projects that augment or provide additional functionality (such as cloud-providers) may have their own roadmaps in future. The functionality for **kube-vip** has grown either been developed organically or through real-world needs, and this is the first attempt to put into words a plan for the future of **kube-vip** and will additional evolve over time. This means that items listed or detailed here are not necessarily set in stone and the roadmap can grow/shrink as the project matures. We definitely welcome suggestions and ideas from everyone about the roadmap and **kube-vip** features. Reach us through Issues, Slack or email <catch-all>@kube-vip.io.
+
+## Release methodology
+
+The **kube-vip** project attempts to follow a tick-tock release cycle, this typically means that one release will come **packed** with new features where the following release will come with fixes, code sanitation and performance enhancements.
+
+## Roadmap
+
+The **kube-vip** project offers two main areas of functionality:
+
+- HA Kubernetes clusters through a control-plane VIP
+- Kubernetes `service type:LoadBalancer`
+
+Whilst both of these functions share underlying technologies and code they will have slightly differing roadmaps.
+
+### HA Kubernetes Control Plane
+
+- **Re-implement LoadBalancing** - due to a previous request the HTTP loadbalancing was removed leaving just HA for the control plane. This functionality will be re-implemented either through the original round-robin HTTP requests or utilising IPVS.
+- **Utilise the Kubernetes API to determine additional Control Plane members** - Once a single node cluster is running **kube-vip** could use the API to determine the additional members, at this time a Cluster-API provider needs to drop a static manifest per CP node.
+- **Re-evaluate raft** - **kube-vip** is mainly designed to run within a Kubernetes cluster, however it's original design was a raft cluster external to Kubernetes. Unfortunately given some of the upgrade paths identified in things like CAPV moving to leaderElection within Kubernetes became a better idea.
+
+## Kubernetes `service type:LoadBalancer`
+
+- **`ARP` LeaderElection per loadBalancer** - Currently only one pod that is elected leader will field all traffic for a VIP.. extending this to generate a leaderElection token per service would allow services to proliferate across all pods across the cluster
+- **Aligning of `service` and `manager`** - The move to allow hybrid (be both HA control plane and offer load-balancer services at the same time) introduced a duplicate code path.. these need to converge as it's currently confusing for contributors.
+
+## Global **Kube-Vip** items
+
+- **Improved metrics** - At this time the scaffolding for monitoring exists, however this needs drastically extending to provide greater observability to what is happening within **kube-vip**
+- **Windows support** - The Go SDK didn't support the capability for low-levels sockets for ARP originally, this should be revisited.
+- **Additional BGP features** :
+  - Communities
+  - BFD

cmd/kube-vip-config.go

@@ -1,146 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-
-	"github.com/ghodss/yaml"
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	appv1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-// [sample configuration] - flags
-var cliConfig kubevip.Config
-var cliConfigLB kubevip.LoadBalancer
-var cliLocalPeer string
-var cliRemotePeers, cliBackends []string
-
-func init() {
-	kubeVipSampleConfig.Flags().StringVar(&cliConfig.Interface, "interface", "eth0", "Name of the interface to bind to")
-	kubeVipSampleConfig.Flags().StringVar(&cliConfig.VIP, "vip", "192.168.0.1", "The Virtual IP address")
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfig.SingleNode, "singleNode", false, "Start this instance as a single node")
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfig.StartAsLeader, "startAsLeader", false, "Start this instance as the cluster leader")
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfig.GratuitousARP, "arp", true, "Use ARP broadcasts to improve VIP re-allocations")
-	kubeVipSampleConfig.Flags().StringVar(&cliLocalPeer, "localPeer", "server1:192.168.0.1:10000", "Settings for this peer, format: id:address:port")
-	kubeVipSampleConfig.Flags().StringSliceVar(&cliRemotePeers, "remotePeers", []string{"server2:192.168.0.2:10000", "server3:192.168.0.3:10000"}, "Comma seperated remotePeers, format: id:address:port")
-	// Load Balancer flags
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfigLB.BindToVip, "lbBindToVip", false, "Bind example load balancer to VIP")
-	kubeVipSampleConfig.Flags().StringVar(&cliConfigLB.Type, "lbType", "tcp", "Type of load balancer instance (TCP/HTTP)")
-	kubeVipSampleConfig.Flags().StringVar(&cliConfigLB.Name, "lbName", "Example Load Balancer", "The name of a load balancer instance")
-	kubeVipSampleConfig.Flags().IntVar(&cliConfigLB.Port, "lbPort", 8080, "Port that load balancer will expose on")
-	kubeVipSampleConfig.Flags().StringSliceVar(&cliBackends, "lbBackends", []string{"192.168.0.1:8080", "192.168.0.2:8080"}, "Comma seperated backends, format: address:port")
-}
-
-var kubeVipSampleConfig = &cobra.Command{
-	Use:   "config",
-	Short: "Generate a Sample configuration",
-	Run: func(cmd *cobra.Command, args []string) {
-
-		// // Parse localPeer
-		// p, err := kubevip.ParsePeerConfig(cliLocalPeer)
-		// if err != nil {
-		// 	cmd.Help()
-		// 	log.Fatalln(err)
-		// }
-		// cliConfig.LocalPeer = *p
-
-		// // Parse remotePeers
-		// //Iterate backends
-		// for i := range cliRemotePeers {
-		// 	p, err := kubevip.ParsePeerConfig(cliRemotePeers[i])
-		// 	if err != nil {
-		// 		cmd.Help()
-		// 		log.Fatalln(err)
-		// 	}
-		// 	cliConfig.RemotePeers = append(cliConfig.RemotePeers, *p)
-		// }
-
-		// //Iterate backends
-		// for i := range cliBackends {
-		// 	b, err := kubevip.ParseBackendConfig(cliBackends[i])
-		// 	if err != nil {
-		// 		cmd.Help()
-		// 		log.Fatalln(err)
-		// 	}
-		// 	cliConfigLB.Backends = append(cliConfigLB.Backends, *b)
-		// }
-
-		// Add the basic Load-Balancer to the configuration
-		cliConfig.LoadBalancers = append(cliConfig.LoadBalancers, cliConfigLB)
-
-		err := cliConfig.ParseFlags(cliLocalPeer, cliRemotePeers, cliBackends)
-		if err != nil {
-			cmd.Help()
-			log.Fatalln(err)
-		}
-
-		err = kubevip.ParseEnvironment(&cliConfig)
-		if err != nil {
-			cmd.Help()
-			log.Fatalln(err)
-		}
-
-		cliConfig.PrintConfig()
-	},
-}
-
-var kubeVipSampleManifest = &cobra.Command{
-	Use:   "manifest",
-	Short: "Generate a Sample kubernetes manifest",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Generate the sample manifest specification
-		p := &appv1.Pod{
-			TypeMeta: metav1.TypeMeta{
-				Kind:       "Pod",
-				APIVersion: "v1",
-			},
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      "kube-vip",
-				Namespace: "kube-system",
-			},
-			Spec: appv1.PodSpec{
-				Containers: []appv1.Container{
-					{
-						Name:  "kube-vip",
-						Image: fmt.Sprintf("docker.io/plndr/kube-vip:%s", Release.Version),
-						SecurityContext: &appv1.SecurityContext{
-							Capabilities: &appv1.Capabilities{
-								Add: []appv1.Capability{
-									"NET_ADMIN",
-									"SYS_TIME",
-								},
-							},
-						},
-						Args: []string{
-							"start",
-							"-c",
-							"/etc/kube-vip/config.yaml",
-						},
-						VolumeMounts: []appv1.VolumeMount{
-							{
-								Name:      "config",
-								MountPath: "/etc/kube-vip/",
-							},
-						},
-					},
-				},
-				Volumes: []appv1.Volume{
-					{
-						Name: "config",
-						VolumeSource: appv1.VolumeSource{
-							HostPath: &appv1.HostPathVolumeSource{
-								Path: "/etc/kube-vip/",
-							},
-						},
-					},
-				},
-				HostNetwork: true,
-			},
-		}
-
-		b, _ := yaml.Marshal(p)
-		fmt.Printf(string(b))
-	},
-}

cmd/kube-vip-kubeadm.go

@@ -1,18 +1,13 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
-	"net"
 	"os"
 
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
 	"github.com/spf13/cobra"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
 )
 
 // kubeadm adds two subcommands for managing a vip during a kubeadm init/join
@@ -28,134 +23,137 @@ func init() {
 var kubeKubeadm = &cobra.Command{
 	Use:   "kubeadm",
 	Short: "Kubeadm functions",
-	Run: func(cmd *cobra.Command, args []string) {
-		cmd.Help()
-		// TODO - A load of text detailing what's actually happening
+	Long: `This command group provides utilities for generating static Pod manifests specifically tailored for the kubeadm bootstrapping process.
+	It contains two subcommands:
+  		- init:  Generates a manifest to be used during 'kubeadm init' on the first control-plane node.
+  		- join:  Generates a manifest to be used during 'kubeadm join' for additional control-plane nodes.
+
+The generated YAML manifest should be saved to the kubeadm static Pod directory (typically /etc/kubernetes/manifests/) so that kubeadm launches the kube-vip static Pod automatically.`,
+	Run: func(cmd *cobra.Command, _ []string) {
+		_ = cmd.Help()
 	},
 }
 
 var kubeKubeadmInit = &cobra.Command{
 	Use:   "init",
 	Short: "kube-vip init",
-	Long:  "The \"init\" subcommand will generate the Kubernetes manifest that will be started by kubeadm through the kubeadm init process",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Long: `The 'init' subcommand generates a Kubernetes Pod manifest that kubeadm will start as a static Pod during the cluster initialisation phase.
+
+This manifest runs kube-vip on the first control-plane node to advertise the Virtual IP (VIP) for the API server. The VIP is typically configured using ARP (Layer 2) or BGP (dynamic routing).
+
+Required flags for this command:
+  --interface   : The network interface to bind the VIP to (e.g., eth0).
+  --vip or --address : The Virtual IP address or DNS name to use.
+
+Example:
+  kube-vip kubeadm init --interface eth0 --vip 192.168.1.100 --controlplane
+
+The output YAML should be written to the kubeadm manifests directory, e.g.:
+  kube-vip kubeadm init ... > /etc/kubernetes/manifests/kube-vip.yaml`,
+	Run: func(cmd *cobra.Command, _ []string) {
+
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
-		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
+		err := kubevip.ParseEnvironment(&initConfig)
+		if err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+		if err := initConfig.Validate(); err != nil {
+			log.Error("validating configuration", "err", err)
+			return
+		}
+
 		// TODO - check for certain things VIP/interfaces
 		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+			_ = cmd.Help()
+			log.Error("No interface is specified for kube-vip to bind to")
+			return
 		}
 
 		if initConfig.VIP == "" && initConfig.Address == "" {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
-		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version)
 
-		fmt.Println(cfg)
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("generating VIPSubnet", "err", err)
+				return
+			}
+		}
+
+		cfg, err := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
+		if err != nil {
+			log.Error("unable to create manifest", "err", err)
+			return
+		}
+		fmt.Println(cfg) // output manifest to stdout
 	},
 }
 
 var kubeKubeadmJoin = &cobra.Command{
 	Use:   "join",
 	Short: "kube-vip join",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Long: `The 'join' subcommand generates a Kubernetes Pod manifest for additional control-plane nodes joining an existing cluster via 'kubeadm join'.
+
+It functions identically to the 'init' subcommand, but is intended for secondary control-plane nodes. It validates that the kubeconfig file (specified by --config, defaulting to /etc/kubernetes/admin.conf) exists on the node to ensure the node can authenticate with the cluster.
+
+Required flags for this command:
+  --interface   : The network interface to bind the VIP to.
+  --vip or --address : The Virtual IP address or DNS name (must match the VIP used during 'init').
+
+Example:
+  kube-vip kubeadm join --interface eth0 --vip 192.168.1.100
+
+The output YAML should be saved to the kubeadm manifests directory on the joining node.`,
+	Run: func(cmd *cobra.Command, _ []string) {
 
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
-		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
+		err := kubevip.ParseEnvironment(&initConfig)
+		if err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+		if err := initConfig.Validate(); err != nil {
+			log.Error("validating configuration", "err", err)
+			return
+		}
+
 		// TODO - check for certain things VIP/interfaces
 		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+			_ = cmd.Help()
+			log.Error("No interface is specified for kube-vip to bind to")
+			return
 		}
 
 		if initConfig.VIP == "" && initConfig.Address == "" {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
 
 		if _, err := os.Stat(kubeConfigPath); os.IsNotExist(err) {
-			log.Fatalf("Unable to find file [%s]", kubeConfigPath)
+			log.Error("kubeConfig not found", "Path", kubeConfigPath)
+			return
 		}
 
-		// We will use kubeconfig in order to find all the master nodes
-		// use the current context in kubeconfig
-		config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
-		if err != nil {
-			log.Fatal(err.Error())
-		}
-
-		// create the clientset
-		clientset, err := kubernetes.NewForConfig(config)
-		if err != nil {
-			log.Fatal(err.Error())
-		}
-
-		opts := metav1.ListOptions{}
-		opts.LabelSelector = "node-role.kubernetes.io/master"
-		nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), opts)
-
-		// Iterate over all nodes that are masters and find the details to build a peer list
-		for x := range nodes.Items {
-			// Get hostname and address
-			var nodeAddress, nodeHostname string
-			for y := range nodes.Items[x].Status.Addresses {
-				switch nodes.Items[x].Status.Addresses[y].Type {
-				case corev1.NodeHostName:
-					nodeHostname = nodes.Items[x].Status.Addresses[y].Address
-				case corev1.NodeInternalIP:
-					nodeAddress = nodes.Items[x].Status.Addresses[y].Address
-				}
-			}
-
-			newPeer, err := kubevip.ParsePeerConfig(fmt.Sprintf("%s:%s:%d", nodeHostname, nodeAddress, 10000))
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
 			if err != nil {
-				panic(err.Error())
+				log.Error("generating VIPSubnet", "err", err)
+				return
 			}
-			initConfig.RemotePeers = append(initConfig.RemotePeers, *newPeer)
-
 		}
-		// Generate manifest and print
-		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version)
-		fmt.Println(cfg)
+
+		cfg, err := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
+		if err != nil {
+			log.Error("unable to create manifest", "err", err)
+			return
+		}
+		fmt.Println(cfg) // output manifest to stdout
 	},
 }
-
-func autoGenLocalPeer() (*kubevip.RaftPeer, error) {
-	// hostname // address // defaultport
-	h, err := os.Hostname()
-	if err != nil {
-		return nil, err
-	}
-
-	var a string
-	addrs, err := net.InterfaceAddrs()
-	if err != nil {
-		return nil, err
-	}
-	for _, address := range addrs {
-		// check the address type and if it is not a loopback the display it
-		if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
-			if ipnet.IP.To4() != nil {
-				a = ipnet.IP.String()
-				break
-			}
-		}
-	}
-	if a == "" {
-		return nil, fmt.Errorf("Unable to find local address")
-	}
-	return &kubevip.RaftPeer{
-		ID:      h,
-		Address: a,
-		Port:    10000,
-	}, nil
-
-}

cmd/kube-vip-manifests.go

@@ -3,8 +3,9 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
 	"github.com/spf13/cobra"
 )
 
@@ -12,68 +13,200 @@ import (
 // manifests will be used to generate:
 // - Pod spec manifest, mainly used for a static pod (kubeadm)
 // - Daemonset manifest, mainly used to run kube-vip as a deamonset within Kubernetes (k3s/rke)
+// - RBAC manifest, used to generate the RBAC permissions for kube-vip
+
+var taint, role, rolebinding bool
 
 func init() {
+	kubeManifest.PersistentFlags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
+	kubeManifest.PersistentFlags().StringVar(&image, "image", "ghcr.io/kube-vip/kube-vip", "Define a hardcoded image with or without tag for the manifest")
+	kubeManifestDaemon.PersistentFlags().BoolVar(&taint, "taint", false, "Taint the manifest for only running on control planes")
+	kubeManifestRbac.PersistentFlags().BoolVar(&role, "role", false, "Generate only a Role inside the serviceNamespace access")
+	kubeManifestRbac.PersistentFlags().BoolVar(&rolebinding, "rolebinding", false, "Generate only a RoleBinding for namespaced access")
 
 	kubeManifest.AddCommand(kubeManifestPod)
 	kubeManifest.AddCommand(kubeManifestDaemon)
+	kubeManifest.AddCommand(kubeManifestRbac)
 }
 
 var kubeManifest = &cobra.Command{
 	Use:   "manifest",
 	Short: "Manifest functions",
-	Run: func(cmd *cobra.Command, args []string) {
-		cmd.Help()
-		// TODO - A load of text detailing what's actually happening
+	Long: `This command group provides flexible manifest generation for deploying kube-vip in various Kubernetes environments.
+
+Unlike the 'kubeadm' subcommands, which are tightly coupled to kubeadm's static Pod requirements, these generators produce standard Kubernetes manifests (Pod, DaemonSet, RBAC) that can be used with any Kubernetes distribution (e.g., k3s, RKE, or vanilla Kubernetes).
+
+Subcommands:
+  pod        : Generates a standalone Pod manifest (similar to a static pod).
+  daemonset  : Generates a DaemonSet manifest to run kube-vip on selected nodes.
+  rbac       : Generates the necessary ServiceAccount, Role/ClusterRole, and Binding manifests.
+
+All output is written to stdout as YAML, typically piped to 'kubectl apply -f -' or saved to a file.`,
+	Run: func(cmd *cobra.Command, _ []string) {
+		_ = cmd.Help()
 	},
 }
 
 var kubeManifestPod = &cobra.Command{
 	Use:   "pod",
 	Short: "Generate a Pod Manifest",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Long: `Generate a standalone Pod manifest for kube-vip.
+
+This is ideal for environments that do not use DaemonSets or where you want to run kube-vip as a static Pod (similar to the 'kubeadm' subcommand, but without kubeadm-specific assumptions). It includes all the necessary container specifications, volumes, and environment variables derived from the provided flags.
+
+Key flags:
+  --interface : Network interface for the VIP.
+  --vip or --address : The Virtual IP address or DNS name.
+  --image     : Override the container image (default: ghcr.io/kube-vip/kube-vip).
+
+The manifest is generated based on the current configuration flags set on the root command.
+
+Example:
+  kube-vip manifest pod --interface eth0 --vip 10.0.0.100 --controlplane | kubectl apply -f -`,
+	Run: func(cmd *cobra.Command, _ []string) {
+		var err error
+
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
-		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
-		// TODO - check for certain things VIP/interfaces
-		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+		if err := initConfig.Validate(); err != nil {
+			log.Error("validating configuration", "err", err)
+			return
 		}
 
-		if initConfig.VIP == "" {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+		// The control plane has a requirement for a VIP being specified
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+			_ = cmd.Help()
+			log.Error("no address is specified for kube-vip to expose services on")
+			return
 		}
-		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version)
 
-		fmt.Println(cfg)
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("config parse", "err", err)
+				return
+			}
+		}
+
+		cfg, err := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
+		if err != nil {
+			log.Error("unable to create manifest", "err", err)
+			return
+		}
+		fmt.Println(cfg) // output manifest to stdout
 	},
 }
 
 var kubeManifestDaemon = &cobra.Command{
 	Use:   "daemonset",
 	Short: "Generate a Daemonset Manifest",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Long: `Generate a DaemonSet manifest to run kube-vip across multiple nodes.
+
+This is the recommended deployment method for production clusters running kube-vip as a service. It ensures that kube-vip runs on all control-plane nodes (or selected nodes via tolerations) and can handle both control-plane HA and service load-balancing.
+
+Flags specific to this subcommand:
+  --taint  : Adds a toleration to the DaemonSet so that pods are scheduled only on nodes with the control-plane taint (node-role.kubernetes.io/control-plane:NoSchedule). This is essential for control-plane-only deployments.
+
+All other standard kube-vip flags (--interface, --vip, --enableARP, --enableBGP, etc.) are respected and embedded into the DaemonSet pod template.
+
+Example:
+  kube-vip manifest daemonset --interface eth0 --vip 192.168.1.100 --controlplane --taint | kubectl apply -f -`,
+	Run: func(cmd *cobra.Command, _ []string) {
+		var err error
+
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
-		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
-		// TODO - check for certain things VIP/interfaces
-		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+		if err := initConfig.Validate(); err != nil {
+			log.Error("validating configuration", "err", err)
+			return
+		}
+		// The control plane has a requirement for a VIP being specified
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+			_ = cmd.Help()
+			log.Error("no address is specified for kube-vip to expose services on")
+			return
 		}
 
-		if initConfig.VIP == "" {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("config parse", "err", err)
+				return
+			}
 		}
-		cfg := kubevip.GenerateDeamonsetManifestFromConfig(&initConfig, Release.Version)
 
-		fmt.Println(cfg)
+		cfg, err := kubevip.GenerateDaemonsetManifestFromConfig(&initConfig, image, Release.Version, inCluster, taint)
+		if err != nil {
+			log.Error("unable to create manifest", "err", err)
+			return
+		}
+		fmt.Println(cfg) // output manifest to stdout
+	},
+}
+
+var kubeManifestRbac = &cobra.Command{
+	Use:   "rbac",
+	Short: "Generate an RBAC Manifest",
+	Long: `Generate the RBAC (Role-Based Access Control) manifests required for kube-vip to interact with the Kubernetes API.
+
+kube-vip needs permissions to watch services, endpoints, configmaps, and manage leader election leases. This command outputs the minimum required ServiceAccount, Role (or ClusterRole), and the corresponding binding.
+
+Flags:
+  --role        : If true, generates a namespaced Role instead of a ClusterRole. The namespace is taken from the root --namespace flag (default: kube-system).
+  --rolebinding : If true, generates a RoleBinding (if --role is also true). If --role is false, a ClusterRoleBinding is generated automatically.
+
+The output is a multi-document YAML (separated by '---'). It is safe to apply directly:
+  kube-vip manifest rbac --role --rolebinding | kubectl apply -f -
+
+Without --role, it generates a ClusterRole and ClusterRoleBinding, which is the default behaviour and suitable for most cluster-wide deployments.`,
+	Run: func(cmd *cobra.Command, _ []string) {
+		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
+		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+		if err := initConfig.Validate(); err != nil {
+			log.Error("validating configuration", "err", err)
+			return
+		}
+
+		// The control plane has a requirement for a VIP being specified
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+			_ = cmd.Help()
+			log.Error("no address is specified for kube-vip to expose services on")
+			return
+		}
+
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			var err error
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("generating VIPSubnet", "err", err)
+				return
+			}
+		}
+		saCfg := kubevip.GenerateSA(&initConfig)
+		roleCfg := kubevip.GenerateRole(&initConfig, role)
+		if role {
+			rolebinding = true
+		}
+		roleBindingCfg := kubevip.GenerateRoleBinding(rolebinding, saCfg, roleCfg)
+
+		// Output the YAML manifests to stdout
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(saCfg))
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(roleCfg))
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(roleBindingCfg))
 	},
 }

cmd/kube-vip-start.go

@@ -1,127 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"os"
-	"os/signal"
-
-	"github.com/plunder-app/kube-vip/pkg/cluster"
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	"github.com/plunder-app/kube-vip/pkg/vip"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-)
-
-// Start as a single node (no cluster), start as a leader in the cluster
-var startConfig kubevip.Config
-var startConfigLB kubevip.LoadBalancer
-var startLocalPeer, startKubeConfigPath string
-var startRemotePeers, startBackends []string
-var inCluster bool
-
-func init() {
-	// Get the configuration file
-	kubeVipStart.Flags().StringVarP(&configPath, "config", "c", "", "Path to a kube-vip configuration")
-	kubeVipStart.Flags().BoolVarP(&disableVIP, "disableVIP", "d", false, "Disable the VIP functionality")
-
-	// Pointers so we can see if they're nil (and not called)
-	kubeVipStart.Flags().StringVar(&startConfig.Interface, "interface", "eth0", "Name of the interface to bind to")
-	kubeVipStart.Flags().StringVar(&startConfig.VIP, "vip", "192.168.0.1", "The Virtual IP address")
-	kubeVipStart.Flags().StringVar(&startConfig.Address, "address", "", "an address (IP or DNS name) to use as a VIP")
-	kubeVipStart.Flags().BoolVar(&startConfig.SingleNode, "singleNode", false, "Start this instance as a single node")
-	kubeVipStart.Flags().BoolVar(&startConfig.StartAsLeader, "startAsLeader", false, "Start this instance as the cluster leader")
-	kubeVipStart.Flags().BoolVar(&startConfig.GratuitousARP, "arp", false, "Use ARP broadcasts to improve VIP re-allocations")
-	kubeVipStart.Flags().StringVar(&startLocalPeer, "localPeer", "server1:192.168.0.1:10000", "Settings for this peer, format: id:address:port")
-	kubeVipStart.Flags().StringSliceVar(&startRemotePeers, "remotePeers", []string{"server2:192.168.0.2:10000", "server3:192.168.0.3:10000"}, "Comma seperated remotePeers, format: id:address:port")
-	// Load Balancer flags
-	kubeVipStart.Flags().BoolVar(&startConfigLB.BindToVip, "lbBindToVip", false, "Bind example load balancer to VIP")
-	kubeVipStart.Flags().StringVar(&startConfigLB.Type, "lbType", "tcp", "Type of load balancer instance (TCP/HTTP)")
-	kubeVipStart.Flags().StringVar(&startConfigLB.Name, "lbName", "Example Load Balancer", "The name of a load balancer instance")
-	kubeVipStart.Flags().IntVar(&startConfigLB.Port, "lbPort", 8080, "Port that load balancer will expose on")
-	kubeVipStart.Flags().IntVar(&startConfigLB.BackendPort, "lbBackEndPort", 6443, "A port that all backends may be using (optional)")
-	kubeVipStart.Flags().StringSliceVar(&startBackends, "lbBackends", []string{"192.168.0.1:8080", "192.168.0.2:8080"}, "Comma seperated backends, format: address:port")
-
-	// Cluster configuration
-	kubeVipStart.Flags().StringVar(&startKubeConfigPath, "kubeConfig", "/etc/kubernetes/admin.conf", "The path of a kubernetes configuration file")
-	kubeVipStart.Flags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
-	kubeVipStart.Flags().BoolVar(&startConfig.EnableLeaderElection, "leaderElection", false, "Use the Kubernetes leader election mechanism for clustering")
-
-}
-
-var kubeVipStart = &cobra.Command{
-	Use:   "start",
-	Short: "Start the Virtual IP / Load balancer",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
-		var err error
-
-		// If a configuration file is loaded, then it will overwrite flags
-
-		if configPath != "" {
-			c, err := kubevip.OpenConfig(configPath)
-			if err != nil {
-				log.Fatalf("%v", err)
-			}
-			startConfig = *c
-		}
-
-		// parse environment variables, these will overwrite anything loaded or flags
-		err = kubevip.ParseEnvironment(&startConfig)
-		if err != nil {
-			log.Fatalln(err)
-		}
-
-		newCluster, err := cluster.InitCluster(&startConfig, disableVIP)
-		if err != nil {
-			log.Fatalf("%v", err)
-		}
-
-		// start the dns updater if the address flag is used and the address isn't an IP
-		if startConfig.Address != "" && !vip.IsIP(startConfig.Address) {
-			log.Infof("starting the DNS updater for the address %s", startConfig.Address)
-
-			ipUpdater := vip.NewIPUpdater(startConfig.Address, newCluster.Network)
-
-			ipUpdater.Run(context.Background())
-		}
-
-		if startConfig.SingleNode {
-			// If the Virtual IP isn't disabled then create the netlink configuration
-			// Start a single node cluster
-			newCluster.StartSingleNode(&startConfig, disableVIP)
-		} else {
-			if disableVIP {
-				log.Fatalln("Cluster mode requires the Virtual IP to be enabled, use single node with no VIP")
-			}
-
-			if startConfig.EnableLeaderElection {
-				cm, err := cluster.NewManager(startKubeConfigPath, inCluster)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-
-				// Leader Cluster will block
-				err = newCluster.StartLeaderCluster(&startConfig, cm)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-			} else {
-
-				// // Start a multi-node (raft) cluster, this doesn't block so will wait on signal
-				err = newCluster.StartRaftCluster(&startConfig)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-				signalChan := make(chan os.Signal, 1)
-				signal.Notify(signalChan, os.Interrupt)
-
-				<-signalChan
-
-				newCluster.Stop()
-			}
-
-		}
-
-	},
-}

cmd/kube-vip.go

@@ -1,30 +1,43 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
+	"net"
+	"net/http"
 	"os"
+	"slices"
 	"strconv"
+	"strings"
+	"sync"
+	"time"
 
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	"github.com/plunder-app/kube-vip/pkg/service"
-	log "github.com/sirupsen/logrus"
+	log "log/slog"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+
+	"github.com/kube-vip/kube-vip/pkg/debouncer"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/manager"
+	"github.com/kube-vip/kube-vip/pkg/metrics"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
 )
 
-// Path to the configuration file
-var configPath string
+// Is an option to set the image
+var image string
 
-// Disable the Virtual IP (bind to the existing network stack)
-var disableVIP bool
-
-// Run as a load balancer service (within a pod / kubernetes)
-var serviceArp bool
+// Is kube-vip running within cluster
+var inCluster bool
 
 // ConfigMap name within a Kubernetes cluster
 var configMap string
 
-// Configure the level of loggin
-var logLevel uint32
+// Points to a kubernetes configuration file
+var kubeConfigPath string
 
 // Release - this struct contains the release information populated when building kube-vip
 var Release struct {
@@ -33,95 +46,164 @@ var Release struct {
 }
 
 // Structs used via the various subcommands
-var initConfig kubevip.Config
-var initLoadBalancer kubevip.LoadBalancer
-
-// Points to a kubernetes configuration file
-var kubeConfigPath string
+var (
+	initConfig       kubevip.Config
+	initLoadBalancer kubevip.LoadBalancer
+)
 
 var kubeVipCmd = &cobra.Command{
-	Use:   "kube-vip",
-	Short: "This is a server for providing a Virtual IP and load-balancer for the Kubernetes control-plane",
+	Use:           "kube-vip",
+	Short:         "This is a server for providing a Virtual IP and load-balancer for the Kubernetes control-plane",
+	SilenceErrors: true,
 }
 
 func init() {
-
-	localpeer, err := autoGenLocalPeer()
-	if err != nil {
-		log.Fatalln(err)
-	}
-	initConfig.LocalPeer = *localpeer
-	//initConfig.Peers = append(initConfig.Peers, *localpeer)
+	// Basic flags
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Interface, "interface", "", "Name of the interface to bind to")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesInterface, "serviceInterface", "", "Name of the interface to bind to (for services)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.AllowInterfaceNotUp, "allowInterfaceNotUp", false, "Allow kube-vip to start even if the interface is not up")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIP, "vip", "", "The Virtual IP address")
-	kubeVipCmd.PersistentFlags().StringVar(&startConfig.Address, "address", "", "an address (IP or DNS name) to use as a VIP")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPCIDR, "cidr", "", "The CIDR range for the virtual IP address")
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.GratuitousARP, "arp", true, "Enable Arp for Vip changes")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPSubnet, "vipSubnet", "", "The Virtual IP address subnet e.g. /32 /24 /8 etc.. (Default to 32 for IPv4 and 128 for IPv6)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.NodeName, "nodeName", "", "Name to be used for lease holder. Must be unique for each node/instance")
+
+	// VIP flags
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Address, "address", "", "an address (IP or DNS name) to use as a VIP")
+	kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.Port, "port", 6443, "Port for the VIP")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for VIP changes")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableWireguard, "wireguard", false, "Enable Wireguard for services VIPs")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableRoutingTable, "table", false, "Enable Routing Table for services VIPs")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.PreserveVIPOnLeadershipLoss, "preserveVipOnLeadershipLoss", false, "Preserve ARP VIP addresses on interface when leadership is lost (default: false for backward compatibility)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoseLeadership, "loseLeadership", false, "Lose leadership when VIP interface goes down")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.LoseLeadershipTimeoutSeconds, "loseLeadershiptTimeoutSeconds", 30, "Timeout before re-electing a leader when the VIP interface is down")
+
+	// LoadBalancer flags
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "enableLoadBalancer", false, "enable loadbalancing on the VIP with IPVS")
+	kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.LoadBalancerPort, "lbPort", 6443, "loadbalancer port for the VIP")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerForwardingMethod, "lbForwardingMethod", "local", "loadbalancer forwarding method")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DDNS, "ddns", false, "use Dynamic DNS + DHCP to allocate VIP for address")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MirrorDestInterface, "mirrorDestInterface", "", "network interface where all traffic that traverses the service interface will be mirrored to. Source interface will use default interface is servicesInterface is not set.")
 
 	// Clustering type (leaderElection)
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLeaderElection, "leaderElection", false, "Use the Kubernetes leader election mechanism for clustering")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 5, "Length of time a Kubernetes leader lease can be held for")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time a Kubernetes leader can attempt to renew its lease")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Number of times the host will retry to hold a lease")
-
-	// Clustering type (raft)
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.StartAsLeader, "startAsLeader", false, "Start this instance as the cluster leader")
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.AddPeersAsBackends, "addPeersToLB", true, "Add raft peers to the load-balancer")
-
-	// Packet flags
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnablePacket, "packet", false, "This will use the Packet API (requires the token ENV) to update the EIP <-> VIP")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.PacketAPIKey, "packetKey", "", "The API token for authenticating with the Packet API")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.PacketProject, "packetProject", "", "The name of project already created within Packet")
-
-	// Load Balancer flags
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "lbEnable", false, "Enable a load-balancer on the VIP")
-	kubeVipCmd.PersistentFlags().BoolVar(&initLoadBalancer.BindToVip, "lbBindToVip", true, "Bind example load balancer to VIP")
-	kubeVipCmd.PersistentFlags().StringVar(&initLoadBalancer.Type, "lbType", "tcp", "Type of load balancer instance (TCP/HTTP)")
-	kubeVipCmd.PersistentFlags().StringVar(&initLoadBalancer.Name, "lbName", "Kubeadm Load Balancer", "The name of a load balancer instance")
-	kubeVipCmd.PersistentFlags().IntVar(&initLoadBalancer.Port, "lbPort", 6443, "Port that load balancer will expose on")
-	kubeVipCmd.PersistentFlags().IntVar(&initLoadBalancer.BackendPort, "lbBackEndPort", 6444, "A port that all backends may be using (optional)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaderElectionType, "leaderElectionType", "kubernetes", "Defines the backend to run the leader election: kubernetes or etcd. Defaults to kubernetes.")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaseName, "leaseName", "plndr-cp-lock", "Name of the lease that is used for leader election")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 15, "Length of time (in seconds) a Kubernetes leader lease can be held for")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 10, "Length of time (in seconds) a Kubernetes leader can attempt to renew its lease")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 2, "Length of time (in seconds) the LeaderElector clients should wait between tries of actions")
 
 	// BGP flags
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableBGP, "bgp", false, "This will enable BGP support within kube-vip")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPAttachIPToInterface, "bgpAttachIPToInterface", false, "Assign BGP service VIPs to the configured interface")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.RouterID, "bgpRouterID", "", "The routerID for the bgp server")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.SourceIF, "sourceIF", "", "The source interface for bgp peering (not to be used with sourceIP)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.SourceIP, "sourceIP", "", "The source address for bgp peering (not to be used with sourceIF)")
 	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.AS, "localAS", 65000, "The local AS number for the bgp server")
+	kubeVipCmd.PersistentFlags().Uint64Var(&initConfig.BGPConfig.HoldTime, "bgpHoldTimer", 30, "The hold timer for all bgp peers (it defines the time a session is held)")
+	kubeVipCmd.PersistentFlags().Uint64Var(&initConfig.BGPConfig.KeepaliveInterval, "bgpKeepAliveInterval", 10, "The keepalive interval for all bgp peers (it defines the heartbeat of keepalive messages)")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPPeerConfig.Address, "peerAddress", "", "The address of a BGP peer")
 	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPPeerConfig.AS, "peerAS", 65000, "The AS number for a BGP peer")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPPeerConfig.Password, "peerPass", "", "The md5 password for a BGP peer")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPPeerConfig.MultiHop, "multihop", false, "This will enable BGP multihop support")
+	kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.BGPPeers, "bgppeers", []string{}, "Comma separated BGP Peer, format: address:as:password:multihop")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Annotations, "annotations", "", "Set Node annotations prefix for parsing")
+
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPConfig.Zebra.Enabled, "zebra", false, "This will enable Zebra support within kube-vip")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.URL, "zebraUrl", "unix:/var/run/frr/zserv.api", "Path to the unix domain socket for connecting to Zebra daemon")
+	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.Zebra.Version, "zebraVersion", 6, "Zebra API Version")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.SoftwareName, "zebraSoftwareName", "frr8.3", "Software Name for Zebra")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ControlPlaneHealthCheck.Address, "controlPlaneHealthCheckAddress", "", "URL to poll for the control-plane health check when using BGP without leader election")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.ControlPlaneHealthCheck.PeriodSeconds, "controlPlaneHealthCheckPeriodSeconds", 5, "Seconds between control-plane health checks")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.ControlPlaneHealthCheck.TimeoutSeconds, "controlPlaneHealthCheckTimeoutSeconds", 3, "Timeout for each control-plane health check request")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.ControlPlaneHealthCheck.FailureThreshold, "controlPlaneHealthCheckFailureThreshold", 3, "Consecutive control-plane health check failures before withdrawing the BGP route")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ControlPlaneHealthCheck.CAPath, "controlPlaneHealthCheckCAPath", "", "Path to CA certificate for TLS verification when the control-plane health check URL is HTTPS")
+
+	// Namespace for kube-vip
+	kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The namespace for the configmap defined within the cluster")
 
 	// Manage logging
-	kubeVipCmd.PersistentFlags().Uint32Var(&logLevel, "log", 4, "Set the level of logging")
+	kubeVipCmd.PersistentFlags().Int32Var(&initConfig.Logging, "log", 0, "Set the level of logging")
 
 	// Service flags
-	kubeVipService.Flags().StringVarP(&configMap, "configMap", "c", "kube-vip", "The configuration map defined within the cluster")
-	kubeVipService.Flags().StringVarP(&service.Interface, "interface", "i", "eth0", "Name of the interface to bind to")
-	kubeVipService.Flags().BoolVar(&service.OutSideCluster, "OutSideCluster", false, "Start Controller outside of cluster")
-	kubeVipService.Flags().BoolVar(&service.EnableArp, "arp", false, "Use ARP broadcasts to improve VIP re-allocations")
+	kubeVipService.Flags().StringVarP(&configMap, "configMap", "c", "plndr", "The configuration map defined within the cluster")
+
+	// Routing Table flags
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableID, "tableID", 198, "The routing table used for all table entries")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableType, "tableType", unix.RTN_UNICAST, "The type of route that will be added to the routing table")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingProtocol, "routingProtocol", 248, "The routing protocol value used to create routes")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.CleanRoutingTable, "cleanRoutingTable", false, "Clean routing table of redundant routes on start")
+
+	// Behaviour flags
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableControlPlane, "controlplane", false, "Enable HA for control plane")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DetectControlPlane, "autodetectcp", false, "Determine working address for control plane (from loopback)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServices, "services", false, "Enable Kubernetes services")
+
+	// Extended behaviour flags
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServicesElection, "servicesElection", false, "Enable leader election per kubernetes service")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassOnly, "lbClassOnly", false, fmt.Sprintf("Enable load balancing only for services with LoadBalancerClass %q", kubevip.LBClassName))
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerClassName, "lbClassName", kubevip.LBClassName, fmt.Sprintf("Name of load balancer class for kube-VIP, defaults to %q", kubevip.LBClassName))
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassLegacyHandling, "lbClassNameLegacyHandling", true, "Use legacy LoadBalancer class name handling (e.g. accepting services both with empty and non-empty class)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServiceSecurity, "onlyAllowTrafficServicePorts", false, "Only allow traffic to service ports, others will be dropped, defaults to false")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableNodeLabeling, "enableNodeLabeling", false, fmt.Sprintf("Enable leader node labeling with %q, defaults to false", kubevip.HasIP))
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesLeaseName, "servicesLeaseName", "plndr-svcs-lock", "Name of the lease that is used for leader election for services (in arp mode)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.DNSMode, "dnsMode", "first", "Name of the mode that DNS lookup will be performed (first, ipv4, ipv6, dual)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.DHCPMode, "dhcpMode", "", "Mode DHCP resolving will use to obtain IP addresses (ipv4, ipv6, dual)")
+	kubeVipCmd.PersistentFlags().UintVar(&initConfig.DHCPBackoffAttempts, "dhcpBackoffAttempts", kubevip.DefaultDHCPBackoffAttempts,
+		fmt.Sprintf("number of times DHCP client will try to obtain an IP address (defaults to: %d, 0 for unlimited retries)", kubevip.DefaultDHCPBackoffAttempts))
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DisableServiceUpdates, "disableServiceUpdates", false, "If true, kube-vip will process services as usual, but will not update service's Status.LoadBalancer.Ingress slice")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpoints, "enableEndpoints", false, "If enabled, kube-vip will only advertise services, but will use the (deprecated since v1.33) endpoints for IP addresses")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoInterfaceGlobalScope, "loInterfaceGlobalScope", false, "If true, kube-vip will set global scope when using the lo interface, otherwise a host scope will be used by default")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.HealthCheckPort, "healthCheckPort", 0, "If set to non-zero (> 1024), then this is the port that the healthcheck will listen on")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.DebounceTime, "debounceTime", debouncer.DefaultTime,
+		"Configures the time that the event debouncer will wait for the events arrival (default 0s - debouncer disabled, enable with min. 200ms)")
+
+	// Prometheus HTTP Server
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.PrometheusHTTPServer, "prometheusHTTPServer", ":2112", "Host and port used to expose Prometheus metrics via an HTTP server")
+
+	// Etcd
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.CAFile, "etcdCACert", "", "Verify certificates of TLS-enabled secure servers using this CA bundle file")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.ClientCertFile, "etcdCert", "", "Identify secure client using this TLS certificate file")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.ClientKeyFile, "etcdKey", "", "Identify secure client using this TLS key file")
+	kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.Etcd.Endpoints, "etcdEndpoints", nil, "Etcd member endpoints")
+
+	// Kubernetes client specific flags
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.K8sConfigFile, "k8sConfigPath", "/etc/kubernetes/admin.conf", "Path to the configuration file used with the Kubernetes client")
+
+	// Configuration file flag
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ConfigFile, "config-file", "", "Path to a JSON/YAML configuration file to load settings from")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.InstanceName, "instanceName", "", "Unique name for this kube-vip instance (currently used to isolate nftables egress tables)")
+
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EgressWithNftables, "egressWithNftables", true, "Use nftables-based egress implementation")
+
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.PerServiceElectionOnDemand, "perServiceElectionOnDemand", false, "Allow kube-vip to use per-service election for annotated services")
 
 	kubeVipCmd.AddCommand(kubeKubeadm)
 	kubeVipCmd.AddCommand(kubeManifest)
+	kubeVipCmd.AddCommand(kubeVipManager)
 	kubeVipCmd.AddCommand(kubeVipSample)
 	kubeVipCmd.AddCommand(kubeVipService)
-	kubeVipCmd.AddCommand(kubeVipStart)
 	kubeVipCmd.AddCommand(kubeVipVersion)
 
-	// Sample commands
-	kubeVipSample.AddCommand(kubeVipSampleConfig)
-	kubeVipSample.AddCommand(kubeVipSampleManifest)
-
+	// Set the logging level for all subsequent functions
+	log.SetLogLoggerLevel(log.Level(initConfig.Logging))
 }
 
 // Execute - starts the command parsing process
-func Execute() {
-	if err := kubeVipCmd.Execute(); err != nil {
-		fmt.Println(err)
-		os.Exit(1)
+func Execute() int {
+	cmd, err := kubeVipCmd.ExecuteC()
+	if err != nil {
+		log.Error("command failed", "err", err)
+		if cmd == kubeVipCmd {
+			_ = cmd.Usage()
+		}
+		return 1
 	}
+	return 0
 }
 
 var kubeVipVersion = &cobra.Command{
 	Use:   "version",
 	Short: "Version and Release information about the Kubernetes Virtual IP Server",
-	Run: func(cmd *cobra.Command, args []string) {
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 		fmt.Printf("Kube-VIP Release Information\n")
 		fmt.Printf("Version:  %s\n", Release.Version)
 		fmt.Printf("Build:    %s\n", Release.Build)
@@ -131,56 +213,367 @@ var kubeVipVersion = &cobra.Command{
 var kubeVipSample = &cobra.Command{
 	Use:   "sample",
 	Short: "Generate a Sample configuration",
-	Run: func(cmd *cobra.Command, args []string) {
-		cmd.Help()
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		_ = cmd.Help()
 	},
 }
 
 var kubeVipService = &cobra.Command{
 	Use:   "service",
 	Short: "Start the Virtual IP / Load balancer as a service within a Kubernetes cluster",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	RunE: func(cmd *cobra.Command, args []string) error { //nolint TODO
+		cmd.SilenceUsage = true
 
-		// User Environment variables as an option to make manifest clearer
-		envInterface := os.Getenv("vip_interface")
-		if envInterface != "" {
-			service.Interface = envInterface
+		// Load configuration from file if specified (lowest priority)
+		if initConfig.ConfigFile != "" {
+			err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
+			if err != nil {
+				return fmt.Errorf("loading config file: %w", err)
+			}
 		}
 
+		// parse environment variables, these will overwrite anything loaded from config file
+		err := kubevip.ParseEnvironment(&initConfig)
+		if err != nil {
+			return fmt.Errorf("parsing environment: %w", err)
+		}
+		if err := initConfig.Validate(); err != nil {
+			return fmt.Errorf("validating configuration: %w", err)
+		}
+
+		// Change RTN_UNSPEC to default type
+		if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
+			initConfig.RoutingProtocol = unix.RTN_UNICAST
+		}
+
+		// Set the logging level for all subsequent functions
+		log.SetLogLoggerLevel(log.Level(initConfig.Logging))
+
+		if err := initConfig.CheckInterface(); err != nil {
+			return fmt.Errorf("checking interface: %w", err)
+		}
+
+		// User Environment variables as an option to make manifest clearer
 		envConfigMap := os.Getenv("vip_configmap")
-		if envInterface != "" {
+		if envConfigMap != "" {
 			configMap = envConfigMap
 		}
 
-		envLog := os.Getenv("vip_loglevel")
-		if envLog != "" {
-			logLevel, err := strconv.Atoi(envLog)
-			if err != nil {
-				panic(fmt.Sprintf("Unable to parse environment variable [vip_loglevel], should be int"))
+		// Legacy vip_address requires vip_subnet for control-plane ARP, BGP, and Routing Table modes.
+		if initConfig.EnableControlPlane &&
+			(initConfig.EnableARP || initConfig.EnableBGP || initConfig.EnableRoutingTable) {
+			if err := initConfig.CheckSubnetExists(); err != nil {
+				return fmt.Errorf("checking subnet exists if vip_address defined: %w", err)
 			}
-			log.SetLevel(log.Level(logLevel))
 		}
 
-		envArp := os.Getenv("vip_arp")
-		if envArp != "" {
-			arpBool, err := strconv.ParseBool(envArp)
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
 			if err != nil {
-				panic(fmt.Sprintf("Unable to parse environment variable [arp], should be bool (true/false)"))
+				return fmt.Errorf("generating CIDR: %w", err)
 			}
-			service.EnableArp = arpBool
+		}
+
+		ctx, cancel := context.WithCancel(cmd.Context())
+		defer cancel()
+
+		// Define the new service manager
+		mgr, err := manager.New(ctx, configMap, &initConfig)
+		if err != nil {
+			return fmt.Errorf("new manager: %w", err)
+		}
+
+		// Start the service manager, this will watch the config Map and construct kube-vip services for it
+		err = mgr.Start(ctx)
+		if err != nil {
+			return fmt.Errorf("manager start: %w", err)
+		}
+		return nil
+	},
+}
+
+var kubeVipManager = &cobra.Command{
+	Use:   "manager",
+	Short: "Start the kube-vip manager",
+	RunE: func(cmd *cobra.Command, args []string) error { //nolint TODO
+		cmd.SilenceUsage = true
+
+		// Load configuration from file if specified (lowest priority)
+		if initConfig.ConfigFile != "" {
+			err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
+			if err != nil {
+				return fmt.Errorf("loading config file: %w", err)
+			}
+		}
+
+		// parse environment variables, these will overwrite anything loaded from config file
+		err := kubevip.ParseEnvironment(&initConfig)
+		if err != nil {
+			return fmt.Errorf("parsing environment: %w", err)
+		}
+		if err := initConfig.Validate(); err != nil {
+			return fmt.Errorf("validating configuration: %w", err)
+		}
+
+		// Change RTN_UNSPEC to default type
+		if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
+			initConfig.RoutingProtocol = unix.RTN_UNICAST
+		}
+
+		// Set the logging level for all subsequent functions
+		log.SetLogLoggerLevel(log.Level(initConfig.Logging))
+
+		// Legacy vip_address requires vip_subnet for control-plane ARP, BGP, and Routing Table modes.
+		if initConfig.EnableControlPlane &&
+			(initConfig.EnableARP || initConfig.EnableBGP || initConfig.EnableRoutingTable) {
+			if err := initConfig.CheckSubnetExists(); err != nil {
+				return fmt.Errorf("checking subnet exists if vip_address defined: %w", err)
+			}
+		}
+
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				return fmt.Errorf("generating CIDR: %w", err)
+			}
+		}
+
+		// Welome messages
+		log.Info("kube-vip.io", "version", Release.Version, "build", Release.Build)
+
+		wg := sync.WaitGroup{}
+		defer wg.Wait()
+
+		// create main manager context
+		ctx, cancel := context.WithCancelCause(cmd.Context())
+		defer cancel(nil)
+
+		// start prometheus server
+		if initConfig.PrometheusHTTPServer != "" {
+			wg.Go(func() {
+				servePrometheusHTTPServer(ctx, PrometheusHTTPServerConfig{
+					Addr: initConfig.PrometheusHTTPServer,
+				})
+			})
+		}
+
+		// Determine the kube-vip mode
+		var (
+			mode         string
+			modesEnabled int
+		)
+		if initConfig.EnableARP {
+			mode = "ARP"
+			modesEnabled++
+		}
+
+		if initConfig.EnableBGP {
+			mode = "BGP"
+			modesEnabled++
+		}
+
+		if initConfig.EnableWireguard {
+			mode = "Wireguard"
+			modesEnabled++
+		}
+
+		if initConfig.EnableRoutingTable {
+			mode = "Routing Table"
+			modesEnabled++
+		}
+
+		if mode == "" {
+			return fmt.Errorf("no valid kube-vip mode detected, ensure a supported mode is configured")
+		}
+
+		if modesEnabled > 1 {
+			return fmt.Errorf("multiple kube-vip modes detected, ensure only one mode is configured")
+		}
+
+		// Provide configuration to output/logging
+		log.Info("starting", "namespace", initConfig.Namespace, "Mode", mode, "Control Plane", initConfig.EnableControlPlane, "Services", initConfig.EnableServices)
+
+		// End if nothing is enabled
+		if !initConfig.EnableServices && !initConfig.EnableControlPlane {
+			return fmt.Errorf("no features are enabled")
+		}
+
+		if !initConfig.EnableARP && strings.Contains(initConfig.VIPSubnet, kubevip.Auto) {
+			return fmt.Errorf("auto subnet discovery cannot be used outside ARP mode")
+		}
+
+		if strings.Contains(initConfig.VIPSubnet, kubevip.Auto) && initConfig.Address != "" {
+			return fmt.Errorf("auto subnet discovery cannot be used if VIP address was provided")
+		}
+
+		// If we're using wireguard then all traffic goes through the wg0 interface
+		if initConfig.EnableWireguard {
+			if initConfig.Interface == "" {
+				// Set the vip interface to the wireguard interface
+				initConfig.Interface = "wg0"
+			}
+
+			log.Info("configuring Wireguard networking")
+			l, err := netlink.LinkByName(initConfig.Interface)
+			if err != nil {
+				if strings.Contains(err.Error(), "Link not found") {
+					log.Warn("attempting to create wireguard interface", "interface not found", initConfig.Interface)
+					err = netlink.LinkAdd(&netlink.Wireguard{LinkAttrs: netlink.LinkAttrs{Name: initConfig.Interface}})
+					if err != nil {
+						return fmt.Errorf("adding link: %w", err)
+					}
+					l, err = netlink.LinkByName(initConfig.Interface)
+					if err != nil {
+						return fmt.Errorf("finding link: %w", err)
+					}
+				}
+			}
+			err = netlink.LinkSetUp(l)
+			if err != nil {
+				return fmt.Errorf("setting link UP: %w", err)
+			}
+
+		} else { // if we're not using Wireguard then we'll need to use an actual interface
+			// Check if the interface needs auto-detecting
+			if initConfig.Interface == "" {
+				log.Info("No interface is specified for VIP in config, auto-detecting default Interface")
+				defaultIF, err := vip.GetDefaultGatewayInterface()
+				if err != nil {
+					_ = cmd.Help()
+					return fmt.Errorf("detecting interface: %w", err)
+				}
+				initConfig.Interface = defaultIF.Name
+				log.Info("kube-vip bind", "interface", initConfig.Interface)
+
+				wg.Go(func() {
+					if err := vip.MonitorDefaultInterface(ctx, defaultIF); err != nil {
+						log.Error("interface monitor", "err", err)
+						cancel(err)
+					}
+				})
+			}
+		}
+		// Perform a check on the state of the interface
+		if err := initConfig.CheckInterface(); err != nil {
+			return fmt.Errorf("checking interface: %w", err)
+		}
+
+		// User Environment variables as an option to make manifest clearer
+		envConfigMap := os.Getenv("vip_configmap")
+		if envConfigMap != "" {
+			configMap = envConfigMap
 		}
 
 		// Define the new service manager
-		mgr, err := service.NewManager(configMap)
+		mgr, err := manager.New(ctx, configMap, &initConfig)
 		if err != nil {
-			log.Fatalf("%v", err)
+			return fmt.Errorf("new manager: %w", err)
 		}
+
+		metrics.RegisterPrometheusMetrics()
+		metrics.BuildInfo.WithLabelValues(Release.Version, Release.Build, initConfig.NodeName)
+
 		// Start the service manager, this will watch the config Map and construct kube-vip services for it
-		err = mgr.Start()
+		err = mgr.Start(ctx)
 		if err != nil {
-			log.Fatalf("%v", err)
+			return fmt.Errorf("start manager: %w", err)
 		}
+		return nil
 	},
 }
+
+// PrometheusHTTPServerConfig defines the Prometheus server configuration.
+type PrometheusHTTPServerConfig struct {
+	// Addr sets the http server address used to expose the metric endpoint
+	Addr string
+}
+
+func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerConfig) {
+	var err error
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.Handler())
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { //nolint TODO
+		_, _ = w.Write([]byte(`<html>
+			<head><title>kube-vip</title></head>
+			<body>
+			<h1>kube-vip Metrics</h1>
+			<p><a href="` + "/metrics" + `">Metrics</a></p>
+			</body>
+			</html>`))
+	})
+
+	srv := &http.Server{
+		Addr:              config.Addr,
+		Handler:           mux,
+		ReadHeaderTimeout: 2 * time.Second,
+	}
+
+	wg := sync.WaitGroup{}
+
+	wg.Go(func() {
+		if err = srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			log.Error("prometheus HTTP server", "err", err)
+			return
+		}
+	})
+
+	log.Info("prometheus HTTP server started")
+
+	<-ctx.Done()
+
+	// create prometheus shutdown context (independent of other contexts)
+	ctxShutDown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer func() {
+		cancel()
+	}()
+
+	if err = srv.Shutdown(ctxShutDown); err != nil {
+		log.Error("shutting down prometheus HTTP server", "err", err)
+		return
+	}
+
+	if err == http.ErrServerClosed {
+		err = nil
+	}
+
+	log.Info("prometheus HTTP server stopped")
+
+	wg.Wait()
+}
+
+func GenerateCidrRange(address string, dnsMode string) (string, error) {
+	var cidrs []string
+
+	addresses := strings.Split(address, ",")
+	for _, a := range addresses {
+		ip := net.ParseIP(a)
+		if ip == nil {
+			// we probably are a DNS name
+			ips, err := utils.LookupHost(a, dnsMode, true)
+			if len(ips) == 0 || err != nil {
+				return "", fmt.Errorf("invalid IP address: %s from [%s], %v", a, address, err)
+			}
+			for _, addr := range ips {
+				ip = net.ParseIP(addr)
+				if ip.To4() != nil {
+					cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv4))
+				} else {
+					cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv6))
+				}
+			}
+		} else {
+			if ip.To4() != nil {
+				cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv4))
+			} else {
+				cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv6))
+			}
+		}
+	}
+	// compact as DNS could have a lot of addresses
+	slices.Sort(cidrs)
+	cidrs = slices.Compact(cidrs)
+	slices.Reverse(cidrs)
+	return strings.Join(cidrs, ","), nil
+}

demo/README.md

@@ -0,0 +1,46 @@
+# Demo client-server
+
+This contains some example code to determine how long "failovers" are taking within kube-vip, the server component should live within the cluster and the client should be externally.
+
+## Deploy the server
+
+Simply apply the manifest to a working cluster that has kube-vip deployed:
+
+```
+kubectl apply -f ./demo/server/deploy.yaml
+```
+
+Retrieve the loadBalancer IP that is fronting the service:
+
+```
+kubectl get svc demo-service
+NAME           TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)           AGE
+demo-service   LoadBalancer   10.104.18.147   192.168.0.217   10002:32529/UDP   117m
+```
+
+## Connect the client
+
+From elsewhere, clone the kube-vip repository and connect the client to the server endpoint (loadBalancer IP) with the following command:
+
+```
+go run ./demo/client/main.go -address=<vip>
+```
+
+You will only see output when the client has reconcilled the connection to a pod beneath the service, where it will print the timestamp to reconnection along with the time in milliseconds it took:
+
+```
+15:58:35.916952 3008
+15:58:45.947506 2005
+15:58:57.983151 3007
+15:59:08.013450 2005
+15:59:20.046491 3008
+15:59:30.076341 2507
+15:59:42.110747 3008
+```
+
+## Kill some pods to test
+
+On a machine or control plane that has `kubectl` and has the credentials to speak to the cluster we will run a command to find the demo pod and kill it every 10 seconds:
+
+`while true ; do kubectl delete pod $(kubectl get pods | grep -v NAME | grep vip| awk '{ print $1 }'); sleep 10; done`
+

demo/client/main.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"net"
+	"time"
+)
+
+const udpdata = "a3ViZS12aXAK=kube-vip"
+
+func main() {
+	address := flag.String("address", "127.0.0.1", "The address of the server")
+	port := flag.Int("port", 10002, "the port of the server")
+	interval := flag.Float64("interval", 1000, "Interval in milliseconds")
+	flag.Parse()
+	var errorTime time.Time
+	var errorOccurred bool
+	for {
+		p := make([]byte, 2048)
+		conn, err := net.Dial("udp", net.JoinHostPort(*address, fmt.Sprint(port)))
+		if err != nil {
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			continue
+		}
+
+		err = conn.SetDeadline(time.Now().Add(time.Duration(*interval) * time.Millisecond))
+		if err != nil {
+			//fmt.Printf("Connectivity error [%v]", err)
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			if err = conn.Close(); err != nil {
+				fmt.Printf("Error closing connection [%v]", err)
+			}
+			continue
+		}
+
+		_, err = fmt.Fprint(conn, udpdata)
+		if err != nil {
+			fmt.Printf("Error writing data [%v]", err)
+		}
+
+		_, err = bufio.NewReader(conn).Read(p)
+		if err != nil {
+			//fmt.Printf("read error %v\n", err)
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			if err = conn.Close(); err != nil {
+				fmt.Printf("Error closing connection [%v]", err)
+			}
+			continue
+		}
+		time.Sleep(time.Duration(*interval) * time.Millisecond)
+		if errorOccurred {
+			finishTime := time.Since(errorTime)
+			//fmt.Printf("connectivity reconciled in %dms\n", finishTime.Milliseconds())
+			//t :=time.Now().Format("15:04:05.000000")
+			fmt.Printf("%s %d\n", time.Now().Format("15:04:05.000000"), finishTime.Milliseconds())
+
+			errorOccurred = false
+		}
+	}
+}

demo/go.mod

@@ -1,3 +0,0 @@
-module github.com/plunder-app/kube-vip/demo
-
-go 1.13

demo/server/Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:experimental
 
-FROM golang:1.13-alpine as dev
+FROM golang:1.19-alpine as dev
 RUN apk add --no-cache git ca-certificates
 RUN adduser -D appuser
 COPY main.go /src/
@@ -13,4 +13,4 @@ RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
 
 FROM scratch
 COPY --from=dev /src/demo /
-CMD ["/demo"]
+CMD ["/demo"]

demo/server/Makefile

@@ -45,7 +45,7 @@ fmt:
 	@gofmt -l -w $(SRC)
 
 docker:
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 simplify:

demo/server/deploy.yaml

@@ -6,7 +6,7 @@ metadata:
     app: kube-vip-demo
   name: kube-vip-demo
 spec:
-  replicas: 3
+  replicas: 1
   selector:
     matchLabels:
       app: kube-vip-demo
@@ -30,4 +30,25 @@ spec:
         ports:
             - containerPort: 10001
             - containerPort: 10002
-status: {}
+status: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: demo-service
+  namespace: default
+  labels:
+    app: demo-service
+  annotations:
+    kube-vip.io/egress: "true"
+spec:
+  type: LoadBalancer
+  # "Local" preserves the client source IP and avoids a second hop for
+  # LoadBalancer and NodePort
+  externalTrafficPolicy: Local
+  ports:
+  - name: demo-udp
+    port: 10002
+    protocol: UDP
+  selector:
+    app: kube-vip-demo

demo/server/go.mod

@@ -0,0 +1,3 @@
+module github.com/kube-vip/kube-vip/demo
+
+go 1.13

demo/server/main.go

@@ -32,7 +32,7 @@ func main() {
 		}
 	}
 
-	if serverType == strings.ToLower("udp") {
+	if serverType == "udp" {
 		// Start the UDP echo server
 
 		ServerAddr, err := net.ResolveUDPAddr("udp", ":10002")

example/deploy/0.1.2.yaml

@@ -1,74 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-cluster
-  name: kube-vip-cluster
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-cluster
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-cluster
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-cluster
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: plndr/kube-vip:0.1.2
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        - --configMap
-        - plndr-configmap
-        - --arp
-        - --interface
-        - ens192
-        - --log
-        - "5"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-status: {}
----
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: lease-access
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "create", "update", "list", "put"]
-  - apiGroups: [""]
-    resources: ["configMap"]
-    verbs: ["get"]
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: lease-access
-subjects:
-  - kind: User
-    name: system:serviceaccount:default:default
-roleRef:
-  kind: Role
-  name: lease-access
-  apiGroup: rbac.authorization.k8s.io

example/deploy/0.1.3.yaml

@@ -1,83 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: vip
----
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: vip-role
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "create", "update", "list", "put"]
-  - apiGroups: [""]
-    resources: ["configmaps", "endpoints"]
-    verbs: ["watch", "get"]
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: vip-role-bind
-subjects:
-  - kind: ServiceAccount
-    name: vip 
-    apiGroup: ""
-roleRef:
-  kind: Role
-  name: vip-role
-  apiGroup: "" 
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-cluster
-  name: kube-vip-cluster
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-cluster
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-cluster
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-cluster
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: plndr/kube-vip:0.1.3
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        env:
-          - name: vip_interface
-            value: "ens192"
-          - name: vip_configmap
-            value: "plndr" 
-          - name: vip_arp
-            value: "true"
-          - name: vip_loglevel
-            value: "5"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-      serviceAccountName: vip
-status: {}

example/deploy/0.1.4.yaml

@@ -1,83 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: vip
----
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: vip-role
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "create", "update", "list", "put"]
-  - apiGroups: [""]
-    resources: ["configmaps", "endpoints"]
-    verbs: ["watch", "get"]
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: vip-role-bind
-subjects:
-  - kind: ServiceAccount
-    name: vip 
-    apiGroup: ""
-roleRef:
-  kind: Role
-  name: vip-role
-  apiGroup: "" 
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-cluster
-  name: kube-vip-cluster
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-cluster
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-cluster
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-cluster
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: plndr/kube-vip:0.1.4
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        env:
-          - name: vip_interface
-            value: "ens192"
-          - name: vip_configmap
-            value: "plndr" 
-          - name: vip_arp
-            value: "true"
-          - name: vip_loglevel
-            value: "5"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-      serviceAccountName: vip
-status: {}

example/deployment.yaml

@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:latest
+        ports:
+        - containerPort: 80

example/service-interface-annotation.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-interface-ens192-service
+  annotations:
+    kube-vip.io/serviceInterface: ens192
+spec:
+  selector:
+    app: nginx
+  type: LoadBalancer
+  ports:
+  - name: http
+    port: 80
+    targetPort: 80
+    protocol: TCP

example/service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-service
+spec:
+  selector:
+    app: nginx
+  type: LoadBalancer
+  ports:
+  - name: http
+    port: 80
+    targetPort: 80
+    protocol: TCP

go.mod

@@ -1,60 +1,154 @@
-module github.com/plunder-app/kube-vip
+module github.com/kube-vip/kube-vip
 
-go 1.14
+go 1.26.4
 
 require (
-	github.com/armon/go-metrics v0.3.4 // indirect
-	github.com/davecgh/go-spew v1.1.1
-	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
-	github.com/digineo/go-dhclient v1.0.2
-	github.com/eapache/queue v1.1.0 // indirect
-	github.com/fatih/color v1.9.0 // indirect
-	github.com/ghodss/yaml v1.0.0
-	github.com/go-logr/logr v0.2.1 // indirect
-	github.com/golang/protobuf v1.4.2
-	github.com/google/go-cmp v0.5.2 // indirect
-	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/gopacket v1.1.18 // indirect
-	github.com/google/uuid v1.1.2 // indirect
-	github.com/googleapis/gnostic v0.5.1 // indirect
-	github.com/hashicorp/go-hclog v0.14.1 // indirect
-	github.com/hashicorp/go-immutable-radix v1.2.0 // indirect
-	github.com/hashicorp/go-msgpack v1.1.5 // indirect
-	github.com/hashicorp/go-retryablehttp v0.6.7 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
-	github.com/hashicorp/raft v1.1.2
-	github.com/imdario/mergo v0.3.11 // indirect
-	github.com/k-sone/critbitgo v1.4.0 // indirect
-	github.com/magiconair/properties v1.8.3 // indirect
-	github.com/mattn/go-colorable v0.1.7 // indirect
-	github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065 // indirect
-	github.com/mitchellh/mapstructure v1.3.3 // indirect
-	github.com/osrg/gobgp v2.0.0+incompatible
-	github.com/packethost/packngo v0.3.0
-	github.com/pelletier/go-toml v1.8.1 // indirect
+	github.com/cloudflare/ipvs v0.12.0
+	github.com/containernetworking/plugins v1.9.1
+	github.com/docker/docker v28.5.2+incompatible
+	github.com/florianl/go-conntrack v0.7.0
+	github.com/google/go-cmp v0.7.0
+	github.com/google/go-containerregistry v0.22.1
+	github.com/google/nftables v0.3.0
+	github.com/gookit/slog v0.7.1
+	github.com/huin/goupnp v1.3.0
+	github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82
+	github.com/jpillora/backoff v1.0.0
+	github.com/mdlayher/ndp v1.1.0
+	github.com/onsi/ginkgo/v2 v2.32.2
+	github.com/onsi/gomega v1.43.0
+	github.com/osrg/gobgp/v4 v4.9.0
 	github.com/pkg/errors v0.9.1
-	github.com/sirupsen/logrus v1.6.0
-	github.com/spf13/afero v1.4.0 // indirect
-	github.com/spf13/cast v1.3.1 // indirect
-	github.com/spf13/cobra v1.0.0
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/viper v1.7.1 // indirect
-	github.com/vishvananda/netlink v1.1.0
-	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect
-	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
-	golang.org/x/net v0.0.0-20200904194848-62affa334b73
-	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
-	golang.org/x/sys v0.0.0-20200917073148-efd3b9a0ff20 // indirect
-	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
-	google.golang.org/genproto v0.0.0-20200917134801-bb4cff56e0d0 // indirect
-	google.golang.org/grpc v1.32.0 // indirect
-	gopkg.in/ini.v1 v1.61.0 // indirect
-	k8s.io/api v0.19.2
-	k8s.io/apimachinery v0.19.2
-	k8s.io/client-go v0.19.0
-	k8s.io/klog v1.0.0
-	k8s.io/klog/v2 v2.3.0 // indirect
-	k8s.io/utils v0.0.0-20200912215256-4140de9c8800 // indirect
+	github.com/prometheus/client_golang v1.24.1
+	github.com/spf13/cobra v1.10.2
+	github.com/stretchr/testify v1.12.1
+	github.com/vishvananda/netlink v1.3.2-0.20260830232854-cf01b55a4a4b
+	github.com/vishvananda/netns v0.0.5
+	go.etcd.io/etcd/api/v3 v3.7.1
+	go.etcd.io/etcd/client/pkg/v3 v3.7.1
+	go.etcd.io/etcd/client/v3 v3.7.1
+	go.uber.org/zap v1.28.0
+	golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
+	golang.org/x/sync v0.23.0
+	golang.org/x/sys v0.48.0
+	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
+	google.golang.org/grpc v1.83.2
+	gopkg.in/yaml.v3 v3.0.1
+	k8s.io/api v0.36.4
+	k8s.io/apimachinery v0.36.4
+	k8s.io/client-go v0.36.4
+	k8s.io/klog/v2 v2.140.0
+	sigs.k8s.io/kind v0.33.0
+	sigs.k8s.io/yaml v1.6.0
 )
 
-replace github.com/osrg/gobgp v2.0.0+incompatible => github.com/osrg/gobgp v0.0.0-20191101114856-a42a1a5f6bf0
+require (
+	al.essio.dev/pkg/shellescape v1.5.1 // indirect
+	github.com/BurntSushi/toml v1.5.0 // indirect
+	github.com/Masterminds/semver/v3 v3.4.0 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/coreos/go-semver v0.3.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.7.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/go-connections v0.7.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/eapache/channels v1.1.0 // indirect
+	github.com/eapache/queue v1.1.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.13.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
+	github.com/gaissmai/bart v0.26.1 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/gnostic-models v0.7.0 // indirect
+	github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gookit/color v1.6.1 // indirect
+	github.com/gookit/goutil v0.7.6 // indirect
+	github.com/gookit/gsr v0.1.1 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/josharian/native v1.1.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/k-sone/critbitgo v1.4.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/mailru/easyjson v0.9.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mdlayher/genetlink v1.3.2 // indirect
+	github.com/mdlayher/netlink v1.8.0 // indirect
+	github.com/mdlayher/packet v1.1.2 // indirect
+	github.com/mdlayher/socket v0.5.1 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/sys/atomicwriter v0.1.0 // indirect
+	github.com/moby/term v0.5.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
+	github.com/morikuni/aec v1.1.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect
+	github.com/pelletier/go-toml v1.9.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.70.1 // indirect
+	github.com/prometheus/procfs v0.21.1 // indirect
+	github.com/sagikazarmark/locafero v0.7.0 // indirect
+	github.com/segmentio/fasthash v1.0.3 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.12.0 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/spf13/pflag v1.0.10 // indirect
+	github.com/spf13/viper v1.20.1 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
+	go.opentelemetry.io/otel v1.44.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect
+	go.opentelemetry.io/otel/metric v1.44.0 // indirect
+	go.opentelemetry.io/otel/trace v1.44.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.4 // indirect
+	go.yaml.in/yaml/v3 v3.0.5 // indirect
+	golang.org/x/crypto v0.55.0 // indirect
+	golang.org/x/mod v0.39.0 // indirect
+	golang.org/x/net v0.58.0 // indirect
+	golang.org/x/oauth2 v0.36.0 // indirect
+	golang.org/x/term v0.45.0 // indirect
+	golang.org/x/text v0.41.0 // indirect
+	golang.org/x/time v0.14.0 // indirect
+	golang.org/x/tools v0.49.0 // indirect
+	golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
+	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect
+	k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect
+	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect
+)

kubernetes-control-plane.md

@@ -1,147 +0,0 @@
-# Load Balancing a Kubernetes Cluster (Control-Plane)
-
-This document covers all of the details for using `kube-vip` to build a HA Kubernetes cluster
-
-`tl;dr version`
-- Generate/modify first node `kube-vip` config/manifest
-- `init` first node
-- `join` remaining nodes
-- Add remaining config/manifests
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.75 |
-| controlPlane01 | 10.0.0.70 |
-| controlPlane02 | 10.0.0.71 |
-| controlPlane03 | 10.0.0.72 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.17.0.
-
-### Generate the `kube-vip` configuration
-
-Make sure that the config directory exists: `sudo mkdir -p /etc/kube-vip/`, this directory can be any directory however the `hostPath` in the manifest will need modifying to point to the correct path.
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1.5 sample config | sudo tee /etc/kube-vip/config.yaml
-```
-
-### Modify the configuration
-
-**Cluster Configuration**
-Modify the `remotePeers` to point to the correct addresses of the other two nodes, ensure that their `id` is unique otherwise this will confuse the raft algorithm. The `localPeer` should be the configuration of the current node (`controlPlane01`), which is where this instance of the cluster will run. 
-
-As this node will be the first node, it will need to elect itself leader as until this occurs the VIP won’t be activated!
-
-`startAsLeader: true`
-
-**VIP Config**
-We will need to set our VIP address to `192.168.0.75` and to ensure all hosts are updated when the VIP moves we will enable ARP broadcasts `gratuitousARP: true`
-
-**Load Balancer**
-We will configure the load balancer to sit on the standard API-Server port `6443` and we will configure the backends to point to the API-servers that will be configured to run on port `6444`. Also for the Kubernetes Control Plane we will configure the load balancer to be of `type: tcp`.
-
-We can also use `6443` for both the VIP and the API-Servers, in order to do this we need to specify that the api-server is bound to it's local IP. To do this we use the `--apiserver-advertise-address` flag as part of the `init`, this means that we can then bind the same port to the VIP and we wont have a port conflict.
-
-**config.yaml**
-
-`user@controlPlane01:/etc/kube-vip$ cat config.yaml`
-
-...
-
-``` 
-remotePeers:
-- id: server2
-  address: 192.168.0.71
-  port: 10000
-- id: server3
-  address: 192.168.0.72
-  port: 10000
-localPeer:
-  id: server1
-  address: 192.168.0.70
-  port: 10000
-vip: 192.168.0.75
-gratuitousARP: true
-singleNode: false
-startAsLeader: true
-interface: ens192
-loadBalancers:
-- name: Kubernetes Control Plane
-  type: tcp
-  port: 6443
-  bindToVip: true
-  backends:
-  - port: 6444
-    address: 192.168.0.70
-  - port: 6444
-    address: 192.168.0.71
-  - port: 6444
-    address: 192.168.0.72
-```
-
-### First Node
-
-To generate the basic Kubernetes static pod `yaml` configuration:
-
-Make sure that the manifest directory exists: `sudo mkdir -p /etc/kubernetes/manifests/`
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1.5 sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Ensure that `image: plndr/kube-vip:<x>` is modified to point to a specific version (`0.1.5` at the time of writing), refer to [docker hub](https://hub.docker.com/r/plndr/kube-vip/tags) for details. Also ensure that the `hostPath` points to the correct `kube-vip` configuration, if it isn’t the above path.
-
-The **vip** is set to `192.168.0.75` and this first node will elect itself as leader, and as part of the `kubeadm init` it will use the VIP in order to speak back to the initialising api-server.
-
-`sudo kubeadm init --control-plane-endpoint “192.168.0.75:6443” --apiserver-bind-port 6444 --upload-certs --kubernetes-version “v1.17.0”`
-
-Once this node is up and running we will be able to see the control-plane pods, including the `kube-vip` pod:
-
-```
-$ kubectl get pods -A
-NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
-<...>
-kube-system   kube-vip-controlplane01                  1/1     Running   0          10m
-```
-
-### Remaining Nodes
-
-We first will need to create the `kube-vip` configuration that resides in `/etc/kube-vip/config.yaml` or we can regenerate it from scratch using the above example. Ensure that the configuration is almost identical with the `localPeer` and `remotePeers` sections are updated for each node. Finally, ensure that the remaining nodes will behave as standard cluster nodes by setting `startAsLeader: false`.
-
-At this point **DON’T** generate the manifests, this is due to some bizarre `kubeadm/kubelet` behaviour.
-
-```
-  kubeadm join 192.168.0.75:6443 --token <tkn> \
-    --discovery-token-ca-cert-hash sha256:<hash> \
-    --control-plane --certificate-key <key> 
-
-```
-
-**After** this node has been added to the cluster, we can add the manifest to also add this node as a `kube-vip` member. (Adding the manifest afterwards doesn’t interfere with `kubeadm`). 
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1.5 sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Once this node is added we will be able to see that the `kube-vip` pod is up and running as expected:
-
-```
-user@controlPlane01:~$ kubectl get pods -A | grep vip
-kube-system   kube-vip-controlplane01                  1/1     Running             1          16m
-kube-system   kube-vip-controlplane02                  1/1     Running             0          18m
-kube-system   kube-vip-controlplane03                  1/1     Running             0          20m
-
-```
-
-If we look at the logs, we can see that the VIP is running on the second node and we’re waiting for our third node to join the cluster:
-
-```
-$ kubectl logs kube-vip-controlplane02  -n kube-system
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-
-```

main.go

@@ -1,6 +1,10 @@
 package main
 
-import "github.com/plunder-app/kube-vip/cmd"
+import (
+	"os"
+
+	"github.com/kube-vip/kube-vip/cmd"
+)
 
 // Version is populated from the Makefile and is tied to the release TAG
 var Version string
@@ -9,7 +13,8 @@ var Version string
 var Build string
 
 func main() {
+
 	cmd.Release.Version = Version
 	cmd.Release.Build = Build
-	cmd.Execute()
+	os.Exit(cmd.Execute())
 }

pkg/arp/arp.go

@@ -0,0 +1,296 @@
+package arp
+
+import (
+	"context"
+	"fmt"
+	log "log/slog"
+	"sync"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	"github.com/vishvananda/netlink"
+)
+
+const linkSubscriptionBuffer = 64
+
+type Manager struct {
+	mu        sync.Mutex
+	instances map[string]*Instance
+	config    *kubevip.Config
+}
+
+type Instance struct {
+	network vip.Network
+	ndp     *vip.NdpResponder
+	counter int
+}
+
+func NewManager(config *kubevip.Config) *Manager {
+	if config.ArpBroadcastRate < 500 {
+		log.Warn("[ARP manager] arp broadcast rate is too low", "rate (ms)", config.ArpBroadcastRate, "setting to (ms)", "3000")
+		config.ArpBroadcastRate = 3000
+	}
+	return &Manager{
+		instances: make(map[string]*Instance),
+		config:    config,
+	}
+}
+
+func NewInstance(network vip.Network, ndp *vip.NdpResponder) *Instance {
+	return &Instance{
+		ndp:     ndp,
+		network: network,
+		counter: 1,
+	}
+}
+
+func (i *Instance) Name() string {
+	return i.network.ARPName()
+}
+
+func (m *Manager) Insert(instance *Instance) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	existing := m.instances[instance.Name()]
+	if existing == nil {
+		m.instances[instance.Name()] = instance
+		log.Info("[ARP manager] inserting ARP/NDP instance", "name", instance.Name())
+		return
+	}
+	existing.counter++
+}
+
+func (m *Manager) Remove(instance *Instance) {
+	m.RemoveWithIPDelete(instance, true)
+}
+
+// RemoveOnLeadershipLoss removes an ARP instance when leadership is lost
+func (m *Manager) RemoveOnLeadershipLoss(instance *Instance) {
+	// Use the inverse of PreserveVIPOnLeadershipLoss to decide whether to delete the IP
+	// If preserve is true, don't delete IP (deleteIP = false)
+	// If preserve is false, delete IP (deleteIP = true), This is the legacy behavior
+	deleteIP := !m.config.PreserveVIPOnLeadershipLoss
+	m.RemoveWithIPDelete(instance, deleteIP)
+}
+
+func (m *Manager) RemoveWithIPDelete(instance *Instance, deleteIP bool) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	i := m.instances[instance.Name()]
+	if i != nil {
+		i.counter--
+		if i.counter == 0 {
+			log.Info("[ARP manager] removing ARP/NDP instance", "name", instance.Name())
+			if deleteIP {
+				if _, err := instance.network.DeleteIP(); err != nil {
+					log.Error("failed to delete IP", "address", instance.network.IP(), "err", err)
+				}
+			}
+			delete(m.instances, instance.Name())
+		}
+	} else {
+		log.Warn("[ARP manager] unable to remove the instance - instance not found", "name", instance.Name())
+	}
+}
+
+func (m *Manager) Count(name string) int {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	i := m.instances[name]
+	if i != nil {
+		return i.counter
+	}
+	return 0
+}
+
+func (m *Manager) StartAdvertisement(ctx context.Context, killFunc func()) {
+	if m.config.LoseLeadership {
+		var wg sync.WaitGroup
+		defer wg.Wait()
+
+		log.Info("[ARP manager] starting watching network device", "interface", m.config.Interface)
+
+		duration := time.Duration(m.config.LoseLeadershipTimeoutSeconds) * time.Second
+		timeout := time.NewTimer(duration)
+		timeout.Stop()
+
+		wg.Go(func() {
+			select {
+			case <-timeout.C:
+				killFunc()
+			case <-ctx.Done():
+				return
+			}
+		})
+
+		wg.Go(func() {
+			if err := watch(ctx, m.config.Interface, func(s netlink.LinkOperState) {
+				if isUp(s) {
+					timeout.Stop()
+					return
+				}
+				timeout.Reset(duration)
+			}); err != nil {
+				log.Error("[ARP manager] stopped watching interface", "err", err)
+				killFunc()
+			}
+		})
+	}
+
+	log.Info("[ARP manager] starting ARP/NDP advertisement")
+
+	ticker := time.NewTicker(time.Duration(m.config.ArpBroadcastRate) * time.Millisecond)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done(): // if cancel() execute
+			return
+		case <-ticker.C: // send gratuitous ARP/NDP on each tick
+			m.advertiseAll()
+		}
+	}
+}
+
+func (m *Manager) advertiseAll() {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	for _, instance := range m.instances {
+		if instance.counter > 0 {
+			ensureIPAndSendGratuitous(instance)
+		} else if _, err := instance.network.DeleteIP(); err != nil {
+			log.Error("[ARP manager] failed to delete IP", "address", instance.network.IP(), "err", err)
+		}
+	}
+}
+
+// ensureIPAndSendGratuitous - adds IP to the interface if missing, and send
+// either a gratuitous ARP or gratuitous NDP. Re-adds the interface if it is IPv6
+// and in a dadfailed state.
+func ensureIPAndSendGratuitous(instance *Instance) {
+	iface := instance.network.Interface()
+	ipString := instance.network.IP()
+
+	// Check if IP is dadfailed
+	if instance.network.IsDADFAIL() {
+		log.Warn("IP address is in dadfailed state, removing config", "ip", ipString, "interface", iface)
+		deleted, err := instance.network.DeleteIP()
+		if err != nil {
+			log.Warn(err.Error())
+		}
+		if deleted {
+			log.Info("deleted and recreating address with NODAD flag to skip DAD", "IP", ipString, "interface", iface)
+			// Re-add immediately without DAD check since we're recovering from DADFAILED
+			// The AddIP function will set IFA_F_NODAD flag for IPv6 addresses when skipDAD=true
+			if _, err := instance.network.AddIP(false, true); err != nil {
+				log.Error("failed to recreate address after DADFAILED", "IP", ipString, "interface", iface, "err", err)
+			} else {
+				log.Info("successfully recreated address after DADFAILED recovery", "IP", ipString, "interface", iface)
+			}
+		}
+		// Return early after DADFAILED recovery to avoid double IP addition
+		return
+	}
+
+	// Normal case: add IP with precheck and normal DAD process
+	if added, err := instance.network.AddIP(true, false); err != nil {
+		log.Warn(err.Error())
+	} else if added {
+		log.Warn("Re-applied the VIP configuration", "ip", ipString, "interface", iface)
+	}
+
+	if utils.IsIPv6(ipString) {
+		// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
+		if instance.ndp == nil {
+			log.Error("NDP responder was not created")
+		} else {
+			err := instance.ndp.SendGratuitous(ipString)
+			if err != nil {
+				log.Warn(err.Error())
+			}
+		}
+	} else {
+		// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
+		err := vip.ARPSendGratuitous(ipString, iface)
+		if err != nil {
+			log.Warn(err.Error())
+		}
+	}
+}
+
+// watch subscribing to the network interface events and calls handler
+func watch(ctx context.Context, interfaceName string, operStateHandler func(netlink.LinkOperState)) error {
+	ifname, err := netlink.LinkByName(interfaceName)
+	if err != nil {
+		return fmt.Errorf("failed to watch interface %q: %w", interfaceName, err)
+	}
+
+	// verify if this interface is physical device
+	if _, ok := ifname.(*netlink.Device); !ok {
+		return fmt.Errorf("interface %s is not physical, ignoring", interfaceName)
+	}
+
+	// The subscription is buffered and drained on exit: netlink parks its reader
+	// goroutine on an unread send, which closing done alone does not release.
+	events := make(chan netlink.LinkUpdate, linkSubscriptionBuffer)
+	done := make(chan struct{})
+
+	if err := netlink.LinkSubscribe(events, done); err != nil {
+		return fmt.Errorf("failed to subscribe to the interface events: %w", err)
+	}
+	defer func() {
+		close(done)
+		drainLinkUpdates(events)
+	}()
+
+	//  handle initial state
+	operStateHandler(ifname.Attrs().OperState)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case event, ok := <-events:
+			if !ok {
+				return fmt.Errorf("interface events channel closed")
+			}
+
+			attrs := event.Attrs()
+			// LinkSubscribe captures events for all network devices found
+			// so we only care about vip interface
+			if ifname.Attrs().Name != attrs.Name {
+				continue
+			}
+			log.Debug("handling device change", "state", attrs.OperState)
+			operStateHandler(attrs.OperState)
+		}
+	}
+}
+
+func isUp(operState netlink.LinkOperState) bool {
+	return operState == netlink.OperUp
+}
+
+// drainLinkUpdates releases a netlink sender that is parked on an unread update
+// so its goroutine can observe the closed subscription and exit.
+func drainLinkUpdates(events <-chan netlink.LinkUpdate) {
+	timer := time.NewTimer(100 * time.Millisecond)
+	defer timer.Stop()
+
+	for {
+		select {
+		case _, ok := <-events:
+			if !ok {
+				return
+			}
+		case <-timer.C:
+			return
+		}
+	}
+}

pkg/arp/arp_test.go

@@ -0,0 +1,119 @@
+package arp
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/vishvananda/netlink"
+	v1 "k8s.io/api/core/v1"
+)
+
+func TestDrainLinkUpdatesReleasesParkedSender(t *testing.T) {
+	events := make(chan netlink.LinkUpdate)
+	sent := make(chan struct{})
+	go func() {
+		events <- netlink.LinkUpdate{}
+		close(sent)
+	}()
+
+	drainLinkUpdates(events)
+
+	select {
+	case <-sent:
+	case <-time.After(time.Second):
+		t.Fatal("netlink sender is still parked on an unread link update")
+	}
+}
+
+// stubNetwork is a minimal vip.Network implementation; only ARPName matters here.
+type stubNetwork struct {
+	name          string
+	deleteStarted chan struct{}
+	releaseDelete chan struct{}
+}
+
+func (s *stubNetwork) AddIP(bool, bool, ...int) (bool, error) { return false, nil }
+func (s *stubNetwork) AddRoute(bool) (bool, error)            { return false, nil }
+func (s *stubNetwork) ReplaceRoute() error                    { return nil }
+func (s *stubNetwork) DeleteIP() (bool, error) {
+	if s.deleteStarted != nil {
+		close(s.deleteStarted)
+		<-s.releaseDelete
+	}
+	return true, nil
+}
+func (s *stubNetwork) DeleteRoute() error            { return nil }
+func (s *stubNetwork) UpdateRoutes() (bool, error)   { return false, nil }
+func (s *stubNetwork) IsSet() (*netlink.Addr, error) { return nil, nil }
+func (s *stubNetwork) IP() string                    { return "" }
+func (s *stubNetwork) CIDR() string                  { return "" }
+func (s *stubNetwork) IPisLinkLocal() bool           { return false }
+func (s *stubNetwork) PrepareRoute() *netlink.Route  { return nil }
+func (s *stubNetwork) RouteHash() string             { return "" }
+func (s *stubNetwork) SetIP(string) error            { return nil }
+func (s *stubNetwork) SetServicePorts(*v1.Service)   {}
+func (s *stubNetwork) Interface() string             { return "eth0" }
+func (s *stubNetwork) IsDADFAIL() bool               { return false }
+func (s *stubNetwork) IsDNS() bool                   { return false }
+func (s *stubNetwork) IsDDNS() bool                  { return false }
+func (s *stubNetwork) DDNSHostName() string          { return "" }
+func (s *stubNetwork) DNSName() string               { return "" }
+func (s *stubNetwork) SetMask(string) error          { return nil }
+func (s *stubNetwork) SetHasEndpoints(bool)          {}
+func (s *stubNetwork) HasEndpoints() bool            { return false }
+func (s *stubNetwork) ARPName() string               { return s.name }
+func (s *stubNetwork) GetPossibleSubnets() string    { return "" }
+func (s *stubNetwork) DHCPFamily() string            { return "" }
+func (s *stubNetwork) IPVSMark() uint32              { return 0 }
+
+// TestManagerInsertConcurrentFirstRegistrationsDoNotLoseClaims guards the
+// get-then-store race: two never-before-seen instances for the same ARP name
+// registering concurrently must both be counted, not just the last writer.
+func TestManagerInsertConcurrentFirstRegistrationsDoNotLoseClaims(t *testing.T) {
+	m := NewManager(&kubevip.Config{ArpBroadcastRate: 3000})
+	const concurrent = 8
+
+	var wg sync.WaitGroup
+	for range concurrent {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			m.Insert(NewInstance(&stubNetwork{name: "shared"}, nil))
+		}()
+	}
+	wg.Wait()
+
+	if got := m.Count("shared"); got != concurrent {
+		t.Fatalf("Count() = %d, want %d claims registered", got, concurrent)
+	}
+}
+
+func TestManagerInsertDoesNotJoinEntryBeingRemoved(t *testing.T) {
+	m := NewManager(&kubevip.Config{ArpBroadcastRate: 3000})
+	deleteStarted := make(chan struct{})
+	releaseDelete := make(chan struct{})
+	first := NewInstance(&stubNetwork{name: "shared", deleteStarted: deleteStarted, releaseDelete: releaseDelete}, nil)
+	m.Insert(first)
+
+	removeDone := make(chan struct{})
+	go func() {
+		m.Remove(first)
+		close(removeDone)
+	}()
+	<-deleteStarted
+
+	insertDone := make(chan struct{})
+	go func() {
+		m.Insert(NewInstance(&stubNetwork{name: "shared"}, nil))
+		close(insertDone)
+	}()
+	close(releaseDelete)
+	<-removeDone
+	<-insertDone
+
+	if got := m.Count("shared"); got != 1 {
+		t.Fatalf("Count() = %d, want replacement claim registered", got)
+	}
+}

pkg/backend/backend.go

@@ -0,0 +1,107 @@
+package backend
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/k8s"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+type Entry struct {
+	Addr    string
+	Port    uint16
+	IsLocal bool
+}
+
+type Map map[Entry]bool
+
+// kubeConfigPath is an explicitly configured kubeconfig used by Check when
+// set; static pod deployments configure it since neither admin.conf nor
+// in-cluster config are available there.
+var (
+	kubeConfigPath string
+	pathMtx        sync.Mutex
+)
+
+// SetKubeConfigPath configures the kubeconfig used by backend health checks.
+func SetKubeConfigPath(path string) {
+	pathMtx.Lock()
+	defer pathMtx.Unlock()
+	kubeConfigPath = path
+}
+
+func (e *Entry) Check() bool {
+	var client *kubernetes.Clientset
+	var err error
+	var config *rest.Config
+
+	adminConfigPath := "/etc/kubernetes/admin.conf"
+	// TODO: add one more switch case of homeConfigPath if there is such scenario in future
+	// homeConfigPath := filepath.Join(os.Getenv("HOME"), ".kube", "config")
+
+	var k8sAddr string
+	if utils.IsIPv6(e.Addr) {
+		k8sAddr = fmt.Sprintf("[%s]:%v", e.Addr, e.Port)
+	} else {
+		k8sAddr = fmt.Sprintf("%s:%v", e.Addr, e.Port)
+	}
+
+	switch {
+	case kubeConfigPath != "" && utils.FileExists(kubeConfigPath):
+		config, err = k8s.NewRestConfig(kubeConfigPath, false, k8sAddr)
+		if err != nil {
+			log.Error("create k8s REST config", "path", kubeConfigPath, "err", err)
+			return false
+		}
+	case utils.FileExists(adminConfigPath):
+		config, err = k8s.NewRestConfig(adminConfigPath, false, k8sAddr)
+		if err != nil {
+			log.Error("create k8s REST config", "path", adminConfigPath, "err", err)
+			return false
+		}
+	default:
+		config, err = k8s.NewRestConfig("", true, k8sAddr)
+		if err != nil {
+			log.Error("create k8s REST config", "err", err)
+			return false
+		}
+	}
+
+	client, err = k8s.NewClientset(config)
+	if err != nil {
+		log.Error("create k8s client", "err", err)
+		return false
+	}
+
+	_, err = client.DiscoveryClient.ServerVersion()
+	if err != nil {
+		log.Error("discover k8s version", "err", err)
+		return false
+	}
+	return true
+}
+
+func Watch(ctx context.Context, interval int, tickAction func()) {
+	if interval <= 0 {
+		interval = 5
+	}
+
+	ticker := time.NewTicker(time.Second * time.Duration(interval))
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			tickAction()
+		}
+	}
+}

pkg/bgp/hosts.go

@@ -2,41 +2,81 @@ package bgp
 
 import (
 	"context"
+	"fmt"
+	log "log/slog"
 	"net"
 
-	api "github.com/osrg/gobgp/api"
+	"github.com/osrg/gobgp/v4/pkg/apiutil"
 )
 
 // AddHost will update peers of a host
-func (b *Server) AddHost(addr string) (err error) {
-	ip, _, err := net.ParseCIDR(addr)
-	if err != nil {
-		return err
-	}
-	p := b.getPath(ip)
-	if p == nil {
-		return
+func (b *Server) AddHost(ctx context.Context, addr string, object string) error {
+	b.mtx.Lock()
+	defer b.mtx.Unlock()
+
+	objects, exists := b.tracker[addr]
+
+	if !exists {
+		b.tracker[addr] = make(map[string]bool)
+		objects = b.tracker[addr]
+
+		ip, _, err := net.ParseCIDR(addr)
+		if err != nil {
+			return err
+		}
+
+		p := b.getPath(ip)
+		if p == nil {
+			return fmt.Errorf("failed to get path for %v", ip)
+		}
+
+		if _, err := b.s.AddPath(apiutil.AddPathRequest{
+			Paths: []*apiutil.Path{p},
+		}); err != nil {
+			return err
+		}
+		log.Debug("[BGP] added host", "addr", addr, "cnt", len(objects)+1, "object", object)
 	}
 
-	_, err = b.s.AddPath(context.Background(), &api.AddPathRequest{
-		Path: p,
-	})
+	objects[object] = true
 
-	return
+	return nil
 }
 
 // DelHost will inform peers to remove a host
-func (b *Server) DelHost(addr string) (err error) {
+func (b *Server) DelHost(ctx context.Context, addr string, object string) error {
+	b.mtx.Lock()
+	defer b.mtx.Unlock()
+
+	objects, exists := b.tracker[addr]
+	if !exists {
+		log.Debug("[BGP] deleting host - nothing to delete", "addr", addr, "object", object)
+		return nil
+	}
+
 	ip, _, err := net.ParseCIDR(addr)
 	if err != nil {
 		return err
 	}
-	p := b.getPath(ip)
-	if p == nil {
-		return
+
+	delete(objects, object)
+
+	if len(objects) == 0 {
+		p := b.getPath(ip)
+		if p == nil {
+			return nil
+		}
+
+		if err := b.s.DeletePath(apiutil.DeletePathRequest{
+			Paths: []*apiutil.Path{p},
+		}); err != nil {
+			return err
+		}
+		delete(b.tracker, addr)
+		log.Debug("[BGP] deleted host", "addr", addr, "cnt", len(objects), "object", object)
+	} else {
+		log.Debug("[BGP] deleting from tracker only", "addr", addr, "object", object)
 	}
 
-	return b.s.DeletePath(context.Background(), &api.DeletePathRequest{
-		Path: p,
-	})
+	return nil
 }

pkg/bgp/peers.go

@@ -3,97 +3,325 @@ package bgp
 import (
 	"context"
 	"fmt"
+	log "log/slog"
 	"net"
+	"net/netip"
 	"strconv"
-	"strings"
+	"time"
 
-	"github.com/golang/protobuf/ptypes"
-	"github.com/golang/protobuf/ptypes/any"
-	api "github.com/osrg/gobgp/api"
+	"github.com/jpillora/backoff"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	api "github.com/osrg/gobgp/v4/api"
+
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/osrg/gobgp/v4/pkg/apiutil"
+	"github.com/osrg/gobgp/v4/pkg/config/oc"
+	bgp "github.com/osrg/gobgp/v4/pkg/packet/bgp"
+	"github.com/osrg/gobgp/v4/pkg/server"
 )
 
-//AddPeer will add peers to the BGP configuration
-func (b *Server) AddPeer(peer Peer) (err error) {
-	port := 179
+const defaultBGPPort uint32 = 179
 
-	if t := strings.SplitN(peer.Address, ":", 2); len(t) == 2 {
-		peer.Address = t[0]
-
-		if port, err = strconv.Atoi(t[1]); err != nil {
-			return fmt.Errorf("Unable to parse port '%s' as int: %s", t[1], err)
-		}
+// AddPeer will add peers to the BGP configuration
+func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error) {
+	remotePort := defaultBGPPort
+	if peer.Port != 0 {
+		remotePort = uint32(peer.Port)
 	}
 
 	p := &api.Peer{
 		Conf: &api.PeerConf{
-			NeighborAddress: peer.Address,
-			PeerAs:          peer.AS,
+			NeighborAddress:   peer.Address,
+			PeerAsn:           peer.AS,
+			NeighborInterface: peer.Interface,
+			AuthPassword:      peer.Password,
 		},
 
 		Timers: &api.Timers{
 			Config: &api.TimersConfig{
-				ConnectRetry: 10,
+				ConnectRetry:      10,
+				HoldTime:          b.c.HoldTime,
+				KeepaliveInterval: b.c.KeepaliveInterval,
 			},
 		},
 
+		// This enables routes to be sent to routers across multiple hops
+		EbgpMultihop: &api.EbgpMultihop{
+			Enabled:     peer.MultiHop,
+			MultihopTtl: 50,
+		},
+
 		Transport: &api.Transport{
 			MtuDiscovery:  true,
 			RemoteAddress: peer.Address,
-			RemotePort:    uint32(port),
+			RemotePort:    remotePort,
 		},
 	}
 
-	// if b.c.SourceIP != "" {
-	// 	p.Transport.LocalAddress = b.c.SourceIP
-	// }
+	if peer.BFDEnabled {
+		p.Bfd = &api.BfdPeerConfig{
+			Enabled:                  true,
+			DesiredMinimumTxInterval: peer.BFDTransmitInterval,
+			RequiredMinimumReceive:   peer.BFDReceiveInterval,
+			DetectionMultiplier:      peer.BFDDetectMultiplier,
+			Port:                     3784, // TODO: Should this be configurable??
+		}
+	}
 
-	// if b.c.SourceIF != "" {
-	// 	p.Transport.BindInterface = b.c.SourceIF
-	// }
+	if peer.Interface != "" {
+		neighborAddress, err := getIPv6LinkLocalNeighborAddress(ctx, peer.Interface)
+		if err != nil {
+			return fmt.Errorf("failed to get link-local address of interface %s: %w", peer.Interface, err)
+		}
 
-	return b.s.AddPeer(context.Background(), &api.AddPeerRequest{
-		Peer: p,
-	})
+		p.State = &api.PeerState{
+			NeighborAddress: neighborAddress,
+		}
+	}
+
+	mpBGP := b.c.MpbgpNexthop
+
+	if peer.MpbgpNexthop != "" {
+		mpBGP = peer.MpbgpNexthop
+	}
+
+	if mpBGP != "" {
+		ipv4Address, ipv6Address, err := peer.FindMpbgpAddresses(p, b.c)
+		if err != nil {
+			log.Error("failed to get MP-BGP addresses, will not us MP-BGP for this host", "error", err)
+			b.setPeerSource(p)
+		} else {
+			p.AfiSafis = []*api.AfiSafi{
+				{
+					Config: &api.AfiSafiConfig{
+						Family: &api.Family{
+							Afi:  api.Family_AFI_IP,
+							Safi: api.Family_SAFI_UNICAST,
+						},
+						Enabled: true,
+					},
+				},
+				{
+					Config: &api.AfiSafiConfig{
+						Family: &api.Family{
+							Afi:  api.Family_AFI_IP6,
+							Safi: api.Family_SAFI_UNICAST,
+						},
+						Enabled: true,
+					},
+				},
+			}
+
+			peer.SetMpbgpOptions(b.c)
+
+			mask := strconv.Itoa(vip.DefaultMaskIPv6)
+			address := ipv4Address
+			family := api.Family_AFI_IP
+			if utils.IsIPv4(p.Conf.NeighborAddress) {
+				mask = strconv.Itoa(vip.DefaultMaskIPv4)
+				address = ipv6Address
+				family = api.Family_AFI_IP6
+			}
+
+			err = b.s.AddDefinedSet(ctx, &api.AddDefinedSetRequest{
+				DefinedSet: &api.DefinedSet{
+					DefinedType: api.DefinedType_DEFINED_TYPE_NEIGHBOR,
+					Name:        fmt.Sprintf("peer-%s", p.Conf.NeighborAddress),
+					List:        []string{fmt.Sprintf("%s/%s", p.Conf.NeighborAddress, mask)},
+				},
+			})
+			if err != nil {
+				return fmt.Errorf("failed to add defined set: %v", err)
+			}
+
+			if address != "" {
+				if err := insertPolicy(ctx, b.s, address, p, family); err != nil {
+					return fmt.Errorf("failed to add policy: %w", err)
+				}
+			}
+		}
+	} else {
+		b.setPeerSource(p)
+	}
+
+	if err := b.s.AddPeer(ctx, &api.AddPeerRequest{Peer: p}); err != nil {
+		return fmt.Errorf("failed to add peer: %v", err)
+	}
+	log.Info("[BGP]", "peer", p.Conf.NeighborAddress, "AS", p.Conf.PeerAsn, "BFD", p.Bfd)
+	return nil
 }
 
-func (b *Server) getPath(ip net.IP) *api.Path {
-	var pfxLen uint32 = 32
-	if ip.To4() == nil {
-		if !b.c.IPv6 {
+func (b *Server) setPeerSource(p *api.Peer) {
+	if b.c.SourceIP != "" {
+		p.Transport.LocalAddress = b.c.SourceIP
+	}
+
+	if b.c.SourceIF != "" {
+		p.Transport.BindInterface = b.c.SourceIF
+	}
+}
+
+func (b *Server) getPath(ip net.IP) *apiutil.Path {
+	isV6 := ip.To4() == nil
+
+	if !isV6 {
+		prefix, err := bgp.NewIPAddrPrefix(netip.MustParsePrefix(
+			fmt.Sprintf("%s/%d", ip.String(), vip.DefaultMaskIPv4),
+		))
+		if err != nil {
 			return nil
 		}
 
-		pfxLen = 128
+		nh, err := bgp.NewPathAttributeNextHop(netip.MustParseAddr("0.0.0.0"))
+		if err != nil {
+			return nil
+		}
+
+		return &apiutil.Path{
+			Family: bgp.RF_IPv4_UC,
+			Nlri:   prefix,
+			Attrs: []bgp.PathAttributeInterface{
+				bgp.NewPathAttributeOrigin(0),
+				nh,
+			},
+		}
 	}
 
-	nlri, _ := ptypes.MarshalAny(&api.IPAddressPrefix{
-		Prefix:    ip.String(),
-		PrefixLen: pfxLen,
-	})
-
-	a1, _ := ptypes.MarshalAny(&api.OriginAttribute{
-		Origin: 0,
-	})
-
-	var nh string
-	if b.c.NextHop != "" {
-		nh = b.c.NextHop
-	} else if b.c.SourceIP != "" {
-		nh = b.c.SourceIP
-	} else {
-		nh = b.c.RouterID
+	prefix, err := bgp.NewIPAddrPrefix(netip.MustParsePrefix(
+		fmt.Sprintf("%s/%d", ip.String(), vip.DefaultMaskIPv6),
+	))
+	if err != nil {
+		return nil
 	}
 
-	a2, _ := ptypes.MarshalAny(&api.NextHopAttribute{
-		NextHop: nh,
-	})
+	mpReach, err := bgp.NewPathAttributeMpReachNLRI(
+		bgp.RF_IPv6_UC,
+		[]bgp.PathNLRI{{NLRI: prefix}},
+		netip.MustParseAddr("::"),
+	)
+	if err != nil {
+		return nil
+	}
 
-	return &api.Path{
-		Family: &api.Family{
-			Afi:  api.Family_AFI_IP,
-			Safi: api.Family_SAFI_UNICAST,
+	return &apiutil.Path{
+		Family: bgp.RF_IPv6_UC,
+		Nlri:   prefix,
+		Attrs: []bgp.PathAttributeInterface{
+			bgp.NewPathAttributeOrigin(0),
+			mpReach,
 		},
-		Nlri:   nlri,
-		Pattrs: []*any.Any{a1, a2},
+	}
+}
+func insertPolicy(ctx context.Context, s *server.BgpServer, address string, p *api.Peer, family api.Family_Afi) error {
+	familyType := "v4"
+	if family == api.Family_AFI_IP6 {
+		familyType = "v6"
+	}
+
+	setName := fmt.Sprintf("peer-%s", p.Conf.NeighborAddress)
+	policyName := fmt.Sprintf("%s-%s", setName, familyType)
+
+	policy := &api.Policy{
+		Name: policyName,
+		Statements: []*api.Statement{
+			{
+				Conditions: &api.Conditions{
+					AfiSafiIn: []*api.Family{
+						{
+							Afi:  family,
+							Safi: api.Family_SAFI_UNICAST,
+						},
+					},
+					NeighborSet: &api.MatchSet{
+						Type: api.MatchSet_TYPE_ANY,
+						Name: setName,
+					},
+				},
+				Actions: &api.Actions{
+					RouteAction: api.RouteAction_ROUTE_ACTION_ACCEPT,
+					Nexthop: &api.NexthopAction{
+						Address: address,
+					},
+				},
+			},
+			{
+				Conditions: &api.Conditions{
+					NeighborSet: &api.MatchSet{
+						Type: api.MatchSet_TYPE_ANY,
+						Name: setName,
+					},
+				},
+				Actions: &api.Actions{
+					RouteAction: api.RouteAction_ROUTE_ACTION_ACCEPT,
+				},
+			},
+		},
+	}
+
+	err := s.AddPolicy(ctx, &api.AddPolicyRequest{
+		Policy: policy,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to add policy: %w", err)
+	}
+
+	err = s.AddPolicyAssignment(ctx, &api.AddPolicyAssignmentRequest{
+		Assignment: &api.PolicyAssignment{
+			Name:      "global",
+			Direction: api.PolicyDirection_POLICY_DIRECTION_EXPORT,
+			Policies: []*api.Policy{
+				{
+					Name: policy.Name,
+				},
+			},
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("failed to add policy assignment: %v", err)
+	}
+
+	return nil
+}
+
+func getIPv6LinkLocalNeighborAddress(ctx context.Context, peerInterface string) (string, error) {
+	neighCtx, neighCancel := context.WithTimeout(ctx, time.Minute)
+	defer neighCancel()
+
+	bo := backoff.Backoff{
+		Factor: 2,
+		Jitter: true,
+		Min:    1 * time.Second,
+		Max:    5 * time.Second,
+	}
+
+	maxAttempts := 20.0
+
+	var err error
+	for {
+		select {
+		case <-neighCtx.Done():
+			if err != nil {
+				return "", fmt.Errorf("failed to get link-local address of interface %s: %w", peerInterface, err)
+			}
+			return "", fmt.Errorf("failed to get link-local address of interface %s: %w", peerInterface, neighCtx.Err())
+		default:
+			dur := bo.Duration()
+			var neighborAddress string
+			neighborAddress, err = oc.GetIPv6LinkLocalNeighborAddress(peerInterface)
+			if err != nil && bo.Attempt() >= maxAttempts {
+				return "", fmt.Errorf("failed to get link-local address of interface %s: %w", peerInterface, err)
+			}
+			if neighborAddress != "" {
+				return neighborAddress, nil
+			}
+			t := time.NewTimer(dur)
+			select {
+			case <-neighCtx.Done():
+				t.Stop()
+			case <-t.C:
+			}
+		}
+
 	}
 }

pkg/bgp/peers_config_test.go

@@ -0,0 +1,146 @@
+package bgp
+
+import (
+	"context"
+	"sync"
+	"testing"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	api "github.com/osrg/gobgp/v4/api"
+	gobgp "github.com/osrg/gobgp/v4/pkg/server"
+)
+
+func TestAddPeerConfiguresTransportOptions(t *testing.T) {
+	tests := []struct {
+		name          string
+		newServer     func(*testing.T) *Server
+		peer          kubevip.BGPPeer
+		wantPort      uint32
+		wantLocalAddr string
+		wantInterface string
+	}{
+		{
+			name: "configured remote port",
+			newServer: func(t *testing.T) *Server {
+				return newStartedTestBGPServer(t, kubevip.BGPConfig{
+					AS:       65000,
+					RouterID: "192.0.2.1",
+					Peers:    []kubevip.BGPPeer{{Address: "192.0.2.10", AS: 65001}},
+				})
+			},
+			peer:     kubevip.BGPPeer{Address: "192.0.2.10", AS: 65001, Port: 180},
+			wantPort: 180,
+		},
+		{
+			name: "configured source interface after MP-BGP fallback",
+			newServer: func(t *testing.T) *Server {
+				return newPeerTestServer(t, kubevip.BGPConfig{
+					AS:           65000,
+					RouterID:     "192.0.2.1",
+					SourceIF:     "lo",
+					MpbgpNexthop: "fixed",
+					Peers:        []kubevip.BGPPeer{{Address: "192.0.2.20", AS: 65001}},
+					MpbgpIPv4:    "",
+					MpbgpIPv6:    "",
+				})
+			},
+			peer:          kubevip.BGPPeer{Address: "192.0.2.20", AS: 65001},
+			wantInterface: "lo",
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			server := tt.newServer(t)
+			if err := server.AddPeer(context.Background(), tt.peer); err != nil {
+				t.Fatalf("AddPeer() error = %v", err)
+			}
+
+			peer := listTestPeer(t, server, tt.peer.Address)
+			if peer.GetTransport() == nil {
+				t.Fatal("configured peer has no transport")
+			}
+			if tt.wantPort != 0 && peer.GetTransport().GetRemotePort() != tt.wantPort {
+				t.Fatalf("remote port = %d, want %d", peer.GetTransport().GetRemotePort(), tt.wantPort)
+			}
+			if tt.wantLocalAddr != "" && peer.GetTransport().GetLocalAddress() != tt.wantLocalAddr {
+				t.Fatalf("local address = %q, want %q", peer.GetTransport().GetLocalAddress(), tt.wantLocalAddr)
+			}
+			if tt.wantInterface != "" && peer.GetTransport().GetBindInterface() != tt.wantInterface {
+				t.Fatalf("bind interface = %q, want %q", peer.GetTransport().GetBindInterface(), tt.wantInterface)
+			}
+		})
+	}
+}
+
+func newStartedTestBGPServer(t *testing.T, config kubevip.BGPConfig) *Server {
+	t.Helper()
+
+	server, err := NewBGPServer(config, log.LevelError)
+	if err != nil {
+		t.Fatalf("NewBGPServer() error = %v", err)
+	}
+
+	go server.s.Serve()
+	if err := server.s.StartBgp(context.Background(), &api.StartBgpRequest{
+		Global: &api.Global{
+			Asn:        config.AS,
+			RouterId:   config.RouterID,
+			ListenPort: -1,
+		},
+	}); err != nil {
+		server.s.Stop()
+		t.Fatalf("StartBgp() error = %v", err)
+	}
+	t.Cleanup(server.s.Stop)
+
+	return server
+}
+
+func listTestPeer(t *testing.T, server *Server, address string) *api.Peer {
+	t.Helper()
+
+	var got *api.Peer
+	if err := server.s.ListPeer(context.Background(), &api.ListPeerRequest{Address: address}, func(peer *api.Peer) {
+		got = peer
+	}); err != nil {
+		t.Fatalf("ListPeer() error = %v", err)
+	}
+	if got == nil {
+		t.Fatalf("ListPeer() returned no peer for %s", address)
+	}
+	return got
+}
+
+func newPeerTestServer(t *testing.T, cfg kubevip.BGPConfig) *Server {
+	t.Helper()
+	raw := startEmbeddedRawBGP(t)
+	return &Server{s: raw, c: &cfg, tracker: make(map[string]map[string]bool)}
+}
+
+func startEmbeddedRawBGP(t *testing.T) *gobgp.BgpServer {
+	t.Helper()
+	raw := gobgp.NewBgpServer()
+	go raw.Serve()
+	if err := raw.StartBgp(context.Background(), &api.StartBgpRequest{
+		Global: &api.Global{
+			Asn:        65000,
+			RouterId:   "192.0.2.1",
+			ListenPort: -1,
+		},
+	}); err != nil {
+		t.Fatalf("starting embedded BGP server: %v", err)
+	}
+	var stopOnce sync.Once
+	t.Cleanup(func() {
+		stopOnce.Do(func() {
+			if err := raw.StopBgp(context.Background(), &api.StopBgpRequest{}); err != nil {
+				t.Logf("stopping embedded BGP server: %v", err)
+			}
+		})
+	})
+	return raw
+}

pkg/bgp/server.go

@@ -3,49 +3,94 @@ package bgp
 import (
 	"context"
 	"fmt"
-	"log"
+	"sync"
 	"time"
 
-	api "github.com/osrg/gobgp/api"
-	gobgp "github.com/osrg/gobgp/pkg/server"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	api "github.com/osrg/gobgp/v4/api"
+	"github.com/osrg/gobgp/v4/pkg/apiutil"
+	bgp "github.com/osrg/gobgp/v4/pkg/packet/bgp"
+	gobgp "github.com/osrg/gobgp/v4/pkg/server"
 )
 
+type BGPManager interface {
+	AddHost(ctx context.Context, addr string, object string) error
+	DelHost(ctx context.Context, addr string, object string) error
+}
+
+// Server manages a server object
+type Server struct {
+	s       *gobgp.BgpServer
+	c       *kubevip.BGPConfig
+	mtx     sync.Mutex
+	tracker map[string]map[string]bool
+}
+
 // NewBGPServer takes a configuration and returns a running BGP server instance
-func NewBGPServer(c *Config) (b *Server, err error) {
+func NewBGPServer(c kubevip.BGPConfig, logLevel log.Level) (b *Server, err error) {
 	if c.AS == 0 {
-		return nil, fmt.Errorf("You need to provide AS")
+		return nil, fmt.Errorf("you need to provide AS")
 	}
 
-	// if c.SourceIP != "" && c.SourceIF != "" {
-	// 	return nil, fmt.Errorf("SourceIP and SourceIF are mutually exclusive")
-	// }
+	if c.SourceIP != "" && c.SourceIF != "" {
+		return nil, fmt.Errorf("sourceIP and SourceIF are mutually exclusive")
+	}
 
 	if len(c.Peers) == 0 {
-		return nil, fmt.Errorf("You need to provide at least one peer")
+		return nil, fmt.Errorf("you need to provide at least one peer")
 	}
+	bgpLogger := log.Default()
+	lvl := &log.LevelVar{}
+	lvl.Set(logLevel)
 
 	b = &Server{
-		s: gobgp.NewBgpServer(),
-		c: c,
+		s:       gobgp.NewBgpServer(gobgp.LoggerOption(bgpLogger, lvl)),
+		c:       &c,
+		tracker: make(map[string]map[string]bool),
 	}
+	return
+}
+
+// Start starts the BGP server
+func (b *Server) Start(ctx context.Context, peerStateChangeCallback func(*apiutil.WatchEventMessage_PeerEvent)) (err error) {
 	go b.s.Serve()
 
-	if err = b.s.StartBgp(context.Background(), &api.StartBgpRequest{
+	if err = b.s.StartBgp(ctx, &api.StartBgpRequest{
 		Global: &api.Global{
-			As:         c.AS,
-			RouterId:   c.RouterID,
+			Asn:        b.c.AS,
+			RouterId:   b.c.RouterID,
 			ListenPort: -1,
 		},
 	}); err != nil {
 		return
 	}
 
-	if err = b.s.MonitorPeer(context.Background(), &api.MonitorPeerRequest{}, func(p *api.Peer) { log.Println(p) }); err != nil {
+	if err = b.s.WatchEvent(ctx, gobgp.WatchEventMessageCallbacks{
+		OnPeerUpdate: func(p *apiutil.WatchEventMessage_PeerEvent, _ time.Time) {
+			log.Info("[BGP]", "peer", fmt.Sprintf("%+v", p))
+			if peerStateChangeCallback != nil {
+				peerStateChangeCallback(p)
+			}
+		},
+	}, gobgp.WatchPeer()); err != nil {
 		return
 	}
 
-	for _, p := range c.Peers {
-		if err = b.AddPeer(p); err != nil {
+	for _, p := range b.c.Peers {
+		if err = b.AddPeer(ctx, p); err != nil {
+			return
+		}
+	}
+
+	if b.c.Zebra.Enabled {
+		if err = b.s.EnableZebra(ctx, &api.EnableZebraRequest{
+			Url:          b.c.Zebra.URL,
+			Version:      b.c.Zebra.Version,
+			SoftwareName: b.c.Zebra.SoftwareName,
+		}); err != nil {
+			log.Error(err.Error())
 			return
 		}
 	}
@@ -55,7 +100,51 @@ func NewBGPServer(c *Config) (b *Server, err error) {
 
 // Close will stop a running BGP Server
 func (b *Server) Close() error {
-	ctx, cf := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cf()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
 	return b.s.StopBgp(ctx, &api.StopBgpRequest{})
 }
+
+// ListAdvertisedRoutes retrieves all active routes inside GoBGP's local RIB.
+// It queries the GLOBAL table type to find routes that kube-vip has requested GoBGP to advertise.
+func (b *Server) ListAdvertisedRoutes(ctx context.Context, isIPv6 bool) ([]*api.Destination, error) {
+	afi := bgp.AFI_IP
+
+	if isIPv6 {
+		afi = bgp.AFI_IP6
+	}
+
+	family := bgp.NewFamily(uint16(afi), bgp.SAFI_UNICAST)
+
+	var destinations []*api.Destination
+
+	req := apiutil.ListPathRequest{
+		TableType: api.TableType_TABLE_TYPE_GLOBAL,
+		Family:    family,
+	}
+
+	// GoBGP's embedded server API uses a callback function to stream results
+	// locally without requiring a gRPC client stream setup.
+	err := b.s.ListPath(req, func(prefix bgp.NLRI, paths []*apiutil.Path) {
+		var newPaths []*api.Path
+		for _, p := range paths {
+			np, err := apiutil.NewPath(p.Family, p.Nlri, p.Withdrawal, p.Attrs, time.Unix(p.Age, 0))
+			if err != nil {
+				log.Error("failed to create BGP path details", "err", err)
+				continue
+			}
+			newPaths = append(newPaths, np)
+		}
+		d := &api.Destination{
+			Prefix: prefix.String(),
+			Paths:  newPaths,
+		}
+		destinations = append(destinations, d)
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to extract local RIB: %w", err)
+	}
+
+	return destinations, nil
+}

pkg/bgp/types.go

@@ -1,27 +0,0 @@
-package bgp
-
-import gobgp "github.com/osrg/gobgp/pkg/server"
-
-// Peer defines a BGP Peer
-type Peer struct {
-	Address string
-	AS      uint32
-}
-
-// Config defines the BGP server configuration
-type Config struct {
-	AS       uint32
-	RouterID string
-	NextHop  string
-	SourceIP string
-	SourceIF string
-
-	Peers []Peer
-	IPv6  bool
-}
-
-// Server manages a server object
-type Server struct {
-	s *gobgp.BgpServer
-	c *Config
-}

pkg/cluster/cluster.go

@@ -1,53 +1,288 @@
 package cluster
 
 import (
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	"github.com/plunder-app/kube-vip/pkg/vip"
-)
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"net/http"
+	"os"
+	"sync"
+	"time"
 
-const leaderLogcount = 5
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/networkinterface"
+	"github.com/kube-vip/kube-vip/pkg/node"
+	"github.com/kube-vip/kube-vip/pkg/route"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
 
 // Cluster - The Cluster object manages the state of the cluster for a particular node
 type Cluster struct {
-	stateMachine FSM
-	stop         chan bool
-	completed    chan bool
-	Network      vip.Network
+	stop                  chan struct{}
+	stopMu                sync.Mutex
+	service               *servicesWorker
+	Network               []vip.Network
+	arpMgr                *arp.Manager
+	routeMgr              *route.Manager
+	nodeLabelMgr          node.Labeler
+	labelAdded            bool
+	healthCheckHTTPClient *http.Client
+}
+
+type servicesWorker struct {
+	stop         chan struct{}
+	done         chan struct{}
+	stopping     bool
+	preserveVIPs map[string]struct{}
 }
 
 // InitCluster - Will attempt to initialise all of the required settings for the cluster
-func InitCluster(c *kubevip.Config, disableVIP bool) (*Cluster, error) {
-
-	// TODO - Check for root (needed to netlink)
-	var network vip.Network
+func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.Manager, arpMgr *arp.Manager,
+	routeMgr *route.Manager, nodeLabelMgr node.Labeler) (*Cluster, error) {
+	var networks []vip.Network
+	var healthCheckHTTPClient *http.Client
 	var err error
 
 	if !disableVIP {
 		// Start the Virtual IP Networking configuration
-		network, err = startNetworking(c)
+		networks, err = startNetworking(c, intfMgr)
 		if err != nil {
 			return nil, err
 		}
 	}
+
+	if c.ControlPlaneHealthCheck.Address != "" {
+		healthCheckHTTPClient, err = newHealthCheckHTTPClient(c)
+		if err != nil {
+			return nil, fmt.Errorf("initializing BGP health check client: %w", err)
+		}
+	}
+
 	// Initialise the Cluster structure
 	newCluster := &Cluster{
-		Network: network,
+		Network:               networks,
+		arpMgr:                arpMgr,
+		stop:                  make(chan struct{}),
+		routeMgr:              routeMgr,
+		nodeLabelMgr:          nodeLabelMgr,
+		healthCheckHTTPClient: healthCheckHTTPClient,
 	}
 
+	log.Debug("service security", "enabled", c.EnableServiceSecurity)
+
 	return newCluster, nil
 }
 
-
-func startNetworking(c *kubevip.Config) (vip.Network, error) {
+func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vip.Network, error) {
 	address := c.VIP
 
 	if c.Address != "" {
 		address = c.Address
 	}
 
-	network, err := vip.NewConfig(address, c.Interface)
-	if err != nil {
-		return nil, err
+	addresses := vip.Split(address)
+
+	networks := []vip.Network{}
+	for _, addr := range addresses {
+		network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.DHCPMode,
+			c.RequireDualStack, c.IsDualStack, c.RoutingTableID, c.RoutingTableType, c.RoutingProtocol, c.DNSMode,
+			c.LoadBalancerForwardingMethod, c.IptablesBackend, c.EnableLoadBalancer, c.LoadBalancerPort,
+			c.EnableServiceSecurity, intfMgr, c.EgressWithNftables, c.SkipDAD)
+		if err != nil {
+			return nil, err
+		}
+		networks = append(networks, network...)
+	}
+
+	return networks, nil
+}
+
+// Stop - Will stop the Cluster and release VIP if needed
+func (cluster *Cluster) Stop() {
+	cluster.stopMu.Lock()
+	defer cluster.stopMu.Unlock()
+	if cluster.service != nil {
+		workers := cluster.service
+		if workers.stopping {
+			return
+		}
+		workers.stopping = true
+		cluster.stop = make(chan struct{})
+		close(workers.stop)
+		return
+	}
+	stop := cluster.stop
+	cluster.stop = make(chan struct{})
+	close(stop)
+}
+
+// StopAndWait signals the current Service worker generation and waits until it
+// has finished its datapath cleanup.
+func (cluster *Cluster) StopAndWait() {
+	cluster.stopAndWait(nil)
+}
+
+// StopAndWaitPreserving stops the current Service worker generation while
+// preserving the supplied VIPs for another Service that shares the same lease.
+func (cluster *Cluster) StopAndWaitPreserving(addresses ...string) {
+	preserve := make(map[string]struct{}, len(addresses))
+	for _, address := range addresses {
+		preserve[address] = struct{}{}
+	}
+	cluster.stopAndWait(preserve)
+}
+
+func (cluster *Cluster) stopAndWait(preserveVIPs map[string]struct{}) {
+	workers, signal := cluster.prepareServiceStop(preserveVIPs)
+	if workers == nil {
+		return
+	}
+	if signal {
+		close(workers.stop)
+	}
+	<-workers.done
+}
+
+func (cluster *Cluster) prepareServiceStop(preserveVIPs map[string]struct{}) (*servicesWorker, bool) {
+	cluster.stopMu.Lock()
+	defer cluster.stopMu.Unlock()
+	workers := cluster.service
+	if workers == nil {
+		return nil, false
+	}
+	if workers.stopping {
+		workers.preserveVIPs = mergeVIPs(workers.preserveVIPs, preserveVIPs)
+		return workers, false
+	}
+	workers.stopping = true
+	workers.preserveVIPs = preserveVIPs
+	cluster.stop = make(chan struct{})
+	return workers, true
+}
+
+func (cluster *Cluster) startServicesWorker() (<-chan struct{}, chan struct{}, error) {
+	cluster.stopMu.Lock()
+	defer cluster.stopMu.Unlock()
+	if cluster.service != nil {
+		return nil, nil, fmt.Errorf("load balancer workers already running")
+	}
+	workers := &servicesWorker{stop: cluster.stop, done: make(chan struct{})}
+	cluster.service = workers
+	return workers.stop, workers.done, nil
+}
+
+func (cluster *Cluster) preserveServiceVIP(done chan struct{}, address string) bool {
+	cluster.stopMu.Lock()
+	defer cluster.stopMu.Unlock()
+	if cluster.service == nil || cluster.service.done != done {
+		return false
+	}
+	_, preserve := cluster.service.preserveVIPs[address]
+	return preserve
+}
+
+func mergeVIPs(existing, addresses map[string]struct{}) map[string]struct{} {
+	if len(addresses) == 0 {
+		return existing
+	}
+	if existing == nil {
+		existing = make(map[string]struct{}, len(addresses))
+	}
+	for address := range addresses {
+		existing[address] = struct{}{}
+	}
+	return existing
+}
+
+func (cluster *Cluster) finishServicesWorker(done chan struct{}) {
+	cluster.stopMu.Lock()
+	defer cluster.stopMu.Unlock()
+	if cluster.service == nil || cluster.service.done != done {
+		return
+	}
+	cluster.service = nil
+	close(done)
+}
+
+func (cluster *Cluster) StopChannel() <-chan struct{} {
+	cluster.stopMu.Lock()
+	defer cluster.stopMu.Unlock()
+	return cluster.stop
+}
+
+func newHealthCheckHTTPClient(c *kubevip.Config) (*http.Client, error) {
+	defaultTransport, ok := http.DefaultTransport.(*http.Transport)
+	if !ok {
+		return nil, fmt.Errorf("unexpected default HTTP transport type %T", http.DefaultTransport)
+	}
+
+	transport := defaultTransport.Clone()
+	if c.ControlPlaneHealthCheck.CAPath != "" {
+		caCert, err := os.ReadFile(c.ControlPlaneHealthCheck.CAPath)
+		if err != nil {
+			return nil, fmt.Errorf("reading health check CA cert %q: %w", c.ControlPlaneHealthCheck.CAPath, err)
+		}
+
+		rootCAs, err := x509.SystemCertPool()
+		if err != nil || rootCAs == nil {
+			rootCAs = x509.NewCertPool()
+		}
+		if !rootCAs.AppendCertsFromPEM(caCert) {
+			return nil, fmt.Errorf("health check CA cert %q contains no valid certificates", c.ControlPlaneHealthCheck.CAPath)
+		}
+
+		tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12}
+		if transport.TLSClientConfig != nil {
+			tlsConfig = transport.TLSClientConfig.Clone()
+		}
+		tlsConfig.RootCAs = rootCAs
+		transport.TLSClientConfig = tlsConfig
+	}
+
+	return &http.Client{
+		Timeout:   time.Duration(c.ControlPlaneHealthCheck.TimeoutSeconds) * time.Second,
+		Transport: transport,
+	}, nil
+}
+
+// cleanupVIPs releases the control plane VIPs after leadership was lost.
+// Nothing waits for the control plane layer2Update goroutine to observe the
+// cancelled context, so this caller usually still holds its own ARP claim and
+// has to delete the address itself.
+func (cluster *Cluster) cleanupVIPs(c *kubevip.Config) {
+	for i := range cluster.Network {
+		cluster.cleanupVIP(c, cluster.Network[i], 1)
+	}
+}
+
+// cleanupServiceVIPs releases the service VIPs once the services worker has
+// drained. layer2Update already removed this instance's own claim by then, so
+// any remaining claim belongs to another service sharing the VIP.
+func (cluster *Cluster) cleanupServiceVIPs(c *kubevip.Config, done chan struct{}) {
+	for i := range cluster.Network {
+		if cluster.preserveServiceVIP(done, cluster.Network[i].IP()) {
+			continue
+		}
+		cluster.cleanupVIP(c, cluster.Network[i], 0)
+	}
+}
+
+// cleanupVIP deletes the VIP unless somebody else still advertises it.
+// ownClaims is the number of ARP claims the caller may still hold itself.
+func (cluster *Cluster) cleanupVIP(c *kubevip.Config, network vip.Network, ownClaims int) {
+	if c.EnableARP && cluster.arpMgr.Count(network.ARPName()) > ownClaims {
+		return
+	}
+
+	log.Info("[VIP] Deleting VIP", "ip", network.IP())
+	deleted, err := network.DeleteIP()
+	if err != nil {
+		log.Warn(err.Error())
+	}
+	if deleted {
+		log.Info("deleted address", "IP", network.IP(), "interface", network.Interface())
 	}
-	return network, nil
 }

pkg/cluster/clusterDDNS.go

@@ -0,0 +1,27 @@
+package cluster
+
+import (
+	"context"
+	"sync"
+
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
+
+// StartDDNS should start go routine for dhclient to hold the lease for the IP
+// StartDDNS should wait until IP is allocated from DHCP, set it to cluster.Network
+// so the OnStartedLeading can continue to configure the VIP initially
+// during runtime if IP changes, startDDNS don't have to do reconfigure because
+// dnsUpdater already have the functionality to keep trying resolve the IP
+// and update the VIP configuration if it changes
+func (cluster *Cluster) StartDDNS(ctx context.Context, network vip.Network, backoffAttempts uint, wg *sync.WaitGroup) error {
+	ddnsMgr := vip.NewDDNSManager(network, backoffAttempts)
+	ip, err := ddnsMgr.Start(ctx, wg)
+	if err != nil {
+		return err
+	}
+	if err = network.SetIP(ip); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/cluster/clusterLeader.go

@@ -1,329 +0,0 @@
-package cluster
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/signal"
-	"path/filepath"
-	"syscall"
-	"time"
-
-	"github.com/plunder-app/kube-vip/pkg/bgp"
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	leaderelection "github.com/plunder-app/kube-vip/pkg/leaderElection"
-	"github.com/plunder-app/kube-vip/pkg/loadbalancer"
-	"github.com/plunder-app/kube-vip/pkg/packet"
-
-	"github.com/plunder-app/kube-vip/pkg/vip"
-
-	"github.com/packethost/packngo"
-
-	log "github.com/sirupsen/logrus"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/tools/leaderelection/resourcelock"
-)
-
-const plunderLock = "plunder-lock"
-const namespace = "kube-system"
-
-// Manager degines the manager of the load-balancing services
-type Manager struct {
-	clientSet *kubernetes.Clientset
-}
-
-// NewManager will create a new managing object
-func NewManager(path string, inCluster bool) (*Manager, error) {
-	var clientset *kubernetes.Clientset
-	if inCluster {
-		// This will attempt to load the configuration when running within a POD
-		cfg, err := rest.InClusterConfig()
-		if err != nil {
-			return nil, fmt.Errorf("error creating kubernetes client config: %s", err.Error())
-		}
-		clientset, err = kubernetes.NewForConfig(cfg)
-
-		if err != nil {
-			return nil, fmt.Errorf("error creating kubernetes client: %s", err.Error())
-		}
-		// use the current context in kubeconfig
-	} else {
-		if path == "" {
-			path = filepath.Join(os.Getenv("HOME"), ".kube", "config")
-		}
-		config, err := clientcmd.BuildConfigFromFlags("", path)
-		if err != nil {
-			panic(err.Error())
-		}
-
-		// We modify the config so that we can always speak to the correct host
-		id, err := os.Hostname()
-		if err != nil {
-			return nil, err
-		}
-
-		// TODO - we need to make the host/port configurable
-
-		config.Host = fmt.Sprintf("%s:6443", id)
-		clientset, err = kubernetes.NewForConfig(config)
-
-		if err != nil {
-			return nil, fmt.Errorf("error creating kubernetes client: %s", err.Error())
-		}
-	}
-
-	return &Manager{
-		clientSet: clientset,
-	}, nil
-}
-
-// StartLeaderCluster - Begins a running instance of the Raft cluster
-func (cluster *Cluster) StartLeaderCluster(c *kubevip.Config, sm *Manager) error {
-
-	id, err := os.Hostname()
-	if err != nil {
-		return err
-	}
-
-	log.Infof("Beginning cluster membership, namespace [%s], lock name [%s], id [%s]", namespace, plunderLock, id)
-
-	// we use the Lease lock type since edits to Leases are less common
-	// and fewer objects in the cluster watch "all Leases".
-	lock := &resourcelock.LeaseLock{
-		LeaseMeta: metav1.ObjectMeta{
-			Name:      plunderLock,
-			Namespace: namespace,
-		},
-		Client: sm.clientSet.CoordinationV1(),
-		LockConfig: resourcelock.ResourceLockConfig{
-			Identity: id,
-		},
-	}
-
-	// use a Go context so we can tell the leaderelection code when we
-	// want to step down
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	// use a Go context so we can tell the arp loop code when we
-	// want to step down
-	ctxArp, cancelArp := context.WithCancel(context.Background())
-	defer cancelArp()
-
-	// listen for interrupts or the Linux SIGTERM signal and cancel
-	// our context, which the leader election code will observe and
-	// step down
-	signalChan := make(chan os.Signal, 1)
-	// Add Notification for Userland interrupt
-	signal.Notify(signalChan, syscall.SIGINT)
-
-	// Add Notification for SIGTERM (sent from Kubernetes)
-	signal.Notify(signalChan, syscall.SIGTERM)
-
-	// Add Notification for SIGKILL (sent from Kubernetes)
-	signal.Notify(signalChan, syscall.SIGKILL)
-
-	go func() {
-		<-signalChan
-		log.Info("Received termination, signaling shutdown")
-		// Cancel the context, which will in turn cancel the leadership
-		cancel()
-		// Cancel the arp context, which will in turn stop any broadcasts
-	}()
-
-	// (attempt to) Remove the virtual IP, incase it already exists
-	cluster.Network.DeleteIP()
-
-	// Managers for Vip load balancers and none-vip loadbalancers
-	nonVipLB := loadbalancer.LBManager{}
-	VipLB := loadbalancer.LBManager{}
-
-	// BGP server
-	var bgpServer *bgp.Server
-	// Defer a function to check if the bgpServer has been created and if so attempt to close it
-	defer func() {
-		if bgpServer != nil {
-			bgpServer.Close()
-		}
-	}()
-
-	// If Packet is enabled then we can begin our preperation work
-	var packetClient *packngo.Client
-	if c.EnablePacket {
-		packetClient, err = packngo.NewClient()
-		if err != nil {
-			log.Error(err)
-		}
-
-		// We're using Packet with BGP, popuplate the Peer information from the API
-		if c.EnableBGP {
-			log.Infoln("Looking up the BGP configuration from packet")
-			err = packet.BGPLookup(packetClient, c)
-			if err != nil {
-				log.Error(err)
-			}
-		}
-	}
-
-	if c.EnableBGP {
-		// Lets start BGP
-		log.Info("Starting the BGP server to adverise VIP routes to VGP peers")
-		bgpServer, err = bgp.NewBGPServer(&c.BGPConfig)
-		if err != nil {
-			log.Error(err)
-		}
-	}
-
-	if c.EnableLoadBalancer {
-
-		// Iterate through all Configurations
-		if len(c.LoadBalancers) != 0 {
-			for x := range c.LoadBalancers {
-				// If the load balancer doesn't bind to the VIP
-				if c.LoadBalancers[x].BindToVip == false {
-					err = nonVipLB.Add("", &c.LoadBalancers[x])
-					if err != nil {
-						log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-					}
-				}
-			}
-		}
-	}
-	// start the leader election code loop
-	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
-		Lock: lock,
-		// IMPORTANT: you MUST ensure that any code you have that
-		// is protected by the lease must terminate **before**
-		// you call cancel. Otherwise, you could have a background
-		// loop still running and another process could
-		// get elected before your background loop finished, violating
-		// the stated goal of the lease.
-		ReleaseOnCancel: true,
-		LeaseDuration:   time.Duration(c.LeaseDuration) * time.Second,
-		RenewDeadline:   time.Duration(c.RenewDeadline) * time.Second,
-		RetryPeriod:     time.Duration(c.RetryPeriod) * time.Second,
-		Callbacks: leaderelection.LeaderCallbacks{
-			OnStartedLeading: func(ctx context.Context) {
-
-				// we're notified when we start
-				log.Info("This node is starting with leadership of the cluster")
-				err = cluster.Network.AddIP()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				if c.EnablePacket {
-					// We're not using Packet with BGP
-					if !c.EnableBGP {
-						// Attempt to attach the EIP in the standard manner
-						log.Debugf("Attaching the Packet EIP through the API to this host")
-						err = packet.AttachEIP(packetClient, c, id)
-						if err != nil {
-							log.Error(err)
-						}
-					}
-				}
-
-				if c.EnableBGP {
-					// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
-					cidrVip := fmt.Sprintf("%s/%s", cluster.Network.IP(), c.VIPCIDR)
-					log.Debugf("Attempting to advertise the address [%s] over BGP", cidrVip)
-
-					err = bgpServer.AddHost(cidrVip)
-					if err != nil {
-						log.Error(err)
-					}
-				}
-
-				if c.EnableLoadBalancer {
-					// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-					for x := range c.LoadBalancers {
-
-						if c.LoadBalancers[x].BindToVip == true {
-							err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-							if err != nil {
-								log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-
-								// Stop all load balancers associated with the VIP
-								err = VipLB.StopAll()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-
-								err = cluster.Network.DeleteIP()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-							}
-						}
-					}
-				}
-
-				if c.GratuitousARP == true {
-					ctxArp, cancelArp = context.WithCancel(context.Background())
-
-					go func(ctx context.Context) {
-						for {
-							select {
-							case <-ctx.Done(): // if cancel() execute
-								return
-							default:
-								// Gratuitous ARP, will broadcast to new MAC <-> IP
-								err = vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-							}
-							time.Sleep(3 * time.Second)
-						}
-					}(ctxArp)
-				}
-			},
-			OnStoppedLeading: func() {
-				// we can do cleanup here
-				log.Info("This node is becoming a follower within the cluster")
-
-				// Stop the Arp context if it is running
-				cancelArp()
-
-				// Stop the BGP server
-				if bgpServer != nil {
-					err = bgpServer.Close()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-
-				// Stop all load balancers associated with the VIP
-				err = VipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				err = cluster.Network.DeleteIP()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-			},
-			OnNewLeader: func(identity string) {
-				// we're notified when new leader elected
-				log.Infof("Node [%s] is assuming leadership of the cluster", identity)
-
-				if identity == id {
-					// We have the lock
-				}
-			},
-		},
-	})
-
-	//<-signalChan
-	log.Infof("Shutting down Kube-Vip Leader Election cluster")
-
-	// Force a removal of the VIP (ignore the error if we don't have it)
-	cluster.Network.DeleteIP()
-
-	return nil
-}

pkg/cluster/clusterLeaderElection.go

@@ -0,0 +1,169 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/election"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/lease"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+
+	log "log/slog"
+)
+
+// StartCluster - Begins a running instance of the Leader Election cluster
+func (cluster *Cluster) StartCluster(ctx context.Context, c *kubevip.Config,
+	em *election.Manager, bgpServer *bgp.Server, leaseMgr *lease.Manager, killFunc func()) error {
+
+	ns, leaseName := lease.NamespaceName(c.LeaseName, c)
+
+	leaseID := lease.NewID(c.LeaderElectionType, ns, leaseName)
+
+	log.Info("cluster membership", "namespace", leaseID.Namespace(), "lock", leaseID.Name(), "id", c.NodeName)
+
+	objectName := lease.ObjectName(leaseID, "cp")
+	objLease, _ := leaseMgr.Acquire(context.Background(), leaseID, objectName)
+	defer leaseMgr.Delete(leaseID, objectName, objLease)
+
+	wg := sync.WaitGroup{}
+	defer wg.Wait()
+
+	electionCtx, cancelElection := objLease.NewElectionContext(ctx)
+	defer cancelElection()
+
+	stop := cluster.StopChannel()
+	wg.Go(func() {
+		select {
+		case <-stop:
+			cancelElection()
+		case <-electionCtx.Done():
+		}
+	})
+
+	// (attempt to) Remove the virtual IP, in case it already exists
+
+	for i := range cluster.Network {
+		deleted, err := cluster.Network[i].DeleteIP()
+		if err != nil {
+			log.Error("could not delete virtualIP", "err", err)
+		}
+		if deleted {
+			log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+		}
+	}
+
+	for {
+		if !objLease.BeginElection() {
+			log.Debug("this election was already done, shared lease", "lease", leaseName)
+			leaderGeneration, elected := objLease.WaitForLeaderGeneration(electionCtx)
+			if !elected {
+				if electionCtx.Err() != nil {
+					return nil
+				}
+				// The runner that owned this shared lease's election ended it
+				// without ever being elected; take over the campaign ourselves
+				// instead of leaving the lease without an active runner.
+				continue
+			}
+
+			leaderCtx, cancelLeader := context.WithCancel(electionCtx)
+			leaderWG := sync.WaitGroup{}
+			leaderWG.Go(func() {
+				cluster.OnStartedLeading(leaderCtx, c, em, bgpServer, killFunc, true)
+			})
+
+			log.Debug("cluster waiting for shared election to finish", "lease", leaseName)
+			objLease.WaitForElectionEndAfter(electionCtx, leaderGeneration)
+			cancelLeader()
+			leaderWG.Wait()
+
+			cluster.OnStoppedLeading(c, bgpServer)
+
+			return nil
+		}
+		break
+	}
+	defer objLease.ElectionStopped()
+
+	run := &election.RunConfig{
+		Config:           c,
+		LeaseID:          leaseID,
+		LeaseAnnotations: c.LeaseAnnotations,
+		VIPs:             controlPlaneElectionVIPs(c),
+		Mgr:              em,
+		OnStartedLeading: func(ctx context.Context) {
+			objLease.ElectionStarted()
+			cluster.OnStartedLeading(ctx, c, em, bgpServer, killFunc, false)
+		},
+		OnStoppedLeading: func() {
+			objLease.ElectionStopped()
+			cluster.OnStoppedLeading(c, bgpServer)
+		},
+		OnNewLeader: func(identity string) {
+			cluster.OnNewLeader(identity, c)
+		},
+	}
+
+	if err := election.RunOrDie(electionCtx, run, c); err != nil {
+		cluster.Stop()
+		return fmt.Errorf("leaderelection failed: %w", err)
+	}
+
+	return nil
+}
+
+func controlPlaneElectionVIPs(config *kubevip.Config) []string {
+	configured := config.VIP
+	if config.Address != "" {
+		configured = config.Address
+	}
+	return vip.Split(configured)
+}
+
+func (cluster *Cluster) OnStartedLeading(ctx context.Context, c *kubevip.Config,
+	em *election.Manager, bgpServer *bgp.Server, killFunc func(), _ bool) {
+	labels := generateLabelsFromConfig(c.Address, kubevip.HasIP)
+	if err := cluster.nodeLabelMgr.AddLabel(labels); err != nil {
+		log.Error("error adding label to node", "err", err)
+	}
+	cluster.labelAdded = true
+
+	// As we're leading lets start the vip service
+	err := cluster.StartVipService(ctx, c, em, bgpServer, killFunc)
+	if err != nil {
+		log.Error("starting VIP service on leader", "err", err)
+		killFunc()
+	}
+}
+
+func (cluster *Cluster) OnStoppedLeading(c *kubevip.Config, bgpServer *bgp.Server) {
+	// we can do cleanup here
+	log.Info("This node is becoming a follower within the cluster")
+
+	if cluster.labelAdded {
+		labels := generateLabelsFromConfig(c.Address, kubevip.HasIP)
+		if err := cluster.nodeLabelMgr.RemoveLabel(labels); err != nil {
+			log.Error("error removing label from node", "err", err)
+		}
+		cluster.labelAdded = false
+	}
+
+	cluster.cleanupVIPs(c)
+
+	log.Error("lost leadership, restarting kube-vip")
+}
+
+func (cluster *Cluster) OnNewLeader(identity string, c *kubevip.Config) {
+	// we're notified when new leader elected
+	log.Info("New leader", "leader", identity)
+}
+
+func generateLabelsFromConfig(addr, labelKey string) map[string]string {
+	return map[string]string{
+		labelKey: utils.SanitizeIPForLabel(addr),
+	}
+}

pkg/cluster/clusterRaft.go

@@ -1,296 +0,0 @@
-package cluster
-
-import (
-	"fmt"
-	"net"
-	"time"
-
-	"github.com/hashicorp/raft"
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	"github.com/plunder-app/kube-vip/pkg/loadbalancer"
-	"github.com/plunder-app/kube-vip/pkg/vip"
-	log "github.com/sirupsen/logrus"
-)
-
-// StartRaftCluster - Begins a running instance of the Raft cluster
-func (cluster *Cluster) StartRaftCluster(c *kubevip.Config) error {
-
-	// Create local configuration address
-	localAddress := fmt.Sprintf("%s:%d", c.LocalPeer.Address, c.LocalPeer.Port)
-
-	// Begin the Raft configuration
-	config := raft.DefaultConfig()
-	config.LocalID = raft.ServerID(c.LocalPeer.ID)
-	logger := log.StandardLogger().Writer()
-	config.LogOutput = logger
-
-	// Initialize communication
-	address, err := net.ResolveTCPAddr("tcp", localAddress)
-	if err != nil {
-		return err
-	}
-
-	// Create transport
-	transport, err := raft.NewTCPTransport(localAddress, address, 3, 10*time.Second, logger)
-	if err != nil {
-		return err
-	}
-
-	// Create Raft structures
-	snapshots := raft.NewInmemSnapshotStore()
-	logStore := raft.NewInmemStore()
-	stableStore := raft.NewInmemStore()
-
-	// Cluster configuration
-	configuration := raft.Configuration{}
-
-	// Add Local Peer
-	configuration.Servers = append(configuration.Servers, raft.Server{
-		ID:      raft.ServerID(c.LocalPeer.ID),
-		Address: raft.ServerAddress(fmt.Sprintf("%s:%d", c.LocalPeer.Address, c.LocalPeer.Port))})
-
-	// If we want to start a node as leader then we will not add any remote peers, this will leave this as a cluster of one
-	// The remotePeers will add themselves to the cluster as they're added
-	if c.StartAsLeader != true {
-		for x := range c.RemotePeers {
-			// Make sure that we don't add in this server twice
-			if c.LocalPeer.Address != c.RemotePeers[x].Address {
-
-				// Build the address from the peer configuration
-				peerAddress := fmt.Sprintf("%s:%d", c.RemotePeers[x].Address, c.RemotePeers[x].Port)
-
-				// Set this peer into the raft configuration
-				configuration.Servers = append(configuration.Servers, raft.Server{
-					ID:      raft.ServerID(c.RemotePeers[x].ID),
-					Address: raft.ServerAddress(peerAddress)})
-			}
-		}
-		log.Info("This node will attempt to start as Follower")
-	} else {
-		log.Info("This node will attempt to start as Leader")
-	}
-
-	// Bootstrap cluster
-	if err := raft.BootstrapCluster(config, logStore, stableStore, snapshots, transport, configuration); err != nil {
-		return err
-	}
-
-	// Create RAFT instance
-	raftServer, err := raft.NewRaft(config, cluster.stateMachine, logStore, stableStore, snapshots, transport)
-	if err != nil {
-		return err
-	}
-
-	cluster.stop = make(chan bool, 1)
-	cluster.completed = make(chan bool, 1)
-	ticker := time.NewTicker(time.Second)
-	isLeader := c.StartAsLeader
-
-	// (attempt to) Remove the virtual IP, incase it already exists
-	cluster.Network.DeleteIP()
-
-	// leader log broadcast - this counter is used to stop flooding STDOUT with leader log entries
-	var leaderbroadcast int
-	// Managers for Vip load balancers and none-vip loadbalancers
-	nonVipLB := loadbalancer.LBManager{}
-	VipLB := loadbalancer.LBManager{}
-
-	// Iterate through all Configurations
-	for x := range c.LoadBalancers {
-		// If the load balancer doesn't bind to the VIP
-		if c.LoadBalancers[x].BindToVip == false {
-			err = nonVipLB.Add("", &c.LoadBalancers[x])
-			if err != nil {
-				log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-			}
-		}
-	}
-
-	// On a cold start the node will sleep for 5 seconds to ensure that leader elections are complete
-	log.Infoln("This instance will wait approximately 5 seconds, from cold start to ensure cluster elections are complete")
-	time.Sleep(time.Second * 5)
-
-	go func() {
-		for {
-			if c.AddPeersAsBackends == true {
-				// Get addresses and change backends
-
-				// c.LoadBalancers[0].Backends
-				// for x := range raftServer.GetConfiguration().Configuration().Servers {
-				// 	raftServer.GetConfiguration().Configuration().Servers[x].Address
-				// }
-
-			}
-			// Broadcast the current leader on this node if it's the correct time (every leaderLogcount * time.Second)
-			if leaderbroadcast == leaderLogcount {
-				log.Infof("The Node [%s] is leading", raftServer.Leader())
-				// Reset the timer
-				leaderbroadcast = 0
-
-				// ensure that if this node is the leader, it is set as the leader
-				if localAddress == string(raftServer.Leader()) {
-					// Re-broadcast arp to ensure network stays up to date
-					if c.GratuitousARP == true {
-						// Gratuitous ARP, will broadcast to new MAC <-> IP
-						err = vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-					}
-					if !isLeader {
-						log.Infoln("This node is leading, but isnt the leader (correcting)")
-						isLeader = true
-					}
-				} else {
-					// (attempt to) Remove the virtual IP, incase it already exists to keep nodes clean
-					cluster.Network.DeleteIP()
-					isLeader = false
-				}
-
-			}
-			leaderbroadcast++
-
-			select {
-			case leader := <-raftServer.LeaderCh():
-				log.Infoln("New Election event")
-				if leader {
-					isLeader = true
-
-					log.Info("This node is assuming leadership of the cluster")
-					err = cluster.Network.AddIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-
-					// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-
-					for x := range c.LoadBalancers {
-
-						if c.LoadBalancers[x].BindToVip == true {
-							err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-							if err != nil {
-								log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-								log.Errorf("Dropping Leadership to another node in the cluster")
-								raftServer.LeadershipTransfer()
-
-								// Stop all load balancers associated with the VIP
-								err = VipLB.StopAll()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-
-								err = cluster.Network.DeleteIP()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-							}
-						}
-					}
-
-					if c.GratuitousARP == true {
-						// Gratuitous ARP, will broadcast to new MAC <-> IP
-						err = vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-					}
-				} else {
-					isLeader = false
-
-					log.Info("This node is becoming a follower within the cluster")
-
-					// Stop all load balancers associated with the VIP
-					err = VipLB.StopAll()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-
-					err = cluster.Network.DeleteIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-
-			case <-ticker.C:
-
-				if isLeader {
-
-					result, err := cluster.Network.IsSet()
-					if err != nil {
-						log.WithFields(log.Fields{"error": err, "ip": cluster.Network.IP(), "interface": cluster.Network.Interface()}).Error("Could not check ip")
-					}
-
-					if result == false {
-						log.Error("This node is leader and is adopting the virtual IP")
-
-						err = cluster.Network.AddIP()
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-						// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-
-						for x := range c.LoadBalancers {
-
-							if c.LoadBalancers[x].BindToVip == true {
-								err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-								if err != nil {
-									log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-								}
-							}
-						}
-						if c.GratuitousARP == true {
-							// Gratuitous ARP, will broadcast to new MAC <-> IP
-							err = vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-							if err != nil {
-								log.Warnf("%v", err)
-							}
-						}
-					}
-				}
-
-			case <-cluster.stop:
-				log.Info("[RAFT] Stopping this node")
-				log.Info("[LOADBALANCER] Stopping load balancers")
-
-				// Stop all load balancers associated with the VIP
-				err = VipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				// Stop all load balancers associated with the Host
-				err = nonVipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				if isLeader {
-					log.Info("[VIP] Releasing the Virtual IP")
-					err = cluster.Network.DeleteIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-
-				close(cluster.completed)
-
-				return
-			}
-		}
-	}()
-
-	log.Info("Started")
-
-	return nil
-}
-
-// Stop - Will stop the Cluster and release VIP if needed
-func (cluster *Cluster) Stop() {
-	// Close the stop chanel, which will shut down the VIP (if needed)
-	close(cluster.stop)
-
-	// Wait until the completed channel is closed, signallign all shutdown tasks completed
-	<-cluster.completed
-
-	log.Info("Stopped")
-}

pkg/cluster/cluster_internal_test.go

@@ -0,0 +1,196 @@
+package cluster
+
+import (
+	"context"
+	"slices"
+	"testing"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/lease"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	"github.com/vishvananda/netlink"
+	v1 "k8s.io/api/core/v1"
+)
+
+func TestControlPlaneElectionVIPsPreservesConfigOrder(t *testing.T) {
+	config := &kubevip.Config{Address: "2001:db8::10,192.0.2.10"}
+	want := []string{"2001:db8::10", "192.0.2.10"}
+	if got := controlPlaneElectionVIPs(config); !slices.Equal(got, want) {
+		t.Fatalf("controlPlaneElectionVIPs() = %v, want %v", got, want)
+	}
+}
+
+type recordingLabeler struct {
+	added   chan struct{}
+	removed chan struct{}
+}
+
+func (l *recordingLabeler) AddLabel(map[string]string) error {
+	l.added <- struct{}{}
+	return nil
+}
+
+func (l *recordingLabeler) RemoveLabel(map[string]string) error {
+	l.removed <- struct{}{}
+	return nil
+}
+
+// stubNetwork is a minimal vip.Network implementation for exercising
+// cleanupVIP without a real interface.
+type stubNetwork struct {
+	ip            string
+	deleteIPCalls int
+}
+
+func (s *stubNetwork) AddIP(bool, bool, ...int) (bool, error) { return false, nil }
+func (s *stubNetwork) AddRoute(bool) (bool, error)            { return false, nil }
+func (s *stubNetwork) ReplaceRoute() error                    { return nil }
+func (s *stubNetwork) DeleteIP() (bool, error)                { s.deleteIPCalls++; return true, nil }
+func (s *stubNetwork) DeleteRoute() error                     { return nil }
+func (s *stubNetwork) UpdateRoutes() (bool, error)            { return false, nil }
+func (s *stubNetwork) IsSet() (*netlink.Addr, error)          { return nil, nil }
+func (s *stubNetwork) IP() string                             { return s.ip }
+func (s *stubNetwork) CIDR() string                           { return s.ip + "/32" }
+func (s *stubNetwork) IPisLinkLocal() bool                    { return false }
+func (s *stubNetwork) PrepareRoute() *netlink.Route           { return nil }
+func (s *stubNetwork) RouteHash() string                      { return "" }
+func (s *stubNetwork) SetIP(string) error                     { return nil }
+func (s *stubNetwork) SetServicePorts(*v1.Service)            {}
+func (s *stubNetwork) Interface() string                      { return "eth0" }
+func (s *stubNetwork) IsDADFAIL() bool                        { return false }
+func (s *stubNetwork) IsDNS() bool                            { return false }
+func (s *stubNetwork) IsDDNS() bool                           { return false }
+func (s *stubNetwork) DDNSHostName() string                   { return "" }
+func (s *stubNetwork) DNSName() string                        { return "" }
+func (s *stubNetwork) SetMask(string) error                   { return nil }
+func (s *stubNetwork) SetHasEndpoints(bool)                   {}
+func (s *stubNetwork) HasEndpoints() bool                     { return false }
+func (s *stubNetwork) ARPName() string                        { return "shared-vip" }
+func (s *stubNetwork) GetPossibleSubnets() string             { return "" }
+func (s *stubNetwork) DHCPFamily() string                     { return "" }
+func (s *stubNetwork) IPVSMark() uint32                       { return 0 }
+
+// TestCleanupVIPRetainsSharedVIPWithOneSiblingLeft reproduces the off-by-one:
+// layer2Update already removes its own ARP claim before cleanupVIP runs, so a
+// single remaining sibling must still block deletion.
+func TestCleanupVIPRetainsSharedVIPWithOneSiblingLeft(t *testing.T) {
+	arpMgr := arp.NewManager(&kubevip.Config{ArpBroadcastRate: 3000})
+	netA := &stubNetwork{ip: "192.0.2.10"}
+	netB := &stubNetwork{ip: "192.0.2.10"}
+
+	instA := arp.NewInstance(netA, nil)
+	instB := arp.NewInstance(netB, nil)
+	arpMgr.Insert(instA)
+	arpMgr.Insert(instB)
+
+	// Cluster A's layer2Update goroutine ends first and drops its own claim,
+	// leaving only sibling B registered.
+	arpMgr.Remove(instA)
+
+	c := &Cluster{arpMgr: arpMgr}
+	c.cleanupVIP(&kubevip.Config{EnableARP: true}, netA, 0)
+
+	if netA.deleteIPCalls != 0 {
+		t.Fatalf("cleanupVIP deleted the shared VIP while a sibling was still registered")
+	}
+}
+
+// TestCleanupVIPsDeletesControlPlaneVIPHoldingItsOwnARPClaim covers the
+// leadership loss path: OnStoppedLeading runs concurrently with the control
+// plane layer2Update goroutine, so the VIP's only ARP claim is still the
+// caller's own and the address must still be removed before the process exits.
+func TestCleanupVIPsDeletesControlPlaneVIPHoldingItsOwnARPClaim(t *testing.T) {
+	arpMgr := arp.NewManager(&kubevip.Config{ArpBroadcastRate: 3000})
+	network := &stubNetwork{ip: "2001:db8::10"}
+	arpMgr.Insert(arp.NewInstance(network, nil))
+
+	c := &Cluster{arpMgr: arpMgr, Network: []vip.Network{network}}
+	c.cleanupVIPs(&kubevip.Config{EnableARP: true})
+
+	if network.deleteIPCalls != 1 {
+		t.Fatalf("cleanupVIPs made %d DeleteIP calls, want 1", network.deleteIPCalls)
+	}
+}
+
+func TestControlPlaneFollowsSharedServiceElection(t *testing.T) {
+	config := &kubevip.Config{KubernetesLeaderElection: kubevip.KubernetesLeaderElection{LeaseName: "default/shared"}}
+	leaseID := lease.NewID(config.LeaderElectionType, "default", "shared")
+	leaseMgr := lease.NewManager()
+	sharedLease, _ := leaseMgr.Acquire(context.Background(), leaseID, "service")
+	if !sharedLease.BeginElection() {
+		t.Fatal("Service election did not start")
+	}
+	sharedLease.ElectionStarted()
+
+	labels := &recordingLabeler{added: make(chan struct{}, 1), removed: make(chan struct{}, 1)}
+	cluster := &Cluster{stop: make(chan struct{}), nodeLabelMgr: labels}
+	done := make(chan error, 1)
+	go func() {
+		done <- cluster.StartCluster(context.Background(), config, nil, nil, leaseMgr, func() {})
+	}()
+	select {
+	case <-labels.added:
+	case <-time.After(time.Second):
+		t.Fatal("control plane did not activate under the shared Service election")
+	}
+
+	sharedLease.ElectionStopped()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Fatalf("shared control-plane follower returned an error: %v", err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("control plane did not stop after shared Service leadership ended")
+	}
+	select {
+	case <-labels.removed:
+	default:
+		t.Fatal("control-plane label was not removed after shared leadership ended")
+	}
+	if sharedLease.Ctx.Err() != nil || leaseMgr.Get(leaseID) != sharedLease {
+		t.Fatal("control-plane cleanup cancelled the surviving Service lease")
+	}
+	leaseMgr.Delete(leaseID, "service", sharedLease)
+}
+
+func TestStopAndWaitPreservingUpgradesInProgressStop(t *testing.T) {
+	done := make(chan struct{})
+	service := &Cluster{
+		stop: make(chan struct{}),
+		service: &servicesWorker{
+			stop:     make(chan struct{}),
+			done:     done,
+			stopping: true,
+		},
+	}
+
+	returned := make(chan struct{})
+	go func() {
+		service.StopAndWaitPreserving("192.0.2.10")
+		close(returned)
+	}()
+
+	deadline := time.Now().Add(time.Second)
+	for {
+		service.stopMu.Lock()
+		_, preserving := service.service.preserveVIPs["192.0.2.10"]
+		service.stopMu.Unlock()
+		if preserving {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("preserving stop did not update the in-progress worker shutdown")
+		}
+		time.Sleep(time.Millisecond)
+	}
+
+	service.finishServicesWorker(done)
+	select {
+	case <-returned:
+	case <-time.After(time.Second):
+		t.Fatal("preserving stop did not return after worker cleanup completed")
+	}
+}

pkg/cluster/cluster_stop_test.go

@@ -0,0 +1,32 @@
+package cluster
+
+import (
+	"sync"
+	"sync/atomic"
+	"testing"
+)
+
+func TestStopConcurrentDoesNotRaceOrPanic(t *testing.T) {
+	c := &Cluster{stop: make(chan struct{})}
+	start := make(chan struct{})
+	var wg sync.WaitGroup
+	var panics atomic.Int64
+
+	for range 128 {
+		wg.Go(func() {
+			<-start
+			defer func() {
+				if recover() != nil {
+					panics.Add(1)
+				}
+			}()
+			c.Stop()
+		})
+	}
+
+	close(start)
+	wg.Wait()
+	if got := panics.Load(); got != 0 {
+		t.Fatalf("concurrent Stop panicked %d time(s)", got)
+	}
+}

pkg/cluster/cluster_test.go

@@ -0,0 +1,125 @@
+package cluster_test
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"encoding/pem"
+	"math/big"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/cluster"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+)
+
+func TestInitCluster_HealthCheckClientNoCA(t *testing.T) {
+	t.Parallel()
+	cfg := &kubevip.Config{
+		EnableBGP: true,
+		ControlPlaneHealthCheck: kubevip.HealthCheck{
+			Address:        "http://localhost:6443/livez",
+			TimeoutSeconds: 5,
+		},
+	}
+
+	_, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
+func TestInitCluster_HealthCheckClientValidCA(t *testing.T) {
+	t.Parallel()
+	caPEM := generateTestCACert(t)
+	caFile := filepath.Join(t.TempDir(), "ca.crt")
+	if err := os.WriteFile(caFile, caPEM, 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	cfg := &kubevip.Config{
+		EnableBGP: true,
+		ControlPlaneHealthCheck: kubevip.HealthCheck{
+			Address:        "https://localhost:6443/livez",
+			TimeoutSeconds: 3,
+			CAPath:         caFile,
+		},
+	}
+
+	_, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
+func TestInitCluster_HealthCheckClientInvalidCAPath(t *testing.T) {
+	t.Parallel()
+	cfg := &kubevip.Config{
+		EnableBGP: true,
+		ControlPlaneHealthCheck: kubevip.HealthCheck{
+			Address: "https://localhost:6443/livez",
+			CAPath:  "/nonexistent/ca.crt",
+		},
+	}
+
+	_, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
+	if err == nil {
+		t.Fatal("expected error for invalid CA path")
+	}
+	if !strings.Contains(err.Error(), "reading health check CA cert") {
+		t.Errorf("expected error about reading CA cert, got: %v", err)
+	}
+}
+
+func TestInitCluster_HealthCheckClientInvalidCAContent(t *testing.T) {
+	t.Parallel()
+	caFile := filepath.Join(t.TempDir(), "bad-ca.crt")
+	if err := os.WriteFile(caFile, []byte("not a certificate"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	cfg := &kubevip.Config{
+		EnableBGP: true,
+		ControlPlaneHealthCheck: kubevip.HealthCheck{
+			Address: "https://localhost:6443/livez",
+			CAPath:  caFile,
+		},
+	}
+
+	_, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
+	if err == nil {
+		t.Fatal("expected error for invalid CA content")
+	}
+	if !strings.Contains(err.Error(), "contains no valid certificates") {
+		t.Errorf("expected error about invalid certificates, got: %v", err)
+	}
+}
+
+// generateTestCACert creates a self-signed CA certificate in PEM format for testing.
+func generateTestCACert(t *testing.T) []byte {
+	t.Helper()
+
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	template := &x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		NotBefore:             time.Now(),
+		NotAfter:              time.Now().Add(time.Hour),
+		IsCA:                  true,
+		BasicConstraintsValid: true,
+	}
+
+	certDER, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
+}

pkg/cluster/service.go

@@ -0,0 +1,626 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
+	"github.com/kube-vip/kube-vip/pkg/backend"
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/election"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config, em *election.Manager, bgpServer bgp.BGPManager, killFunc func()) error {
+
+	var err error
+
+	var wg sync.WaitGroup
+	defer wg.Wait()
+
+	wg.Go(func() {
+		<-ctx.Done()
+		killFunc()
+	})
+
+	loadbalancers := []*loadbalancer.IPVSLoadBalancer{}
+
+	for i := range cluster.Network {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		network := cluster.Network[i]
+
+		if network.IsDDNS() {
+			if err := cluster.StartDDNS(ctx, cluster.Network[i], c.DHCPBackoffAttempts, &wg); err != nil {
+				log.Error("failed to start DDNS", "err", err)
+			}
+		}
+
+		if err := network.SetMask(c.VIPSubnet); err != nil {
+			killFunc()
+			return fmt.Errorf("failed to set mask for subnet %q: %w", c.VIPSubnet, err)
+		}
+
+		// start the dns updater if address is dns
+		if network.IsDNS() {
+			log.Info("starting the DNS updater", "address", network.DNSName())
+			ipUpdater := vip.NewIPUpdater(network)
+			wg.Go(func() {
+				ipUpdater.Run(ctx)
+			})
+		}
+
+		if !c.EnableRoutingTable {
+			// Normal VIP addition, use skipDAD=false for normal DAD process
+			if _, err = network.AddIP(false, false); err != nil {
+				log.Error("failed to add IP", "address", network.IP(), "error", err)
+			}
+		}
+
+		if c.EnableBGP {
+			if c.ControlPlaneHealthCheck.Address != "" {
+				// The health check loop owns route advertisement/withdrawal when configured.
+				wg.Go(func() {
+					cluster.bgpHealthCheckLoop(ctx, c, bgpServer, network.CIDR())
+				})
+			} else {
+				// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation.
+				log.Debug("Attempting to advertise over BGP", "address", network.CIDR())
+				err = bgpServer.AddHost(ctx, network.CIDR(), c.NodeName)
+				if err != nil {
+					log.Error(err.Error())
+				}
+			}
+		}
+
+		if c.EnableLoadBalancer {
+			lb, err := loadbalancer.NewIPVSLB(ctx, network, c.LoadBalancerPort, c.LoadBalancerForwardingMethod,
+				c.BackendHealthCheckInterval, c.EgressWithNftables, killFunc, &wg)
+			if err != nil {
+				killFunc()
+				return fmt.Errorf("creating IPVS LoadBalancer: %w", err)
+			}
+
+			wg.Go(func() {
+				for {
+					select {
+					case <-ctx.Done():
+						return
+					default:
+						err = em.NodeWatcher(ctx, lb, c.Port)
+						if err != nil {
+							log.Error("Error watching node labels", "err", err)
+							if utils.IsPanicError(err) {
+								killFunc()
+								return
+							}
+						}
+					}
+				}
+			})
+
+			loadbalancers = append(loadbalancers, lb)
+		}
+
+		if c.EnableARP {
+			wg.Go(func() {
+				cluster.layer2Update(ctx, network, c)
+			})
+		}
+	}
+
+	if c.EnableLoadBalancer {
+		// Shutdown function that will wait on this signal, unless we call it ourselves
+		<-ctx.Done()
+		for _, lb := range loadbalancers {
+			err = lb.RemoveIPVSLB()
+			if err != nil {
+				log.Error("Error stopping IPVS LoadBalancer", "err", err)
+			}
+		}
+	}
+
+	if c.EnableRoutingTable {
+		backendMapV4 := backend.Map{}
+		backendMapV6 := backend.Map{}
+		// only check localhost
+
+		// An explicitly configured Kubernetes API address (static-pod
+		// deployments point it at the local API server, whose loopback
+		// listener is often the only certificate-valid local endpoint)
+		// takes precedence over the Node object's addresses: the check
+		// answers "is the local API server healthy" for every VIP family,
+		// regardless of the transport family of the override itself.
+		if entry := kubernetesAddrBackendEntry(c.KubernetesAddr, c.Port); entry != nil {
+			log.Info("using configured Kubernetes address for backend health checks", "address", c.KubernetesAddr)
+			backendMapV4[*entry] = false
+			backendMapV6[*entry] = false
+		} else {
+			ips := []string{}
+			if c.NodeName != "" {
+				if ips, err = getNodeIPs(ctx, c.NodeName, em.KubernetesClient); err != nil && !apierrors.IsNotFound(err) {
+					log.Error("failed to get IP of control-plane node", "err", err)
+				}
+			}
+
+			if len(ips) == 0 {
+				if !utils.IsIPv6(cluster.Network[0].IP()) {
+					ips = append(ips, "127.0.0.1")
+				} else {
+					ips = append(ips, "::1")
+				}
+
+				log.Info("no IP address found for node - will fallback to use localhost address", "addresses", ips)
+			}
+
+			for _, ip := range ips {
+				entry := backend.Entry{Addr: ip, Port: c.Port}
+				if !utils.IsIPv6(ip) {
+					backendMapV4[entry] = false
+				} else {
+					backendMapV6[entry] = false
+				}
+			}
+		}
+
+		backend.SetKubeConfigPath(c.K8sConfigFile)
+		backend.Watch(ctx, c.BackendHealthCheckInterval, func() {
+			for i := range cluster.Network {
+				network := cluster.Network[i]
+				networkIP := network.IP()
+				isNetworkV6 := utils.IsIPv6(networkIP)
+				log.Debug("current ip to process", "ip", networkIP)
+
+				backendMap := &backendMapV4
+				if isNetworkV6 {
+					backendMap = &backendMapV6
+				}
+
+				for entry := range *backendMap {
+					log.Debug("entry.Check() for entry", "entry", entry)
+					var healthy bool
+					if c.ControlPlaneHealthCheck.Address != "" {
+						req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, c.ControlPlaneHealthCheck.Address, nil)
+						if reqErr != nil {
+							log.Error("create health check request", "err", reqErr)
+						} else if resp, doErr := cluster.healthCheckHTTPClient.Do(req); doErr != nil {
+							log.Error("health check request failed", "url", c.ControlPlaneHealthCheck.Address, "err", doErr)
+						} else {
+							resp.Body.Close()
+							healthy = resp.StatusCode == http.StatusOK
+							if !healthy {
+								log.Warn("health check returned non-200 status", "url", c.ControlPlaneHealthCheck.Address, "status", resp.StatusCode)
+							}
+						}
+					} else {
+						healthy = entry.Check()
+					}
+					if healthy {
+						log.Debug("entry.Check() true")
+						// Normal VIP addition with precheck, use skipDAD=false for normal DAD process
+						_, err = network.AddIP(true, false)
+						if err != nil {
+							log.Error("error adding address", "err", err)
+						}
+						if !(*backendMap)[entry] {
+							log.Info("added backend", "ip", network.IP())
+						}
+
+						err = cluster.routeMgr.Add(c.NodeName, network, true, false)
+						if err != nil && !errors.Is(err, fs.ErrExist) && !errors.Is(err, syscall.ESRCH) {
+							log.Warn(err.Error())
+						} else if err == nil && !(*backendMap)[entry] {
+							log.Info("added route", "route", network.PrepareRoute())
+						} else if err == nil || errors.Is(err, fs.ErrExist) {
+							// Re-assert the route on every healthy cycle: routing daemons
+							// (e.g. zebra) can miss the single netlink event for the route,
+							// leaving it unadvertised even though it exists in the kernel.
+							// RouteReplace is idempotent and regenerates that event.
+							if replaceErr := network.ReplaceRoute(); replaceErr != nil {
+								log.Warn("re-asserting route", "err", replaceErr)
+							} else {
+								log.Debug("re-asserted route", "route", network.PrepareRoute())
+							}
+						}
+
+						(*backendMap)[entry] = true
+						break
+					}
+					(*backendMap)[entry] = false
+				}
+
+				deleteAddress := true
+				for entry := range *backendMap {
+					if (*backendMap)[entry] {
+						deleteAddress = false
+						break
+					}
+				}
+
+				if deleteAddress {
+					err = cluster.routeMgr.Delete(c.NodeName, network)
+					if err != nil {
+						log.Warn("deleting route", "err", err)
+					}
+
+					deleted, err := network.DeleteIP()
+					if err != nil {
+						log.Error("error deleting IP", "err", err)
+						killFunc()
+						return
+					}
+					if deleted {
+						log.Info("deleted address", "IP", network.IP(), "interface", network.Interface())
+					}
+				}
+			}
+		})
+	}
+
+	if c.EnableBGP {
+		<-ctx.Done()
+	}
+
+	return nil
+}
+
+func (cluster *Cluster) bgpHealthCheck(ctx context.Context, c *kubevip.Config) (bool, error) {
+	statusCode := 0
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.ControlPlaneHealthCheck.Address, nil)
+	if err != nil {
+		return false, fmt.Errorf("building request %v: %w", req, err)
+	} else {
+		resp, err := cluster.healthCheckHTTPClient.Do(req)
+		if err != nil {
+			return false, fmt.Errorf("checking control-plane: %w", err)
+		}
+		defer resp.Body.Close()
+		statusCode = resp.StatusCode
+	}
+	healthy := statusCode == http.StatusOK
+	if !healthy {
+		return healthy, fmt.Errorf("wrong status code: %d", statusCode)
+	}
+	return healthy, nil
+}
+
+func (cluster *Cluster) bgpHealthCheckLoop(ctx context.Context, c *kubevip.Config, bgpServer bgp.BGPManager, vipCIDR string) {
+	period := time.Duration(c.ControlPlaneHealthCheck.PeriodSeconds) * time.Second
+
+	consecutiveFailures := 0
+	routeAnnounced := false
+	ticker := time.NewTicker(period)
+	defer ticker.Stop()
+
+	log.Info("Starting BGP health check",
+		"address", c.ControlPlaneHealthCheck.Address,
+		"cidr", vipCIDR,
+		"period", period,
+		"timeout", cluster.healthCheckHTTPClient.Timeout,
+		"threshold", c.ControlPlaneHealthCheck.FailureThreshold,
+	)
+
+	for {
+		healthy, healthErr := cluster.bgpHealthCheck(ctx, c)
+		if healthy {
+			consecutiveFailures = 0
+			if !routeAnnounced {
+				log.Info("BGP health check passed, announcing route", "cidr", vipCIDR)
+				if err := bgpServer.AddHost(ctx, vipCIDR, c.NodeName); err != nil {
+					log.Error("BGP health check: failed to announce route", "cidr", vipCIDR, "err", err)
+				} else {
+					routeAnnounced = true
+				}
+			}
+		} else {
+			consecutiveFailures++
+			if healthErr != nil {
+				log.Warn("BGP health check failed", "address", c.ControlPlaneHealthCheck.Address, "consecutive", consecutiveFailures, "err", healthErr)
+			}
+			if consecutiveFailures >= c.ControlPlaneHealthCheck.FailureThreshold && routeAnnounced {
+				log.Warn("BGP health check threshold reached, withdrawing route", "failureThreshold", c.ControlPlaneHealthCheck.FailureThreshold, "cidr", vipCIDR)
+				if err := bgpServer.DelHost(ctx, vipCIDR, c.NodeName); err != nil {
+					log.Error("BGP health check: failed to withdraw route", "cidr", vipCIDR, "err", err)
+				} else {
+					routeAnnounced = false
+				}
+			}
+		}
+
+		select {
+		case <-ctx.Done():
+			if routeAnnounced {
+				if err := bgpServer.DelHost(ctx, vipCIDR, c.NodeName); err != nil {
+					log.Error("BGP health check: failed to withdraw route", "cidr", vipCIDR, "err", err)
+				}
+			}
+			return
+		case <-ticker.C:
+		}
+	}
+}
+
+// kubernetesAddrBackendEntry converts an explicitly configured Kubernetes
+// API address override (config.KubernetesAddr, e.g. "https://127.0.0.1:6443"
+// on static-pod deployments) into a backend health-check entry. Returns nil
+// when no usable override is configured.
+func kubernetesAddrBackendEntry(kubernetesAddr string, defaultPort uint16) *backend.Entry {
+	if kubernetesAddr == "" {
+		return nil
+	}
+	u, err := url.Parse(kubernetesAddr)
+	if err != nil || u.Hostname() == "" {
+		return nil
+	}
+	port := defaultPort
+	if p := u.Port(); p != "" {
+		if parsed, err := strconv.ParseUint(p, 10, 16); err == nil {
+			port = uint16(parsed)
+		}
+	}
+	return &backend.Entry{Addr: u.Hostname(), Port: port}
+}
+
+func getNodeIPs(ctx context.Context, nodename string, client *kubernetes.Clientset) ([]string, error) {
+	node, err := client.CoreV1().Nodes().Get(ctx, nodename, metav1.GetOptions{})
+	if err != nil && !apierrors.IsNotFound(err) {
+		return []string{}, fmt.Errorf("failed to get data about '%s' node: %w", nodename, err)
+	}
+	ips := []string{}
+	for _, addr := range node.Status.Addresses {
+		if addr.Type == corev1.NodeInternalIP {
+			ips = append(ips, addr.Address)
+		}
+	}
+	return ips, nil
+}
+
+// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
+func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp bgp.BGPManager, name string, wg *sync.WaitGroup) error {
+	// use a Go context so we can tell the arp loop code when we
+	// want to step down
+	//nolint
+	lbCtx, lbCancel := context.WithCancel(ctx)
+	var lbWg sync.WaitGroup
+	stop, done, err := cluster.startServicesWorker()
+	if err != nil {
+		lbCancel()
+		return err
+	}
+	type startedNetwork struct {
+		network    vip.Network
+		routeAdded bool
+		ipAdded    bool
+		bgpAdded   bool
+	}
+	startedNetworks := make([]startedNetwork, 0, len(cluster.Network))
+	servicesWorkerStarted := false
+	defer func() {
+		if !servicesWorkerStarted {
+			lbCancel()
+			lbWg.Wait()
+			cleanupCtx := context.WithoutCancel(ctx)
+			for index := len(startedNetworks) - 1; index >= 0; index-- {
+				started := startedNetworks[index]
+				if started.bgpAdded && bgp != nil {
+					if err := bgp.DelHost(cleanupCtx, started.network.CIDR(), name); err != nil {
+						log.Warn("failed to withdraw BGP host after startup failure", "address", started.network.CIDR(), "err", err)
+					}
+				}
+				if started.routeAdded && cluster.routeMgr != nil {
+					if err := cluster.routeMgr.Delete(name, started.network); err != nil {
+						log.Warn("failed to delete route after startup failure", "address", started.network.CIDR(), "err", err)
+					}
+				}
+				if started.ipAdded {
+					if _, err := started.network.DeleteIP(); err != nil {
+						log.Warn("failed to delete VIP after startup failure", "address", started.network.IP(), "err", err)
+					}
+				}
+			}
+			cluster.finishServicesWorker(done)
+		}
+	}()
+
+	for i := range cluster.Network {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		network := cluster.Network[i]
+		startedNetworks = append(startedNetworks, startedNetwork{network: network})
+		started := &startedNetworks[len(startedNetworks)-1]
+
+		if network.IsDDNS() {
+			ddnsReady := make(chan struct{})
+			lbWg.Go(func() {
+				// start the DDNS if requested
+				log.Debug("(svcs) start DDNS", "name", network.DNSName())
+				if err := cluster.StartDDNS(lbCtx, cluster.Network[i], c.DHCPBackoffAttempts, &lbWg); err != nil {
+					log.Error("failed to start DDNS", "err", err)
+				}
+
+				close(ddnsReady)
+				<-lbCtx.Done()
+			})
+			<-ddnsReady
+		}
+
+		log.Debug("current ip to process", "ip", network.IP(), "mask", c.VIPSubnet)
+		if err := network.SetMask(c.VIPSubnet); err != nil {
+			log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
+			lbCancel()
+			return utils.WrapPanicError(err, "failed to set mask for subnet %q", c.VIPSubnet)
+		}
+		existing, err := network.IsSet()
+		if err != nil {
+			lbCancel()
+			return fmt.Errorf("check existing VIP %q: %w", network.IP(), err)
+		}
+		log.Debug("config flags", "enable_routing_table", c.EnableRoutingTable, "enable_leader_election", c.EnableLeaderElection, "enable_services_election", c.EnableServicesElection)
+
+		if c.EnableRoutingTable && (c.EnableLeaderElection || c.EnableServicesElection) {
+			err = cluster.routeMgr.Add(name, network, false, false)
+			if err != nil {
+				log.Warn(err.Error())
+			} else {
+				started.routeAdded = true
+				log.Info("successful add Route")
+			}
+		}
+
+		if shouldAddServiceIP(c) {
+			// Normal VIP addition, use skipDAD=false for normal DAD process
+			// Note: When WireGuard is enabled, the VIP is added to the tunnel interface
+			// instead of lo, so we skip adding it here.
+			added, addErr := network.AddIP(false, false)
+			started.ipAdded = existing == nil && added
+			if addErr != nil {
+				log.Warn(addErr.Error())
+			} else {
+				log.Info("successful add IP", "address", network.IP())
+			}
+		}
+
+		if c.EnableARP {
+			lbWg.Go(func() {
+				cluster.layer2Update(lbCtx, network, c)
+			})
+		}
+
+		if c.EnableBGP && (c.EnableLeaderElection || c.EnableServicesElection) {
+			// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
+			log.Debug("(svcs) attempting to advertise over BGP", "address", network.CIDR())
+			err = bgp.AddHost(lbCtx, network.CIDR(), name)
+			if err != nil {
+				log.Error(err.Error())
+			} else {
+				started.bgpAdded = true
+			}
+		}
+	}
+
+	wg.Go(func() {
+		defer cluster.finishServicesWorker(done)
+		for i := range cluster.Network {
+			network := cluster.Network[i]
+
+			// start the dns updater if address is dns
+			if network.IsDNS() {
+				log.Info("(svcs) starting the DNS updater", "address", network.DNSName(), "ip", network.IP())
+				ipUpdater := vip.NewIPUpdater(network)
+				wg.Go(func() {
+					ipUpdater.Run(lbCtx)
+				})
+			}
+		}
+
+		select {
+		case <-stop:
+		case <-ctx.Done():
+		}
+
+		// Stop the loadbalancer context if it is running
+		lbCancel()
+
+		lbWg.Wait() // wait for all cluster ARP/NDP to be finished
+
+		log.Info("[LOADBALANCER] Stopping load balancers", "name", name)
+
+		if c.EnableRoutingTable {
+			for i := range cluster.Network {
+				if err := cluster.routeMgr.Delete(name, cluster.Network[i]); err != nil {
+					log.Warn(err.Error())
+				}
+			}
+
+			return
+		}
+
+		cluster.cleanupServiceVIPs(c, done)
+	})
+	servicesWorkerStarted = true
+
+	return nil
+}
+
+func shouldAddServiceIP(c *kubevip.Config) bool {
+	return !c.EnableRoutingTable && (!c.EnableBGP || c.BGPAttachIPToInterface) && !c.EnableWireguard
+}
+
+// Layer2Update, handles the creation of the
+func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config) {
+	var ndp *vip.NdpResponder
+	var err error
+	ipString := network.IP()
+	if utils.IsIPv6(ipString) {
+		if network.IPisLinkLocal() {
+			log.Error("layer2 is link-local can't use NDP", "address", ipString)
+		} else {
+			ndp, err = waitNDPResponder(ctx, network.Interface())
+			if err != nil {
+				log.Error("failed to create new NDP Responder", "error", err)
+			} else {
+				if ndp != nil {
+					defer ndp.Close()
+				}
+			}
+		}
+	}
+
+	log.Info("layer 2 broadcaster starting", "IP", network.IP(), "device", network.Interface())
+	log.Debug("layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
+
+	arpInstance := arp.NewInstance(network, ndp)
+	cluster.arpMgr.Insert(arpInstance)
+
+	<-ctx.Done() // if cancel() execute
+	log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
+	cluster.arpMgr.RemoveOnLeadershipLoss(arpInstance)
+}
+
+func waitNDPResponder(ctx context.Context, ifaceName string) (*vip.NdpResponder, error) {
+	ndp, err := vip.NewNDPResponder(ifaceName)
+	if err != nil && strings.Contains(err.Error(), "no such device") {
+		log.Warn("unable to create NDP responder at first try", "interface", ifaceName, "err", err)
+		ndpCreateCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
+		defer cancel()
+		ticker := time.NewTicker(time.Second)
+
+		for {
+			select {
+			case <-ndpCreateCtx.Done():
+				return nil, fmt.Errorf("failed to create NDP responder for interface %q: %w", ifaceName, ndpCreateCtx.Err())
+			case <-ticker.C:
+				ndp, err = vip.NewNDPResponder(ifaceName)
+				if err != nil {
+					log.Warn("unable to create NDP responder on retry", "interface", ifaceName, "err", err)
+				} else {
+					return ndp, nil
+				}
+			}
+		}
+	} else if err != nil {
+		return nil, fmt.Errorf("unable to create NDP responder for interface %q: %w", ifaceName, err)
+	}
+	return ndp, nil
+}

pkg/cluster/service_config_test.go

@@ -0,0 +1,55 @@
+package cluster
+
+import (
+	"testing"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+)
+
+func TestShouldAddServiceIP(t *testing.T) {
+	tests := []struct {
+		name   string
+		config *kubevip.Config
+		want   bool
+	}{
+		{
+			name:   "BGP default does not attach IP",
+			config: &kubevip.Config{EnableBGP: true},
+			want:   false,
+		},
+		{
+			name: "BGP opt-in attaches IP",
+			config: &kubevip.Config{
+				EnableBGP:              true,
+				BGPAttachIPToInterface: true,
+			},
+			want: true,
+		},
+		{
+			name: "routing table takes precedence",
+			config: &kubevip.Config{
+				EnableBGP:              true,
+				BGPAttachIPToInterface: true,
+				EnableRoutingTable:     true,
+			},
+			want: false,
+		},
+		{
+			name: "WireGuard takes precedence",
+			config: &kubevip.Config{
+				EnableBGP:              true,
+				BGPAttachIPToInterface: true,
+				EnableWireguard:        true,
+			},
+			want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := shouldAddServiceIP(tt.config); got != tt.want {
+				t.Fatalf("shouldAddServiceIP() = %t, want %t", got, tt.want)
+			}
+		})
+	}
+}

pkg/cluster/service_internal_test.go

@@ -0,0 +1,61 @@
+package cluster
+
+import (
+	"testing"
+)
+
+func TestKubernetesAddrBackendEntry(t *testing.T) {
+	cases := []struct {
+		name     string
+		addr     string
+		port     uint16
+		wantAddr string
+		wantPort uint16
+		wantNil  bool
+	}{
+		{
+			name:     "explicit v4 loopback with port",
+			addr:     "https://127.0.0.1:6443",
+			port:     9999,
+			wantAddr: "127.0.0.1",
+			wantPort: 6443,
+		},
+		{
+			name:     "hostname without port falls back to config port",
+			addr:     "https://localhost",
+			port:     6443,
+			wantAddr: "localhost",
+			wantPort: 6443,
+		},
+		{
+			name:    "empty override",
+			addr:    "",
+			port:    6443,
+			wantNil: true,
+		},
+		{
+			name:    "garbage override",
+			addr:    "://not-a-url",
+			port:    6443,
+			wantNil: true,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			entry := kubernetesAddrBackendEntry(tc.addr, tc.port)
+			if tc.wantNil {
+				if entry != nil {
+					t.Fatalf("expected nil entry, got %+v", entry)
+				}
+				return
+			}
+			if entry == nil {
+				t.Fatal("expected an entry, got nil")
+			}
+			if entry.Addr != tc.wantAddr || entry.Port != tc.wantPort {
+				t.Fatalf("got %+v, want addr %q port %d", entry, tc.wantAddr, tc.wantPort)
+			}
+		})
+	}
+}

pkg/cluster/service_test.go

@@ -0,0 +1,548 @@
+package cluster_test
+
+import (
+	"context"
+	"encoding/pem"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/cluster"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/route"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	"github.com/vishvananda/netlink"
+	corev1 "k8s.io/api/core/v1"
+)
+
+const testCIDR = "10.0.0.34/32"
+
+func TestBGPHealthCheckLoop_AnnouncesOnHealthy(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusOK)
+	t.Cleanup(healthcheck.server.Close)
+
+	bgpManager := newMockBGPRouteManager()
+	startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
+
+	expectEventually(t, func() bool { return bgpManager.isAnnounced() },
+		"route should be announced")
+}
+
+func TestServicesWorkerStopAndWaitDrainsBeforeRestart(t *testing.T) {
+	config := &kubevip.Config{}
+	serviceCluster, err := cluster.InitCluster(config, true, nil, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("InitCluster() error = %v", err)
+	}
+	var workers sync.WaitGroup
+	if err := serviceCluster.StartLoadBalancerService(context.Background(), config, nil, "service", &workers); err != nil {
+		t.Fatalf("first StartLoadBalancerService() error = %v", err)
+	}
+	if err := serviceCluster.StartLoadBalancerService(context.Background(), config, nil, "service", &workers); err == nil {
+		t.Fatal("second StartLoadBalancerService() started while the first workers were active")
+	}
+
+	serviceCluster.StopAndWait()
+	if err := serviceCluster.StartLoadBalancerService(context.Background(), config, nil, "service", &workers); err != nil {
+		t.Fatalf("StartLoadBalancerService() after StopAndWait error = %v", err)
+	}
+	serviceCluster.StopAndWait()
+	workers.Wait()
+}
+
+func TestServicesWorkerStopAndWaitPreservingDrainsBeforeRestart(t *testing.T) {
+	config := &kubevip.Config{}
+	serviceCluster, err := cluster.InitCluster(config, true, nil, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("InitCluster() error = %v", err)
+	}
+	var workers sync.WaitGroup
+	if err := serviceCluster.StartLoadBalancerService(context.Background(), config, nil, "service", &workers); err != nil {
+		t.Fatalf("StartLoadBalancerService() error = %v", err)
+	}
+	serviceCluster.StopAndWait()
+	if err := serviceCluster.StartLoadBalancerService(context.Background(), config, nil, "service", &workers); err != nil {
+		t.Fatalf("StartLoadBalancerService() after preserving stop error = %v", err)
+	}
+	serviceCluster.StopAndWait()
+	workers.Wait()
+}
+
+func TestStartLoadBalancerServiceRollsBackEarlierNetwork(t *testing.T) {
+	first := &mockNetwork{ip: "192.0.2.10", cidr: "192.0.2.10/32"}
+	second := &mockNetwork{ip: "192.0.2.11", cidr: "192.0.2.11/32", setMaskErr: errors.New("set mask")}
+	serviceCluster, err := cluster.InitCluster(&kubevip.Config{}, true, nil, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("InitCluster() error = %v", err)
+	}
+	serviceCluster.Network = []vip.Network{first, second}
+
+	if err := serviceCluster.StartLoadBalancerService(context.Background(), &kubevip.Config{VIPSubnet: "32"}, nil, "service", &sync.WaitGroup{}); err == nil {
+		t.Fatal("StartLoadBalancerService() error = nil, want second-network failure")
+	}
+	first.mu.Lock()
+	addCalls, deleteCalls, present := first.addIPCalls, first.deleteIPCalls, first.present
+	first.mu.Unlock()
+	if addCalls != 1 || deleteCalls != 1 || present {
+		t.Fatalf("first network rollback = add %d, delete %d, present %t; want 1, 1, false", addCalls, deleteCalls, present)
+	}
+	serviceCluster.StopAndWait()
+}
+
+func TestStartLoadBalancerServiceRollbackPreservesExistingVIP(t *testing.T) {
+	first := &mockNetwork{ip: "192.0.2.10", cidr: "192.0.2.10/32", present: true}
+	second := &mockNetwork{ip: "192.0.2.11", cidr: "192.0.2.11/32", setMaskErr: errors.New("set mask")}
+	serviceCluster, err := cluster.InitCluster(&kubevip.Config{}, true, nil, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("InitCluster() error = %v", err)
+	}
+	serviceCluster.Network = []vip.Network{first, second}
+
+	if err := serviceCluster.StartLoadBalancerService(context.Background(), &kubevip.Config{VIPSubnet: "32"}, nil, "service", &sync.WaitGroup{}); err == nil {
+		t.Fatal("StartLoadBalancerService() error = nil, want second-network failure")
+	}
+	first.mu.Lock()
+	addCalls, deleteCalls, present := first.addIPCalls, first.deleteIPCalls, first.present
+	first.mu.Unlock()
+	if addCalls != 1 || deleteCalls != 0 || !present {
+		t.Fatalf("existing VIP rollback = add %d, delete %d, present %t; want 1, 0, true", addCalls, deleteCalls, present)
+	}
+	serviceCluster.StopAndWait()
+}
+
+func TestStartLoadBalancerServiceCancelledContextDoesNotConfigureVIP(t *testing.T) {
+	network := &mockNetwork{ip: "192.0.2.10", cidr: "192.0.2.10/32"}
+	serviceCluster, err := cluster.InitCluster(&kubevip.Config{}, true, nil, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("InitCluster() error = %v", err)
+	}
+	serviceCluster.Network = []vip.Network{network}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	err = serviceCluster.StartLoadBalancerService(ctx, &kubevip.Config{VIPSubnet: "32"}, nil, "service", &sync.WaitGroup{})
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("StartLoadBalancerService() error = %v, want context.Canceled", err)
+	}
+	network.mu.Lock()
+	addCalls := network.addIPCalls
+	network.mu.Unlock()
+	if addCalls != 0 {
+		t.Fatalf("AddIP calls = %d, want 0 after context cancellation", addCalls)
+	}
+}
+
+func TestBGPHealthCheckLoop_NoAnnouncementUntilHealthy(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusInternalServerError)
+	t.Cleanup(healthcheck.server.Close)
+
+	bgpManager := newMockBGPRouteManager()
+	startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
+
+	expectConsistently(t, func() bool { return !bgpManager.isAnnounced() },
+		2*time.Second, "route should not be announced while unhealthy")
+
+	healthcheck.setStatus(http.StatusOK)
+	expectEventually(t, func() bool { return bgpManager.isAnnounced() },
+		"route should be announced after recovery")
+}
+
+func TestBGPHealthCheckLoop_WithdrawsAfterThreshold(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusOK)
+	t.Cleanup(healthcheck.server.Close)
+
+	bgpManager := newMockBGPRouteManager()
+	cfg := newBGPConfig(healthcheck.server.URL, healthcheck.caPath)
+	cfg.ControlPlaneHealthCheck.FailureThreshold = 3
+	startVipService(t, cfg, bgpManager)
+
+	expectEventually(t, func() bool { return bgpManager.isAnnounced() },
+		"route should be announced")
+
+	healthcheck.setStatus(http.StatusServiceUnavailable)
+
+	expectConsistently(t, func() bool { return bgpManager.isAnnounced() },
+		1500*time.Millisecond, "route should stay announced before threshold is reached")
+
+	expectEventually(t, func() bool { return !bgpManager.isAnnounced() },
+		"route should be withdrawn after threshold")
+}
+
+func TestBGPHealthCheckLoop_ReAnnouncesOnRecovery(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusOK)
+	t.Cleanup(healthcheck.server.Close)
+
+	bgpManager := newMockBGPRouteManager()
+	cfg := newBGPConfig(healthcheck.server.URL, healthcheck.caPath)
+	cfg.ControlPlaneHealthCheck.FailureThreshold = 1
+	startVipService(t, cfg, bgpManager)
+
+	expectEventually(t, func() bool { return bgpManager.isAnnounced() },
+		"route should be announced")
+
+	healthcheck.setStatus(http.StatusServiceUnavailable)
+	expectEventually(t, func() bool { return !bgpManager.isAnnounced() },
+		"route should be withdrawn")
+
+	healthcheck.setStatus(http.StatusOK)
+	expectEventually(t, func() bool { return bgpManager.isAnnounced() },
+		"route should be re-announced")
+}
+
+func TestBGPHealthCheckLoop_StopsOnContextCancel(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusOK)
+	t.Cleanup(healthcheck.server.Close)
+
+	bgpManager := newMockBGPRouteManager()
+	cancelContext, vipServiceDone := startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
+
+	expectEventually(t, func() bool { return bgpManager.isAnnounced() },
+		"route should be announced")
+
+	cancelContext()
+
+	select {
+	case <-vipServiceDone:
+	case <-time.After(5 * time.Second):
+		t.Fatal("vipService did not stop after context cancellation")
+	}
+}
+
+func TestBGPHealthCheckLoop_RetriesAddHostOnFailure(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusOK)
+	t.Cleanup(healthcheck.server.Close)
+
+	bgpManager := newMockBGPRouteManager()
+	bgpManager.setAddErr(errTestAddHost)
+	startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
+
+	expectConsistently(t, func() bool { return !bgpManager.isAnnounced() },
+		2*time.Second, "route should not be announced while AddHost errors")
+
+	bgpManager.setAddErr(nil)
+	expectEventually(t, func() bool { return bgpManager.isAnnounced() },
+		"route should be announced after clearing AddHost error")
+}
+
+func TestBGPHealthCheckLoop_RetriesDelHostOnFailure(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusOK)
+	t.Cleanup(healthcheck.server.Close)
+
+	bgpManager := newMockBGPRouteManager()
+	cfg := newBGPConfig(healthcheck.server.URL, healthcheck.caPath)
+	cfg.ControlPlaneHealthCheck.FailureThreshold = 1
+	startVipService(t, cfg, bgpManager)
+
+	expectEventually(t, func() bool { return bgpManager.isAnnounced() },
+		"route should be announced")
+
+	bgpManager.setDelErr(errTestDelHost)
+	healthcheck.setStatus(http.StatusServiceUnavailable)
+
+	expectConsistently(t, func() bool { return bgpManager.isAnnounced() },
+		1500*time.Millisecond, "route should stay announced while DelHost errors")
+
+	bgpManager.setDelErr(nil)
+	expectEventually(t, func() bool { return !bgpManager.isAnnounced() },
+		"route should be withdrawn after clearing DelHost error")
+}
+
+func TestRoutingTableHealthCheck_AddsVIPWhenHealthy(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusOK)
+	t.Cleanup(healthcheck.server.Close)
+
+	network := &mockNetwork{ip: "10.0.0.1", cidr: testCIDR}
+	startRoutingTableVipService(t, newRoutingTableConfig(healthcheck.server.URL, healthcheck.caPath), network)
+
+	expectEventually(t, network.isPresent,
+		"VIP should be added while health check is healthy")
+}
+
+func TestRoutingTableHealthCheck_RemovesVIPWhenUnhealthy(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusOK)
+	t.Cleanup(healthcheck.server.Close)
+
+	network := &mockNetwork{ip: "10.0.0.1", cidr: testCIDR}
+	startRoutingTableVipService(t, newRoutingTableConfig(healthcheck.server.URL, healthcheck.caPath), network)
+
+	expectEventually(t, network.isPresent,
+		"VIP should be added while health check is healthy")
+
+	healthcheck.setStatus(http.StatusServiceUnavailable)
+	expectEventually(t, func() bool { return !network.isPresent() },
+		"VIP should be removed once health check becomes unhealthy")
+}
+
+var (
+	errTestAddHost = &testError{msg: "mock AddHost error"}
+	errTestDelHost = &testError{msg: "mock DelHost error"}
+)
+
+type testError struct{ msg string }
+
+func (e *testError) Error() string { return e.msg }
+
+// startVipService launches vipService in a goroutine with a mock network and
+// registers a cleanup to cancel the context and wait for it to finish.
+// Uses InitCluster so the real code parses certs for the BGP health check client.
+func startVipService(t *testing.T, cfg *kubevip.Config, bgpServer bgp.BGPManager) (context.CancelFunc, <-chan struct{}) {
+	t.Helper()
+	c, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("InitCluster: %v", err)
+	}
+	c.Network = []vip.Network{&mockNetwork{ip: "10.0.0.1", cidr: testCIDR}}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+
+	go func() {
+		_ = c.StartVipService(ctx, cfg, nil, bgpServer, func() {})
+		close(done)
+	}()
+
+	t.Cleanup(func() {
+		cancel()
+		<-done
+	})
+
+	return cancel, done
+}
+
+// startRoutingTableVipService launches vipService in routing-table mode with a
+// mock network and a real route.Manager (which only drives the mock network's
+// route methods, so no netlink calls happen). Registers cleanup to stop it.
+func startRoutingTableVipService(t *testing.T, cfg *kubevip.Config, network *mockNetwork) {
+	t.Helper()
+
+	c, err := cluster.InitCluster(cfg, true, nil, nil, route.NewManager(), nil)
+	if err != nil {
+		t.Fatalf("InitCluster: %v", err)
+	}
+	c.Network = []vip.Network{network}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+
+	go func() {
+		_ = c.StartVipService(ctx, cfg, nil, nil, func() {})
+		close(done)
+	}()
+
+	t.Cleanup(func() {
+		cancel()
+		<-done
+	})
+}
+
+func newRoutingTableConfig(url, caPath string) *kubevip.Config {
+	cfg := newBGPConfig(url, caPath)
+	cfg.EnableBGP = false
+	cfg.EnableRoutingTable = true
+	cfg.BackendHealthCheckInterval = 1
+	return cfg
+}
+
+func newBGPConfig(url, caPath string) *kubevip.Config {
+	return &kubevip.Config{
+		EnableBGP: true,
+		ControlPlaneHealthCheck: kubevip.HealthCheck{
+			Address:          url,
+			CAPath:           caPath,
+			PeriodSeconds:    1,
+			TimeoutSeconds:   2,
+			FailureThreshold: 1,
+		},
+	}
+}
+
+// mockBGPRouteManager tracks announced addresses as a set.
+// AddHost adds, DelHost removes. Errors prevent state changes.
+type mockBGPRouteManager struct {
+	mu        sync.Mutex
+	announced map[string]bool
+	addErr    error
+	delErr    error
+}
+
+func newMockBGPRouteManager() *mockBGPRouteManager {
+	return &mockBGPRouteManager{announced: make(map[string]bool)}
+}
+
+func (m *mockBGPRouteManager) AddHost(_ context.Context, addr string, _ string) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.addErr != nil {
+		return m.addErr
+	}
+	m.announced[addr] = true
+	return nil
+}
+
+func (m *mockBGPRouteManager) DelHost(_ context.Context, addr string, _ string) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.delErr != nil {
+		return m.delErr
+	}
+	delete(m.announced, addr)
+	return nil
+}
+
+func (m *mockBGPRouteManager) isAnnounced() bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.announced[testCIDR]
+}
+
+func (m *mockBGPRouteManager) setAddErr(err error) {
+	m.mu.Lock()
+	m.addErr = err
+	m.mu.Unlock()
+}
+
+func (m *mockBGPRouteManager) setDelErr(err error) {
+	m.mu.Lock()
+	m.delErr = err
+	m.mu.Unlock()
+}
+
+// mockNetwork implements vip.Network with no-op operations.
+type mockNetwork struct {
+	ip   string
+	cidr string
+
+	mu            sync.Mutex
+	present       bool
+	setMaskErr    error
+	addIPCalls    int
+	deleteIPCalls int
+}
+
+func (m *mockNetwork) AddIP(bool, bool, ...int) (bool, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.addIPCalls++
+	m.present = true
+	return true, nil
+}
+func (m *mockNetwork) DeleteIP() (bool, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.deleteIPCalls++
+	deleted := m.present
+	m.present = false
+	return deleted, nil
+}
+func (m *mockNetwork) isPresent() bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.present
+}
+func (m *mockNetwork) AddRoute(bool) (bool, error) { return false, nil }
+func (m *mockNetwork) ReplaceRoute() error         { return nil }
+func (m *mockNetwork) DeleteRoute() error          { return nil }
+func (m *mockNetwork) UpdateRoutes() (bool, error) { return false, nil }
+func (m *mockNetwork) IsSet() (*netlink.Addr, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.present {
+		return &netlink.Addr{}, nil
+	}
+	return nil, nil
+}
+func (m *mockNetwork) IP() string                      { return m.ip }
+func (m *mockNetwork) CIDR() string                    { return m.cidr }
+func (m *mockNetwork) IPisLinkLocal() bool             { return false }
+func (m *mockNetwork) PrepareRoute() *netlink.Route    { return nil }
+func (m *mockNetwork) RouteHash() string               { return "" }
+func (m *mockNetwork) SetIP(string) error              { return nil }
+func (m *mockNetwork) SetServicePorts(*corev1.Service) {}
+func (m *mockNetwork) Interface() string               { return "eth0" }
+func (m *mockNetwork) IsDADFAIL() bool                 { return false }
+func (m *mockNetwork) IsDNS() bool                     { return false }
+func (m *mockNetwork) IsDDNS() bool                    { return false }
+func (m *mockNetwork) DDNSHostName() string            { return "" }
+func (m *mockNetwork) DNSName() string                 { return "" }
+func (m *mockNetwork) SetMask(string) error            { return m.setMaskErr }
+func (m *mockNetwork) SetHasEndpoints(bool)            {}
+func (m *mockNetwork) HasEndpoints() bool              { return false }
+func (m *mockNetwork) ARPName() string                 { return "" }
+func (m *mockNetwork) GetPossibleSubnets() string      { return "" }
+func (m *mockNetwork) DHCPFamily() string              { return "" }
+func (m *mockNetwork) IPVSMark() uint32                { return 0 }
+
+// testHealthServer wraps an HTTPS httptest.Server with an atomic status code.
+// caPath is the path to the server's CA cert for client verification.
+type testHealthServer struct {
+	server     *httptest.Server
+	statusCode atomic.Int64
+	caPath     string
+}
+
+func newTestHealthServer(t *testing.T, status int) *testHealthServer {
+	t.Helper()
+	healthcheck := &testHealthServer{}
+	healthcheck.statusCode.Store(int64(status))
+	healthcheck.server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(int(healthcheck.statusCode.Load()))
+	}))
+
+	cert := healthcheck.server.Certificate()
+	if cert == nil {
+		t.Fatal("TLS server has no certificate")
+	}
+	caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})
+	caFile := filepath.Join(t.TempDir(), "ca.crt")
+	if err := os.WriteFile(caFile, caPEM, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	healthcheck.caPath = caFile
+	return healthcheck
+}
+
+func (ths *testHealthServer) setStatus(code int) {
+	ths.statusCode.Store(int64(code))
+}
+
+// expectConsistently continuously checks that condition remains true for the given duration.
+// Fails immediately if the condition becomes false at any point.
+func expectConsistently(t *testing.T, condition func() bool, duration time.Duration, msg string) {
+	t.Helper()
+	deadline := time.Now().Add(duration)
+	for time.Now().Before(deadline) {
+		if !condition() {
+			t.Fatalf("condition violated: %s", msg)
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+}
+
+// expectEventually polls condition until it returns true or 5s timeout is reached.
+func expectEventually(t *testing.T, condition func() bool, msg string) {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		if condition() {
+			return
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	t.Fatalf("timeout: %s", msg)
+}

pkg/cluster/singleNode.go

@@ -1,103 +0,0 @@
-package cluster
-
-import (
-	log "github.com/sirupsen/logrus"
-
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	"github.com/plunder-app/kube-vip/pkg/loadbalancer"
-	"github.com/plunder-app/kube-vip/pkg/vip"
-)
-
-// StartSingleNode will start a single node cluster
-func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) error {
-	// Start kube-vip as a single node server
-
-	// TODO - Split all this code out as a seperate function
-	log.Infoln("Starting kube-vip as a single node cluster")
-
-	log.Info("This node is assuming leadership of the cluster")
-
-	cluster.stop = make(chan bool, 1)
-	cluster.completed = make(chan bool, 1)
-
-	// Managers for Vip load balancers and none-vip loadbalancers
-	nonVipLB := loadbalancer.LBManager{}
-	VipLB := loadbalancer.LBManager{}
-
-	// Iterate through all Configurations
-	for x := range c.LoadBalancers {
-		// If the load balancer doesn't bind to the VIP
-		if c.LoadBalancers[x].BindToVip == false {
-			err := nonVipLB.Add("", &c.LoadBalancers[x])
-			if err != nil {
-				log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-			}
-
-		}
-	}
-
-	if !disableVIP {
-		err := cluster.Network.DeleteIP()
-		if err != nil {
-			log.Warnf("Attempted to clean existing VIP => %v", err)
-		}
-
-		err = cluster.Network.AddIP()
-		if err != nil {
-			log.Warnf("%v", err)
-		}
-
-		// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-		for x := range c.LoadBalancers {
-
-			if c.LoadBalancers[x].BindToVip == true {
-				err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-				if err != nil {
-					log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-				}
-			}
-		}
-	}
-
-	if c.GratuitousARP == true {
-		// Gratuitous ARP, will broadcast to new MAC <-> IP
-		err := vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-		if err != nil {
-			log.Warnf("%v", err)
-		}
-	}
-
-	go func() {
-		for {
-			select {
-			case <-cluster.stop:
-				log.Info("[LOADBALANCER] Stopping load balancers")
-
-				// Stop all load balancers associated with the VIP
-				err := VipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				// Stop all load balancers associated with the Host
-				err = nonVipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				if !disableVIP {
-
-					log.Info("[VIP] Releasing the Virtual IP")
-					err = cluster.Network.DeleteIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-				close(cluster.completed)
-				return
-			}
-		}
-	}()
-	log.Infoln("Started Load Balancer and Virtual IP")
-	return nil
-}

pkg/cluster/state.go

@@ -1,39 +0,0 @@
-package cluster
-
-import (
-	"io"
-
-	"github.com/hashicorp/raft"
-)
-
-// FSM - Finite State Machine for Raft
-type FSM struct {
-}
-
-// Apply - TODO
-func (fsm FSM) Apply(log *raft.Log) interface{} {
-	return nil
-}
-
-// Restore - TODO
-func (fsm FSM) Restore(snap io.ReadCloser) error {
-	return nil
-}
-
-// Snapshot - TODO, returns an empty snapshot
-func (fsm FSM) Snapshot() (raft.FSMSnapshot, error) {
-	return Snapshot{}, nil
-}
-
-// Snapshot -
-type Snapshot struct {
-}
-
-// Persist -
-func (snapshot Snapshot) Persist(sink raft.SnapshotSink) error {
-	return nil
-}
-
-// Release -
-func (snapshot Snapshot) Release() {
-}

pkg/debouncer/debouncer.go

@@ -0,0 +1,277 @@
+package debouncer
+
+import (
+	"context"
+	"fmt"
+	log "log/slog"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	"k8s.io/apimachinery/pkg/watch"
+)
+
+const (
+	DefaultTime = "0s"
+	minimalTime = time.Millisecond * 200
+)
+
+type debouncer struct {
+	input    <-chan watch.Event
+	output   chan watch.Event
+	stopChan chan any
+	stopOnce sync.Once
+	// events holds event per namespace
+	namespaces   sync.Map
+	debounceTime time.Duration
+}
+
+type ns struct {
+	sync.Map
+	cnt atomic.Int64
+}
+
+func (n *ns) get(name string) (*object, bool) {
+	value, exists := n.Load(name)
+	if !exists {
+		return nil, false
+	}
+	i, ok := value.(*object)
+	if !ok {
+		return nil, false
+	}
+	return i, true
+}
+
+func (n *ns) add(name string, output chan<- watch.Event) *object {
+	i := newObject(output)
+	n.Store(name, i)
+	n.cnt.Add(1)
+	return i
+}
+
+func (n *ns) del(name string, object *object) {
+	if n.CompareAndDelete(name, object) {
+		n.cnt.Add(-1)
+	}
+}
+
+func New(input <-chan watch.Event, debounceTime string) (*debouncer, error) {
+	dt, err := time.ParseDuration(debounceTime)
+	if err != nil {
+		// debouncer was configured with invalid unparsable value, return error
+		return nil, fmt.Errorf("failed to parse debounce time configuration: %w", err)
+	}
+	if dt < minimalTime {
+		if dt > 0 {
+			log.Warn("configured debounce time is less than the minimal threshold of 200ms, debouncer will remain disabled", "config value", dt.String())
+		}
+		return nil, nil
+	}
+	return &debouncer{
+		input:        input,
+		output:       make(chan watch.Event),
+		stopChan:     make(chan any),
+		debounceTime: dt,
+	}, nil
+}
+
+func (d *debouncer) Start(ctx context.Context) error {
+	wg := sync.WaitGroup{}
+	debouncerCtx, cancel := context.WithCancel(ctx)
+	defer func() {
+		cancel()
+		wg.Wait()
+		close(d.output)
+	}()
+
+	for {
+		select {
+		case <-debouncerCtx.Done():
+			// return if debouncer context was cancelled
+			return nil
+		case <-d.stopChan:
+			// return if Stop() was called
+			return nil
+		case tmp := <-d.input:
+			// event has no type, probably error
+			if tmp.Type == "" {
+				return fmt.Errorf("get undefined object (input channel probably closed)")
+			}
+
+			var namespace, name string
+
+			// type switch event object
+			switch v := tmp.Object.(type) {
+			case *discoveryv1.EndpointSlice:
+				namespace = v.Namespace
+				name = v.Name
+			case *v1.Endpoints: //nolint:staticcheck
+				namespace = v.Namespace
+				name = v.Name
+			case *v1.Service:
+				namespace = v.Namespace
+				name = v.Name
+			default:
+				return fmt.Errorf("objects of type %T are not supported", v)
+			}
+
+		processEvent:
+			for {
+				eventNs, exists := d.getNs(namespace)
+				if !exists {
+					// if not, create new map for the namespace
+					eventNs = d.addNs(namespace)
+				}
+
+				// check if the object was previously reconciled
+				eventObject, exists := eventNs.get(name)
+
+				// if not and the event is not of type 'Deleted', create new object
+				if !exists && tmp.Type != watch.Deleted {
+					eventObject = eventNs.add(name, d.output)
+
+					workerObject := eventObject
+					workerNs := eventNs
+					workerName := name
+					workerNamespace := namespace
+					workerObject.onStop = func() {
+						// Remove the object before its worker can become receiver-less.
+						workerNs.del(workerName, workerObject)
+					}
+
+					wg.Go(func() {
+						// start deboucing events for this object
+						workerObject.start(debouncerCtx, d.debounceTime)
+						// if debouncer for the object ended - e.g. object was deleted - clean the map of objects
+						workerNs.del(workerName, workerObject)
+						// if namespace is empty, delete the namespace map
+						if workerNs.cnt.Load() == 0 {
+							d.delNs(workerNamespace, workerNs)
+						}
+					})
+				}
+
+				if eventObject == nil {
+					break processEvent
+				}
+
+				// pass the watch event to the debouncer object
+				select {
+				case eventObject.input <- tmp:
+					break processEvent
+				case <-eventObject.stopChan:
+					// The object stopped after the map lookup. Retry the event
+					// against the newly-created object instead of dropping it.
+					continue processEvent
+				case <-debouncerCtx.Done():
+					return nil
+				}
+			}
+		}
+	}
+}
+
+func (d *debouncer) Stop() {
+	d.stopOnce.Do(func() {
+		close(d.stopChan)
+	})
+}
+
+func (d *debouncer) Output() chan watch.Event {
+	return d.output
+}
+
+func (d *debouncer) getNs(namespace string) (*ns, bool) {
+	value, exists := d.namespaces.Load(namespace)
+	if !exists {
+		return nil, false
+	}
+	n, ok := value.(*ns)
+	if !ok {
+		return nil, false
+	}
+	return n, true
+}
+
+func (d *debouncer) addNs(namespace string) *ns {
+	n := ns{}
+	d.namespaces.Store(namespace, &n)
+	return &n
+}
+
+func (d *debouncer) delNs(namespace string, ns *ns) {
+	d.namespaces.CompareAndDelete(namespace, ns)
+}
+
+type object struct {
+	input    chan watch.Event
+	output   chan<- watch.Event
+	stopChan chan any
+	stopOnce sync.Once
+	onStop   func()
+}
+
+func newObject(output chan<- watch.Event) *object {
+	return &object{
+		input:    make(chan watch.Event),
+		output:   output,
+		stopChan: make(chan any),
+	}
+}
+
+func (o *object) start(ctx context.Context, debounceTime time.Duration) {
+	t := time.NewTicker(debounceTime)
+
+	var last *watch.Event
+
+	defer func() {
+		if last != nil {
+			o.output <- *last
+			last = nil
+		}
+	}()
+
+	for {
+		select {
+		case <-ctx.Done():
+			// if context is done, return
+			return
+		case <-o.stopChan:
+			// return if Stop() was called
+			return
+		case tmp := <-o.input:
+			// if last event is known, but an event of another type arrived,
+			// send out the previous event
+			if last != nil && last.Type != tmp.Type {
+				o.output <- *last
+			}
+			// save current event as the last event
+			last = &tmp
+			// reset the ticker to wait for more events
+			t.Reset(debounceTime)
+		case <-t.C:
+			if last != nil {
+				// on tick, if we have an event, send it out
+				o.output <- *last
+				// if the event is of type 'Deleted', stop the debouncer for the object
+				if last.Type == watch.Deleted {
+					o.stop()
+				}
+				// reset last known event, so it won't be send out twice
+				last = nil
+			}
+		}
+	}
+}
+
+func (o *object) stop() {
+	o.stopOnce.Do(func() {
+		if o.onStop != nil {
+			o.onStop()
+		}
+		close(o.stopChan)
+	})
+}

pkg/debouncer/debouncer_deadlock_test.go

@@ -0,0 +1,162 @@
+package debouncer
+
+import (
+	"context"
+	"runtime"
+	"testing"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+)
+
+func TestStartReturnsWhenCancellationInterruptsObjectForwarding(t *testing.T) {
+	input := make(chan watch.Event)
+	d := &debouncer{
+		input:        input,
+		output:       make(chan watch.Event),
+		stopChan:     make(chan any),
+		debounceTime: 200 * time.Millisecond,
+	}
+
+	// Leave the object without a receiver. This is the state reached when its
+	// worker exits on context cancellation just before Start forwards an event.
+	ns := d.addNs("default")
+	ns.add("example", d.output)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- d.Start(ctx) }()
+
+	event := watch.Event{
+		Type: watch.Modified,
+		Object: &v1.Service{ObjectMeta: metav1.ObjectMeta{
+			Name: "example", Namespace: "default",
+		}},
+	}
+	sent := make(chan struct{})
+	go func() {
+		input <- event
+		close(sent)
+	}()
+	select {
+	case <-sent:
+	case <-time.After(250 * time.Millisecond):
+		cancel()
+		t.Fatal("debouncer did not receive the test event")
+	}
+	cancel()
+
+	// Nothing ever receives from the object, so Start can only return by
+	// abandoning the blocked forward when the context is cancelled. Receiving
+	// here instead would make the forward succeed and the assertion racy.
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("debouncer remained blocked forwarding an event after context cancellation")
+	}
+}
+
+func TestStartRecreatesObjectAfterDeletionWithoutCancellation(t *testing.T) {
+	previousProcs := runtime.GOMAXPROCS(1)
+	t.Cleanup(func() { runtime.GOMAXPROCS(previousProcs) })
+
+	input := make(chan watch.Event)
+	d, err := New(input, "200ms")
+	if err != nil {
+		t.Fatalf("failed to create debouncer: %s", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+
+	done := make(chan error, 1)
+	go func() { done <- d.Start(ctx) }()
+
+	service := func(eventType watch.EventType, resourceVersion string) watch.Event {
+		return watch.Event{
+			Type: eventType,
+			Object: &v1.Service{ObjectMeta: metav1.ObjectMeta{
+				Name:            "example",
+				Namespace:       "default",
+				ResourceVersion: resourceVersion,
+			}},
+		}
+	}
+
+	send := func(event watch.Event) {
+		t.Helper()
+		select {
+		case input <- event:
+		case <-time.After(time.Second):
+			t.Fatal("debouncer did not receive the test event")
+		}
+	}
+
+	send(service(watch.Added, "initial"))
+	select {
+	case event := <-d.output:
+		if event.Type != watch.Added {
+			t.Fatalf("expected initial Added event, got %s", event.Type)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("debouncer did not emit the initial event")
+	}
+
+	eventNs, exists := d.getNs("default")
+	if !exists {
+		t.Fatal("debouncer did not create the namespace map")
+	}
+	oldObject, exists := eventNs.get("example")
+	if !exists {
+		t.Fatal("debouncer did not create the object")
+	}
+
+	send(service(watch.Deleted, "deleted"))
+	select {
+	case event := <-d.output:
+		if event.Type != watch.Deleted {
+			t.Fatalf("expected Deleted event, got %s", event.Type)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("debouncer did not emit the Deleted event")
+	}
+
+	select {
+	case <-oldObject.stopChan:
+	case <-time.After(time.Second):
+		t.Fatal("object did not self-terminate")
+	}
+	if _, exists := eventNs.get("example"); exists {
+		t.Fatal("self-terminated object remained in the namespace map")
+	}
+
+	send(service(watch.Modified, "fresh"))
+	select {
+	case event := <-d.output:
+		if event.Type != watch.Modified {
+			t.Fatalf("expected fresh Modified event, got %s", event.Type)
+		}
+		service, ok := event.Object.(*v1.Service)
+		if !ok {
+			t.Fatalf("expected a Service event, got %T", event.Object)
+		}
+		if service.ResourceVersion != "fresh" {
+			t.Fatalf("expected the fresh event, got resource version %q", service.ResourceVersion)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("debouncer did not process the fresh event after object deletion")
+	}
+
+	cancel()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Fatalf("debouncer returned an error: %s", err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("debouncer did not stop")
+	}
+}

pkg/debouncer/debouncer_test.go

@@ -0,0 +1,394 @@
+package debouncer
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+)
+
+func TestTimeSetting(t *testing.T) {
+	tcs := []struct {
+		name       string
+		configured string
+		expected   string
+	}{
+		{
+			name:       "configured proper value 10s",
+			configured: "10s",
+			expected:   "10s",
+		},
+		{
+			name:       "configured value less than 200ms",
+			configured: "0s",
+			expected:   "disabled",
+		},
+		{
+			name:       "configured proper value 1s",
+			configured: "1s",
+			expected:   "1s",
+		},
+		{
+			name:       "configured proper value 1500ms",
+			configured: "1500ms",
+			expected:   "1.5s",
+		},
+		{
+			name:       "configured to value greater than 0s but lower than 200ms",
+			configured: "150ms",
+			expected:   "disabled",
+		},
+		{
+			name:       "configured invalid value that cannot be parsed",
+			configured: "invalid",
+			expected:   "error",
+		},
+		{
+			name:       "configured negative value",
+			configured: "-1s",
+			expected:   "disabled",
+		},
+	}
+
+	input := make(chan watch.Event)
+	defer close(input)
+
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			d, err := New(input, tc.configured)
+
+			switch tc.expected {
+			case "disabled":
+				if err != nil {
+					t.Fatalf("failed to create debouncer with debounce time %q", tc.configured)
+				}
+				if d != nil {
+					t.Fatalf("debouncer was created but should be disabled for value %q", tc.configured)
+				}
+			case "error":
+				if err == nil {
+					t.Fatalf("debouncer was created but should error for value %q", tc.configured)
+				}
+			default:
+				if d == nil {
+					t.Fatalf("debouncer was not created for value %q", tc.configured)
+				}
+
+				if d.debounceTime.String() != tc.expected {
+					t.Fatalf("invalid debounce time %q was configured instead of expected %q", d.debounceTime.String(), tc.expected)
+				}
+			}
+
+		})
+	}
+}
+
+func TestStartStop(t *testing.T) {
+	t.Run("Run and stop the debouncer without issues", func(t *testing.T) {
+		input := make(chan watch.Event)
+		defer close(input)
+
+		expected := "200ms"
+
+		d, err := New(input, expected)
+
+		if err != nil {
+			t.Fatalf("failed to create debouncer with debounce time %q", expected)
+		}
+
+		if d.debounceTime.String() != expected {
+			t.Fatalf("invalid debounce time %q was configured instead of expected %q", d.debounceTime.String(), expected)
+		}
+
+		ctx, cancel := context.WithCancel(context.Background())
+
+		wg := sync.WaitGroup{}
+
+		wg.Go(func() {
+			if err := d.Start(ctx); err != nil {
+				t.Fatalf("debouncer error: %s", err.Error())
+			}
+		})
+
+		cancel()
+
+		timedOut := waitTimeout(&wg, time.Second*3)
+
+		if timedOut {
+			t.Fatal("debouncer was not closed before timeout")
+		}
+	})
+}
+
+func TestDebouncing(t *testing.T) {
+	tcs := []string{"endpointslices", "endpoints", "services"}
+
+	for _, tc := range tcs {
+		t.Run(fmt.Sprintf("Get the newest event as the only one when using %s", tc), func(t *testing.T) {
+			expected := "500ms"
+
+			fw := watch.NewFake()
+			defer fw.Stop()
+
+			d, err := New(fw.ResultChan(), expected)
+
+			if err != nil {
+				t.Fatalf("failed to create debouncer with debounce time %q", expected)
+			}
+
+			if d.debounceTime.String() != expected {
+				t.Fatalf("invalid debounce time %q was configured instead of expected %q", d.debounceTime.String(), expected)
+			}
+
+			ctx, cancel := context.WithCancel(context.Background())
+
+			wg := sync.WaitGroup{}
+
+			wg.Go(func() {
+				if err := d.Start(ctx); err != nil {
+					t.Fatalf("debouncer error: %s", err.Error())
+				}
+			})
+
+			numOfUpdates := 100
+
+			switch tc {
+			case "endpointslices":
+				epslice := &discoveryv1.EndpointSlice{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "test",
+						Namespace: "test",
+					},
+					Endpoints: make([]discoveryv1.Endpoint, 1),
+				}
+
+				addrEpslices := []string{}
+
+				for i := range numOfUpdates {
+					addrEpslices = append(addrEpslices, strconv.Itoa(i))
+					epslice.Endpoints[0].Addresses = addrEpslices
+					fw.Add(epslice)
+				}
+			case "endpoints":
+				ep := &v1.Endpoints{ //nolint:staticcheck
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "test",
+						Namespace: "test",
+					},
+					Subsets: make([]v1.EndpointSubset, 1), //nolint:staticcheck
+				}
+
+				addrEp := []v1.EndpointAddress{}
+
+				for i := range numOfUpdates {
+					addrEp = append(addrEp, v1.EndpointAddress{IP: strconv.Itoa(i)})
+					ep.Subsets[0].Addresses = addrEp
+					fw.Add(ep)
+				}
+			case "services":
+				svc := &v1.Service{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "test",
+						Namespace: "test",
+					},
+				}
+
+				svcPorts := []v1.ServicePort{}
+
+				for i := range numOfUpdates {
+					svcPorts = append(svcPorts, v1.ServicePort{Port: int32(i)})
+					svc.Spec.Ports = svcPorts
+					fw.Add(svc)
+				}
+
+			default:
+				t.Fatal("unknown test", "type", tc)
+			}
+
+			out := <-d.output
+
+			switch tc {
+			case "endpointslices":
+				outEps, ok := out.Object.(*discoveryv1.EndpointSlice)
+				if !ok {
+					t.Fatal("got different type of object than EndpointSlice, failed to cast")
+				}
+
+				if len(outEps.Endpoints[0].Addresses) != numOfUpdates {
+					t.Fatalf("expected to aggregate %d events, but got %d", numOfUpdates, len(outEps.Endpoints[0].Addresses))
+				}
+			case "endpoints":
+				outEps, ok := out.Object.(*v1.Endpoints) //nolint:staticcheck
+				if !ok {
+					t.Fatal("got different type of object than EndpointSlice, failed to cast")
+				}
+
+				if len(outEps.Subsets[0].Addresses) != numOfUpdates {
+					t.Fatalf("expected to aggregate %d events, but got %d", numOfUpdates, len(outEps.Subsets[0].Addresses))
+				}
+			case "services":
+				outSvc, ok := out.Object.(*v1.Service) //nolint:staticcheck
+				if !ok {
+					t.Fatal("got different type of object than EndpointSlice, failed to cast")
+				}
+
+				if len(outSvc.Spec.Ports) != numOfUpdates {
+					t.Fatalf("expected to aggregate %d events, but got %d", numOfUpdates, len(outSvc.Spec.Ports))
+				}
+			default:
+				t.Fatal("unknown test", "type", tc)
+			}
+
+			cancel()
+
+			timedOut := waitTimeout(&wg, time.Second*3)
+
+			if timedOut {
+				t.Fatal("debouncer was not closed before timeout")
+			}
+		})
+	}
+}
+
+func TestTypeChange(t *testing.T) {
+	tcs := []string{"endpointslices", "endpoints", "services"}
+
+	for _, tc := range tcs {
+		t.Run(fmt.Sprintf("Get the newest event as the only one when using %s", tc), func(t *testing.T) {
+			expected := "500ms"
+
+			fw := watch.NewFake()
+			defer fw.Stop()
+
+			d, err := New(fw.ResultChan(), expected)
+
+			if err != nil {
+				t.Fatalf("failed to create debouncer with debounce time %q", expected)
+			}
+
+			if d.debounceTime.String() != expected {
+				t.Fatalf("invalid debounce time %q was configured instead of expected %q", d.debounceTime.String(), expected)
+			}
+
+			ctx, cancel := context.WithCancel(context.Background())
+
+			wg := sync.WaitGroup{}
+
+			wg.Go(func() {
+				if err := d.Start(ctx); err != nil {
+					t.Fatalf("debouncer error: %s", err.Error())
+				}
+			})
+
+			numOfUpdates := 100
+
+			switch tc {
+			case "endpointslices":
+				epslice := &discoveryv1.EndpointSlice{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "test",
+						Namespace: "test",
+					},
+					Endpoints: make([]discoveryv1.Endpoint, 1),
+				}
+
+				addrEpslices := []string{}
+
+				for i := range numOfUpdates {
+					addrEpslices = append(addrEpslices, strconv.Itoa(i))
+					epslice.Endpoints[0].Addresses = addrEpslices
+					if i < numOfUpdates-1 {
+						fw.Add(epslice)
+					} else {
+						fw.Delete(epslice)
+					}
+				}
+			case "endpoints":
+				ep := &v1.Endpoints{ //nolint:staticcheck
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "test",
+						Namespace: "test",
+					},
+					Subsets: make([]v1.EndpointSubset, 1), //nolint:staticcheck
+				}
+
+				addrEp := []v1.EndpointAddress{}
+
+				for i := range numOfUpdates {
+					addrEp = append(addrEp, v1.EndpointAddress{IP: strconv.Itoa(i)})
+					ep.Subsets[0].Addresses = addrEp
+					if i < numOfUpdates-1 {
+						fw.Add(ep)
+					} else {
+						fw.Delete(ep)
+					}
+				}
+			case "services":
+				svc := &v1.Service{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "test",
+						Namespace: "test",
+					},
+				}
+
+				svcPorts := []v1.ServicePort{}
+
+				for i := range numOfUpdates {
+					svcPorts = append(svcPorts, v1.ServicePort{Port: int32(i)})
+					svc.Spec.Ports = svcPorts
+					if i < numOfUpdates-1 {
+						fw.Add(svc)
+					} else {
+						fw.Delete(svc)
+					}
+				}
+
+			default:
+				t.Fatal("unknown test", "type", tc)
+			}
+
+			out := <-d.output
+
+			if out.Type != watch.Added {
+				t.Fatalf("expected to get add event, but got %s event", out.Type)
+			}
+
+			out = <-d.output
+
+			if out.Type != watch.Deleted {
+				t.Fatalf("expected to get delete event, but got %s event", out.Type)
+			}
+
+			cancel()
+
+			timedOut := waitTimeout(&wg, time.Second*3)
+
+			if timedOut {
+				t.Fatal("debouncer was not closed before timeout")
+			}
+		})
+	}
+}
+
+// waitTimeout waits for the waitgroup for the specified max timeout.
+// Returns true if waiting timed out.
+func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
+	c := make(chan struct{})
+	go func() {
+		defer close(c)
+		wg.Wait()
+	}()
+	select {
+	case <-c:
+		return false // completed normally
+	case <-time.After(timeout):
+		return true // timed out
+	}
+}

pkg/detector/interfaces.go

@@ -22,12 +22,12 @@ func FindIPAddress(addrName string) (string, string, error) {
 		}
 		for _, a := range addrs {
 			if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
-				if ipnet.IP.To4() != nil {
+				if ipnet.IP.To16() != nil {
 					address = ipnet.IP.String()
 					// If we're not searching for a specific adapter return the first one
 					if addrName == "" {
 						return iface.Name, address, nil
-					} else
+					}
 					// If this is the correct adapter return the details
 					if iface.Name == addrName {
 						return iface.Name, address, nil
@@ -37,5 +37,5 @@ func FindIPAddress(addrName string) (string, string, error) {
 		}
 
 	}
-	return "", "", fmt.Errorf("Unknown interface [%s]", addrName)
+	return "", "", fmt.Errorf("unknown interface [%s]", addrName)
 }

pkg/egress/egress.go

@@ -0,0 +1,98 @@
+package egress
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/kube-vip/kube-vip/pkg/iptables"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/nftables"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
+
+func Teardown(podIP, vipIP, namespace, serviceUUID string, annotations map[string]string, useNftables bool) error {
+	// Look up the destination ports from the annotations on the service
+	destinationPorts := annotations[kubevip.EgressDestinationPorts]
+	deniedNetworks := annotations[kubevip.EgressDeniedNetworks]
+	allowedNetworks := annotations[kubevip.EgressAllowedNetworks]
+	internalEgress := annotations[kubevip.EgressInternal]
+
+	protocol := iptables.ProtocolIPv4
+	if utils.IsIPv6(podIP) {
+		protocol = iptables.ProtocolIPv6
+	}
+
+	// Use the internal egress implementation
+	if internalEgress != "" || useNftables {
+		return nftables.DeleteSNATFromAllTables(serviceUUID)
+	}
+
+	i, err := vip.CreateIptablesClient(useNftables, namespace, protocol)
+	if err != nil {
+		return fmt.Errorf("error Creating iptables client [%s]", err)
+	}
+
+	if deniedNetworks != "" {
+		networks := strings.Split(deniedNetworks, ",")
+		for x := range networks {
+			err = i.DeleteMangleReturnForNetwork(vip.MangleChainName, networks[x])
+			if err != nil {
+				return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
+			}
+		}
+	}
+
+	if allowedNetworks != "" {
+		networks := strings.Split(allowedNetworks, ",")
+		for x := range networks {
+			err = i.DeleteMangleMarkingForNetwork(podIP, vip.MangleChainName, networks[x])
+			if err != nil {
+				return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
+			}
+		}
+	} else {
+		// Remove the marking of egress packets
+		err = i.DeleteMangleMarking(podIP, vip.MangleChainName)
+		if err != nil {
+			return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+		}
+	}
+
+	// Clear up SNAT rules
+	if destinationPorts != "" {
+		fixedPorts := strings.Split(destinationPorts, ",")
+
+		for _, fixedPort := range fixedPorts {
+			var proto, port string
+
+			data := strings.Split(fixedPort, ":")
+			if len(data) == 0 {
+				continue
+			} else if len(data) == 1 {
+				proto = "tcp"
+				port = data[0]
+			} else {
+				proto = data[0]
+				port = data[1]
+			}
+
+			err = i.DeleteSourceNatForDestinationPort(podIP, vipIP, port, proto)
+			if err != nil {
+				return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+			}
+
+		}
+	} else {
+		err = i.DeleteSourceNat(podIP, vipIP)
+		if err != nil {
+			return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+		}
+	}
+
+	err = vip.DeleteExistingSessions(podIP, false, destinationPorts, "")
+	if err != nil {
+		return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+	}
+	return nil
+}

pkg/election/election.go

@@ -0,0 +1,268 @@
+package election
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/etcd"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/lease"
+	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	watchtools "k8s.io/client-go/tools/watch"
+)
+
+type Manager struct {
+	KubernetesClient   *kubernetes.Clientset
+	RetryWatcherClient *kubernetes.Clientset
+	// This channel is used to signal a shutdown
+
+	EtcdClient *clientv3.Client
+}
+
+// NewManager will create a new managing object
+func NewManager(config *kubevip.Config, k8sClientset, rwClientset *kubernetes.Clientset) (*Manager, error) {
+	m := &Manager{}
+
+	switch config.LeaderElectionType {
+	case "kubernetes", "":
+		if k8sClientset == nil || rwClientset == nil {
+			return nil, fmt.Errorf("provided nil clientset")
+		}
+		m.KubernetesClient = k8sClientset
+		m.RetryWatcherClient = rwClientset
+	case "etcd":
+		client, err := etcd.NewClient(config)
+		if err != nil {
+			return nil, err
+		}
+		m.EtcdClient = client
+	default:
+		return nil, fmt.Errorf("invalid LeaderElectionMode %s not supported", config.LeaderElectionType)
+	}
+
+	return m, nil
+}
+
+func RunOrDie(ctx context.Context, run *RunConfig, c *kubevip.Config) error {
+	switch c.LeaderElectionType {
+	case "kubernetes", "":
+		return runKubernetesLeaderElectionOrDie(ctx, run)
+	case "etcd":
+		if err := runEtcdLeaderElectionOrDie(ctx, run); err != nil {
+			return err
+		}
+	default:
+		log.Info("LeaderElectionMode not supported, exiting", "mode", c.LeaderElectionType)
+	}
+
+	return nil
+}
+
+func runKubernetesLeaderElectionOrDie(ctx context.Context, run *RunConfig) error {
+	annotations, err := kubevip.WithLeaseVIPs(run.LeaseAnnotations, run.Config.InstanceName, run.Config.RoutingProtocol, run.VIPs)
+	if err != nil {
+		return err
+	}
+	leaseClient := run.Mgr.KubernetesClient.CoordinationV1().Leases(run.LeaseID.Namespace())
+	// we use the Lease lock type since edits to Leases are less common
+	// and fewer objects in the cluster watch "all Leases".
+	baseLock := &resourcelock.LeaseLock{
+		LeaseMeta: metav1.ObjectMeta{
+			Name:      run.LeaseID.Name(),
+			Namespace: run.LeaseID.Namespace(),
+		},
+		Client: run.Mgr.KubernetesClient.CoordinationV1(),
+		LockConfig: resourcelock.ResourceLockConfig{
+			Identity: run.Config.NodeName,
+		},
+	}
+	lock := newAnnotatedLeaseLock(baseLock, leaseClient, run.LeaseID.Name(), annotations)
+
+	// start the leader election code loop
+	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
+		Lock: lock,
+		// IMPORTANT: you MUST ensure that any code you have that
+		// is protected by the lease must terminate **before**
+		// you call cancel. Otherwise, you could have a background
+		// loop still running and another process could
+		// get elected before your background loop finished, violating
+		// the stated goal of the lease.
+		ReleaseOnCancel: true,
+		LeaseDuration:   time.Duration(run.Config.LeaseDuration) * time.Second,
+		RenewDeadline:   time.Duration(run.Config.RenewDeadline) * time.Second,
+		RetryPeriod:     time.Duration(run.Config.RetryPeriod) * time.Second,
+		Callbacks: leaderelection.LeaderCallbacks{
+			OnStartedLeading: run.OnStartedLeading,
+			OnStoppedLeading: run.OnStoppedLeading,
+			OnNewLeader:      run.OnNewLeader,
+		},
+	})
+	return nil
+}
+
+func runEtcdLeaderElectionOrDie(ctx context.Context, run *RunConfig) error {
+	if err := etcd.RunElectionOrDie(ctx, &etcd.LeaderElectionConfig{
+		EtcdConfig:           etcd.ClientConfig{Client: run.Mgr.EtcdClient},
+		Name:                 run.LeaseID.NamespacedName(),
+		MemberID:             run.Config.NodeName,
+		LeaseDurationSeconds: int64(run.Config.LeaseDuration),
+		Callbacks: etcd.LeaderCallbacks{
+			OnStartedLeading: run.OnStartedLeading,
+			OnStoppedLeading: run.OnStoppedLeading,
+			OnNewLeader:      run.OnNewLeader,
+		},
+	}); err != nil {
+		return fmt.Errorf("etcd leaderelection: %w", err)
+	}
+	return nil
+}
+
+type Actions interface {
+	OnStartedLeading(ctx context.Context)
+	OnStoppedLeading()
+	OnNewLeader(identity string)
+}
+
+type RunConfig struct {
+	Config           *kubevip.Config
+	LeaseID          lease.ID
+	Mgr              *Manager
+	LeaseAnnotations map[string]string
+	VIPs             []string
+
+	// onStartedLeading is called when this member starts leading.
+	OnStartedLeading func(context.Context)
+	// onStoppedLeading is called when this member stops leading.
+	OnStoppedLeading func()
+	// onNewLeader is called when the client observes a leader that is
+	// not the previously observed leader. This includes the first observed
+	// leader when the client starts.
+	OnNewLeader func(identity string)
+}
+
+func (em *Manager) NodeWatcher(ctx context.Context, lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
+	// Use a restartable watcher, as this should help in the event of etcd or timeout issues
+	log.Info("Kube-Vip is watching nodes for control-plane labels")
+
+	listOptions := metav1.ListOptions{
+		LabelSelector: "node-role.kubernetes.io/control-plane",
+	}
+
+	wg := sync.WaitGroup{}
+	defer wg.Wait()
+
+	watchCtx, watchCancel := context.WithCancel(ctx)
+	defer watchCancel()
+
+	rw, err := watchtools.NewRetryWatcherWithContext(watchCtx, "1", &cache.ListWatch{
+		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
+			return utils.WatchWithAuthRetry(ctx, func(ctx context.Context) (watch.Interface, error) {
+				return em.RetryWatcherClient.CoreV1().Nodes().Watch(watchCtx, listOptions)
+			})
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("error creating label watcher: %w", err)
+	}
+
+	wg.Go(func() {
+		<-watchCtx.Done()
+		log.Info("Node watcher context cancelled, stopping")
+		// Stop the retrywatcher
+		rw.Stop()
+	})
+
+	ch := rw.ResultChan()
+
+	var watchErr error
+	for event := range ch {
+		// We need to inspect the event and get ResourceVersion out of it
+		switch event.Type {
+		case watch.Added, watch.Modified:
+			node, ok := event.Object.(*v1.Node)
+			if !ok {
+				return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
+			}
+			// Find the node IP address (this isn't foolproof)
+			for x := range node.Status.Addresses {
+				if node.Status.Addresses[x].Type == v1.NodeInternalIP {
+					if checkIfNodeIsReady(node) {
+						err = lb.AddBackend(node.Status.Addresses[x].Address, port)
+						if err != nil {
+							log.Error("adding node to load balancer", "node", node.Name, "ip", node.Status.Addresses[x].Address, "err", err)
+							if utils.IsPanicError(err) {
+								return fmt.Errorf("add IPVS backend: %w", err)
+							}
+						}
+					} else {
+						err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
+						if err != nil {
+							log.Error("removing node from load balancer", "node", node.Name, "ip", node.Status.Addresses[x].Address, "err", err)
+						}
+					}
+				}
+			}
+		case watch.Deleted:
+			node, ok := event.Object.(*v1.Node)
+			if !ok {
+				return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
+			}
+
+			// Find the node IP address (this isn't foolproof)
+			for x := range node.Status.Addresses {
+				if node.Status.Addresses[x].Type == v1.NodeInternalIP {
+					err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
+					if err != nil {
+						log.Error("removing node from load balancer", "node", node.Name, "ip", node.Status.Addresses[x].Address, "err", err)
+					}
+				}
+			}
+
+			log.Info("Node deleted", "name", node.Name)
+
+		case watch.Bookmark:
+			// Un-used
+		case watch.Error:
+			log.Error("Error attempting to watch Kubernetes Nodes")
+			watchErr = fmt.Errorf("node watcher error: %w", utils.WatchError(event.Object))
+			log.Error("watcher", "err", watchErr)
+		default:
+		}
+	}
+
+	log.Info("Exiting Node watcher")
+	if watchErr != nil {
+		return watchErr
+	}
+	if ctx.Err() != nil {
+		return nil
+	}
+	return utils.NewPanicError("node watcher channel closed unexpectedly")
+}
+
+func checkIfNodeIsReady(node *v1.Node) bool {
+	if node == nil {
+		return false
+	}
+	for _, condition := range node.Status.Conditions {
+		if condition.Type == v1.NodeReady {
+			if condition.Status == v1.ConditionTrue {
+				return true
+			}
+		}
+	}
+	return false
+}

pkg/election/lease_lock.go

@@ -0,0 +1,90 @@
+package election
+
+import (
+	"context"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	coordinationv1client "k8s.io/client-go/kubernetes/typed/coordination/v1"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	"k8s.io/client-go/util/retry"
+)
+
+type annotatedLeaseLock struct {
+	resourcelock.Interface
+	leases      coordinationv1client.LeaseInterface
+	name        string
+	annotations map[string]string
+}
+
+func newAnnotatedLeaseLock(lock resourcelock.Interface, leases coordinationv1client.LeaseInterface,
+	name string, annotations map[string]string) resourcelock.Interface {
+	return &annotatedLeaseLock{Interface: lock, leases: leases, name: name, annotations: annotations}
+}
+
+func (lock *annotatedLeaseLock) Get(ctx context.Context) (*resourcelock.LeaderElectionRecord, []byte, error) {
+	return lock.Interface.Get(ctx)
+}
+
+func (lock *annotatedLeaseLock) Create(ctx context.Context, record resourcelock.LeaderElectionRecord) error {
+	if err := lock.Interface.Create(ctx, record); err != nil {
+		return err
+	}
+	if record.HolderIdentity != lock.Identity() {
+		return nil
+	}
+	changed, err := lock.ensureAnnotations(ctx)
+	if err != nil {
+		return err
+	}
+	if changed {
+		_, _, err = lock.Interface.Get(ctx)
+	}
+	return err
+}
+
+func (lock *annotatedLeaseLock) Update(ctx context.Context, record resourcelock.LeaderElectionRecord) error {
+	if err := lock.Interface.Update(ctx, record); err != nil {
+		return err
+	}
+	if record.HolderIdentity != lock.Identity() {
+		return nil
+	}
+	changed, err := lock.ensureAnnotations(ctx)
+	if err != nil {
+		return err
+	}
+	if changed {
+		_, _, err = lock.Interface.Get(ctx)
+	}
+	return err
+}
+
+func (lock *annotatedLeaseLock) ensureAnnotations(ctx context.Context) (bool, error) {
+	changed := false
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		resource, err := lock.leases.Get(ctx, lock.name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		if resource.Annotations == nil {
+			resource.Annotations = make(map[string]string, len(lock.annotations))
+		}
+		resourceChanged := false
+		for key, value := range lock.annotations {
+			if resource.Annotations[key] == value {
+				continue
+			}
+			resource.Annotations[key] = value
+			resourceChanged = true
+		}
+		if !resourceChanged {
+			return nil
+		}
+		_, err = lock.leases.Update(ctx, resource, metav1.UpdateOptions{})
+		if err == nil {
+			changed = true
+		}
+		return err
+	})
+	return changed, err
+}

pkg/election/lease_lock_test.go

@@ -0,0 +1,139 @@
+package election
+
+import (
+	"context"
+	"testing"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+)
+
+func TestAnnotatedLeaseLockPersistsAnnotationsOnCreateAndUpdate(t *testing.T) {
+	client := fake.NewSimpleClientset()
+	leaseClient := client.CoordinationV1().Leases("default")
+	base := &resourcelock.LeaseLock{
+		LeaseMeta: metav1.ObjectMeta{Name: "lease", Namespace: "default"},
+		Client:    client.CoordinationV1(),
+		LockConfig: resourcelock.ResourceLockConfig{
+			Identity: "node-a",
+		},
+	}
+	annotations, err := kubevip.WithLeaseVIPs(map[string]string{"example.test/preserved": "true"},
+		"release_a", 248, []string{"192.0.2.10"})
+	if err != nil {
+		t.Fatalf("WithLeaseVIPs() error = %v", err)
+	}
+	lock := newAnnotatedLeaseLock(base, leaseClient, "lease", annotations)
+	record := resourcelock.LeaderElectionRecord{HolderIdentity: "node-a"}
+	if err := lock.Create(context.Background(), record); err != nil {
+		t.Fatalf("Create() error = %v", err)
+	}
+	if err := lock.Update(context.Background(), record); err != nil {
+		t.Fatalf("Update() error = %v", err)
+	}
+
+	resource, err := leaseClient.Get(context.Background(), "lease", metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("get Lease: %v", err)
+	}
+	value, err := kubevip.ParseLeaseVIPs(resource.Annotations[kubevip.LeaseVIPs])
+	if err != nil {
+		t.Fatalf("ParseLeaseVIPs() error = %v", err)
+	}
+	if value.InstanceName != "release_a" || value.IFAProto != 248 || len(value.VIPs) != 1 ||
+		value.VIPs[0] != (kubevip.LeaseVIP{Index: 0, Value: "192.0.2.10"}) {
+		t.Fatalf("Lease VIP metadata = %+v", value)
+	}
+	if resource.Annotations["example.test/preserved"] != "true" {
+		t.Fatal("Lease update dropped a configured annotation")
+	}
+}
+
+func TestAnnotatedLeaseLockFollowerDoesNotOverwriteAnnotations(t *testing.T) {
+	client := fake.NewSimpleClientset()
+	leaseClient := client.CoordinationV1().Leases("default")
+	newBase := func(identity string) *resourcelock.LeaseLock {
+		return &resourcelock.LeaseLock{
+			LeaseMeta:  metav1.ObjectMeta{Name: "lease", Namespace: "default"},
+			Client:     client.CoordinationV1(),
+			LockConfig: resourcelock.ResourceLockConfig{Identity: identity},
+		}
+	}
+	ownerBase := newBase("node-a")
+	followerBase := newBase("node-b")
+	active, err := kubevip.WithLeaseVIPs(nil, "release_a", 248, []string{"192.0.2.10"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	creator := newAnnotatedLeaseLock(ownerBase, leaseClient, "lease", active)
+	if err := creator.Create(context.Background(), resourcelock.LeaderElectionRecord{HolderIdentity: "node-a"}); err != nil {
+		t.Fatalf("Create() error = %v", err)
+	}
+
+	follower, err := kubevip.WithLeaseVIPs(nil, "release_b", 249, []string{"192.0.2.20"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	observer := newAnnotatedLeaseLock(followerBase, leaseClient, "lease", follower)
+	if _, _, err := observer.Get(context.Background()); err != nil {
+		t.Fatalf("Get() error = %v", err)
+	}
+	resource, err := leaseClient.Get(context.Background(), "lease", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	metadata, err := kubevip.ParseLeaseVIPs(resource.Annotations[kubevip.LeaseVIPs])
+	if err != nil {
+		t.Fatal(err)
+	}
+	if metadata.InstanceName != "release_a" || metadata.IFAProto != 248 {
+		t.Fatalf("follower overwrote active metadata: %+v", metadata)
+	}
+}
+
+func TestAnnotatedLeaseLockReleaseDoesNotOverwriteSuccessorMetadata(t *testing.T) {
+	client := fake.NewSimpleClientset()
+	leaseClient := client.CoordinationV1().Leases("default")
+	newLock := func(identity, instanceName string, protocol int, vip string) resourcelock.Interface {
+		base := &resourcelock.LeaseLock{
+			LeaseMeta:  metav1.ObjectMeta{Name: "lease", Namespace: "default"},
+			Client:     client.CoordinationV1(),
+			LockConfig: resourcelock.ResourceLockConfig{Identity: identity},
+		}
+		annotations, err := kubevip.WithLeaseVIPs(nil, instanceName, protocol, []string{vip})
+		if err != nil {
+			t.Fatal(err)
+		}
+		return newAnnotatedLeaseLock(base, leaseClient, "lease", annotations)
+	}
+
+	first := newLock("node-a", "release_a", 248, "192.0.2.10")
+	if err := first.Create(context.Background(), resourcelock.LeaderElectionRecord{HolderIdentity: "node-a"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := first.Update(context.Background(), resourcelock.LeaderElectionRecord{}); err != nil {
+		t.Fatal(err)
+	}
+
+	second := newLock("node-b", "release_b", 249, "192.0.2.20")
+	if _, _, err := second.Get(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if err := second.Update(context.Background(), resourcelock.LeaderElectionRecord{HolderIdentity: "node-b"}); err != nil {
+		t.Fatal(err)
+	}
+
+	resource, err := leaseClient.Get(context.Background(), "lease", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	metadata, err := kubevip.ParseLeaseVIPs(resource.Annotations[kubevip.LeaseVIPs])
+	if err != nil {
+		t.Fatal(err)
+	}
+	if metadata.InstanceName != "release_b" || metadata.IFAProto != 249 || metadata.VIPs[0].Value != "192.0.2.20" {
+		t.Fatalf("successor metadata = %+v", metadata)
+	}
+}

pkg/endpoints/cleanup.go

@@ -0,0 +1,152 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	log "log/slog"
+
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/egress"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/lease"
+	"github.com/kube-vip/kube-vip/pkg/nftables"
+	"github.com/kube-vip/kube-vip/pkg/route"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/wireguard"
+)
+
+// CleanupService stops one Service's datapath before its instance is detached.
+// The service processor owns labels and instance bookkeeping; this package owns
+// endpoint-dependent networking and waits for worker shutdown to complete.
+func CleanupService(ctx context.Context, config *kubevip.Config, bgpServer *bgp.Server, routeMgr *route.Manager,
+	tunnelMgr *wireguard.TunnelManager, serviceInstance *instance.Instance, remaining []*instance.Instance) error {
+	if serviceInstance == nil || serviceInstance.ServiceSnapshot == nil {
+		return nil
+	}
+	service := serviceInstance.ServiceSnapshot
+	for _, serviceCluster := range serviceInstance.Clusters {
+		for _, network := range serviceCluster.Network {
+			network.SetHasEndpoints(false)
+		}
+	}
+
+	if config.EnableBGP {
+		ClearBGPHostsByInstance(ctx, serviceInstance, bgpServer)
+	}
+	if config.EnableRoutingTable {
+		for _, err := range ClearRoutesByInstance(service, serviceInstance, &remaining, routeMgr) {
+			log.Error("unable to clear routes", "err", err)
+		}
+	}
+
+	internalNftablesEgress := service.Annotations[kubevip.EgressInternal] != "" || config.EgressWithNftables
+	if service.Annotations[kubevip.Egress] == "true" && internalNftablesEgress {
+		if err := nftables.DeleteSNATFromAllTables(string(serviceInstance.UID())); err != nil {
+			log.Error("[service] nftables egress teardown", "service", service.Name, "err", err)
+		}
+	}
+
+	sharedVIPs := sharedServiceVIPs(config, serviceInstance, remaining)
+	for _, serviceCluster := range serviceInstance.Clusters {
+		preserve := make([]string, 0, len(serviceCluster.Network))
+		for _, network := range serviceCluster.Network {
+			if _, shared := sharedVIPs[network.IP()]; shared {
+				preserve = append(preserve, network.IP())
+			}
+		}
+		if len(preserve) != 0 {
+			serviceCluster.StopAndWaitPreserving(preserve...)
+		} else {
+			serviceCluster.StopAndWait()
+		}
+	}
+	if err := serviceInstance.CleanupLinkAttachments(remaining...); err != nil {
+		return fmt.Errorf("clean Service link attachments: %w", err)
+	}
+	if service.Annotations[kubevip.Egress] == "true" && !internalNftablesEgress && service.Annotations[kubevip.ActiveEndpoint] != "" {
+		if err := egress.Teardown(service.Annotations[kubevip.ActiveEndpoint], service.Spec.LoadBalancerIP, service.Namespace,
+			string(serviceInstance.UID()), service.Annotations, config.EgressWithNftables); err != nil {
+			log.Error("[service] egress teardown", "err", err)
+		}
+	}
+	if config.EnableWireguard {
+		cleanupWireguardService(tunnelMgr, service)
+	}
+	return nil
+}
+
+// StartService starts a Service's cluster datapath after endpoint handling has
+// made the Service eligible for activation.
+func StartService(ctx context.Context, service *v1.Service, serviceInstance *instance.Instance, bgpServer *bgp.Server,
+	wg *sync.WaitGroup) error {
+	if serviceInstance == nil {
+		return fmt.Errorf("missing service instance for %s/%s", service.Namespace, service.Name)
+	}
+	for index := range serviceInstance.VIPConfigs {
+		if err := serviceInstance.Clusters[index].StartLoadBalancerService(ctx, serviceInstance.VIPConfigs[index], bgpServer,
+			lease.ServiceNamespacedName(service), wg); err != nil {
+			return fmt.Errorf("start load balancer: %w", err)
+		}
+	}
+	return nil
+}
+
+func sharedServiceVIPs(config *kubevip.Config, serviceInstance *instance.Instance, remaining []*instance.Instance) map[string]struct{} {
+	shared := make(map[string]struct{})
+	if serviceInstance.ServiceSnapshot == nil ||
+		serviceInstance.ServiceSnapshot.Spec.ExternalTrafficPolicy != v1.ServiceExternalTrafficPolicyTypeCluster {
+		return shared
+	}
+	serviceNamespace, serviceLeaseName := lease.ServiceName(serviceInstance.ServiceSnapshot)
+	serviceLease := lease.NewID(config.LeaderElectionType, serviceNamespace, serviceLeaseName).NamespacedName()
+	addresses := serviceInstance.Addresses()
+	for _, candidate := range remaining {
+		candidateInfo, ok := candidate.CleanupInfo()
+		if !ok || candidateInfo.ExternalTrafficPolicy != v1.ServiceExternalTrafficPolicyTypeCluster {
+			continue
+		}
+		candidateNamespace, candidateLeaseName := lease.ServiceNameFor(candidateInfo.Namespace, candidateInfo.Name, candidateInfo.Lease)
+		candidateLease := lease.NewID(config.LeaderElectionType, candidateNamespace, candidateLeaseName).NamespacedName()
+		if config.EnableServicesElection && candidateLease != serviceLease {
+			continue
+		}
+		for _, address := range candidate.Addresses() {
+			for _, serviceAddress := range addresses {
+				if address == serviceAddress {
+					shared[address] = struct{}{}
+				}
+			}
+		}
+	}
+	return shared
+}
+
+func cleanupWireguardService(tunnelMgr *wireguard.TunnelManager, service *v1.Service) {
+	if tunnelMgr == nil {
+		return
+	}
+	forEachServiceDNATChain(service, func(ipv6 bool, serviceID string) {
+		if err := nftables.DeleteIngressChains(ipv6, serviceID); err != nil {
+			log.Error("[wireguard] failed to delete DNAT chains", "ipv6", ipv6, "service", service.Name, "err", err)
+		}
+	})
+	releaseWireguardServiceTunnels(tunnelMgr, service)
+}
+
+type wireguardTunnelReleaser interface {
+	ReleaseTunnelForVIP(vip, owner string) error
+}
+
+func releaseWireguardServiceTunnels(tunnelMgr wireguardTunnelReleaser, service *v1.Service) {
+	serviceIPs, _ := utils.FetchServiceIPs(service)
+	for _, serviceIP := range serviceIPs {
+		if err := tunnelMgr.ReleaseTunnelForVIP(serviceIP, string(service.UID)); err != nil {
+			log.Error("[wireguard] failed to tear down tunnel", "service", service.Name, "vip", serviceIP, "err", err)
+		}
+	}
+}

pkg/endpoints/endpoints.go

@@ -0,0 +1,355 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strings"
+	"sync"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/lease"
+	"github.com/kube-vip/kube-vip/pkg/route"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/wireguard"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+)
+
+type Processor struct {
+	config         *kubevip.Config
+	provider       providers.Provider
+	bgpServer      *bgp.Server
+	worker         endpointWorker
+	instances      *[]*instance.Instance
+	instancesMutex *sync.RWMutex
+	leaseMgr       *lease.Manager
+	lockService    func(types.UID) func()
+}
+
+func NewEndpointProcessor(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
+	instances *[]*instance.Instance, instancesMutex *sync.RWMutex, leaseMgr *lease.Manager, tunnelMgr *wireguard.TunnelManager, routeMgr *route.Manager,
+	lockService func(types.UID) func()) *Processor {
+	return &Processor{
+		config:         config,
+		provider:       provider,
+		bgpServer:      bgpServer,
+		instances:      instances,
+		instancesMutex: instancesMutex,
+		leaseMgr:       leaseMgr,
+		lockService:    lockService,
+		worker:         newEndpointWorker(config, provider, bgpServer, leaseMgr, tunnelMgr, routeMgr),
+	}
+}
+
+// Reconcile applies a watch event to the provider and reconciles the service
+// against the endpoints that remain afterwards. A deleted object is only one of
+// potentially several backing the service, so deletions are recomputed rather
+// than assumed to empty it. It reports whether the caller should skip this event
+// and wait for the next one.
+func (p *Processor) Reconcile(svcCtx *servicecontext.Context, event watch.Event,
+	lastKnownGoodEndpoint *string, service *v1.Service, id string,
+	wg *sync.WaitGroup,
+	clientSet *kubernetes.Clientset,
+	egressUpdateFunc func(context.Context, *v1.Service, *instance.Instance) error) (bool, error) {
+	if p.lockService == nil {
+		return false, fmt.Errorf("service operation lock is not configured")
+	}
+	endpointCount := 0
+	var readinessLossGeneration uint64
+	clearNoEndpoints := false
+	updatedService, inst, changed, skip, err := func() (*v1.Service, *instance.Instance, bool, bool, error) {
+		unlockService := p.lockService(service.UID)
+		defer unlockService()
+
+		if err := p.applyEvent(svcCtx, event); err != nil {
+			return nil, nil, false, false, err
+		}
+
+		endpoints, err := p.worker.getEndpoints(service, id)
+		if err != nil {
+			return nil, nil, false, false, fmt.Errorf("[%s] error getting endpoints: %w", p.provider.GetLabel(), err)
+		}
+		if service.Annotations[kubevip.EgressIPv6] == "true" && !hasV6(endpoints) {
+			endpoints = nil
+		}
+		endpointCount = len(endpoints)
+
+		inst := p.findServiceInstance(service)
+
+		if err := p.worker.setInstanceEndpointsStatus(service, inst, endpoints); err != nil {
+			log.Error("updating instance", "err", err)
+		}
+
+		allowReconcileWithoutEndpoints := shouldAllowReconcileWithoutEndpoints(service)
+
+		if len(endpoints) != 0 {
+			p.updateLastKnownGoodEndpoint(lastKnownGoodEndpoint, endpoints, service)
+
+			if err := p.startServiceHandlingIfNeeded(svcCtx, service, inst, wg); err != nil {
+				return nil, nil, false, true, err
+			}
+
+			svcCtx.SignalReadiness()
+
+			if p.shouldProcessInstance() {
+				if err := p.worker.processInstance(svcCtx.Ctx, &svcCtx.ConfiguredNetworks, service, inst); err != nil {
+					return nil, nil, false, false, fmt.Errorf("failed to process non-empty instance: %w", err)
+				}
+			}
+		} else {
+			if allowReconcileWithoutEndpoints {
+				// Explicit opt-in for controllers that create LoadBalancer services without endpoints
+				if err := p.startServiceHandlingIfNeeded(svcCtx, service, inst, wg); err != nil {
+					return nil, nil, false, true, err
+				}
+				svcCtx.SignalReadiness()
+
+				if p.shouldProcessInstance() {
+					if err := p.worker.processInstance(svcCtx.Ctx, &svcCtx.ConfiguredNetworks, service, inst); err != nil {
+						return nil, nil, false, false, fmt.Errorf("failed to process endpointless instance: %w", err)
+					}
+				}
+			} else if svcCtx.IsReady() {
+				readinessLossGeneration, _, _, _ = svcCtx.ReadinessState()
+				clearNoEndpoints = true
+			}
+		}
+
+		updatedService, changed := p.updateAnnotations(service, inst, lastKnownGoodEndpoint, clientSet)
+		return updatedService, inst, changed, false, nil
+	}()
+	if err != nil || skip {
+		return skip, err
+	}
+	if clearNoEndpoints && svcCtx.ResetReadinessGeneration(readinessLossGeneration) {
+		unlockService := p.lockService(service.UID)
+		p.handleNoEndpoints(svcCtx, service, inst, lastKnownGoodEndpoint)
+		unlockService()
+	}
+
+	if changed && egressUpdateFunc != nil {
+		if err := egressUpdateFunc(context.Background(), updatedService, inst); err != nil {
+			log.Error("failed to reconfigure egress", "service", service.Name, "namespace", service.Namespace, "err", err)
+		}
+	}
+
+	log.Debug("watcher", "provider",
+		p.provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", endpointCount, "last endpoint", *lastKnownGoodEndpoint)
+
+	return false, nil
+}
+
+// applyEvent updates the provider's view of the objects backing this service.
+func (p *Processor) applyEvent(svcCtx *servicecontext.Context, event watch.Event) error {
+	if event.Type == watch.Deleted {
+		if err := p.provider.DeleteObject(event.Object); err != nil {
+			return fmt.Errorf("[%s] error deleting k8s object: %w", p.provider.GetLabel(), err)
+		}
+		return nil
+	}
+
+	if err := p.provider.LoadObject(event.Object, svcCtx.Cancel); err != nil {
+		return fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
+	}
+	return nil
+}
+
+// shouldProcessInstance reports whether this node has to program the datapath
+// itself, rather than waiting to be told to by a leader election callback.
+// WireGuard always reprograms, because its DNAT rules are per-endpoint.
+func (p *Processor) shouldProcessInstance() bool {
+	return (!p.config.EnableServicesElection && !p.config.EnableLeaderElection) || p.config.EnableWireguard
+}
+
+// handleNoEndpoints tears down everything backing a service that no longer has
+// any usable endpoints.
+func (p *Processor) handleNoEndpoints(svcCtx *servicecontext.Context, service *v1.Service, inst *instance.Instance, lastKnownGoodEndpoint *string) {
+	p.worker.clear(svcCtx.Ctx, &svcCtx.ConfiguredNetworks, lastKnownGoodEndpoint, service, inst)
+	stopWorkers := p.config.EnableARP || (p.config.EnableRoutingTable && p.config.EnableLeaderElection)
+	if stopWorkers && !p.config.EnableServicesElection {
+		if inst != nil {
+			for _, c := range inst.Clusters {
+				c.StopAndWait()
+			}
+		}
+	}
+}
+
+func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service) {
+	// if we haven't populated one, then do so
+	family := utils.IPv4Family
+	if service.Annotations[kubevip.EgressIPv6] == "true" {
+		family = utils.IPv6Family
+	}
+
+	ep := getEndpoint(endpoints, family)
+
+	if *lastKnownGoodEndpoint == "" {
+		*lastKnownGoodEndpoint = ep
+		return
+	}
+
+	// check out previous endpoint exists
+	stillExists := false
+
+	for x := range endpoints {
+		if endpoints[x] == *lastKnownGoodEndpoint {
+			stillExists = true
+		}
+	}
+	// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
+	if !stillExists {
+		ip := net.ParseIP(*lastKnownGoodEndpoint)
+		if (ip.To4() != nil && service.Annotations[kubevip.Egress] == "true") ||
+			(ip.To4() == nil && service.Annotations[kubevip.EgressIPv6] == "true") {
+			p.worker.removeEgress(service, lastKnownGoodEndpoint)
+		}
+		// Set our active endpoint to an existing one
+		*lastKnownGoodEndpoint = ep
+	}
+}
+
+func (p *Processor) updateAnnotations(service *v1.Service, inst *instance.Instance, lastKnownGoodEndpoint *string,
+	clientSet *kubernetes.Clientset) (*v1.Service, bool) {
+	// Set the service accordingly
+	if service.Annotations[kubevip.Egress] == "true" {
+		if *lastKnownGoodEndpoint != "" {
+			ip := net.ParseIP(*lastKnownGoodEndpoint)
+			expectIPv6 := service.Annotations[kubevip.EgressIPv6] == "true"
+			if ip == nil || (ip.To4() == nil) != expectIPv6 {
+				log.Warn("ignoring active endpoint with unexpected address family",
+					"service", service.Name,
+					"namespace", service.Namespace,
+					"endpoint", *lastKnownGoodEndpoint,
+					"expected_ipv6", expectIPv6)
+				return nil, false
+			}
+		}
+
+		// Store old values from ServiceSnapshot to detect if annotation actually changed
+		// We use the ServiceSnapshot instead of the service parameter because the service parameter
+		// may have stale annotations if the last update failed
+		var oldEndpoint, oldEndpointIPv6 string
+		snapshotFound := false
+		if inst != nil {
+			if inst.ServiceSnapshot != nil {
+				snapshotFound = true
+				oldEndpoint = inst.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint]
+				oldEndpointIPv6 = inst.ServiceSnapshot.Annotations[kubevip.ActiveEndpointIPv6]
+			}
+		}
+		// Empty annotations in an existing snapshot are meaningful after a zero-endpoint transition.
+		if !snapshotFound {
+			oldEndpoint = service.Annotations[kubevip.ActiveEndpoint]
+			oldEndpointIPv6 = service.Annotations[kubevip.ActiveEndpointIPv6]
+		}
+
+		// Determine which annotation to update based on IP version
+		var endpoint, endpointIPv6 string
+		if service.Annotations[kubevip.EgressIPv6] == "true" && !p.config.EnableEndpoints {
+			// IPv6
+			endpointIPv6 = *lastKnownGoodEndpoint
+			endpoint = oldEndpoint // Preserve existing IPv4 if any
+		} else {
+			// IPv4
+			endpoint = *lastKnownGoodEndpoint
+			endpointIPv6 = oldEndpointIPv6 // Preserve existing IPv6 if any
+		}
+
+		// Check if annotation actually changed
+		annotationChanged := (oldEndpoint != endpoint) || (oldEndpointIPv6 != endpointIPv6)
+		if !annotationChanged {
+			return nil, false
+		}
+
+		// Persist to Kubernetes
+		ctx := context.Background()
+
+		if err := p.provider.UpdateServiceAnnotation(ctx, endpoint, endpointIPv6, service, clientSet); err != nil {
+			log.Warn("failed to update service annotation", "service", service.Name, "namespace", service.Namespace, "err", err)
+			return nil, false
+		}
+
+		log.Debug("updated active endpoint annotation", "service", service.Name, "namespace", service.Namespace, "endpoint", *lastKnownGoodEndpoint)
+
+		svcCopy := service.DeepCopy()
+		svcCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
+		svcCopy.Annotations[kubevip.ActiveEndpointIPv6] = endpointIPv6
+		return svcCopy, true
+	}
+	return nil, false
+}
+
+func (p *Processor) startServiceHandlingIfNeeded(svcCtx *servicecontext.Context, service *v1.Service,
+	inst *instance.Instance, wg *sync.WaitGroup) error {
+	if p.config.EnableServicesElection {
+		return nil
+	}
+
+	if p.config.EnableARP || (p.config.EnableRoutingTable && p.config.EnableLeaderElection) {
+		if !svcCtx.IsReady() {
+			if inst == nil {
+				return fmt.Errorf("[%s] failed to find an instance for service %s/%s", p.provider.GetLabel(), service.Namespace, service.Name)
+			}
+			if err := StartService(svcCtx.Ctx, service, inst, p.bgpServer, wg); err != nil {
+				return fmt.Errorf("start service datapath: %w", err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (p *Processor) findServiceInstance(service *v1.Service) *instance.Instance {
+	if p.instances == nil {
+		return nil
+	}
+	if p.instancesMutex != nil {
+		p.instancesMutex.RLock()
+		defer p.instancesMutex.RUnlock()
+	}
+	inst := instance.FindServiceInstance(service, *p.instances)
+	return inst
+}
+
+func shouldAllowReconcileWithoutEndpoints(service *v1.Service) bool {
+	if service == nil || service.Spec.ExternalTrafficPolicy != v1.ServiceExternalTrafficPolicyTypeCluster {
+		return false
+	}
+
+	return strings.EqualFold(service.Annotations[kubevip.AllowReconcileWithoutEndpoints], "true")
+}
+
+func hasV6(endpoints []string) bool {
+	for _, e := range endpoints {
+		ip := net.ParseIP(e)
+		if ip != nil {
+			if ip.To4() == nil {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func getEndpoint(endpoints []string, family string) string {
+	for _, e := range endpoints {
+		ip := net.ParseIP(e)
+		if family == utils.IPv4Family && ip.To4() != nil {
+			return e
+		}
+		if family == utils.IPv6Family && ip.To4() == nil {
+			return e
+		}
+	}
+	return ""
+}

pkg/endpoints/endpoints_bgp.go

@@ -0,0 +1,101 @@
+package endpoints
+
+import (
+	"context"
+	log "log/slog"
+	"sync"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/lease"
+	v1 "k8s.io/api/core/v1"
+)
+
+type BGP struct {
+	generic
+	bgpServer *bgp.Server
+}
+
+func newBGP(generic generic, bgpServer *bgp.Server) endpointWorker {
+	return &BGP{
+		generic:   generic,
+		bgpServer: bgpServer,
+	}
+}
+
+func (b *BGP) processInstance(ctx context.Context, configuredNetworks *sync.Map, service *v1.Service, inst *instance.Instance) error {
+	if inst != nil {
+		for _, cluster := range inst.Clusters {
+			for i := range cluster.Network {
+				if _, configured := configuredNetworks.Load(cluster.Network[i].IP()); !configured {
+					log.Debug("attempting to advertise BGP service", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP())
+					err := b.bgpServer.AddHost(ctx, cluster.Network[i].CIDR(), lease.ServiceNamespacedName(service))
+					if err != nil {
+						log.Error("error adding BGP host", "provider", b.provider.GetLabel(), "err", err)
+					} else {
+						log.Info("added BGP host", "provider",
+							b.provider.GetLabel(), "ip", cluster.Network[i].CIDR(), "service name", service.Name, "namespace", service.Namespace)
+						configuredNetworks.Store(cluster.Network[i].IP(), true)
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (b *BGP) clear(ctx context.Context, configuredNetworks *sync.Map, lastKnownGoodEndpoint *string, service *v1.Service, inst *instance.Instance) {
+	if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
+		// If BGP mode is enabled - routes should be deleted
+		if inst != nil {
+			for _, cluster := range inst.Clusters {
+				for i := range cluster.Network {
+					err := b.bgpServer.DelHost(ctx, cluster.Network[i].CIDR(), lease.ServiceNamespacedName(service))
+					if err != nil {
+						log.Error("deleting BGP host", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "err", err)
+					} else {
+						log.Info("deleted BGP host", "provider",
+							b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace)
+						configuredNetworks.Delete(cluster.Network[i].IP())
+					}
+				}
+			}
+
+		}
+	}
+
+	b.clearEgress(lastKnownGoodEndpoint, service)
+}
+
+func (b *BGP) getEndpoints(service *v1.Service, id string) ([]string, error) {
+	return b.getAllEndpoints(service, id)
+}
+
+func (b *BGP) setInstanceEndpointsStatus(_ *v1.Service, _ *instance.Instance, _ []string) error {
+	return nil
+}
+
+func ClearBGPHosts(ctx context.Context, service *v1.Service, instances *[]*instance.Instance, bgpServer *bgp.Server) {
+	if instance := instance.FindServiceInstance(service, *instances); instance != nil {
+		ClearBGPHostsByInstance(ctx, instance, bgpServer)
+	}
+}
+
+func ClearBGPHostsByInstance(ctx context.Context, instance *instance.Instance, bgpServer *bgp.Server) {
+	if instance == nil {
+		log.Error("failed to clear BGP host for nil instance")
+		return
+	}
+	for _, cluster := range instance.Clusters {
+		for i := range cluster.Network {
+			network := cluster.Network[i]
+			err := bgpServer.DelHost(ctx, network.CIDR(), lease.ServiceNamespacedName(instance.ServiceSnapshot))
+			if err != nil {
+				log.Error("[endpoint] error deleting BGP host", "err", err)
+			} else {
+				log.Debug("[endpoint] deleted BGP host", "ip",
+					network.CIDR(), "service name", instance.ServiceSnapshot.Name, "namespace", instance.ServiceSnapshot.Namespace)
+			}
+		}
+	}
+}

pkg/endpoints/endpoints_generic.go

@@ -0,0 +1,104 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	log "log/slog"
+	"sync"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/egress"
+	"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/lease"
+	"github.com/kube-vip/kube-vip/pkg/route"
+	"github.com/kube-vip/kube-vip/pkg/wireguard"
+	v1 "k8s.io/api/core/v1"
+)
+
+type endpointWorker interface {
+	processInstance(ctx context.Context, configuredNetworks *sync.Map, service *v1.Service, inst *instance.Instance) error
+	clear(ctx context.Context, configuredNetworks *sync.Map, lastKnownGoodEndpoint *string, service *v1.Service, inst *instance.Instance)
+	getEndpoints(service *v1.Service, id string) ([]string, error)
+	removeEgress(service *v1.Service, lastKnownGoodEndpoint *string)
+	setInstanceEndpointsStatus(service *v1.Service, inst *instance.Instance, endpoints []string) error
+}
+
+func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
+	leaseMgr *lease.Manager, tunnelMgr *wireguard.TunnelManager, routeMgr *route.Manager) endpointWorker {
+	generic := newGeneric(config, provider, leaseMgr)
+
+	if config.EnableWireguard {
+		return newWireguardWorker(config, provider, tunnelMgr)
+	}
+	if config.EnableRoutingTable {
+		return newRoutingTable(generic, routeMgr)
+	}
+	if config.EnableBGP {
+		return newBGP(generic, bgpServer)
+	}
+
+	return &generic
+}
+
+type generic struct {
+	config   *kubevip.Config
+	provider providers.Provider
+	leaseMgr *lease.Manager
+}
+
+func newGeneric(config *kubevip.Config, provider providers.Provider, leaseMgr *lease.Manager) generic {
+	return generic{
+		config:   config,
+		provider: provider,
+		leaseMgr: leaseMgr,
+	}
+}
+
+func (g *generic) processInstance(_ context.Context, _ *sync.Map, _ *v1.Service, _ *instance.Instance) error {
+	return nil
+}
+
+func (g *generic) clear(_ context.Context, _ *sync.Map, lastKnownGoodEndpoint *string, service *v1.Service, _ *instance.Instance) {
+	g.clearEgress(lastKnownGoodEndpoint, service)
+}
+
+func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service) {
+	if *lastKnownGoodEndpoint != "" {
+		log.Warn("existing endpoint has been removed, no remaining endpoints for leaderElection", "provider", g.provider.GetLabel(), "endpoint", lastKnownGoodEndpoint)
+		if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP, service.Namespace, string(service.UID), service.Annotations, g.config.EgressWithNftables); err != nil {
+			log.Error("error removing redundant egress rules", "err", err)
+		}
+
+		*lastKnownGoodEndpoint = "" // reset endpoint
+	}
+}
+
+func (g *generic) getEndpoints(service *v1.Service, id string) ([]string, error) {
+	return g.getAllEndpoints(service, id)
+}
+
+func (g *generic) getAllEndpoints(service *v1.Service, id string) ([]string, error) {
+	// Build endpoints
+	var err error
+	var endpoints []string
+	if service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
+		if endpoints, err = g.provider.GetAllEndpoints(); err != nil {
+			return nil, fmt.Errorf("[%s] error getting all endpoints: %w", g.provider.GetLabel(), err)
+		}
+	} else {
+		if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
+			return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
+		}
+	}
+
+	return endpoints, nil
+}
+
+func (g *generic) removeEgress(_ *v1.Service, _ *string) {
+}
+
+func (g *generic) setInstanceEndpointsStatus(_ *v1.Service, _ *instance.Instance, _ []string) error {
+	return nil
+}

pkg/endpoints/endpoints_routing_table.go

@@ -0,0 +1,129 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"sync"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/egress"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/lease"
+	"github.com/kube-vip/kube-vip/pkg/route"
+	v1 "k8s.io/api/core/v1"
+)
+
+type RoutingTable struct {
+	generic
+	routeMgr *route.Manager
+}
+
+func newRoutingTable(generic generic, routeMgr *route.Manager) endpointWorker {
+	return &RoutingTable{
+		generic:  generic,
+		routeMgr: routeMgr,
+	}
+}
+
+func (rt *RoutingTable) processInstance(_ context.Context, configuredNetworks *sync.Map, service *v1.Service, inst *instance.Instance) error {
+	if inst != nil {
+		for _, cluster := range inst.Clusters {
+			for i := range cluster.Network {
+				if _, configured := configuredNetworks.Load(cluster.Network[i].IP()); !configured && cluster.Network[i].HasEndpoints() {
+					if err := rt.routeMgr.Add(lease.ServiceNamespacedName(service), cluster.Network[i], false, true); err != nil {
+						return fmt.Errorf("[%s] error adding route: %s", rt.provider.GetLabel(), err.Error())
+					} else {
+						log.Info("added route", "provider",
+							rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
+							service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
+						configuredNetworks.Store(cluster.Network[i].IP(), true)
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (rt *RoutingTable) clear(_ context.Context, configuredNetworks *sync.Map, lastKnownGoodEndpoint *string, service *v1.Service, inst *instance.Instance) {
+	if !rt.config.EnableServicesElection {
+		if errs := ClearRoutesByInstance(service, inst, nil, rt.routeMgr); len(errs) == 0 {
+			configuredNetworks.Clear()
+		} else {
+			for _, err := range errs {
+				log.Error("error while clearing routes", "err", err)
+			}
+		}
+	}
+
+	rt.clearEgress(lastKnownGoodEndpoint, service)
+}
+
+func (rt *RoutingTable) getEndpoints(service *v1.Service, id string) ([]string, error) {
+	return rt.getAllEndpoints(service, id)
+}
+
+func (rt *RoutingTable) removeEgress(service *v1.Service, lastKnownGoodEndpoint *string) {
+	if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
+		service.Namespace, string(service.UID), service.Annotations, rt.config.EgressWithNftables); err != nil {
+		log.Warn("removing redundant egress rules", "err", err)
+	}
+}
+
+func (rt *RoutingTable) setInstanceEndpointsStatus(service *v1.Service, inst *instance.Instance, endpoints []string) error {
+	if inst == nil {
+		log.Error("failed to find the instance", "namespace", service.Namespace, "name", service.Name, "uid", service.UID, "provider", rt.provider.GetLabel())
+	} else {
+		for _, c := range inst.Clusters {
+			for n := range c.Network {
+				// if there are no endpoints set HasEndpoints false just in case
+				if len(endpoints) < 1 {
+					c.Network[n].SetHasEndpoints(false)
+				} else {
+					// check if endpoint are available and are of same IP family as service
+					for _, ep := range endpoints {
+						if (net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(ep).To4() == nil) {
+							c.Network[n].SetHasEndpoints(true)
+							break
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func ClearRoutes(service *v1.Service, instances *[]*instance.Instance, routeMgr *route.Manager) []error {
+	errs := []error{}
+	if svcInst := instance.FindServiceInstance(service, *instances); svcInst != nil {
+		clearErrs := ClearRoutesByInstance(service, svcInst, instances, routeMgr)
+		errs = append(errs, clearErrs...)
+	}
+	return errs
+}
+
+func ClearRoutesByInstance(service *v1.Service, svcInst *instance.Instance, instances *[]*instance.Instance, routeMgr *route.Manager) []error {
+	if svcInst == nil {
+		return []error{fmt.Errorf("failed to remove routes for nil instance of service %s/%s, uid: %s", service.Namespace, service.Name, service.UID)}
+	}
+	errs := []error{}
+	for _, cluster := range svcInst.Clusters {
+		for i := range cluster.Network {
+			err := routeMgr.Delete(lease.ServiceNamespacedName(service), cluster.Network[i])
+			if err != nil {
+				log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
+				errs = append(errs, err)
+			}
+			log.Debug("deleted route", "ip",
+				cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface())
+
+		}
+	}
+
+	return errs
+}

pkg/endpoints/endpoints_test.go

@@ -0,0 +1,579 @@
+package endpoints
+
+import (
+	"context"
+	"sync"
+	"testing"
+
+	"github.com/kube-vip/kube-vip/pkg/cluster"
+	"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/lease"
+	"github.com/kube-vip/kube-vip/pkg/route"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	v1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+)
+
+func TestShouldAllowReconcileWithoutEndpoints(t *testing.T) {
+	if shouldAllowReconcileWithoutEndpoints(nil) {
+		t.Fatal("nil service should not be allowed")
+	}
+
+	clusterOptIn := &v1.Service{
+		Spec:       v1.ServiceSpec{ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeCluster},
+		ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{kubevip.AllowReconcileWithoutEndpoints: "true"}},
+	}
+	if !shouldAllowReconcileWithoutEndpoints(clusterOptIn) {
+		t.Fatal("cluster service with opt-in annotation should be allowed")
+	}
+
+	localOptIn := &v1.Service{
+		Spec:       v1.ServiceSpec{ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal},
+		ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{kubevip.AllowReconcileWithoutEndpoints: "true"}},
+	}
+	if shouldAllowReconcileWithoutEndpoints(localOptIn) {
+		t.Fatal("local service should not be allowed")
+	}
+}
+
+type fakeWorker struct {
+	endpoints     []string
+	clearCalled   bool
+	processCalled bool
+	processHook   func()
+}
+
+type annotationUpdate struct {
+	endpoint     string
+	endpointIPv6 string
+}
+
+type recordingProvider struct {
+	providers.Provider
+	updates []annotationUpdate
+}
+
+func (p *recordingProvider) UpdateServiceAnnotation(_ context.Context, endpoint, endpointIPv6 string,
+	_ *v1.Service, _ *kubernetes.Clientset) error {
+	p.updates = append(p.updates, annotationUpdate{endpoint: endpoint, endpointIPv6: endpointIPv6})
+	return nil
+}
+
+func TestUpdateAnnotationsZeroEndpointsThenSameEndpoint(t *testing.T) {
+	for _, enableEndpoints := range []bool{true, false} {
+		providerName := "EndpointSlices"
+		provider := providers.NewEndpointslices()
+		if enableEndpoints {
+			providerName = "Endpoints"
+			provider = providers.NewEndpoints()
+		}
+
+		for _, family := range []struct {
+			name       string
+			endpoint   string
+			other      string
+			egressIPv6 bool
+		}{
+			{name: "IPv4", endpoint: "10.0.0.1", other: "fd00::1"},
+			{name: "IPv6", endpoint: "fd00::1", other: "10.0.0.1", egressIPv6: true},
+		} {
+			t.Run(providerName+"/"+family.name, func(t *testing.T) {
+				annotations := map[string]string{kubevip.Egress: "true"}
+				if family.egressIPv6 {
+					annotations[kubevip.EgressIPv6] = "true"
+				}
+				if !enableEndpoints {
+					if family.egressIPv6 {
+						annotations[kubevip.ActiveEndpoint] = family.other
+						annotations[kubevip.ActiveEndpointIPv6] = family.endpoint
+					} else {
+						annotations[kubevip.ActiveEndpoint] = family.endpoint
+						annotations[kubevip.ActiveEndpointIPv6] = family.other
+					}
+				} else {
+					annotations[kubevip.ActiveEndpoint] = family.endpoint
+				}
+				service := &v1.Service{ObjectMeta: metav1.ObjectMeta{
+					Name: "test-service", Namespace: "default", UID: "test-uid", Annotations: annotations,
+				}}
+				serviceInstance := &instance.Instance{ServiceUID: service.UID, ServiceSnapshot: service.DeepCopy()}
+				instances := []*instance.Instance{serviceInstance}
+				recorder := &recordingProvider{Provider: provider}
+				processor := &Processor{
+					config:    &kubevip.Config{EnableEndpoints: enableEndpoints},
+					provider:  recorder,
+					instances: &instances,
+				}
+
+				noEndpoint := ""
+				updated, changed := processor.updateAnnotations(service, serviceInstance, &noEndpoint, nil)
+				if changed {
+					serviceInstance.ServiceSnapshot = updated
+				}
+				repopulatedEndpoint := family.endpoint
+				updated, changed = processor.updateAnnotations(service, serviceInstance, &repopulatedEndpoint, nil)
+				if changed {
+					serviceInstance.ServiceSnapshot = updated
+				}
+
+				cleared := annotationUpdate{}
+				repopulated := annotationUpdate{endpoint: family.endpoint}
+				if !enableEndpoints {
+					if family.egressIPv6 {
+						cleared = annotationUpdate{endpoint: family.other}
+						repopulated = annotationUpdate{endpoint: family.other, endpointIPv6: family.endpoint}
+					} else {
+						cleared = annotationUpdate{endpointIPv6: family.other}
+						repopulated = annotationUpdate{endpoint: family.endpoint, endpointIPv6: family.other}
+					}
+				}
+				want := []annotationUpdate{cleared, repopulated}
+				if len(recorder.updates) != len(want) {
+					t.Fatalf("annotation updates = %+v, want %+v", recorder.updates, want)
+				}
+				for index := range want {
+					if recorder.updates[index] != want[index] {
+						t.Errorf("annotation update %d = %+v, want %+v", index, recorder.updates[index], want[index])
+					}
+				}
+			})
+		}
+	}
+}
+
+func TestUpdateAnnotationsEndpointSlicesClearsConfiguredFamily(t *testing.T) {
+	for _, test := range []struct {
+		name       string
+		egressIPv6 bool
+		want       annotationUpdate
+	}{
+		{name: "IPv4", want: annotationUpdate{endpointIPv6: "fd00::1"}},
+		{name: "IPv6", egressIPv6: true, want: annotationUpdate{endpoint: "10.0.0.1"}},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			annotations := map[string]string{
+				kubevip.Egress:             "true",
+				kubevip.ActiveEndpoint:     "10.0.0.1",
+				kubevip.ActiveEndpointIPv6: "fd00::1",
+			}
+			if test.egressIPv6 {
+				annotations[kubevip.EgressIPv6] = "true"
+			}
+			service := &v1.Service{ObjectMeta: metav1.ObjectMeta{
+				Name: "test-service", Namespace: "default", UID: "test-uid", Annotations: annotations,
+			}}
+			instances := []*instance.Instance{{ServiceUID: service.UID, ServiceSnapshot: service.DeepCopy()}}
+			recorder := &recordingProvider{Provider: providers.NewEndpointslices()}
+			processor := &Processor{
+				config:    &kubevip.Config{EnableEndpoints: false},
+				provider:  recorder,
+				instances: &instances,
+			}
+
+			noEndpoint := ""
+			processor.updateAnnotations(service, instances[0], &noEndpoint, nil)
+
+			if len(recorder.updates) != 1 || recorder.updates[0] != test.want {
+				t.Fatalf("annotation updates = %+v, want [%+v]", recorder.updates, test.want)
+			}
+		})
+	}
+}
+
+func TestUpdateAnnotationsValidatesEndpointFamily(t *testing.T) {
+	tests := []struct {
+		name       string
+		endpoint   string
+		egressIPv6 bool
+		want       annotationUpdate
+		wantUpdate bool
+	}{
+		{name: "invalid address", endpoint: "not-an-ip"},
+		{name: "IPv6 endpoint for IPv4 egress", endpoint: "fd00::1"},
+		{name: "IPv4 endpoint for IPv6 egress", endpoint: "10.0.0.1", egressIPv6: true},
+		{name: "IPv4 endpoint", endpoint: "10.0.0.2", want: annotationUpdate{endpoint: "10.0.0.2", endpointIPv6: "fd00::1"}, wantUpdate: true},
+		{name: "IPv6 endpoint", endpoint: "fd00::2", egressIPv6: true, want: annotationUpdate{endpoint: "10.0.0.1", endpointIPv6: "fd00::2"}, wantUpdate: true},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			annotations := map[string]string{
+				kubevip.Egress:             "true",
+				kubevip.ActiveEndpoint:     "10.0.0.1",
+				kubevip.ActiveEndpointIPv6: "fd00::1",
+			}
+			if test.egressIPv6 {
+				annotations[kubevip.EgressIPv6] = "true"
+			}
+			service := &v1.Service{ObjectMeta: metav1.ObjectMeta{
+				Name: "test-service", Namespace: "default", Annotations: annotations,
+			}}
+			recorder := &recordingProvider{Provider: providers.NewEndpointslices()}
+			processor := &Processor{
+				config:   &kubevip.Config{EnableEndpoints: false},
+				provider: recorder,
+			}
+
+			processor.updateAnnotations(service, nil, &test.endpoint, nil)
+
+			if !test.wantUpdate {
+				if len(recorder.updates) != 0 {
+					t.Fatalf("annotation updates = %+v, want none", recorder.updates)
+				}
+				return
+			}
+			if len(recorder.updates) != 1 || recorder.updates[0] != test.want {
+				t.Fatalf("annotation updates = %+v, want [%+v]", recorder.updates, test.want)
+			}
+		})
+	}
+}
+
+func (f *fakeWorker) processInstance(_ context.Context, _ *sync.Map, _ *v1.Service, _ *instance.Instance) error {
+	if f.processHook != nil {
+		f.processHook()
+	}
+	f.processCalled = true
+	return nil
+}
+
+func (f *fakeWorker) clear(_ context.Context, _ *sync.Map, _ *string, _ *v1.Service, _ *instance.Instance) {
+	f.clearCalled = true
+}
+
+func (f *fakeWorker) getEndpoints(_ *v1.Service, _ string) ([]string, error) { return f.endpoints, nil }
+func (f *fakeWorker) removeEgress(_ *v1.Service, _ *string)                  {}
+func (f *fakeWorker) setInstanceEndpointsStatus(_ *v1.Service, _ *instance.Instance, _ []string) error {
+	return nil
+}
+
+func noOpServiceLock(types.UID) func() {
+	return func() {}
+}
+
+// TestReconcile_RecomputesRemainingEndpoints asserts that deleting one EndpointSlice
+// reconciles against the endpoints that remain, instead of assuming the service
+// lost all of them.
+func TestReconcile_RecomputesRemainingEndpoints(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name              string
+		remaining         []string
+		lastKnown         string
+		expectReady       bool
+		expectClear       bool
+		expectProcess     bool
+		expectedLastKnown string
+	}{
+		{
+			name:              "remaining endpoints keep the service up",
+			remaining:         []string{"10.0.0.2"},
+			lastKnown:         "10.0.0.2",
+			expectReady:       true,
+			expectProcess:     true,
+			expectedLastKnown: "10.0.0.2",
+		},
+		{
+			name:              "stale last known endpoint moves to a survivor",
+			remaining:         []string{"10.0.0.2"},
+			lastKnown:         "10.0.0.1",
+			expectReady:       true,
+			expectProcess:     true,
+			expectedLastKnown: "10.0.0.2",
+		},
+		{
+			name:        "last endpoint removed tears the service down",
+			remaining:   nil,
+			lastKnown:   "10.0.0.1",
+			expectReady: false,
+			expectClear: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			worker := &fakeWorker{endpoints: test.remaining}
+			p := &Processor{
+				config:      &kubevip.Config{},
+				provider:    providers.NewEndpointslices(),
+				worker:      worker,
+				lockService: noOpServiceLock,
+			}
+
+			svcCtx := servicecontext.New(context.Background())
+			svcCtx.SignalReadiness()
+
+			lastKnown := test.lastKnown
+			restart, err := p.Reconcile(
+				svcCtx,
+				watch.Event{
+					Type:   watch.Deleted,
+					Object: &discoveryv1.EndpointSlice{ObjectMeta: metav1.ObjectMeta{Name: "slice-1"}},
+				},
+				&lastKnown,
+				&v1.Service{Spec: v1.ServiceSpec{ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal}},
+				"node-1",
+				&sync.WaitGroup{},
+				nil,
+				nil,
+			)
+			if err != nil {
+				t.Fatalf("Reconcile returned error: %v", err)
+			}
+			if restart {
+				t.Fatal("Reconcile unexpectedly requested restart")
+			}
+
+			if ready := svcCtx.IsReady(); ready != test.expectReady {
+				t.Fatalf("readiness mismatch: expected %v, got %v", test.expectReady, ready)
+			}
+			if worker.clearCalled != test.expectClear {
+				t.Fatalf("clearCalled mismatch: expected %v, got %v", test.expectClear, worker.clearCalled)
+			}
+			if worker.processCalled != test.expectProcess {
+				t.Fatalf("processCalled mismatch: expected %v, got %v", test.expectProcess, worker.processCalled)
+			}
+			if test.expectedLastKnown != "" && lastKnown != test.expectedLastKnown {
+				t.Fatalf("lastKnownGoodEndpoint mismatch: expected %q, got %q", test.expectedLastKnown, lastKnown)
+			}
+		})
+	}
+}
+
+func TestReconcile_ZeroEndpointsBehavior(t *testing.T) {
+	t.Parallel()
+
+	run := func(t *testing.T, service *v1.Service, presetSignalled bool, expectReady bool, expectClear bool, expectProcess bool) {
+		t.Helper()
+
+		worker := &fakeWorker{endpoints: []string{}}
+		p := &Processor{
+			config:      &kubevip.Config{},
+			provider:    providers.NewEndpointslices(),
+			worker:      worker,
+			lockService: noOpServiceLock,
+		}
+
+		svcCtx := servicecontext.New(context.Background())
+		if presetSignalled {
+			svcCtx.SignalReadiness()
+		}
+
+		restart, err := p.Reconcile(
+			svcCtx,
+			watch.Event{Type: watch.Modified, Object: &discoveryv1.EndpointSlice{}},
+			new(string),
+			service,
+			"node-1",
+			&sync.WaitGroup{},
+			nil,
+			nil,
+		)
+		if err != nil {
+			t.Fatalf("Reconcile returned error: %v", err)
+		}
+		if restart {
+			t.Fatal("Reconcile unexpectedly requested restart")
+		}
+
+		if ready := svcCtx.IsReady(); ready != expectReady {
+			t.Fatalf("readiness mismatch: expected %v, got %v", expectReady, ready)
+		}
+		if worker.clearCalled != expectClear {
+			t.Fatalf("clearCalled mismatch: expected %v, got %v", expectClear, worker.clearCalled)
+		}
+		if worker.processCalled != expectProcess {
+			t.Fatalf("processCalled mismatch: expected %v, got %v", expectProcess, worker.processCalled)
+		}
+	}
+
+	t.Run("cluster opt-in keeps readiness and skips clear", func(t *testing.T) {
+		service := &v1.Service{
+			ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{kubevip.AllowReconcileWithoutEndpoints: "true"}},
+			Spec:       v1.ServiceSpec{ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeCluster},
+		}
+		run(t, service, false, true, false, true)
+	})
+
+	t.Run("cluster without opt-in resets and clears when pre-signalled", func(t *testing.T) {
+		service := &v1.Service{
+			ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}},
+			Spec:       v1.ServiceSpec{ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeCluster},
+		}
+		run(t, service, true, false, true, false)
+	})
+
+	t.Run("local opt-in still resets and clears when pre-signalled", func(t *testing.T) {
+		service := &v1.Service{
+			ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{kubevip.AllowReconcileWithoutEndpoints: "true"}},
+			Spec:       v1.ServiceSpec{ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal},
+		}
+		run(t, service, true, false, true, false)
+	})
+}
+
+func TestHandleNoEndpointsStopsGlobalRoutingTableWorkers(t *testing.T) {
+	config := &kubevip.Config{EnableRoutingTable: true, KubernetesLeaderElection: kubevip.KubernetesLeaderElection{EnableLeaderElection: true}}
+	serviceCluster, err := cluster.InitCluster(&kubevip.Config{}, true, nil, nil, route.NewManager(), nil)
+	if err != nil {
+		t.Fatalf("initializing Service cluster: %v", err)
+	}
+	var workers sync.WaitGroup
+	if err := serviceCluster.StartLoadBalancerService(context.Background(), config, nil, "service", &workers); err != nil {
+		t.Fatalf("starting Service cluster: %v", err)
+	}
+	instance := &instance.Instance{Clusters: []*cluster.Cluster{serviceCluster}}
+	processor := &Processor{config: config, worker: &fakeWorker{}}
+
+	processor.handleNoEndpoints(servicecontext.New(context.Background()), &v1.Service{}, instance, new(string))
+	if err := serviceCluster.StartLoadBalancerService(context.Background(), config, nil, "service", &workers); err != nil {
+		t.Fatalf("starting Service cluster after endpoint loss: %v", err)
+	}
+	serviceCluster.StopAndWait()
+	workers.Wait()
+}
+
+func TestReconcileIPv6EgressWithoutIPv6EndpointsClearsReadiness(t *testing.T) {
+	worker := &fakeWorker{endpoints: []string{"192.0.2.10"}}
+	processor := &Processor{
+		config:      &kubevip.Config{},
+		provider:    providers.NewEndpointslices(),
+		worker:      worker,
+		lockService: noOpServiceLock,
+	}
+	service := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{kubevip.EgressIPv6: "true"}},
+		Spec:       v1.ServiceSpec{ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal},
+	}
+	svcCtx := servicecontext.New(context.Background())
+	svcCtx.SignalReadiness()
+
+	restart, err := processor.Reconcile(svcCtx, watch.Event{Type: watch.Modified, Object: &discoveryv1.EndpointSlice{}},
+		new(string), service, "node", &sync.WaitGroup{}, nil, nil)
+	if err != nil {
+		t.Fatalf("Reconcile() error = %v", err)
+	}
+	if restart {
+		t.Fatal("Reconcile requested a restart for an unusable endpoint family")
+	}
+	if svcCtx.IsReady() {
+		t.Fatal("IPv6 egress remained ready with only IPv4 endpoints")
+	}
+	if !worker.clearCalled {
+		t.Fatal("IPv6 egress did not clear the worker with only IPv4 endpoints")
+	}
+}
+
+func TestSharedServiceVIPRequiresClusterPolicyAndElectionLease(t *testing.T) {
+	service := &v1.Service{ObjectMeta: metav1.ObjectMeta{
+		Name: "first", Namespace: "default", Annotations: map[string]string{kubevip.ServiceLease: "shared"},
+	}, Spec: v1.ServiceSpec{LoadBalancerIP: "192.0.2.10", ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeCluster}}
+	candidate := service.DeepCopy()
+	candidate.Name = "second"
+	first := &instance.Instance{ServiceSnapshot: service}
+	second := &instance.Instance{ServiceSnapshot: candidate}
+
+	if len(sharedServiceVIPs(&kubevip.Config{EnableServicesElection: true}, first, []*instance.Instance{second})) == 0 {
+		t.Fatal("shared lease Services with Cluster traffic policy did not share their VIP")
+	}
+	if len(sharedServiceVIPs(&kubevip.Config{}, first, []*instance.Instance{second})) == 0 {
+		t.Fatal("Cluster policy Services without per-Service election did not share their VIP")
+	}
+	candidate.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal
+	if len(sharedServiceVIPs(&kubevip.Config{}, first, []*instance.Instance{second})) != 0 {
+		t.Fatal("Local traffic policy Services shared a VIP")
+	}
+	candidate.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeCluster
+	candidate.Annotations[kubevip.ServiceLease] = "different"
+	if len(sharedServiceVIPs(&kubevip.Config{EnableServicesElection: true}, first, []*instance.Instance{second})) != 0 {
+		t.Fatal("Services with different leases shared a VIP")
+	}
+}
+
+func TestSharedServiceVIPsReturnsOnlyOverlappingAddresses(t *testing.T) {
+	first := &v1.Service{ObjectMeta: metav1.ObjectMeta{
+		Name: "first", Namespace: "default", Annotations: map[string]string{kubevip.ServiceLease: "shared"},
+	}, Spec: v1.ServiceSpec{LoadBalancerIP: "192.0.2.10", ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeCluster}}
+	second := first.DeepCopy()
+	second.Name = "second"
+	second.Spec.LoadBalancerIP = "192.0.2.11"
+	firstInstance := &instance.Instance{ServiceSnapshot: first, ServiceAddresses: []string{"192.0.2.10", "2001:db8::10"}}
+	secondInstance := &instance.Instance{ServiceSnapshot: second, ServiceAddresses: []string{"192.0.2.10", "2001:db8::11"}}
+
+	shared := sharedServiceVIPs(&kubevip.Config{EnableServicesElection: true}, firstInstance, []*instance.Instance{secondInstance})
+	if len(shared) != 1 {
+		t.Fatalf("shared VIPs = %v, want exactly one", shared)
+	}
+	if _, found := shared["192.0.2.10"]; !found {
+		t.Fatalf("shared VIPs = %v, missing overlapping IPv4 address", shared)
+	}
+}
+
+// TestReconcileServicesElectionDoesNotStartElectionLoop asserts endpoint events
+// only update readiness. The services coordinator owns the election loop.
+func TestReconcileServicesElectionDoesNotStartElectionLoop(t *testing.T) {
+	config := &kubevip.Config{
+		EnableServicesElection: true,
+		LeaderElectionType:     "kubernetes",
+	}
+
+	service := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{Name: "test-svc", Namespace: "default", UID: "test-uid"},
+		Spec:       v1.ServiceSpec{Type: v1.ServiceTypeLoadBalancer},
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	leaseMgr := lease.NewManager()
+	leaseNamespace, serviceLease := lease.ServiceName(service)
+	svcLease := leaseMgr.Add(ctx, lease.NewID(config.LeaderElectionType, leaseNamespace, serviceLease))
+
+	svcCtx := servicecontext.New(svcLease.Ctx)
+
+	wg := &sync.WaitGroup{}
+	defer wg.Wait()
+	defer svcCtx.Cancel()
+
+	p := &Processor{
+		config:      config,
+		provider:    providers.NewEndpointslices(),
+		worker:      &fakeWorker{endpoints: []string{"10.0.0.1"}},
+		leaseMgr:    leaseMgr,
+		lockService: noOpServiceLock,
+	}
+
+	// Three endpoint events, as a flapping backend pod would produce.
+	for range 3 {
+		restart, err := p.Reconcile(svcCtx, watch.Event{Type: watch.Modified, Object: &discoveryv1.EndpointSlice{}},
+			new(string), service, "node-1", wg, nil, nil)
+		if err != nil {
+			t.Fatalf("Reconcile returned error: %v", err)
+		}
+		if restart {
+			t.Fatal("Reconcile unexpectedly requested restart")
+		}
+	}
+
+	generation, ready, lost, isReady := svcCtx.ReadinessState()
+	if generation != 1 || !isReady {
+		t.Fatalf("readiness state = generation %d, ready %t; want generation 1 ready", generation, isReady)
+	}
+	select {
+	case <-ready:
+	default:
+		t.Fatal("endpoint reconciliation did not signal readiness")
+	}
+	select {
+	case <-lost:
+		t.Fatal("endpoint reconciliation unexpectedly reset readiness")
+	default:
+	}
+}

pkg/endpoints/endpoints_wireguard.go

@@ -0,0 +1,253 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/nftables"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/wireguard"
+	v1 "k8s.io/api/core/v1"
+)
+
+// wireguardWorker handles endpoint changes for WireGuard-based services
+type wireguardWorker struct {
+	config    *kubevip.Config
+	provider  providers.Provider
+	tunnelMgr *wireguard.TunnelManager
+}
+
+func newWireguardWorker(config *kubevip.Config, provider providers.Provider, tunnelMgr *wireguard.TunnelManager) *wireguardWorker {
+	return &wireguardWorker{
+		config:    config,
+		provider:  provider,
+		tunnelMgr: tunnelMgr,
+	}
+}
+
+// processInstance updates nftables DNAT rules when endpoints change
+// This is called by the endpoint watcher when endpoints are added/modified
+func (w *wireguardWorker) processInstance(ctx context.Context, _ *sync.Map, service *v1.Service, inst *instance.Instance) error {
+	log.Debug("[wireguard] processing instance for endpoint change", "service", service.Name, "namespace", service.Namespace)
+
+	// Get the target endpoint for this service
+	// For ExternalTrafficPolicy=Local, only use local endpoints
+	// For ExternalTrafficPolicy=Cluster, use all endpoints
+	var endpoints []string
+	var err error
+	if service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
+		endpoints, err = w.provider.GetLocalEndpoints(w.config.NodeName, w.config)
+	} else {
+		endpoints, err = w.provider.GetAllEndpoints()
+	}
+	if err != nil {
+		return fmt.Errorf("failed to get endpoints: %w", err)
+	}
+
+	if len(endpoints) == 0 {
+		log.Debug("[wireguard] no endpoints available", "service", service.Name)
+		w.clear(ctx, nil, nil, service, inst)
+		return nil
+	}
+
+	// Get service VIPs
+	serviceIPs, err := utils.FetchServiceIPs(service)
+	if err != nil {
+		return fmt.Errorf("failed to get service IPs: %w", err)
+	}
+	if len(service.Spec.Ports) != 0 {
+		if err := w.ensureTunnels(service, serviceIPs); err != nil {
+			return err
+		}
+	}
+	w.clearDNAT(service)
+
+	log.Info("[wireguard] updating DNAT rules for endpoint change",
+		"service", service.Name,
+		"namespace", service.Namespace,
+		"endpoints", endpoints,
+		"vips", serviceIPs)
+
+	for _, port := range service.Spec.Ports {
+		if port.Protocol != v1.ProtocolTCP && port.Protocol != v1.ProtocolUDP {
+			continue
+		}
+		targetPort := w.provider.ResolvePort(port)
+		log.Info("[wireguard] resolved port", "service", service.Name, "servicePort", port.Port, "targetPort", targetPort, "targetPortName", port.TargetPort.StrVal)
+
+		targets := make([]nftables.DNATTarget, len(endpoints))
+		for index, endpoint := range endpoints {
+			targets[index] = nftables.DNATTarget{
+				IP:   endpoint,
+				Port: uint16(targetPort), //nolint:gosec // Port range validated by Kubernetes
+			}
+		}
+		portServiceID, _ := wireguard.ServicePortIDs(service.Namespace, service.Name, port)
+
+		for _, serviceIP := range serviceIPs {
+			vipAddress := utils.StripCIDR(serviceIP)
+			if w.tunnelMgr == nil {
+				return fmt.Errorf("WireGuard tunnel manager not configured")
+			}
+			tunnelConfig := w.tunnelMgr.GetConfigForVIP(vipAddress)
+			if tunnelConfig == nil {
+				return fmt.Errorf("wireguard interface name not configured for VIP %s", vipAddress)
+			}
+			if err := nftables.ApplyDNAT(
+				tunnelConfig.InterfaceName,
+				vipAddress,
+				uint16(port.Port), //nolint:gosec // Port range validated by Kubernetes
+				targets,
+				portServiceID,
+				port.Protocol,
+				service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal,
+				tunnelConfig.ListenPort,
+			); err != nil {
+				log.Error("[wireguard] failed to update DNAT rule", "service", service.Name, "vip", vipAddress, "port", port.Port, "err", err)
+				continue
+			}
+		}
+	}
+
+	return nil
+}
+
+func (w *wireguardWorker) ensureTunnels(service *v1.Service, serviceIPs []string) error {
+	if w.tunnelMgr == nil {
+		return fmt.Errorf("WireGuard tunnel manager not configured")
+	}
+	if len(serviceIPs) == 0 {
+		return fmt.Errorf("no service IPs found for service %s/%s", service.Namespace, service.Name)
+	}
+	var successCount int
+	var lastErr error
+	for _, serviceIP := range serviceIPs {
+		if !w.tunnelMgr.HasConfigForVIP(serviceIP) {
+			lastErr = fmt.Errorf("no WireGuard tunnel configuration found for VIP %s", serviceIP)
+			continue
+		}
+		if err := w.tunnelMgr.AcquireTunnelForVIP(serviceIP, string(service.UID)); err != nil {
+			lastErr = fmt.Errorf("bring up WireGuard tunnel for VIP %s: %w", serviceIP, err)
+			continue
+		}
+		successCount++
+	}
+	if successCount == 0 {
+		return fmt.Errorf("failed to setup WireGuard tunnel for any VIP in service %s/%s: %w", service.Namespace, service.Name, lastErr)
+	}
+	return nil
+}
+
+// clear removes DNAT rules when no endpoints are available
+func (w *wireguardWorker) clear(_ context.Context, _ *sync.Map, _ *string, service *v1.Service, _ *instance.Instance) {
+	w.clearDNAT(service)
+}
+
+func (w *wireguardWorker) clearDNAT(service *v1.Service) {
+	log.Info("[wireguard] clearing DNAT rules (no endpoints)", "service", service.Name, "namespace", service.Namespace)
+	forEachServiceDNATChain(service, func(ipv6 bool, serviceID string) {
+		if err := nftables.DeleteIngressChains(ipv6, serviceID); err != nil {
+			family := utils.IPv4Family
+			if ipv6 {
+				family = utils.IPv6Family
+			}
+			log.Warn("[wireguard] failed to delete DNAT chains", "family", family, "service", service.Name, "err", err)
+		}
+	})
+}
+
+func forEachServiceDNATChain(service *v1.Service, visit func(bool, string)) {
+	if service == nil {
+		return
+	}
+	serviceIPs, _ := utils.FetchServiceIPs(service)
+	familyUnknown := len(serviceIPs) == 0
+	hasIPv4, hasIPv6 := familyUnknown, familyUnknown
+	for _, serviceIP := range serviceIPs {
+		if isIPv6Address(serviceIP) {
+			hasIPv6 = true
+		} else {
+			hasIPv4 = true
+		}
+	}
+
+	for _, port := range service.Spec.Ports {
+		if port.Protocol != v1.ProtocolTCP && port.Protocol != v1.ProtocolUDP {
+			continue
+		}
+		portServiceID, legacyServiceID := wireguard.ServicePortIDs(service.Namespace, service.Name, port)
+		// The legacy identifier is visited so an upgrade removes chains written
+		// before rule IDs carried the protocol.
+		for _, serviceID := range []string{portServiceID, legacyServiceID} {
+			if hasIPv4 {
+				visit(false, serviceID)
+			}
+			if hasIPv6 {
+				visit(true, serviceID)
+			}
+		}
+	}
+}
+
+// getEndpoints retrieves the list of endpoints for a service
+// For ExternalTrafficPolicy=Local, only local endpoints are returned
+// For ExternalTrafficPolicy=Cluster, all endpoints are returned
+func (w *wireguardWorker) getEndpoints(service *v1.Service, id string) ([]string, error) {
+	var endpoints []string
+	var err error
+	if service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
+		endpoints, err = w.provider.GetLocalEndpoints(id, w.config)
+	} else {
+		endpoints, err = w.provider.GetAllEndpoints()
+	}
+	if err != nil {
+		return nil, fmt.Errorf("[wireguard] failed to get endpoints: %w", err)
+	}
+
+	log.Debug("[wireguard] retrieved endpoints", "service", service.Name, "count", len(endpoints), "endpoints", endpoints)
+	return endpoints, nil
+}
+
+// removeEgress is a no-op for WireGuard since egress is handled separately
+func (w *wireguardWorker) removeEgress(service *v1.Service, lastKnownGoodEndpoint *string) {
+	// WireGuard doesn't use egress in the same way as other modes
+	log.Debug("[wireguard] removeEgress called (no-op)", "service", service.Name)
+}
+
+// setInstanceEndpointsStatus updates the endpoint status on the service instance
+func (w *wireguardWorker) setInstanceEndpointsStatus(service *v1.Service, inst *instance.Instance, endpoints []string) error {
+	hasEndpoints := len(endpoints) > 0
+
+	log.Debug("[wireguard] setting instance endpoint status",
+		"service", service.Name,
+		"hasEndpoints", hasEndpoints,
+		"endpointCount", len(endpoints))
+
+	if inst != nil {
+		// Update the network status for all clusters
+		for _, cluster := range inst.Clusters {
+			for i := range cluster.Network {
+				cluster.Network[i].SetHasEndpoints(hasEndpoints)
+			}
+		}
+		log.Debug("[wireguard] updated instance endpoint status",
+			"service", service.Name,
+			"hasEndpoints", hasEndpoints)
+		return nil
+	}
+
+	log.Debug("[wireguard] instance not found for endpoint status update", "service", service.Name)
+	return nil
+}
+
+func isIPv6Address(ip string) bool {
+	// Strip CIDR notation if present before checking
+	addr := utils.StripCIDR(ip)
+	return utils.IsIPv6(addr)
+}

pkg/endpoints/endpoints_wireguard_test.go

@@ -0,0 +1,41 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+type recordingTunnelReleaser struct {
+	releases []string
+}
+
+func (r *recordingTunnelReleaser) ReleaseTunnelForVIP(vip, owner string) error {
+	r.releases = append(r.releases, fmt.Sprintf("%s:%s", vip, owner))
+	return nil
+}
+
+func TestWireguardClearDoesNotDereferenceNilServiceContext(t *testing.T) {
+	worker := &wireguardWorker{}
+	service := &v1.Service{}
+
+	worker.clear(context.TODO(), nil, nil, service, nil)
+}
+
+func TestReleaseWireguardServiceTunnelsUsesServiceUIDOwner(t *testing.T) {
+	releaser := &recordingTunnelReleaser{}
+	service := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{UID: types.UID("service-uid")},
+		Spec:       v1.ServiceSpec{LoadBalancerIP: "192.0.2.10"},
+	}
+
+	releaseWireguardServiceTunnels(releaser, service)
+
+	if len(releaser.releases) != 1 || releaser.releases[0] != "192.0.2.10:service-uid" {
+		t.Fatalf("tunnel releases = %v, want [192.0.2.10:service-uid]", releaser.releases)
+	}
+}

pkg/endpoints/providers/endpoints.go

@@ -0,0 +1,158 @@
+package providers
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	watchtools "k8s.io/client-go/tools/watch"
+	"k8s.io/client-go/util/retry"
+
+	log "log/slog"
+)
+
+type Endpoints struct {
+	label string
+	//nolint:staticcheck // SA1019 endpoints are moving to an opt-in only
+	endpoints *v1.Endpoints
+}
+
+func NewEndpoints() Provider {
+	return &Endpoints{
+		label: "endpoints",
+	}
+}
+
+func (ep *Endpoints) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
+	service *v1.Service) (*watchtools.RetryWatcher, error) {
+	opts := metav1.ListOptions{
+		FieldSelector: fields.OneTermEqualSelector("metadata.name", service.Name).String(),
+	}
+
+	rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
+		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
+			return clientSet.CoreV1().Endpoints(service.Namespace).Watch(ctx, opts)
+		},
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("error creating endpoint watcher: %s", err.Error())
+	}
+
+	return rw, nil
+}
+
+func (ep *Endpoints) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
+	//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
+	eps, ok := endpoints.(*v1.Endpoints)
+	if !ok {
+		cancel()
+		return fmt.Errorf("[%s] unable to parse Kubernetes services from API watcher", ep.GetLabel())
+	}
+	ep.endpoints = eps
+	return nil
+}
+
+// DeleteObject drops the tracked object. A service is backed by exactly one
+// v1.Endpoints object, so there is nothing to match on and the cache is reset.
+func (ep *Endpoints) DeleteObject(endpoints runtime.Object) error {
+	//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
+	if _, ok := endpoints.(*v1.Endpoints); !ok {
+		return fmt.Errorf("[%s] unable to parse Kubernetes object", ep.GetLabel())
+	}
+	//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
+	ep.endpoints = &v1.Endpoints{}
+	return nil
+}
+
+func (ep *Endpoints) GetAllEndpoints() ([]string, error) {
+	result := []string{}
+	for subset := range ep.endpoints.Subsets {
+		for address := range ep.endpoints.Subsets[subset].Addresses {
+			addr := strings.Split(ep.endpoints.Subsets[subset].Addresses[address].IP, "/")
+			result = append(result, addr[0])
+		}
+	}
+
+	return result, nil
+}
+
+func (ep *Endpoints) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
+	var localEndpoints []string
+
+	for _, subset := range ep.endpoints.Subsets {
+		for _, address := range subset.Addresses {
+			log.Debug("processing endpoint", "label", ep.label, "ip", address.IP)
+
+			// 1. Compare the Nodename
+			if address.NodeName != nil && id == *address.NodeName {
+				log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname, "nodename", *address.NodeName)
+				localEndpoints = append(localEndpoints, address.IP)
+				continue
+			}
+			// 2. Compare the Hostname (only useful if address.NodeName is not available)
+			if address.NodeName == nil && id == address.Hostname {
+				log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname)
+				localEndpoints = append(localEndpoints, address.IP)
+				continue
+			}
+		}
+	}
+	return localEndpoints, nil
+}
+
+func (ep *Endpoints) UpdateServiceAnnotation(ctx context.Context, endpoint string, _ string, service *v1.Service, clientSet *kubernetes.Clientset) error {
+	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		// Retrieve the latest version of Deployment before attempting update
+		// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
+		currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(ctx, service.Name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		currentServiceCopy := currentService.DeepCopy()
+		if currentServiceCopy.Annotations == nil {
+			currentServiceCopy.Annotations = make(map[string]string)
+		}
+
+		currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
+
+		_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(ctx, currentServiceCopy, metav1.UpdateOptions{})
+		if err != nil {
+			log.Error("error updating Service Spec", "label", ep.GetLabel(), "name", currentServiceCopy.Name, "err", err)
+			return err
+		}
+		return nil
+	})
+
+	if retryErr != nil {
+		log.Error("failed to set Services", "label", ep.GetLabel(), "err", retryErr)
+		return retryErr
+	}
+	return nil
+}
+
+func (ep *Endpoints) GetLabel() string {
+	return ep.label
+}
+
+func (ep *Endpoints) ResolvePort(servicePort v1.ServicePort) int32 {
+	return ResolvePortWithLookup(servicePort, func(name string) int32 {
+		for _, subset := range ep.endpoints.Subsets {
+			for _, p := range subset.Ports {
+				if p.Name == name {
+					return p.Port
+				}
+			}
+		}
+		return 0
+	})
+}

pkg/endpoints/providers/endpointslices.go

@@ -0,0 +1,179 @@
+package providers
+
+import (
+	"context"
+	"fmt"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	v1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	watchtools "k8s.io/client-go/tools/watch"
+	"k8s.io/client-go/util/retry"
+)
+
+type Endpointslices struct {
+	label  string
+	slices map[string]*discoveryv1.EndpointSlice
+}
+
+func NewEndpointslices() Provider {
+	return &Endpointslices{
+		label:  "endpointslices",
+		slices: make(map[string]*discoveryv1.EndpointSlice),
+	}
+}
+
+func (ep *Endpointslices) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
+	service *v1.Service) (*watchtools.RetryWatcher, error) {
+	labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/service-name": service.Name}}
+
+	opts := metav1.ListOptions{
+		LabelSelector: labels.Set(labelSelector.MatchLabels).String(),
+	}
+
+	rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
+		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
+			return clientSet.DiscoveryV1().EndpointSlices(service.Namespace).Watch(ctx, opts)
+		},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("[%s] error creating endpointslices watcher: %s", ep.label, err.Error())
+	}
+
+	return rw, nil
+}
+
+func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
+	eps, ok := endpoints.(*discoveryv1.EndpointSlice)
+	if !ok {
+		cancel()
+		return fmt.Errorf("[%s] error casting endpoints to v1.Endpoints struct", ep.label)
+	}
+
+	if ep.slices == nil {
+		ep.slices = make(map[string]*discoveryv1.EndpointSlice)
+	}
+	ep.slices[eps.Name] = eps.DeepCopy()
+
+	return nil
+}
+
+func (ep *Endpointslices) DeleteObject(endpoints runtime.Object) error {
+	eps, ok := endpoints.(*discoveryv1.EndpointSlice)
+	if !ok {
+		return fmt.Errorf("[%s] unable to parse Kubernetes object", ep.GetLabel())
+	}
+	delete(ep.slices, eps.Name)
+	return nil
+}
+
+// isServing reports whether an endpoint should receive traffic. Per the
+// EndpointConditions godoc a nil Serving defers to Ready, and a nil Ready is an
+// unknown state that consumers should interpret as ready.
+func isServing(conditions discoveryv1.EndpointConditions) bool {
+	serving := conditions.Serving
+	if serving == nil {
+		serving = conditions.Ready
+	}
+	return serving == nil || *serving
+}
+
+func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
+	result := []string{}
+	for _, eps := range ep.slices {
+		for _, e := range eps.Endpoints {
+			if !isServing(e.Conditions) {
+				continue
+			}
+			result = append(result, e.Addresses...)
+		}
+	}
+	return result, nil
+}
+
+func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
+	var localEndpoints []string
+	for _, eps := range ep.slices {
+		for _, endpoint := range eps.Endpoints {
+			if !isServing(endpoint.Conditions) {
+				continue
+			}
+			for _, address := range endpoint.Addresses {
+				// 1. Compare the Nodename
+				if endpoint.NodeName != nil && id == *endpoint.NodeName {
+					if endpoint.Hostname != nil {
+						log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname, "nodename", *endpoint.NodeName)
+					} else {
+						log.Debug("found endpoint", "provider", ep.label, "ip", address, "nodename", *endpoint.NodeName)
+					}
+					localEndpoints = append(localEndpoints, address)
+					continue
+				}
+
+				// 2. Compare the Hostname (only useful if endpoint.NodeName is not available)
+				if endpoint.NodeName == nil && endpoint.Hostname != nil && id == *endpoint.Hostname {
+					log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname)
+					localEndpoints = append(localEndpoints, address)
+				}
+			}
+		}
+	}
+	return localEndpoints, nil
+}
+
+func (ep *Endpointslices) UpdateServiceAnnotation(ctx context.Context, endpoint, endpointIPv6 string, service *v1.Service, clientSet *kubernetes.Clientset) error {
+	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		// Retrieve the latest version of Deployment before attempting update
+		// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
+		currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(ctx, service.Name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		currentServiceCopy := currentService.DeepCopy()
+		if currentServiceCopy.Annotations == nil {
+			currentServiceCopy.Annotations = make(map[string]string)
+		}
+
+		currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
+		currentServiceCopy.Annotations[kubevip.ActiveEndpointIPv6] = endpointIPv6
+
+		_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(ctx, currentServiceCopy, metav1.UpdateOptions{})
+		if err != nil {
+			log.Error("error updating Service Spec", "provider", ep.label, "service name", currentServiceCopy.Name, "err", err)
+			return err
+		}
+		return nil
+	})
+
+	if retryErr != nil {
+		log.Error("failed to set Services", "provider", ep.label, "err", retryErr)
+		return retryErr
+	}
+	return nil
+}
+
+func (ep *Endpointslices) GetLabel() string {
+	return ep.label
+}
+
+func (ep *Endpointslices) ResolvePort(servicePort v1.ServicePort) int32 {
+	return ResolvePortWithLookup(servicePort, func(name string) int32 {
+		for _, eps := range ep.slices {
+			for _, p := range eps.Ports {
+				if p.Name != nil && *p.Name == name && p.Port != nil {
+					return *p.Port
+				}
+			}
+		}
+		return 0
+	})
+}

pkg/endpoints/providers/endpointslices_test.go

@@ -0,0 +1,149 @@
+package providers
+
+import (
+	"context"
+	"testing"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestEndpointslicesTracksAndDeletesSlices(t *testing.T) {
+	provider := NewEndpointslices().(*Endpointslices)
+	serving := true
+	nodeName := "node-1"
+
+	slice1 := &discoveryv1.EndpointSlice{
+		ObjectMeta:  metav1.ObjectMeta{Name: "slice-1"},
+		AddressType: discoveryv1.AddressTypeIPv4,
+		Endpoints: []discoveryv1.Endpoint{{
+			Addresses:  []string{"10.0.0.1"},
+			Conditions: discoveryv1.EndpointConditions{Serving: &serving},
+			NodeName:   &nodeName,
+		}},
+	}
+	slice2 := &discoveryv1.EndpointSlice{
+		ObjectMeta:  metav1.ObjectMeta{Name: "slice-2"},
+		AddressType: discoveryv1.AddressTypeIPv4,
+		Endpoints: []discoveryv1.Endpoint{{
+			Addresses:  []string{"10.0.0.2"},
+			Conditions: discoveryv1.EndpointConditions{Serving: &serving},
+			NodeName:   &nodeName,
+		}},
+	}
+
+	for _, slice := range []*discoveryv1.EndpointSlice{slice1, slice2} {
+		if err := provider.LoadObject(slice, func() {}); err != nil {
+			t.Fatalf("LoadObject returned error: %v", err)
+		}
+	}
+
+	assertEndpoints(t, provider, []string{"10.0.0.1", "10.0.0.2"})
+	assertLocalEndpoints(t, provider, nodeName, []string{"10.0.0.1", "10.0.0.2"})
+
+	if err := provider.DeleteObject(slice1); err != nil {
+		t.Fatalf("DeleteObject returned error: %v", err)
+	}
+	assertEndpoints(t, provider, []string{"10.0.0.2"})
+	assertLocalEndpoints(t, provider, nodeName, []string{"10.0.0.2"})
+
+	if err := provider.DeleteObject(slice2); err != nil {
+		t.Fatalf("DeleteObject returned error: %v", err)
+	}
+	assertEndpoints(t, provider, nil)
+	assertLocalEndpoints(t, provider, nodeName, nil)
+}
+
+func TestEndpointslicesReplacingSliceUpdatesState(t *testing.T) {
+	provider := NewEndpointslices().(*Endpointslices)
+	first := &discoveryv1.EndpointSlice{
+		ObjectMeta:  metav1.ObjectMeta{Name: "slice-1"},
+		AddressType: discoveryv1.AddressTypeIPv4,
+		Endpoints:   []discoveryv1.Endpoint{{Addresses: []string{"10.0.0.1"}}},
+	}
+	replacement := first.DeepCopy()
+	replacement.Endpoints[0].Addresses = []string{"10.0.0.2"}
+
+	if err := provider.LoadObject(first, context.CancelFunc(func() {})); err != nil {
+		t.Fatalf("LoadObject returned error: %v", err)
+	}
+	if err := provider.LoadObject(replacement, context.CancelFunc(func() {})); err != nil {
+		t.Fatalf("LoadObject returned error: %v", err)
+	}
+
+	assertEndpoints(t, provider, []string{"10.0.0.2"})
+}
+
+func TestEndpointslicesEndpointConditions(t *testing.T) {
+	yes, no := true, false
+	nodeName := "node-1"
+
+	tests := []struct {
+		name       string
+		conditions discoveryv1.EndpointConditions
+		want       []string
+	}{
+		{"serving true", discoveryv1.EndpointConditions{Serving: &yes}, []string{"10.0.0.1"}},
+		{"serving false", discoveryv1.EndpointConditions{Serving: &no}, nil},
+		{"serving false overrides ready true", discoveryv1.EndpointConditions{Serving: &no, Ready: &yes}, nil},
+		{"nil serving defers to ready true", discoveryv1.EndpointConditions{Ready: &yes}, []string{"10.0.0.1"}},
+		{"nil serving defers to ready false", discoveryv1.EndpointConditions{Ready: &no}, nil},
+		{"both nil is treated as ready", discoveryv1.EndpointConditions{}, []string{"10.0.0.1"}},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			provider := NewEndpointslices().(*Endpointslices)
+			slice := &discoveryv1.EndpointSlice{
+				ObjectMeta:  metav1.ObjectMeta{Name: "slice-1"},
+				AddressType: discoveryv1.AddressTypeIPv4,
+				Endpoints: []discoveryv1.Endpoint{{
+					Addresses:  []string{"10.0.0.1"},
+					Conditions: test.conditions,
+					NodeName:   &nodeName,
+				}},
+			}
+			if err := provider.LoadObject(slice, func() {}); err != nil {
+				t.Fatalf("LoadObject returned error: %v", err)
+			}
+			// Cluster and Local policy have to agree on which endpoints are usable.
+			assertEndpoints(t, provider, test.want)
+			assertLocalEndpoints(t, provider, nodeName, test.want)
+		})
+	}
+}
+
+func assertEndpoints(t *testing.T, provider *Endpointslices, want []string) {
+	t.Helper()
+	got, err := provider.GetAllEndpoints()
+	if err != nil {
+		t.Fatalf("GetAllEndpoints returned error: %v", err)
+	}
+	assertStringSet(t, got, want)
+}
+
+func assertLocalEndpoints(t *testing.T, provider *Endpointslices, nodeName string, want []string) {
+	t.Helper()
+	got, err := provider.GetLocalEndpoints(nodeName, &kubevip.Config{})
+	if err != nil {
+		t.Fatalf("GetLocalEndpoints returned error: %v", err)
+	}
+	assertStringSet(t, got, want)
+}
+
+func assertStringSet(t *testing.T, got, want []string) {
+	t.Helper()
+	counts := map[string]int{}
+	for _, value := range got {
+		counts[value]++
+	}
+	for _, value := range want {
+		counts[value]--
+	}
+	for value, count := range counts {
+		if count != 0 {
+			t.Fatalf("endpoint set mismatch for %q: got %v, want %v", value, got, want)
+		}
+	}
+}

pkg/endpoints/providers/interface.go

@@ -0,0 +1,40 @@
+package providers
+
+import (
+	"context"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
+	watchtools "k8s.io/client-go/tools/watch"
+)
+
+type Provider interface {
+	CreateRetryWatcher(context.Context, *kubernetes.Clientset,
+		*v1.Service) (*watchtools.RetryWatcher, error)
+	GetAllEndpoints() ([]string, error)
+	GetLocalEndpoints(string, *kubevip.Config) ([]string, error)
+	GetLabel() string
+	UpdateServiceAnnotation(context.Context, string, string, *v1.Service, *kubernetes.Clientset) error
+	LoadObject(runtime.Object, context.CancelFunc) error
+	DeleteObject(runtime.Object) error
+	// ResolvePort resolves a service port to the actual target port.
+	// For named ports, it looks up the port number from the endpoint.
+	// For numeric ports, it returns the port as-is.
+	ResolvePort(servicePort v1.ServicePort) int32
+}
+
+// ResolvePortWithLookup is a helper that resolves a service port using a lookup function
+// for named ports. This consolidates the common resolution logic.
+func ResolvePortWithLookup(servicePort v1.ServicePort, lookupNamedPort func(string) int32) int32 {
+	if servicePort.TargetPort.IntVal != 0 {
+		return servicePort.TargetPort.IntVal
+	}
+	if servicePort.TargetPort.StrVal != "" {
+		if port := lookupNamedPort(servicePort.TargetPort.StrVal); port != 0 {
+			return port
+		}
+	}
+	return servicePort.Port
+}

pkg/endpoints/providers/providers_test.go

@@ -0,0 +1,319 @@
+package providers
+
+import (
+	"net"
+	"reflect"
+	"testing"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	v1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/kubernetes/fake"
+)
+
+func TestEndpointProvidersParityForLocalAndAllEndpoints(t *testing.T) {
+	t.Parallel()
+
+	nodeA := "node-a"
+	nodeB := "node-b"
+	serving := true
+	v4Addresses := []discoveryv1.Endpoint{
+		{Addresses: []string{"10.0.0.1"}, NodeName: &nodeA, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
+		{Addresses: []string{"10.0.0.2"}, NodeName: &nodeB, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
+	}
+	v6Addresses := []discoveryv1.Endpoint{
+		{Addresses: []string{"2001:db8::1"}, NodeName: &nodeA, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
+		{Addresses: []string{"2001:db8::2"}, NodeName: &nodeB, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
+	}
+
+	legacy := NewEndpoints()
+	//nolint:staticcheck // this test covers the deprecated legacy Endpoints provider on purpose
+	if err := legacy.LoadObject(&v1.Endpoints{
+		ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "default"},
+		//nolint:staticcheck // deprecated legacy Endpoints API is the subject under test
+		Subsets: []v1.EndpointSubset{{
+			Addresses: []v1.EndpointAddress{
+				{IP: "10.0.0.1", NodeName: &nodeA},
+				{IP: "10.0.0.2", NodeName: &nodeB},
+				{IP: "2001:db8::1", NodeName: &nodeA},
+				{IP: "2001:db8::2", NodeName: &nodeB},
+			},
+		}},
+	}, func() {}); err != nil {
+		t.Fatalf("loading legacy Endpoints: %v", err)
+	}
+
+	slices := NewEndpointslices()
+	if err := slices.LoadObject(&discoveryv1.EndpointSlice{
+		ObjectMeta:  metav1.ObjectMeta{Name: "service-v4", Namespace: "default"},
+		AddressType: discoveryv1.AddressTypeIPv4,
+		Endpoints:   v4Addresses,
+	}, func() {}); err != nil {
+		t.Fatalf("loading IPv4 EndpointSlice: %v", err)
+	}
+	if err := slices.LoadObject(&discoveryv1.EndpointSlice{
+		ObjectMeta:  metav1.ObjectMeta{Name: "service-v6", Namespace: "default"},
+		AddressType: discoveryv1.AddressTypeIPv6,
+		Endpoints:   v6Addresses,
+	}, func() {}); err != nil {
+		t.Fatalf("loading IPv6 EndpointSlice: %v", err)
+	}
+
+	legacyAll, err := legacy.GetAllEndpoints()
+	if err != nil {
+		t.Fatalf("legacy GetAllEndpoints() error = %v", err)
+	}
+	sliceAll, err := slices.GetAllEndpoints()
+	if err != nil {
+		t.Fatalf("EndpointSlice GetAllEndpoints() error = %v", err)
+	}
+	wantAll := endpointSet([]string{"10.0.0.1", "10.0.0.2", "2001:db8::1", "2001:db8::2"})
+	if got := endpointSet(legacyAll); !reflect.DeepEqual(got, wantAll) {
+		t.Errorf("legacy all endpoints = %v, want %v", got, wantAll)
+	}
+	if got := endpointSet(sliceAll); !reflect.DeepEqual(got, wantAll) {
+		t.Errorf("EndpointSlice all endpoints = %v, want %v", got, wantAll)
+	}
+
+	legacyLocal, err := legacy.GetLocalEndpoints(nodeA, &kubevip.Config{})
+	if err != nil {
+		t.Fatalf("legacy GetLocalEndpoints() error = %v", err)
+	}
+	sliceLocal, err := slices.GetLocalEndpoints(nodeA, &kubevip.Config{})
+	if err != nil {
+		t.Fatalf("EndpointSlice GetLocalEndpoints() error = %v", err)
+	}
+	wantLocal := endpointSet([]string{"10.0.0.1", "2001:db8::1"})
+	if got := endpointSet(legacyLocal); !reflect.DeepEqual(got, wantLocal) {
+		t.Errorf("legacy local endpoints = %v, want %v", got, wantLocal)
+	}
+	if got := endpointSet(sliceLocal); !reflect.DeepEqual(got, wantLocal) {
+		t.Errorf("EndpointSlice local endpoints = %v, want %v", got, wantLocal)
+	}
+
+	assertEndpointFamilies(t, legacyAll, 2, 2)
+	assertEndpointFamilies(t, sliceAll, 2, 2)
+}
+
+func TestEndpointSlicesLocalFilteringRequiresServingEndpoint(t *testing.T) {
+	t.Parallel()
+
+	node := "node-a"
+	serving := true
+	notServing := false
+	provider := NewEndpointslices()
+	if err := provider.LoadObject(&discoveryv1.EndpointSlice{
+		AddressType: discoveryv1.AddressTypeIPv4,
+		Endpoints: []discoveryv1.Endpoint{
+			{Addresses: []string{"10.0.0.1"}, NodeName: &node, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
+			{Addresses: []string{"10.0.0.2"}, NodeName: &node, Conditions: discoveryv1.EndpointConditions{Serving: &notServing}},
+		},
+	}, func() {}); err != nil {
+		t.Fatalf("loading EndpointSlice: %v", err)
+	}
+
+	local, err := provider.GetLocalEndpoints(node, &kubevip.Config{})
+	if err != nil {
+		t.Fatalf("GetLocalEndpoints() error = %v", err)
+	}
+	if got, want := endpointSet(local), endpointSet([]string{"10.0.0.1"}); !reflect.DeepEqual(got, want) {
+		t.Errorf("local endpoints = %v, want serving endpoints %v", got, want)
+	}
+}
+
+func TestResolvePortFromFakeClientObjects(t *testing.T) {
+	t.Parallel()
+
+	//nolint:staticcheck // deprecated legacy Endpoints API is the subject under test
+	legacyObject := &v1.Endpoints{
+		ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "default"},
+		//nolint:staticcheck // deprecated legacy Endpoints API is the subject under test
+		Subsets: []v1.EndpointSubset{{
+			Ports: []v1.EndpointPort{{Name: "web", Port: 8080}},
+		}},
+	}
+	legacyClient := fake.NewSimpleClientset(legacyObject)
+	legacyLoaded, err := legacyClient.CoreV1().Endpoints("default").Get(t.Context(), "service", metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("getting fake legacy Endpoints: %v", err)
+	}
+	legacy := NewEndpoints()
+	if err := legacy.LoadObject(legacyLoaded, func() {}); err != nil {
+		t.Fatalf("loading fake legacy Endpoints: %v", err)
+	}
+
+	portName := "web"
+	port := int32(8081)
+	sliceObject := &discoveryv1.EndpointSlice{
+		ObjectMeta:  metav1.ObjectMeta{Name: "service-slice", Namespace: "default"},
+		AddressType: discoveryv1.AddressTypeIPv4,
+		Ports:       []discoveryv1.EndpointPort{{Name: &portName, Port: &port}},
+	}
+	sliceClient := fake.NewSimpleClientset(sliceObject)
+	sliceLoaded, err := sliceClient.DiscoveryV1().EndpointSlices("default").Get(t.Context(), "service-slice", metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("getting fake EndpointSlice: %v", err)
+	}
+	slices := NewEndpointslices()
+	if err := slices.LoadObject(sliceLoaded, func() {}); err != nil {
+		t.Fatalf("loading fake EndpointSlice: %v", err)
+	}
+
+	namedPort := v1.ServicePort{Port: 80, TargetPort: intstr.FromString("web")}
+	if got := legacy.ResolvePort(namedPort); got != 8080 {
+		t.Errorf("legacy ResolvePort() = %d, want 8080", got)
+	}
+	if got := slices.ResolvePort(namedPort); got != 8081 {
+		t.Errorf("EndpointSlice ResolvePort() = %d, want 8081", got)
+	}
+}
+
+func TestResolvePortWithLookup(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name   string
+		port   v1.ServicePort
+		lookup func(string) int32
+		want   int32
+	}{
+		{
+			name:   "numeric target port wins",
+			port:   v1.ServicePort{Port: 80, TargetPort: intstr.FromInt(8080)},
+			lookup: func(string) int32 { return 9090 },
+			want:   8080,
+		},
+		{
+			name: "named target port is looked up",
+			port: v1.ServicePort{Port: 80, TargetPort: intstr.FromString("web")},
+			lookup: func(name string) int32 {
+				if name == "web" {
+					return 8081
+				}
+				return 0
+			},
+			want: 8081,
+		},
+		{
+			name:   "missing named target falls back to service port",
+			port:   v1.ServicePort{Port: 80, TargetPort: intstr.FromString("missing")},
+			lookup: func(string) int32 { return 0 },
+			want:   80,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := ResolvePortWithLookup(tt.port, tt.lookup); got != tt.want {
+				t.Errorf("ResolvePortWithLookup() = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+func endpointSet(endpoints []string) map[string]struct{} {
+	result := make(map[string]struct{}, len(endpoints))
+	for _, endpoint := range endpoints {
+		result[endpoint] = struct{}{}
+	}
+	return result
+}
+
+func assertEndpointFamilies(t *testing.T, endpoints []string, wantIPv4, wantIPv6 int) {
+	t.Helper()
+	ipv4, ipv6 := 0, 0
+	for _, endpoint := range endpoints {
+		ip := net.ParseIP(endpoint)
+		if ip == nil {
+			t.Errorf("endpoint %q is not an IP address", endpoint)
+			continue
+		}
+		if ip.To4() != nil {
+			ipv4++
+		} else {
+			ipv6++
+		}
+	}
+	if ipv4 != wantIPv4 || ipv6 != wantIPv6 {
+		t.Errorf("endpoint families = IPv4 %d, IPv6 %d; want IPv4 %d, IPv6 %d", ipv4, ipv6, wantIPv4, wantIPv6)
+	}
+}
+
+func TestEndpointProvidersPreferNodeNameOverHostname(t *testing.T) {
+	t.Parallel()
+
+	nodeA := "node-a"
+	nodeB := "node-b"
+	serving := true
+
+	tests := []struct {
+		name string
+		load func(Provider) error
+	}{
+		{
+			name: "legacy Endpoints",
+			load: func(provider Provider) error {
+				//nolint:staticcheck // the legacy provider is deliberately under test
+				return provider.LoadObject(&v1.Endpoints{
+					Subsets: []v1.EndpointSubset{{
+						Addresses: []v1.EndpointAddress{{
+							IP:       "10.0.0.1",
+							NodeName: &nodeB,
+							Hostname: nodeA,
+						}},
+					}},
+				}, func() {})
+			},
+		},
+		{
+			name: "EndpointSlice",
+			load: func(provider Provider) error {
+				hostname := nodeA
+				return provider.LoadObject(&discoveryv1.EndpointSlice{
+					AddressType: discoveryv1.AddressTypeIPv4,
+					Endpoints: []discoveryv1.Endpoint{{
+						Addresses:  []string{"10.0.0.1"},
+						NodeName:   &nodeB,
+						Hostname:   &hostname,
+						Conditions: discoveryv1.EndpointConditions{Serving: &serving},
+					}},
+				}, func() {})
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			var provider Provider
+			if tt.name == "legacy Endpoints" {
+				provider = NewEndpoints()
+			} else {
+				provider = NewEndpointslices()
+			}
+			if err := tt.load(provider); err != nil {
+				t.Fatalf("LoadObject() error = %v", err)
+			}
+
+			local, err := provider.GetLocalEndpoints(nodeA, &kubevip.Config{})
+			if err != nil {
+				t.Fatalf("GetLocalEndpoints(%q) error = %v", nodeA, err)
+			}
+			if len(local) != 0 {
+				t.Fatalf("GetLocalEndpoints(%q) = %v, want no endpoints", nodeA, local)
+			}
+
+			local, err = provider.GetLocalEndpoints(nodeB, &kubevip.Config{})
+			if err != nil {
+				t.Fatalf("GetLocalEndpoints(%q) error = %v", nodeB, err)
+			}
+			if got, want := endpointSet(local), endpointSet([]string{"10.0.0.1"}); !reflect.DeepEqual(got, want) {
+				t.Fatalf("GetLocalEndpoints(%q) = %v, want %v", nodeB, got, want)
+			}
+		})
+	}
+}

pkg/etcd/client.go

@@ -0,0 +1,26 @@
+package etcd
+
+import (
+	"go.etcd.io/etcd/client/pkg/v3/transport"
+	clientv3 "go.etcd.io/etcd/client/v3"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+)
+
+func NewClient(c *kubevip.Config) (*clientv3.Client, error) {
+	tlsInfo := transport.TLSInfo{
+		TrustedCAFile: c.Etcd.CAFile,
+		CertFile:      c.Etcd.ClientCertFile,
+		KeyFile:       c.Etcd.ClientKeyFile,
+	}
+
+	clientTLS, err := tlsInfo.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	return clientv3.New(clientv3.Config{
+		Endpoints: c.Etcd.Endpoints,
+		TLS:       clientTLS,
+	})
+}

pkg/etcd/election.go

@@ -0,0 +1,237 @@
+package etcd
+
+import (
+	"context"
+	"fmt"
+	"hash/fnv"
+	"sync"
+	"time"
+
+	log "log/slog"
+
+	"github.com/pkg/errors"
+	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/concurrency"
+)
+
+// LeaderElectionConfig allows to configure the leader election params.
+type LeaderElectionConfig struct {
+	// EtcdConfig contains the client to connect to the etcd cluster.
+	EtcdConfig ClientConfig
+
+	// Name uniquely identifies this leader election. All members of the same election
+	// should use the same value here.
+	Name string
+
+	// MemberID identifies uniquely this contestant from other in the leader election.
+	// It will be converted to an int64 using a hash, so theoretically collisions are possible
+	// when using a string. If you want to guarantee safety, us MemberUniqueID to specify a unique
+	// int64 directly.
+	// If two processes start a leader election using the same MemberID, one of them will
+	// fail.
+	MemberID string
+
+	// MemberUniqueID is the int equivalent to MemberID that allows to override the default conversion
+	// from string to int using hashing.
+	MemberUniqueID *uint64
+
+	// LeaseDurationSeconds is the duration that non-leader candidates will
+	// wait to force acquire leadership.
+	// This is just a request to the etcd server but it's not guaranteed, the server
+	// might decide to make the duration longer.
+	LeaseDurationSeconds int64
+
+	// Callbacks are callbacks that are triggered during certain lifecycle
+	// events of the LeaderElector
+	Callbacks LeaderCallbacks
+}
+
+// LeaderCallbacks are callbacks that are triggered during certain
+// lifecycle events of the election.
+type LeaderCallbacks struct {
+	// OnStartedLeading is called when this member starts leading.
+	OnStartedLeading func(context.Context)
+	// OnStoppedLeading is called when this member stops leading.
+	OnStoppedLeading func()
+	// OnNewLeader is called when the client observes a leader that is
+	// not the previously observed leader. This includes the first observed
+	// leader when the client starts.
+	OnNewLeader func(identity string)
+}
+
+// ClientConfig contains the client to connect to the etcd cluster.
+type ClientConfig struct {
+	Client *clientv3.Client
+}
+
+// RunElectionOrDie behaves the same way as RunElection but panics if there is an error.
+func RunElectionOrDie(ctx context.Context, config *LeaderElectionConfig) error {
+	if err := RunElection(ctx, config); err != nil {
+		return fmt.Errorf("leaderelection error: %w", err)
+	}
+	return nil
+}
+
+// RunElection starts a client with the provided config or panics.
+// RunElection blocks until leader election loop is
+// stopped by ctx or it has stopped holding the leader lease.
+func RunElection(ctx context.Context, config *LeaderElectionConfig) error {
+	var memberID uint64
+	if config.MemberUniqueID != nil {
+		memberID = *config.MemberUniqueID
+	} else {
+		h := fnv.New64a()
+		if _, err := h.Write(append([]byte(config.Name), []byte(config.MemberID)...)); err != nil {
+			return err
+		}
+		memberID = h.Sum64()
+	}
+
+	ttl := config.LeaseDurationSeconds
+	r := &pb.LeaseGrantRequest{TTL: ttl, ID: int64(memberID)} //nolint
+	lease, err := clientv3.RetryLeaseClient(
+		config.EtcdConfig.Client,
+	).LeaseGrant(ctx, r)
+	if err != nil {
+		return errors.Wrap(err, "creating lease")
+	}
+
+	leaseID := clientv3.LeaseID(lease.ID)
+
+	s, err := concurrency.NewSession(
+		config.EtcdConfig.Client,
+		concurrency.WithTTL(int(lease.TTL)),
+		concurrency.WithLease(leaseID),
+	)
+	if err != nil {
+		return err
+	}
+
+	election := concurrency.NewElection(s, config.Name)
+
+	m := &member{
+		client:         config.EtcdConfig.Client,
+		election:       election,
+		callbacks:      config.Callbacks,
+		memberID:       config.MemberID,
+		weAreTheLeader: make(chan struct{}, 1),
+		leaseTTL:       lease.TTL,
+	}
+
+	wg := sync.WaitGroup{}
+	defer wg.Wait()
+
+	wg.Go(func() {
+		m.tryToBeLeader(ctx, &wg)
+	})
+	m.watchLeaderChanges(ctx)
+
+	return nil
+}
+
+type member struct {
+	key              string
+	client           *clientv3.Client
+	election         *concurrency.Election
+	isLeader         bool
+	currentLeaderKey string
+	callbacks        LeaderCallbacks
+	memberID         string
+	weAreTheLeader   chan struct{}
+	leaseTTL         int64
+}
+
+func (m *member) watchLeaderChanges(ctx context.Context) {
+	observeCtx, observeCancel := context.WithCancel(ctx)
+	defer observeCancel()
+	changes := m.election.Observe(observeCtx)
+
+watcher:
+	for {
+		select {
+		case <-ctx.Done():
+			break watcher
+		case <-m.weAreTheLeader:
+
+			m.isLeader = true
+			m.key = m.election.Key() // by this time, this should already be set, since Campaign has already returned
+			log.Debug("Marking self as leader with key", "id", m.memberID, "key", m.key)
+		case response := <-changes:
+			log.Debug("Leader Changes", "id", m.memberID, "response", response)
+			if len(response.Kvs) == 0 {
+				// There is a race condition where just after we stop being the leader
+				// if there are no more leaders, we might get a response with no key-values
+				// just before the response channel is closed or the context is cancel
+				// In that case, just continue and let one of those two things happen
+				continue
+			}
+			newLeaderKey := response.Kvs[0].Key
+			if m.isLeader && m.key != string(newLeaderKey) {
+				// We stopped being leaders
+
+				// exit the loop, so we cancel the observe context so we stop watching
+				// for new leaders. That will close the channel and make this function exit,
+				// which also makes the routine to finish and RunElection returns
+				break watcher
+			}
+
+			if m.currentLeaderKey != string(newLeaderKey) {
+				// we observed a leader, this could be us or someone else
+				m.currentLeaderKey = string(newLeaderKey)
+				m.callbacks.OnNewLeader(string(response.Kvs[0].Value))
+			}
+		}
+	}
+
+	// If we are here, either we have stopped being leaders or we lost the watcher
+	// Make sure we call OnStoppedLeading if we were the leader.
+	if m.isLeader {
+		m.callbacks.OnStoppedLeading()
+	}
+
+	log.Debug("Exiting watcher", "id", m.memberID)
+}
+
+func (m *member) tryToBeLeader(ctx context.Context, wg *sync.WaitGroup) {
+	if err := m.election.Campaign(ctx, m.memberID); err != nil {
+		log.Error("Failed trying to become the leader", "err", err)
+		// Resign just in case we acquired leadership just before failing
+		if err := m.election.Resign(m.client.Ctx()); err != nil {
+			log.Warn("Failed to resign after we failed becoming the leader, this might not be a problem if we were never the leader", "err", err)
+		}
+		return
+		// TODO: what to do here?
+		// We probably want watchLeaderChanges to exit as well, since Run
+		// is expecting us to try to become the leader, but if we are here,
+		// we won't. So if we don't panic, we need to signal it somehow
+	}
+
+	// Inform the observer that we are the leader as soon as possible,
+	// so it can detect if we stop being it
+	m.weAreTheLeader <- struct{}{}
+
+	// Once we are the leader, start the routine to resign if context is canceled
+	wg.Go(func() {
+		m.resignOnCancel(ctx)
+	})
+
+	// After becoming the leader, we wait for at least a lease TTL to wait for
+	// the previous leader to detect the new leadership (if there was one) and
+	// stop its processes
+	// TODO: is this too cautious?
+	log.Debug("timeout before OnStartedLeading", "id", m.memberID, "timeout", m.leaseTTL)
+	time.Sleep(time.Second * time.Duration(m.leaseTTL))
+
+	// We are the leader, execute our code
+	m.callbacks.OnStartedLeading(ctx)
+
+	// Here the routine dies if OnStartedLeading doesn't block, there is nothing else to do
+}
+
+func (m *member) resignOnCancel(ctx context.Context) {
+	<-ctx.Done()
+	if err := m.election.Resign(m.client.Ctx()); err != nil && !errors.Is(err, context.Canceled) {
+		log.Error("Failed to resign after the context was canceled", "err", err)
+	}
+}

pkg/etcd/election_test.go

@@ -0,0 +1,196 @@
+//go:build integration
+// +build integration
+
+package etcd_test
+
+import (
+	"context"
+	"log"
+	"math/rand"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/etcd"
+	. "github.com/onsi/gomega"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.uber.org/zap"
+)
+
+func TestRunElectionWithMemberIDCollision(t *testing.T) {
+	t.Parallel()
+	g := NewWithT(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	cli := client(g)
+	defer cli.Close()
+
+	electionName := randomElectionNameForTest("memberIDConflict")
+	log.Printf("Election name %s\n", electionName)
+	memberCtx, cancelMember1 := context.WithCancel(ctx)
+
+	// Use a channel to signal when the first member has observed a new leader
+	// This ensures proper ordering without relying on sleep timing
+	firstMemberObservedLeader := make(chan struct{})
+	var firstMemberObservedOnce sync.Once
+
+	config := &etcd.LeaderElectionConfig{
+		EtcdConfig: etcd.ClientConfig{
+			Client: cli,
+		},
+		Name:                 electionName,
+		MemberID:             randomElectionNameForTest("my-host"),
+		LeaseDurationSeconds: 1,
+		Callbacks: etcd.LeaderCallbacks{
+			OnStartedLeading: func(ctx context.Context) {
+				log.Println("I'm the leader!!!!")
+				log.Println("Renouncing as leader by canceling context")
+				cancelMember1()
+			},
+			OnNewLeader: func(identity string) {
+				log.Printf("New leader: %s\n", identity)
+				// Signal that the first member has observed a leader
+				// This means the lease has been created
+				firstMemberObservedOnce.Do(func() {
+					close(firstMemberObservedLeader)
+				})
+			},
+			OnStoppedLeading: func() {
+				log.Println("I'm not the leader anymore")
+			},
+		},
+	}
+
+	wg := &sync.WaitGroup{}
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		g.Expect(etcd.RunElection(memberCtx, config)).To(Succeed())
+	}()
+
+	go func() {
+		defer wg.Done()
+		// Wait for the first member to observe a leader, which means the lease has been created
+		select {
+		case <-firstMemberObservedLeader:
+			// First member has created the lease, now try to create a conflicting one
+		case <-time.After(5 * time.Second):
+			t.Error("timeout waiting for first member to observe leader")
+			return
+		}
+		// Use a cancellable context to prevent hanging if this goroutine unexpectedly succeeds
+		member2Ctx, cancelMember2 := context.WithTimeout(ctx, 5*time.Second)
+		defer cancelMember2()
+		g.Expect(etcd.RunElection(member2Ctx, config)).Should(MatchError(ContainSubstring("creating lease")))
+	}()
+
+	wg.Wait()
+}
+
+func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
+	t.Parallel()
+	g := NewWithT(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	cli := client(g)
+	defer cli.Close()
+
+	cliMember1 := client(g)
+	defer cliMember1.Close()
+
+	electionName := randomElectionNameForTest("steppingDown")
+	configBase := etcd.LeaderElectionConfig{
+		EtcdConfig: etcd.ClientConfig{
+			Client: cli,
+		},
+		Name:                 electionName,
+		LeaseDurationSeconds: 1,
+	}
+
+	member1Ctx, cancelMember1 := context.WithCancel(ctx)
+	member2Ctx, cancelMember2 := context.WithCancel(ctx)
+
+	config1 := configBase
+	config1.EtcdConfig.Client = cliMember1
+	config1.MemberID = randomElectionNameForTest("my-host")
+	uniqueID := rand.Uint64()
+	config1.MemberUniqueID = &uniqueID
+	config1.Callbacks = baseCallbacksForName(config1.MemberID)
+	syncMembers := make(chan (any))
+	config1.Callbacks.OnStartedLeading = func(_ context.Context) {
+		log.Println("I'm my-host, the new leader!!!!")
+		close(syncMembers)
+		log.Println("Losing the leadership on purpose by stopping renewing the lease")
+		g.Expect(cliMember1.Lease.Close()).To(Succeed())
+		log.Println("Member1 leases closed")
+		cancelMember1()
+	}
+
+	config2 := configBase
+	config2.MemberID = randomElectionNameForTest("my-other-host")
+	config2.Callbacks = baseCallbacksForName(config2.MemberID)
+	config2.Callbacks.OnStartedLeading = func(_ context.Context) {
+		log.Println("I'm my-other-host, the new leader!!!!")
+		log.Println("Renouncing as leader by canceling context")
+		cancelMember2()
+	}
+
+	wg := &sync.WaitGroup{}
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		g.Expect(etcd.RunElection(member1Ctx, &config1)).To(Succeed())
+		log.Printf("%s routine done\n", config1.MemberID)
+	}()
+
+	go func() {
+		defer wg.Done()
+		<-syncMembers
+		g.Expect(etcd.RunElection(member2Ctx, &config2)).To(Succeed())
+		log.Printf("%s routine done\n", config2.MemberID)
+	}()
+
+	wg.Wait()
+
+}
+
+func baseCallbacksForName(name string) etcd.LeaderCallbacks {
+	return etcd.LeaderCallbacks{
+		OnStartedLeading: func(ctx context.Context) {
+			log.Printf("[%s] I'm the new leader!!!!\n", name)
+		},
+		OnNewLeader: func(identity string) {
+			log.Printf("[%s] New leader: %s\n", name, identity)
+		},
+		OnStoppedLeading: func() {
+			log.Printf("[%s] I'm not the leader anymore\n", name)
+		},
+	}
+}
+
+func randomElectionNameForTest(name string) string {
+	return name + "-" + randomString(6)
+}
+
+const charSet = "0123456789abcdefghijklmnopqrstuvwxyz"
+
+var rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
+
+func randomString(n int) string {
+	result := make([]byte, n)
+	for i := range result {
+		result[i] = charSet[rnd.Intn(len(charSet))]
+	}
+	return string(result)
+}
+
+func client(g Gomega) *clientv3.Client {
+	c, err := clientv3.New(clientv3.Config{
+		Endpoints: []string{"localhost:2379"},
+		Logger:    zap.NewNop(),
+	})
+	g.Expect(err).NotTo(HaveOccurred())
+	return c
+}

pkg/etcd/etcd_suite_test.go

@@ -0,0 +1,152 @@
+//go:build integration
+// +build integration
+
+package etcd_test
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	etcdVersion     = "v3.5.9"
+	etcdBinDir      = "bin"
+	etcdBinPath     = etcdBinDir + "/etcd"
+	downloadURL     = "https://storage.googleapis.com/etcd"
+	tmpDownloadFile = "etcd.tar.gz"
+	pidFile         = "etcd.pid"
+)
+
+func TestMain(m *testing.M) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	expectSuccess(startEtcd(ctx), "starting etcd")
+
+	os.Exit(runTestsWithCleanup(m, func() {
+		expectSuccess(stopEtcd(), "stopping etcd")
+	}))
+}
+
+func runTestsWithCleanup(m *testing.M, cleanup func()) int {
+	defer cleanup()
+	return m.Run()
+}
+
+func expectSuccess(err error, msg string) {
+	if err != nil {
+		log.Fatalf("%s: %s\n", msg, err)
+	}
+}
+
+func startEtcd(ctx context.Context) error {
+	if _, err := os.Stat(pidFile); err == nil {
+		log.Println("Etcd already running, reusing")
+		return nil
+	}
+
+	etcdPath, err := installEtcd(ctx)
+	if err != nil {
+		errors.Wrap(err, "installing etcd for tests")
+	}
+
+	etcdCmd := exec.Command(etcdPath, "--data-dir", "./etcd-data")
+	if os.Getenv("ETCD_SERVER_LOGS") == "true" {
+		log.Println("Enabling etcd server logs")
+		etcdCmd.Stdout = os.Stdout
+		etcdCmd.Stderr = os.Stderr
+	}
+	log.Println("Starting etcd")
+	if err := etcdCmd.Start(); err != nil {
+		errors.Wrap(err, "starting etcd for tests")
+	}
+
+	if err := os.WriteFile(pidFile, []byte(strconv.Itoa(etcdCmd.Process.Pid)), 0o600); err != nil {
+		return err
+	}
+
+	log.Println("Waiting for etcd to be up")
+	time.Sleep(time.Second)
+
+	return nil
+}
+
+func installEtcd(ctx context.Context) (string, error) {
+	projectRoot, err := filepath.Abs("../../")
+	if err != nil {
+		return "", err
+	}
+	binDir := filepath.Join(projectRoot, etcdBinDir)
+	etcdPath := filepath.Join(projectRoot, etcdBinPath)
+
+	if _, err := os.Stat(etcdPath); err == nil {
+		log.Println("Etcd already installed, skipping")
+		return etcdPath, nil
+	}
+
+	if err := os.MkdirAll(binDir, 0o755); err != nil {
+		return "", err
+	}
+
+	download := fmt.Sprintf("%s/%s/etcd-%s-linux-amd64.tar.gz", downloadURL, etcdVersion, etcdVersion)
+
+	// Hacky to run bash, but simplifies this code a lot
+	cmd := fmt.Sprintf("curl -sL %s | tar -xzvf - -C %s --strip-components=1", download, binDir)
+	out, err := exec.CommandContext(ctx, "bash", "-c", cmd).CombinedOutput()
+	if err != nil {
+		return "", errors.Wrapf(err, "downloading etcd: %s", string(out))
+	}
+
+	return etcdPath, nil
+}
+
+func stopEtcd() error {
+	if os.Getenv("REUSE_ETCD") == "true" {
+		log.Println("REUSE_ETCD=true, leaving etcd running")
+		return nil
+	}
+
+	if _, err := os.Stat(pidFile); os.IsNotExist(err) {
+		log.Println("Etcd pid file doesn't exit, skipping cleanup")
+		return nil
+	}
+
+	dat, err := os.ReadFile(pidFile)
+	if err != nil {
+		return err
+	}
+	pid, err := strconv.Atoi(string(dat))
+	if err != nil {
+		return err
+	}
+
+	etcdProcess, err := os.FindProcess(pid)
+	if err != nil {
+		return err
+	}
+
+	log.Println("Stopping etcd")
+	if err := etcdProcess.Kill(); err != nil {
+		return errors.Wrap(err, "Failed stopping etcd")
+	}
+
+	log.Println("Deleting etcd data")
+	if err := os.RemoveAll("./etcd-data"); err != nil {
+		return errors.Wrap(err, "deleting etcd data")
+	}
+
+	log.Println("Deleting etcd pid file")
+	if err := os.RemoveAll(pidFile); err != nil {
+		return errors.Wrap(err, "deleting pid file")
+	}
+
+	return nil
+}

pkg/instance/instance.go

@@ -0,0 +1,886 @@
+package instance
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"slices"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+
+	log "log/slog"
+
+	"github.com/vishvananda/netlink"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
+	"github.com/kube-vip/kube-vip/pkg/cluster"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/networkinterface"
+	"github.com/kube-vip/kube-vip/pkg/node"
+	"github.com/kube-vip/kube-vip/pkg/route"
+	"github.com/kube-vip/kube-vip/pkg/sysctl"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
+
+// Instance defines an instance of everything needed to manage vips
+type Instance struct {
+	// Virtual IP / Load Balancer configuration
+	VIPConfigs []*kubevip.Config
+
+	// cluster instances
+	Clusters []*cluster.Cluster
+
+	// Service uses DHCP
+	IsDHCPv4            bool
+	IsDHCPv6            bool
+	DHCPInterface       string
+	DHCPInterfaceHwaddr string
+	DHCPInterfaceIP     string
+	DHCPInterfaceIPv4   string
+	DHCPInterfaceIPv6   string
+	DHCPHostname        string
+	DHCPv4Client        vip.DHCPClient
+	DHCPv6Client        vip.DHCPClient
+	macvlanName         string
+	dhcpBroadcast       bool
+	dhcpInterfaceOwned  atomic.Bool
+
+	// Service use Vlan
+	IsVLAN        bool
+	VLANInterface string
+	vlanOwned     atomic.Bool
+
+	// External Gateway IP the service is forwarded from
+	UPNPGatewayIPs []string
+
+	// Kubernetes service mapping
+	ServiceUID       types.UID
+	ServiceAddresses []string
+	ServiceSnapshot  *v1.Service
+	cleanupInfo      *ServiceCleanupInfo
+
+	// AddCalled determined that ActionAdd was already performed for the instance
+	AddCalled bool
+
+	// LabelAdded determined that node was labeled with
+	// service-provided.kube-vip.io label
+	LabelAdded bool
+}
+
+func (instance *Instance) UID() types.UID {
+	return instance.ServiceUID
+}
+
+func (instance *Instance) Addresses() []string {
+	if instance.ServiceAddresses != nil {
+		return append([]string(nil), instance.ServiceAddresses...)
+	}
+	addresses, _ := FetchServiceAddresses(instance.ServiceSnapshot)
+	return addresses
+}
+
+type ServiceCleanupInfo struct {
+	Namespace             string
+	Name                  string
+	Lease                 string
+	ExternalTrafficPolicy v1.ServiceExternalTrafficPolicy
+}
+
+// CleanupInfo returns Service policy captured when the instance was created.
+func (instance *Instance) CleanupInfo() (ServiceCleanupInfo, bool) {
+	if instance == nil {
+		return ServiceCleanupInfo{}, false
+	}
+	if instance.cleanupInfo != nil {
+		return *instance.cleanupInfo, true
+	}
+	if instance.ServiceSnapshot == nil {
+		return ServiceCleanupInfo{}, false
+	}
+	return serviceCleanupInfo(instance.ServiceSnapshot), true
+}
+
+func serviceCleanupInfo(service *v1.Service) ServiceCleanupInfo {
+	return ServiceCleanupInfo{
+		Namespace:             service.Namespace,
+		Name:                  service.Name,
+		Lease:                 service.Annotations[kubevip.ServiceLease],
+		ExternalTrafficPolicy: service.Spec.ExternalTrafficPolicy,
+	}
+}
+
+type Port struct {
+	Port uint16
+	Type string
+}
+
+func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
+	intfMgr *networkinterface.Manager, arpMgr *arp.Manager, routeMgr *route.Manager,
+	nodeLabelMgr node.Labeler, wg *sync.WaitGroup) (*Instance, error) {
+	instanceAddresses, instanceHostnames := FetchServiceAddresses(svc)
+	log.Info("new instance", "namespace", svc.Namespace, "service", svc.Name, "addresses", instanceAddresses, "hostnames", instanceHostnames)
+
+	cleanupInfo := serviceCleanupInfo(svc)
+	instance := &Instance{
+		ServiceUID:       svc.UID,
+		ServiceAddresses: append([]string(nil), instanceAddresses...),
+		ServiceSnapshot:  svc,
+		cleanupInfo:      &cleanupInfo,
+	}
+	if err := instance.initialize(ctx, svc, config, intfMgr, arpMgr, routeMgr, nodeLabelMgr, wg, instanceAddresses, instanceHostnames); err != nil {
+		return nil, errors.Join(err, instance.CleanupLinkAttachments())
+	}
+	return instance, nil
+}
+
+func (instance *Instance) initialize(ctx context.Context, svc *v1.Service, config *kubevip.Config,
+	intfMgr *networkinterface.Manager, arpMgr *arp.Manager, routeMgr *route.Manager,
+	nodeLabelMgr node.Labeler, wg *sync.WaitGroup, instanceAddresses, instanceHostnames []string) error {
+	var newVips []*kubevip.Config
+	var link netlink.Link
+	var err error
+
+	for _, address := range instanceAddresses {
+		// Detect if we're using a specific interface for services
+		var svcInterface string
+
+		svcInterface = svc.Annotations[kubevip.ServiceVlan]
+		if svcInterface != "" {
+			parent, tag, err := utils.ParseVLANInterface(svcInterface)
+			if err != nil {
+				log.Error("failed to validate VLAN", "err", err)
+			}
+
+			if err := instance.addVLAN(parent, tag); err != nil {
+				log.Error("failed to create VLAN", "err", err)
+			}
+		} else {
+			// If no vlan defined use specific interface from annotation
+			svcInterface = svc.Annotations[kubevip.ServiceInterface]
+		}
+
+		if svcInterface == kubevip.Auto {
+			link, err = autoFindInterface(address)
+			if err != nil {
+				log.Error("automatically discover network interface for annotated IP", "address", address, "err", err)
+			} else {
+				if link == nil {
+					log.Error("automatically discover network interface for annotated IP address", "address", address)
+				}
+			}
+			if link == nil {
+				svcInterface = ""
+			} else {
+				svcInterface = getAutoInterfaceName(link, config.Interface)
+			}
+		}
+		// If it is still blank then use the
+		if svcInterface == "" {
+			switch config.ServicesInterface {
+			case kubevip.Auto:
+				link, err = autoFindInterface(address)
+				if err != nil {
+					log.Error("failed to automatically discover network interface for address", "ip", address, "err", err, "interface", config.Interface)
+				} else if link == nil {
+					log.Error("failed to automatically discover network interface for address", "ip", address, "defaulting to", config.Interface)
+				}
+				svcInterface = getAutoInterfaceName(link, config.Interface)
+			case "":
+				svcInterface = config.Interface
+			default:
+				svcInterface = config.ServicesInterface
+			}
+		}
+
+		if link == nil {
+			if link, err = netlink.LinkByName(svcInterface); err != nil {
+				return fmt.Errorf("failed to get interface %s: %w", svcInterface, err)
+			}
+			if link == nil {
+				return fmt.Errorf("failed to get interface %s", svcInterface)
+			}
+		}
+
+		cidrs := vip.Split(config.VIPSubnet)
+
+		ipv4AutoSubnet := false
+		ipv6AutoSubnet := false
+		if cidrs[0] == kubevip.Auto {
+			ipv4AutoSubnet = true
+		}
+
+		if len(cidrs) > 1 && cidrs[1] == kubevip.Auto {
+			ipv6AutoSubnet = true
+		}
+
+		if (config.Address != "" || config.VIP != "") && (ipv4AutoSubnet || ipv6AutoSubnet) {
+			return fmt.Errorf("auto subnet discovery cannot be used if VIP address was provided")
+		}
+
+		subnet := ""
+		var err error
+		if utils.IsIPv4(address) {
+			if ipv4AutoSubnet {
+				subnet, err = autoFindSubnet(link, address)
+				if err != nil {
+					return fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
+				}
+			} else {
+				if cidrs[0] != "" && cidrs[0] != kubevip.Auto {
+					subnet = cidrs[0]
+				} else {
+					subnet = strconv.Itoa(vip.DefaultMaskIPv4)
+				}
+			}
+		} else {
+			if ipv6AutoSubnet {
+				subnet, err = autoFindSubnet(link, address)
+				if err != nil {
+					return fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
+				}
+			} else {
+				if len(cidrs) > 1 && cidrs[1] != "" && cidrs[1] != kubevip.Auto {
+					subnet = cidrs[1]
+				} else {
+					subnet = strconv.Itoa(vip.DefaultMaskIPv6)
+				}
+			}
+		}
+
+		// Generate new Virtual IP configuration
+		newVips = append(newVips, &kubevip.Config{
+			VIP:                    address,
+			Interface:              svcInterface,
+			SingleNode:             true,
+			EnableARP:              config.EnableARP,
+			EnableBGP:              config.EnableBGP,
+			BGPAttachIPToInterface: config.BGPAttachIPToInterface,
+			VIPSubnet:              subnet,
+			EnableRoutingTable:     config.EnableRoutingTable,
+			RoutingTableID:         config.RoutingTableID,
+			RoutingTableType:       config.RoutingTableType,
+			RoutingProtocol:        config.RoutingProtocol,
+			SkipDAD:                config.SkipDAD,
+			ArpBroadcastRate:       config.ArpBroadcastRate,
+			EnableServiceSecurity:  config.EnableServiceSecurity,
+			DNSMode:                config.DNSMode,
+			DHCPMode:               config.DHCPMode,
+			DHCPBackoffAttempts:    config.DHCPBackoffAttempts,
+			DisableServiceUpdates:  config.DisableServiceUpdates,
+			EnableServicesElection: config.EnableServicesElection,
+			// cleanupVIPs reads this from the per-VIP config, so Service VIPs need it too.
+			PreserveVIPOnLeadershipLoss: config.PreserveVIPOnLeadershipLoss,
+			KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
+				EnableLeaderElection: config.EnableLeaderElection,
+			},
+		})
+	}
+
+	for _, hostname := range instanceHostnames {
+		log.Info("hostname", "addr", hostname)
+		// Detect if we're using a specific interface for services
+		var svcInterface string
+
+		svcInterface = svc.Annotations[kubevip.ServiceVlan]
+		if svcInterface != "" {
+			parent, tag, err := utils.ParseVLANInterface(svcInterface)
+			if err != nil {
+				log.Error("failed to validate VLAN", "err", err)
+			}
+
+			if err := instance.addVLAN(parent, tag); err != nil {
+				log.Error("failed to create VLAN", "err", err)
+			}
+		} else {
+			// If no vlan defined use specific interface from annotation
+			svcInterface = svc.Annotations[kubevip.ServiceInterface]
+		}
+
+		// If it is still blank then use the
+		if svcInterface == "" {
+			switch config.ServicesInterface {
+			case "":
+				svcInterface = config.Interface
+			default:
+				svcInterface = config.ServicesInterface
+			}
+		}
+
+		if link == nil {
+			if link, err = netlink.LinkByName(svcInterface); err != nil {
+				return fmt.Errorf("failed to get interface %s: %w", svcInterface, err)
+			}
+			if link == nil {
+				return fmt.Errorf("failed to get interface %s", svcInterface)
+			}
+		}
+
+		// Generate new Virtual IP configuration
+		newVips = append(newVips, &kubevip.Config{
+			VIP:                    hostname,
+			Interface:              svcInterface,
+			SingleNode:             true,
+			EnableARP:              config.EnableARP,
+			EnableBGP:              config.EnableBGP,
+			BGPAttachIPToInterface: config.BGPAttachIPToInterface,
+			VIPSubnet:              config.VIPSubnet,
+			EnableRoutingTable:     config.EnableRoutingTable,
+			RoutingTableID:         config.RoutingTableID,
+			RoutingTableType:       config.RoutingTableType,
+			RoutingProtocol:        config.RoutingProtocol,
+			SkipDAD:                config.SkipDAD,
+			ArpBroadcastRate:       config.ArpBroadcastRate,
+			EnableServiceSecurity:  config.EnableServiceSecurity,
+			DNSMode:                config.DNSMode,
+			DHCPMode:               config.DHCPMode,
+			DisableServiceUpdates:  config.DisableServiceUpdates,
+			EnableServicesElection: config.EnableServicesElection,
+			KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
+				EnableLeaderElection: config.EnableLeaderElection,
+			},
+		})
+	}
+
+	if svc.Annotations != nil {
+		instance.DHCPInterfaceHwaddr = svc.Annotations[kubevip.HwAddrKey]
+		requestedIP := svc.Annotations[kubevip.RequestedIP]
+		if requestedIP != "" {
+			requestedIPs := strings.Split(requestedIP, ",")
+			if len(requestedIPs) > 2 {
+				return fmt.Errorf("annotation %q cannot request more than one IPv4 and one Ipv6 address", kubevip.RequestedIP)
+			}
+			for _, ip := range requestedIPs {
+				netip := net.ParseIP(ip)
+				if netip.To4() != nil {
+					instance.DHCPInterfaceIPv4 = ip
+				} else {
+					instance.DHCPInterfaceIPv6 = ip
+				}
+			}
+		}
+		instance.DHCPHostname = svc.Annotations[kubevip.LoadbalancerHostname]
+		instance.macvlanName = svc.Annotations[kubevip.MacvlanName]
+		instance.dhcpBroadcast = svc.Annotations[kubevip.DHCPBroadcast] == "true"
+	}
+
+	configPorts := make([]kubevip.Port, 0)
+	for _, p := range svc.Spec.Ports {
+		configPorts = append(configPorts, kubevip.Port{
+			Type: string(p.Protocol),
+			Port: int(p.Port),
+		})
+	}
+	// Generate Load Balancer config
+	newLB := kubevip.LoadBalancer{
+		Name:      fmt.Sprintf("%s-load-balancer", svc.Name),
+		Ports:     configPorts,
+		BindToVip: true,
+	}
+	for _, vip := range newVips {
+		// Add Load Balancer Configuration
+		vip.LoadBalancers = append(vip.LoadBalancers, newLB)
+	}
+	// Create Add configuration to the new service
+	instance.VIPConfigs = newVips
+
+	// If this was purposely created with the address '0.0.0.0', or '::'
+	// we will create a macvlan on the main interface and a DHCP client
+	if len(instanceAddresses) > 2 && (slices.Contains(instanceAddresses, "0.0.0.0") || slices.Contains(instanceAddresses, "::")) {
+		return fmt.Errorf("DHCP cannot be used if more than 2 addresses (one IPv4 and one IPv6) were specified")
+	}
+	for index := range instance.VIPConfigs {
+		if instance.VIPConfigs[index].VIP == "0.0.0.0" {
+			err := instance.startDHCP(ctx, index, config.DHCPBackoffAttempts, wg)
+			if err != nil {
+				return err
+			}
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case err := <-instance.DHCPv4Client.ErrorChannel():
+				return fmt.Errorf("error starting DHCPv4 for %s/%s: error: %s",
+					instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
+			case ip := <-instance.DHCPv4Client.IPChannel():
+				instance.VIPConfigs[index].Interface = instance.DHCPInterface
+				instance.VIPConfigs[index].VIP = ip
+				instance.DHCPInterfaceIPv4 = ip
+			}
+		}
+		if instance.VIPConfigs[index].VIP == "::" {
+			err := instance.startDHCP(ctx, index, config.DHCPBackoffAttempts, wg)
+			if err != nil {
+				return err
+			}
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case err := <-instance.DHCPv6Client.ErrorChannel():
+				return fmt.Errorf("error starting DHCPv6 for %s/%s: error: %s",
+					instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
+			case ip := <-instance.DHCPv6Client.IPChannel():
+				instance.VIPConfigs[index].Interface = instance.DHCPInterface
+				instance.VIPConfigs[index].VIP = ip
+				instance.DHCPInterfaceIPv6 = ip
+			}
+		}
+
+		ddnsAnnotation, exists := svc.Annotations[kubevip.ServiceDDNS]
+
+		if exists {
+			instance.VIPConfigs[index].DDNS, err = strconv.ParseBool(ddnsAnnotation)
+			if err != nil {
+				log.Error("Failed to add service", "err", err)
+				return err
+			}
+		}
+
+		if len(svc.Spec.IPFamilies) > 0 {
+			if len(svc.Spec.IPFamilies) > 1 {
+				instance.VIPConfigs[index].DHCPMode = utils.DualFamily
+				instance.VIPConfigs[index].DNSMode = utils.DualFamily
+				switch *svc.Spec.IPFamilyPolicy {
+				case v1.IPFamilyPolicyRequireDualStack:
+					instance.VIPConfigs[index].IsDualStack = true
+					instance.VIPConfigs[index].RequireDualStack = true
+				case v1.IPFamilyPolicyPreferDualStack:
+					instance.VIPConfigs[index].IsDualStack = true
+					instance.VIPConfigs[index].RequireDualStack = false
+				default:
+					instance.VIPConfigs[index].IsDualStack = false
+					instance.VIPConfigs[index].RequireDualStack = false
+				}
+			} else {
+				if strings.EqualFold(string(svc.Spec.IPFamilies[0]), utils.IPv4Family) {
+					instance.VIPConfigs[index].DHCPMode = strings.ToLower(utils.IPv4Family)
+					instance.VIPConfigs[index].DNSMode = strings.ToLower(utils.IPv4Family)
+				} else {
+					instance.VIPConfigs[index].DHCPMode = strings.ToLower(utils.IPv6Family)
+					instance.VIPConfigs[index].DNSMode = strings.ToLower(utils.IPv6Family)
+				}
+			}
+		}
+
+		instance.VIPConfigs[index].EgressWithNftables = config.EgressWithNftables
+
+		c, err := cluster.InitCluster(instance.VIPConfigs[index], false, intfMgr, arpMgr, routeMgr, nodeLabelMgr)
+		if err != nil {
+			log.Error("failed to add service", "err", err)
+			return err
+		}
+
+		for networkIndex := range c.Network {
+			c.Network[networkIndex].SetServicePorts(svc)
+		}
+
+		instance.Clusters = append(instance.Clusters, c)
+		log.Info("(svcs) adding VIP", "ip", instance.VIPConfigs[index].VIP, "interface", instance.VIPConfigs[index].Interface, "namespace", svc.Namespace, "name", svc.Name)
+	}
+
+	return nil
+}
+
+func autoFindInterface(ip string) (netlink.Link, error) {
+	links, err := netlink.LinkList()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list network interfaces: %w", err)
+	}
+
+	address := net.ParseIP(ip)
+
+	family := netlink.FAMILY_V4
+
+	if address.To4() == nil {
+		family = netlink.FAMILY_V6
+	}
+
+	for _, link := range links {
+		addr, err := netlink.AddrList(link, family)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get IP addresses for interface %s: %w", link.Attrs().Name, err)
+		}
+		for _, a := range addr {
+			if a.IPNet.Contains(address) {
+				return link, nil
+			}
+		}
+	}
+
+	return nil, nil
+}
+
+func autoFindSubnet(link netlink.Link, ip string) (string, error) {
+	address := net.ParseIP(ip)
+
+	family := netlink.FAMILY_V4
+	if address.To4() == nil {
+		family = netlink.FAMILY_V6
+	}
+
+	addr, err := netlink.AddrList(link, family)
+	if err != nil {
+		return "", fmt.Errorf("failed to get IP addresses for interface %s: %w", link.Attrs().Name, err)
+	}
+	for _, a := range addr {
+		if a.IPNet.Contains(address) {
+			m, _ := a.IPNet.Mask.Size()
+			return strconv.Itoa(m), nil
+		}
+	}
+	return "", fmt.Errorf("failed to find suitable subnet for address %s", ip)
+}
+
+func getAutoInterfaceName(link netlink.Link, defaultInterface string) string {
+	if link == nil {
+		return defaultInterface
+	}
+	return link.Attrs().Name
+}
+
+func (instance *Instance) addVLAN(parentInterface string, tag int) error {
+	interfaceName := fmt.Sprintf("%s.%d", parentInterface, tag)
+	parent, err := netlink.LinkByName(parentInterface)
+	if err != nil {
+		return fmt.Errorf("finding VLAN parent interface %s: %w", parentInterface, err)
+	}
+	iface, err := netlink.LinkByName(interfaceName)
+	if err != nil {
+		var notFound netlink.LinkNotFoundError
+		if !errors.As(err, &notFound) {
+			return fmt.Errorf("finding VLAN interface %s: %w", interfaceName, err)
+		}
+
+		log.Info("Creating new VLAN interface", "interface", interfaceName)
+
+		vlan := &netlink.Vlan{
+			LinkAttrs: netlink.LinkAttrs{
+				Name:        interfaceName,
+				ParentIndex: parent.Attrs().Index,
+			},
+			VlanId:       tag,
+			VlanProtocol: netlink.VLAN_PROTOCOL_8021Q,
+		}
+
+		err = netlink.LinkAdd(vlan)
+		if err != nil {
+			return fmt.Errorf("could not add VLAN %s: %v", interfaceName, err)
+		}
+		instance.vlanOwned.Store(true)
+
+		err = netlink.LinkSetUp(vlan)
+		if err != nil {
+			return fmt.Errorf("could not bring up VLAN interface [%s] : %v", interfaceName, err)
+		}
+
+		_, err = net.InterfaceByName(interfaceName)
+		if err != nil {
+			return fmt.Errorf("error finding new VLAN interface by name [%v]", err)
+		}
+	} else {
+		log.Info("Using existing VLAN interface", "interface", interfaceName)
+
+		if err := utils.ValidateVLANInterface(iface, parent, tag); err != nil {
+			return err
+		}
+	}
+
+	instance.VLANInterface = interfaceName
+	instance.IsVLAN = true
+
+	return nil
+}
+
+// CleanupLinkAttachments stops this instance's DHCP clients and removes only
+// VLAN or macvlan links created by this instance that are not used by a
+// remaining Service instance.
+func (instance *Instance) CleanupLinkAttachments(remaining ...*Instance) error {
+	var errs []error
+	if instance.DHCPv4Client != nil {
+		instance.DHCPv4Client.Stop()
+	}
+	if instance.DHCPv6Client != nil {
+		instance.DHCPv6Client.Stop()
+	}
+	if instance.dhcpInterfaceOwned.Load() {
+		if transferLinkAttachmentOwnership(instance.DHCPInterface, remaining, false) {
+			instance.dhcpInterfaceOwned.Store(false)
+		} else if err := deleteOwnedLink(instance.DHCPInterface, "DHCP"); err != nil {
+			errs = append(errs, err)
+		} else {
+			instance.dhcpInterfaceOwned.Store(false)
+		}
+	}
+	if instance.vlanOwned.Load() {
+		if transferLinkAttachmentOwnership(instance.VLANInterface, remaining, true) {
+			instance.vlanOwned.Store(false)
+		} else if err := deleteOwnedLink(instance.VLANInterface, "VLAN"); err != nil {
+			errs = append(errs, err)
+		} else {
+			instance.vlanOwned.Store(false)
+		}
+	}
+	return errors.Join(errs...)
+}
+
+func transferLinkAttachmentOwnership(name string, instances []*Instance, vlan bool) bool {
+	if name == "" {
+		return false
+	}
+	for _, instance := range instances {
+		if instance == nil {
+			continue
+		}
+		if vlan && instance.IsVLAN && instance.VLANInterface == name {
+			instance.vlanOwned.Store(true)
+			return true
+		}
+		if !vlan && (instance.IsDHCPv4 || instance.IsDHCPv6) && instance.DHCPInterface == name {
+			instance.dhcpInterfaceOwned.Store(true)
+			return true
+		}
+	}
+	return false
+}
+
+func deleteOwnedLink(name, kind string) error {
+	if name == "" {
+		return nil
+	}
+	link, err := netlink.LinkByName(name)
+	if err != nil {
+		var notFound netlink.LinkNotFoundError
+		if errors.As(err, &notFound) {
+			return nil
+		}
+		return fmt.Errorf("find %s interface %q: %w", kind, name, err)
+	}
+	if err := netlink.LinkDel(link); err != nil {
+		return fmt.Errorf("delete %s interface %q: %w", kind, name, err)
+	}
+	return nil
+}
+
+func (instance *Instance) startDHCP(ctx context.Context, index int, backoffAttempts uint, wg *sync.WaitGroup) error {
+	if len(instance.VIPConfigs) > 2 {
+		return fmt.Errorf("DHCP can be used with 2 VIP config maximally, got: %v", len(instance.VIPConfigs))
+	}
+	parent, err := netlink.LinkByName(instance.VIPConfigs[index].Interface)
+	if err != nil {
+		return fmt.Errorf("error finding VIP Interface, for building DHCP Link : %v", err)
+	}
+
+	interfaceName := instance.macvlanName
+
+	if interfaceName == "" {
+		// Generate name from UID
+		interfaceName = fmt.Sprintf("vip-%s", instance.UID()[0:8])
+	}
+
+	// Check if the interface doesn't exist first
+	iface, err := net.InterfaceByName(interfaceName)
+	if err != nil {
+		log.Info("creating new macvlan interface for DHCP", "interface", interfaceName)
+
+		hwaddr, err := net.ParseMAC(instance.DHCPInterfaceHwaddr)
+		if instance.DHCPInterfaceHwaddr != "" && err != nil {
+			return err
+		} else if hwaddr == nil {
+			hwaddr, err = net.ParseMAC(vip.GenerateMac())
+			if err != nil {
+				return err
+			}
+		}
+
+		log.Info("new macvlan interface", "interface", interfaceName, "hardware address", hwaddr)
+		mac := &netlink.Macvlan{
+			LinkAttrs: netlink.LinkAttrs{
+				Name:         interfaceName,
+				ParentIndex:  parent.Attrs().Index,
+				HardwareAddr: hwaddr,
+			},
+			Mode: netlink.MACVLAN_MODE_DEFAULT,
+		}
+
+		err = netlink.LinkAdd(mac)
+		if err != nil {
+			return fmt.Errorf("could not add %s: %v", interfaceName, err)
+		}
+		instance.dhcpInterfaceOwned.Store(true)
+
+		err = netlink.LinkSetUp(mac)
+		if err != nil {
+			return fmt.Errorf("could not bring up interface [%s] : %v", interfaceName, err)
+		}
+
+		iface, err = net.InterfaceByName(interfaceName)
+		if err != nil {
+			return fmt.Errorf("error finding new DHCP interface by name [%v]", err)
+		}
+	} else {
+		log.Info("Using existing macvlan interface for DHCP", "interface", interfaceName)
+	}
+
+	var initRebootFlag bool
+	ip := net.ParseIP(instance.VIPConfigs[index].VIP)
+
+	var client vip.DHCPClient
+	if ip.To4() != nil {
+		// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
+		rpfilterSetting := "0"
+
+		// Check if we need to set an override rp_filter value for the interface
+		if instance.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
+			// Check the rp_filter value
+			rpFilter, err := strconv.Atoi(instance.ServiceSnapshot.Annotations[kubevip.RPFilter])
+			if err != nil {
+				log.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
+			} else {
+				if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
+					rpfilterSetting = instance.ServiceSnapshot.Annotations[kubevip.RPFilter]
+				} else {
+					log.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
+				}
+			}
+		}
+
+		err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
+		if err != nil {
+			log.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
+		}
+
+		if instance.DHCPInterfaceIPv4 != "" {
+			initRebootFlag = true
+		}
+
+		client = vip.NewDHCPv4Client(iface, initRebootFlag, instance.DHCPInterfaceIPv4, backoffAttempts, instance.dhcpBroadcast)
+
+		// Add the client so that we can call it to stop function
+		instance.DHCPv4Client = client
+
+		// Set that DHCPv4 is enabled
+		instance.IsDHCPv4 = true
+	} else {
+		if instance.DHCPInterfaceIPv6 != "" {
+			initRebootFlag = true
+		}
+
+		client, err = vip.NewDHCPv6Client(iface, parent, initRebootFlag, instance.DHCPInterfaceIPv6, backoffAttempts)
+		if err != nil {
+			return fmt.Errorf("unable to create client: %w", err)
+		}
+
+		// Add the client so that we can call it to stop function
+		instance.DHCPv6Client = client
+
+		// Set that DHCPv6 is enabled
+		instance.IsDHCPv6 = true
+	}
+
+	// Add hostname to dhcp client if annotated
+	if instance.DHCPHostname != "" {
+		log.Info("Hostname specified for dhcp lease", "interface", interfaceName, "hostname", instance.DHCPHostname)
+		client.WithHostName(instance.DHCPHostname)
+	}
+
+	wg.Go(func() {
+		if err := client.Start(ctx); err != nil {
+			log.Error("[instance] DHCP client", "error", err)
+			client.Stop()
+		}
+	})
+
+	// Set the name of the interface so that it can be removed on Service deletion
+	instance.DHCPInterface = interfaceName
+	instance.DHCPInterfaceHwaddr = iface.HardwareAddr.String()
+
+	return nil
+}
+
+// FetchLoadBalancerIngressAddresses tries to get the addresses from status.loadBalancerIP
+func FetchLoadBalancerIngress(s *v1.Service) ([]string, []string) {
+	// If the service has no status, return empty
+	lbStatusAddresses := []string{}
+	lbStatusHostnames := []string{}
+	if len(s.Status.LoadBalancer.Ingress) == 0 {
+		return lbStatusAddresses, lbStatusHostnames
+	}
+
+	for _, ingress := range s.Status.LoadBalancer.Ingress {
+		if ingress.IP != "" {
+			lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
+		}
+		if ingress.Hostname != "" {
+			lbStatusHostnames = append(lbStatusHostnames, ingress.Hostname)
+		}
+	}
+	return lbStatusAddresses, lbStatusHostnames
+}
+
+// FetchServiceAddresses tries to get the addresses from annotations
+// kube-vip.io/loadbalancerIPs, then from spec.loadbalancerIP
+func FetchServiceAddresses(s *v1.Service) ([]string, []string) {
+	annotationAvailable := false
+	if s.Annotations != nil {
+
+		if v, annotationAvailable := s.Annotations[kubevip.LoadbalancerIPAnnotation]; annotationAvailable {
+			ips := strings.Split(v, ",")
+			var trimmedIPs []string
+			var trimmedHostnames []string
+			for _, a := range ips {
+				a = strings.TrimSpace(a)
+				ip := net.ParseIP(a)
+				if ip == nil {
+					// this is probably a DNS name
+					trimmedHostnames = append(trimmedHostnames, a)
+				} else {
+					trimmedIPs = append(trimmedIPs, ip.String())
+				}
+			}
+			return trimmedIPs, trimmedHostnames
+		}
+	}
+
+	lbStatusAddresses := []string{}
+	lbStatusHostnames := []string{}
+	if !annotationAvailable {
+		lbStatusAddresses, lbStatusHostnames = FetchLoadBalancerIngress(s)
+	}
+
+	// Spec.LoadBalancerIP legacy handling
+	// if the loadBalancerIP is different from Status.LoadBalancer.Ingress IPs
+	// return the legacy LB as spec wins over status.
+	if lbIP := net.ParseIP(s.Spec.LoadBalancerIP); lbIP != nil && len(lbStatusAddresses) > 0 {
+		isLbIPv4 := utils.IsIPv4(s.Spec.LoadBalancerIP)
+		for _, a := range lbStatusAddresses {
+			if lbStatusIP := net.ParseIP(a); lbStatusIP != nil && utils.IsIPv4(a) == isLbIPv4 && !lbIP.Equal(lbStatusIP) {
+				return []string{s.Spec.LoadBalancerIP}, []string{}
+			}
+		}
+	}
+
+	if len(lbStatusAddresses) > 0 || len(lbStatusHostnames) > 0 {
+		return lbStatusAddresses, lbStatusHostnames
+	}
+
+	if s.Spec.LoadBalancerIP != "" {
+		return []string{s.Spec.LoadBalancerIP}, []string{}
+	}
+
+	return []string{}, []string{}
+}
+
+func FindServiceInstance(svc *v1.Service, instances []*Instance) *Instance {
+	log.Debug("finding service", "namespace", svc.Namespace, "name", svc.Name, "UID", svc.UID)
+	for index := range instances {
+		if instances[index].UID() == svc.UID {
+			return instances[index]
+		}
+	}
+	log.Debug("instance not found", "namespace", svc.Namespace, "name", svc.Name, "UID", svc.UID)
+	return nil
+}

pkg/instance/instance_bgp_attach_test.go

@@ -0,0 +1,62 @@
+package instance_test
+
+import (
+	"context"
+	"sync"
+	"testing"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/networkinterface"
+	"github.com/kube-vip/kube-vip/pkg/route"
+)
+
+func TestNewInstance_PropagatesBGPAttachIPToInterface(t *testing.T) {
+	tests := []struct {
+		name   string
+		attach bool
+	}{
+		{name: "attach enabled is propagated", attach: true},
+		{name: "attach disabled is propagated", attach: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			globalConfig := &kubevip.Config{
+				Interface:              "lo",
+				VIPSubnet:              "32",
+				EnableBGP:              true,
+				BGPAttachIPToInterface: tt.attach,
+			}
+
+			svc := &v1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-svc",
+					Namespace: "default",
+					Annotations: map[string]string{
+						kubevip.LoadbalancerIPAnnotation: "10.0.1.2",
+					},
+				},
+			}
+
+			inst, err := instance.NewInstance(context.Background(), svc, globalConfig,
+				networkinterface.NewManager(), arp.NewManager(globalConfig), route.NewManager(),
+				nil, &sync.WaitGroup{})
+			if err != nil {
+				t.Fatalf("NewInstance() error = %v", err)
+			}
+
+			if len(inst.VIPConfigs) != 1 {
+				t.Fatalf("VIPConfigs len = %d, want 1", len(inst.VIPConfigs))
+			}
+
+			if got := inst.VIPConfigs[0].BGPAttachIPToInterface; got != tt.attach {
+				t.Fatalf("BGPAttachIPToInterface = %t, want %t", got, tt.attach)
+			}
+		})
+	}
+}

pkg/instance/instance_test.go

@@ -0,0 +1,65 @@
+package instance
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+func TestUIDUsesImmutableServiceUID(t *testing.T) {
+	serviceUID := types.UID("original-service")
+	instance := &Instance{
+		ServiceUID: serviceUID,
+		ServiceSnapshot: &v1.Service{ObjectMeta: metav1.ObjectMeta{
+			UID: types.UID("replacement-service"),
+		}},
+	}
+
+	if got := instance.UID(); got != serviceUID {
+		t.Fatalf("UID() = %q, want %q", got, serviceUID)
+	}
+
+	instance.ServiceUID = ""
+	if got := instance.UID(); got != "" {
+		t.Fatalf("UID() without ServiceUID = %q, want empty UID", got)
+	}
+}
+
+func TestCleanupStateIsImmutable(t *testing.T) {
+	original := &v1.Service{ObjectMeta: metav1.ObjectMeta{
+		Name: "service", Namespace: "default", Annotations: map[string]string{kubevip.ServiceLease: "shared"},
+	}, Spec: v1.ServiceSpec{LoadBalancerIP: "192.0.2.10", ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeCluster}}
+	immutableInfo := serviceCleanupInfo(original)
+	instance := &Instance{
+		ServiceSnapshot:  original,
+		ServiceAddresses: []string{"192.0.2.10"},
+		cleanupInfo:      &immutableInfo,
+	}
+	instance.ServiceSnapshot = &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "changed"}}
+
+	cleanupInfo, ok := instance.CleanupInfo()
+	if !ok || cleanupInfo.Namespace != "default" || cleanupInfo.Name != "service" || cleanupInfo.Lease != "shared" ||
+		cleanupInfo.ExternalTrafficPolicy != v1.ServiceExternalTrafficPolicyTypeCluster {
+		t.Fatalf("CleanupInfo() = %+v, %t, want creation-time Service policy", cleanupInfo, ok)
+	}
+}
+
+func TestTransferLinkAttachmentOwnershipConcurrent(t *testing.T) {
+	target := &Instance{IsVLAN: true, VLANInterface: "eth0.42"}
+	var wg sync.WaitGroup
+	for range 100 {
+		wg.Go(func() {
+			if !transferLinkAttachmentOwnership("eth0.42", []*Instance{target}, true) {
+				t.Error("transferLinkAttachmentOwnership() did not find target")
+			}
+		})
+	}
+	wg.Wait()
+	if !target.vlanOwned.Load() {
+		t.Fatal("target did not receive VLAN ownership")
+	}
+}

pkg/instance/links_linux_test.go

@@ -0,0 +1,102 @@
+//go:build linux
+
+package instance
+
+import (
+	"errors"
+	"os"
+	"runtime"
+	"testing"
+
+	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netns"
+)
+
+// requireNetworkNamespaces makes the privileged CI job fail instead of silently
+// skipping when it cannot enter a network namespace.
+var requireNetworkNamespaces = os.Getenv("KUBE_VIP_REQUIRE_NETNS") != ""
+
+func TestCleanupLinkAttachmentsOnlyDeletesOwnedVLAN(t *testing.T) {
+	for _, test := range []struct {
+		name      string
+		preexists bool
+		inUse     bool
+	}{
+		{name: "owned VLAN", preexists: false},
+		{name: "adopted VLAN", preexists: true},
+		{name: "owned VLAN used by another Service", inUse: true},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			runtime.LockOSThread()
+			defer runtime.UnlockOSThread()
+
+			originalNamespace, err := netns.Get()
+			if err != nil {
+				t.Fatalf("getting current network namespace: %v", err)
+			}
+			defer originalNamespace.Close()
+			testNamespace, err := netns.New()
+			if err != nil {
+				if requireNetworkNamespaces {
+					t.Fatalf("creating isolated network namespace: %v", err)
+				}
+				t.Skipf("creating isolated network namespace: %v", err)
+			}
+			defer testNamespace.Close()
+			defer func() {
+				if err := netns.Set(originalNamespace); err != nil {
+					t.Errorf("restoring network namespace: %v", err)
+				}
+			}()
+
+			parent := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "kvattach0"}}
+			if err := netlink.LinkAdd(parent); err != nil {
+				t.Fatalf("creating parent interface: %v", err)
+			}
+			if err := netlink.LinkSetUp(parent); err != nil {
+				t.Fatalf("bringing up parent interface: %v", err)
+			}
+			if test.preexists {
+				vlan := &netlink.Vlan{LinkAttrs: netlink.LinkAttrs{Name: "kvattach0.42", ParentIndex: parent.Attrs().Index}, VlanId: 42}
+				if err := netlink.LinkAdd(vlan); err != nil {
+					t.Fatalf("creating existing VLAN: %v", err)
+				}
+			}
+
+			instance := &Instance{}
+			if err := instance.addVLAN(parent.Attrs().Name, 42); err != nil {
+				t.Fatalf("adding VLAN attachment: %v", err)
+			}
+			if instance.vlanOwned.Load() == test.preexists {
+				t.Fatalf("vlanOwned = %t, want %t", instance.vlanOwned.Load(), !test.preexists)
+			}
+			var remaining []*Instance
+			if test.inUse {
+				remaining = []*Instance{{IsVLAN: true, VLANInterface: instance.VLANInterface}}
+			}
+			if err := instance.CleanupLinkAttachments(remaining...); err != nil {
+				t.Fatalf("cleaning attachments: %v", err)
+			}
+			if test.inUse {
+				if !remaining[0].vlanOwned.Load() {
+					t.Fatal("remaining Service did not receive VLAN cleanup ownership")
+				}
+				if _, err := netlink.LinkByName("kvattach0.42"); err != nil {
+					t.Fatalf("VLAN was removed while a Service still used it: %v", err)
+				}
+				if err := remaining[0].CleanupLinkAttachments(); err != nil {
+					t.Fatalf("cleaning transferred attachment: %v", err)
+				}
+			}
+
+			_, err = netlink.LinkByName("kvattach0.42")
+			var notFound netlink.LinkNotFoundError
+			if test.preexists && err != nil {
+				t.Fatalf("adopted VLAN was removed: %v", err)
+			}
+			if !test.preexists && !errors.As(err, &notFound) {
+				t.Fatalf("owned VLAN remains after cleanup: %v", err)
+			}
+		})
+	}
+}

pkg/iptables/iptables.go

@@ -0,0 +1,592 @@
+// Copyright 2015 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iptables
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"syscall"
+)
+
+// Adds the output of stderr to exec.ExitError
+type Error struct {
+	exec.ExitError
+	cmd        exec.Cmd
+	msg        string
+	exitStatus *int //for overriding
+}
+
+func (e *Error) ExitStatus() int {
+	if e.exitStatus != nil {
+		return *e.exitStatus
+	}
+	return e.Sys().(syscall.WaitStatus).ExitStatus()
+}
+
+func (e *Error) Error() string {
+	return fmt.Sprintf("running %v: exit status %v: %v", e.cmd.Args, e.ExitStatus(), e.msg)
+}
+
+// IsNotExist returns true if the error is due to the chain or rule not existing
+func (e *Error) IsNotExist() bool {
+	if e.ExitStatus() != 1 {
+		return false
+	}
+	msgNoRuleExist := "Bad rule (does a matching rule exist in that chain?).\n"
+	msgNoChainExist := "No chain/target/match by that name.\n"
+	return strings.Contains(e.msg, msgNoRuleExist) || strings.Contains(e.msg, msgNoChainExist)
+}
+
+// Protocol to differentiate between IPv4 and IPv6
+type Protocol byte
+
+const (
+	ProtocolIPv4 Protocol = iota
+	ProtocolIPv6
+)
+
+const (
+	TableFilter      = "filter"
+	TableMangle      = "mangle"
+	TableNat         = "nat"
+	ChainInput       = "INPUT"
+	ChainPREROUTING  = "PREROUTING"
+	ChainPOSTROUTING = "POSTROUTING"
+)
+
+type IPTables struct {
+	path              string
+	proto             Protocol
+	hasCheck          bool
+	hasWait           bool
+	waitSupportSecond bool
+	hasRandomFully    bool
+	v1                int
+	v2                int
+	v3                int
+	mode              string // the underlying iptables operating mode, e.g. nf_tables
+	timeout           int    // time to wait for the iptables lock, default waits forever
+
+	nftables bool
+}
+type Option func(*IPTables)
+
+func IPFamily(proto Protocol) Option {
+	return func(ipt *IPTables) {
+		ipt.proto = proto
+	}
+}
+
+func Timeout(timeout int) Option {
+	return func(ipt *IPTables) {
+		ipt.timeout = timeout
+	}
+}
+
+func EnableNFTables(enable bool) Option {
+	return func(ipt *IPTables) {
+		ipt.nftables = enable
+	}
+}
+
+// New creates a new IPTables configured with the options passed as parameter.
+// For backwards compatibility, by default always uses IPv4 and timeout 0.
+// i.e. you can create an IPv6 IPTables using a timeout of 5 seconds passing
+// the IPFamily and Timeout options as follow:
+//
+//	ip6t := New(IPFamily(ProtocolIPv6), Timeout(5))
+func New(opts ...Option) (*IPTables, error) {
+
+	ipt := &IPTables{
+		proto:   ProtocolIPv4,
+		timeout: 0,
+	}
+
+	for _, opt := range opts {
+		opt(ipt)
+	}
+
+	path, err := exec.LookPath(getIptablesCommand(ipt.proto, ipt.nftables))
+	if err != nil {
+		return nil, err
+	}
+	ipt.path = path
+
+	vstring, err := getIptablesVersionString(path)
+	if err != nil {
+		return nil, fmt.Errorf("could not get iptables version: %v", err)
+	}
+	v1, v2, v3, mode, err := extractIptablesVersion(vstring)
+	if err != nil {
+		return nil, fmt.Errorf("failed to extract iptables version from [%s]: %v", vstring, err)
+	}
+	ipt.v1 = v1
+	ipt.v2 = v2
+	ipt.v3 = v3
+	ipt.mode = mode
+
+	checkPresent, waitPresent, waitSupportSecond, randomFullyPresent := getIptablesCommandSupport(v1, v2, v3)
+	ipt.hasCheck = checkPresent
+	ipt.hasWait = waitPresent
+	ipt.waitSupportSecond = waitSupportSecond
+	ipt.hasRandomFully = randomFullyPresent
+
+	return ipt, nil
+}
+
+// New creates a new IPTables for the given proto.
+// The proto will determine which command is used, either "iptables" or "ip6tables".
+func NewWithProtocol(proto Protocol) (*IPTables, error) {
+	return New(IPFamily(proto), Timeout(0))
+}
+
+// Proto returns the protocol used by this IPTables.
+func (ipt *IPTables) Proto() Protocol {
+	return ipt.proto
+}
+
+// Exists checks if given rulespec in specified table/chain exists
+func (ipt *IPTables) Exists(table, chain string, rulespec ...string) (bool, error) {
+	if !ipt.hasCheck {
+		return ipt.existsForOldIptables(table, chain, rulespec)
+
+	}
+	cmd := append([]string{"-t", table, "-C", chain}, rulespec...)
+	err := ipt.run(cmd...)
+	eerr, eok := err.(*Error)
+	switch {
+	case err == nil:
+		return true, nil
+	case eok && eerr.ExitStatus() == 1:
+		return false, nil
+	default:
+		return false, err
+	}
+}
+
+// Insert inserts rulespec to specified table/chain (in specified pos)
+func (ipt *IPTables) Insert(table, chain string, pos int, rulespec ...string) error {
+	cmd := append([]string{"-t", table, "-I", chain, strconv.Itoa(pos)}, rulespec...)
+	return ipt.run(cmd...)
+}
+
+// InsertUnique acts like Insert except that it won't insert a duplicate (no matter the position in the chain)
+func (ipt *IPTables) InsertUnique(table, chain string, pos int, rulespec ...string) error {
+	exists, err := ipt.Exists(table, chain, rulespec...)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		return ipt.Insert(table, chain, pos, rulespec...)
+	}
+
+	return nil
+}
+
+// Append appends rulespec to specified table/chain
+func (ipt *IPTables) Append(table, chain string, rulespec ...string) error {
+	cmd := append([]string{"-t", table, "-A", chain}, rulespec...)
+	return ipt.run(cmd...)
+}
+
+// AppendUnique acts like Append except that it won't add a duplicate
+func (ipt *IPTables) AppendUnique(table, chain string, rulespec ...string) error {
+	exists, err := ipt.Exists(table, chain, rulespec...)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		return ipt.Append(table, chain, rulespec...)
+	}
+
+	return nil
+}
+
+// Delete removes rulespec in specified table/chain
+func (ipt *IPTables) Delete(table, chain string, rulespec ...string) error {
+	cmd := append([]string{"-t", table, "-D", chain}, rulespec...)
+	return ipt.run(cmd...)
+}
+
+func (ipt *IPTables) DeleteIfExists(table, chain string, rulespec ...string) error {
+	exists, err := ipt.Exists(table, chain, rulespec...)
+	if err == nil && exists {
+		err = ipt.Delete(table, chain, rulespec...)
+	}
+	return err
+}
+
+// List rules in specified table/chain
+func (ipt *IPTables) List(table, chain string) ([]string, error) {
+	args := []string{"-t", table, "-S", chain}
+	return ipt.executeList(args)
+}
+
+// List rules (with counters) in specified table/chain
+func (ipt *IPTables) ListWithCounters(table, chain string) ([]string, error) {
+	args := []string{"-t", table, "-v", "-S", chain}
+	return ipt.executeList(args)
+}
+
+// ListChains returns a slice containing the name of each chain in the specified table.
+func (ipt *IPTables) ListChains(table string) ([]string, error) {
+	args := []string{"-t", table, "-S"}
+
+	result, err := ipt.executeList(args)
+	if err != nil {
+		return nil, err
+	}
+
+	// Iterate over rules to find all default (-P) and user-specified (-N) chains.
+	// Chains definition always come before rules.
+	// Format is the following:
+	// -P OUTPUT ACCEPT
+	// -N Custom
+	var chains []string
+	for _, val := range result {
+		if strings.HasPrefix(val, "-P") || strings.HasPrefix(val, "-N") {
+			chains = append(chains, strings.Fields(val)[1])
+		} else {
+			break
+		}
+	}
+	return chains, nil
+}
+
+// '-S' is fine with non existing rule index as long as the chain exists
+// therefore pass index 1 to reduce overhead for large chains
+func (ipt *IPTables) ChainExists(table, chain string) (bool, error) {
+	err := ipt.run("-t", table, "-S", chain, "1")
+	eerr, eok := err.(*Error)
+	switch {
+	case err == nil:
+		return true, nil
+	case eok && eerr.ExitStatus() == 1:
+		return false, nil
+	default:
+		return false, err
+	}
+}
+
+func (ipt *IPTables) executeList(args []string) ([]string, error) {
+	var stdout bytes.Buffer
+	if err := ipt.runWithOutput(args, &stdout); err != nil {
+		return nil, err
+	}
+
+	rules := strings.Split(stdout.String(), "\n")
+
+	// strip trailing newline
+	if len(rules) > 0 && rules[len(rules)-1] == "" {
+		rules = rules[:len(rules)-1]
+	}
+
+	for i, rule := range rules {
+		rules[i] = filterRuleOutput(rule)
+	}
+
+	return rules, nil
+}
+
+// NewChain creates a new chain in the specified table.
+// If the chain already exists, it will result in an error.
+func (ipt *IPTables) NewChain(table, chain string) error {
+	return ipt.run("-t", table, "-N", chain)
+}
+
+const existsErr = 1
+
+// ClearChain flushed (deletes all rules) in the specified table/chain.
+// If the chain does not exist, a new one will be created
+func (ipt *IPTables) ClearChain(table, chain string) error {
+	err := ipt.NewChain(table, chain)
+
+	eerr, eok := err.(*Error)
+	switch {
+	case err == nil:
+		return nil
+	case eok && eerr.ExitStatus() == existsErr:
+		// chain already exists. Flush (clear) it.
+		return ipt.run("-t", table, "-F", chain)
+	default:
+		return err
+	}
+}
+
+// RenameChain renames the old chain to the new one.
+func (ipt *IPTables) RenameChain(table, oldChain, newChain string) error {
+	return ipt.run("-t", table, "-E", oldChain, newChain)
+}
+
+// DeleteChain deletes the chain in the specified table.
+// The chain must be empty
+func (ipt *IPTables) DeleteChain(table, chain string) error {
+	return ipt.run("-t", table, "-X", chain)
+}
+
+func (ipt *IPTables) ClearAndDeleteChain(table, chain string) error {
+	exists, err := ipt.ChainExists(table, chain)
+	if err != nil || !exists {
+		return err
+	}
+	err = ipt.run("-t", table, "-F", chain)
+	if err == nil {
+		err = ipt.run("-t", table, "-X", chain)
+	}
+	return err
+}
+
+func (ipt *IPTables) ClearAll() error {
+	return ipt.run("-F")
+}
+
+func (ipt *IPTables) DeleteAll() error {
+	return ipt.run("-X")
+}
+
+// ChangePolicy changes policy on chain to target
+func (ipt *IPTables) ChangePolicy(table, chain, target string) error {
+	return ipt.run("-t", table, "-P", chain, target)
+}
+
+// Check if the underlying iptables command supports the --random-fully flag
+func (ipt *IPTables) HasRandomFully() bool {
+	return ipt.hasRandomFully
+}
+
+// Return version components of the underlying iptables command
+func (ipt *IPTables) GetIptablesVersion() (int, int, int) {
+	return ipt.v1, ipt.v2, ipt.v3
+}
+
+// run runs an iptables command with the given arguments, ignoring
+// any stdout output
+func (ipt *IPTables) run(args ...string) error {
+	return ipt.runWithOutput(args, nil)
+}
+
+// runWithOutput runs an iptables command with the given arguments,
+// writing any stdout output to the given writer
+func (ipt *IPTables) runWithOutput(args []string, stdout io.Writer) error {
+	args = append([]string{ipt.path}, args...)
+	if ipt.hasWait {
+		args = append(args, "--wait")
+		if ipt.timeout != 0 && ipt.waitSupportSecond {
+			args = append(args, strconv.Itoa(ipt.timeout))
+		}
+	} else {
+		fmu, err := newXtablesFileLock()
+		if err != nil {
+			return err
+		}
+		ul, err := fmu.tryLock()
+		if err != nil {
+			syscall.Close(fmu.fd)
+			return err
+		}
+		defer func() {
+			_ = ul.Unlock()
+		}()
+	}
+
+	var stderr bytes.Buffer
+	cmd := exec.Cmd{
+		Path:   ipt.path,
+		Args:   args,
+		Stdout: stdout,
+		Stderr: &stderr,
+	}
+
+	if err := cmd.Run(); err != nil {
+		switch e := err.(type) {
+		case *exec.ExitError:
+			return &Error{*e, cmd, stderr.String(), nil}
+		default:
+			return err
+		}
+	}
+
+	return nil
+}
+
+// getIptablesCommand returns the correct command for the given protocol, either "iptables" or "ip6tables".
+func getIptablesCommand(proto Protocol, nftables bool) string {
+	if proto == ProtocolIPv6 {
+		if nftables {
+			return "ip6tables-nft"
+		}
+		return "ip6tables-legacy"
+	}
+
+	if nftables {
+		return "iptables-nft"
+	}
+	return "iptables-legacy"
+}
+
+// Checks if iptables has the "-C" and "--wait" flag
+func getIptablesCommandSupport(v1 int, v2 int, v3 int) (bool, bool, bool, bool) {
+	return iptablesHasCheckCommand(v1, v2, v3), iptablesHasWaitCommand(v1, v2, v3), iptablesWaitSupportSecond(v1, v2), iptablesHasRandomFully(v1, v2, v3)
+}
+
+// getIptablesVersion returns the first three components of the iptables version
+// and the operating mode (e.g. nf_tables or legacy)
+// e.g. "iptables v1.3.66" would return (1, 3, 66, legacy, nil)
+func extractIptablesVersion(str string) (int, int, int, string, error) {
+	versionMatcher := regexp.MustCompile(`v([0-9]+)\.([0-9]+)\.([0-9]+)(?:\s+\((\w+))?`)
+	result := versionMatcher.FindStringSubmatch(str)
+	if result == nil {
+		return 0, 0, 0, "", fmt.Errorf("no iptables version found in string: %s", str)
+	}
+
+	v1, err := strconv.Atoi(result[1])
+	if err != nil {
+		return 0, 0, 0, "", err
+	}
+
+	v2, err := strconv.Atoi(result[2])
+	if err != nil {
+		return 0, 0, 0, "", err
+	}
+
+	v3, err := strconv.Atoi(result[3])
+	if err != nil {
+		return 0, 0, 0, "", err
+	}
+
+	mode := "legacy"
+	if result[4] != "" {
+		mode = result[4]
+	}
+	return v1, v2, v3, mode, nil
+}
+
+// Runs "iptables --version" to get the version string
+func getIptablesVersionString(path string) (string, error) {
+	cmd := exec.Command(path, "--version")
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err := cmd.Run()
+	if err != nil {
+		return "", err
+	}
+	return out.String(), nil
+}
+
+// Checks if an iptables version is after 1.4.11, when --check was added
+func iptablesHasCheckCommand(v1 int, v2 int, v3 int) bool {
+	if v1 > 1 {
+		return true
+	}
+	if v1 == 1 && v2 > 4 {
+		return true
+	}
+	if v1 == 1 && v2 == 4 && v3 >= 11 {
+		return true
+	}
+	return false
+}
+
+// Checks if an iptables version is after 1.4.20, when --wait was added
+func iptablesHasWaitCommand(v1 int, v2 int, v3 int) bool { //nolint
+	if v1 > 1 {
+		return true
+	}
+	if v1 == 1 && v2 > 4 {
+		return true
+	}
+	if v1 == 1 && v2 == 4 && v3 >= 20 {
+		return true
+	}
+	return false
+}
+
+// Checks if an iptablse version is after 1.6.0, when --wait support second
+func iptablesWaitSupportSecond(v1 int, v2 int) bool {
+	if v1 > 1 {
+		return true
+	}
+	if v1 == 1 && v2 >= 6 {
+		return true
+	}
+	return false
+}
+
+// Checks if an iptables version is after 1.6.2, when --random-fully was added
+func iptablesHasRandomFully(v1 int, v2 int, v3 int) bool {
+	if v1 > 1 {
+		return true
+	}
+	if v1 == 1 && v2 > 6 {
+		return true
+	}
+	if v1 == 1 && v2 == 6 && v3 >= 2 {
+		return true
+	}
+	return false
+}
+
+// Checks if a rule specification exists for a table
+func (ipt *IPTables) existsForOldIptables(table, chain string, rulespec []string) (bool, error) {
+	rs := strings.Join(append([]string{"-A", chain}, rulespec...), " ")
+	args := []string{"-t", table, "-S"}
+	var stdout bytes.Buffer
+	err := ipt.runWithOutput(args, &stdout)
+	if err != nil {
+		return false, err
+	}
+	return strings.Contains(stdout.String(), rs), nil
+}
+
+// counterRegex is the regex used to detect nftables counter format
+var counterRegex = regexp.MustCompile(`^\[([0-9]+):([0-9]+)\] `)
+
+// filterRuleOutput works around some inconsistencies in output.
+// For example, when iptables is in legacy vs. nftables mode, it produces
+// different results.
+func filterRuleOutput(rule string) string {
+	out := rule
+
+	// work around an output difference in nftables mode where counters
+	// are output in iptables-save format, rather than iptables -S format
+	// The string begins with "[0:0]"
+	//
+	// Fixes #49
+	if groups := counterRegex.FindStringSubmatch(out); groups != nil {
+		// drop the brackets
+		out = out[len(groups[0]):]
+		out = fmt.Sprintf("%s -c %s %s", out, groups[1], groups[2])
+	}
+
+	return out
+}
+
+func GetIPTablesRuleSpecification(rule, specification string) string {
+	parts := strings.Split(rule, " ")
+	for i, part := range parts {
+		if part == specification && i+1 < len(parts) {
+			return parts[i+1]
+		}
+	}
+
+	return ""
+}

pkg/iptables/lock.go

@@ -0,0 +1,84 @@
+// Copyright 2015 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iptables
+
+import (
+	"os"
+	"sync"
+	"syscall"
+)
+
+const (
+	// In earlier versions of iptables, the xtables lock was implemented
+	// via a Unix socket, but now flock is used via this lockfile:
+	// http://git.netfilter.org/iptables/commit/?id=aa562a660d1555b13cffbac1e744033e91f82707
+	// Note the LSB-conforming "/run" directory does not exist on old
+	// distributions, so assume "/var" is symlinked
+	xtablesLockFilePath = "/var/run/xtables.lock"
+
+	defaultFilePerm = 0600
+)
+
+type Unlocker interface {
+	Unlock() error
+}
+
+type nopUnlocker struct{}
+
+func (n nopUnlocker) Unlock() error { return nil }
+
+type fileLock struct {
+	// mu is used to protect against concurrent invocations from within this process
+	mu sync.Mutex
+	fd int
+}
+
+// tryLock takes an exclusive lock on the xtables lock file without blocking.
+// This is best-effort only: if the exclusive lock would block (i.e. because
+// another process already holds it), no error is returned. Otherwise, any
+// error encountered during the locking operation is returned.
+// The returned Unlocker should be used to release the lock when the caller is
+// done invoking iptables commands.
+func (l *fileLock) tryLock() (Unlocker, error) {
+	l.mu.Lock()
+	err := syscall.Flock(l.fd, syscall.LOCK_EX|syscall.LOCK_NB)
+	switch err {
+	case syscall.EWOULDBLOCK:
+		l.mu.Unlock()
+		return nopUnlocker{}, nil
+	case nil:
+		return l, nil
+	default:
+		l.mu.Unlock()
+		return nil, err
+	}
+}
+
+// Unlock closes the underlying file, which implicitly unlocks it as well. It
+// also unlocks the associated mutex.
+func (l *fileLock) Unlock() error {
+	defer l.mu.Unlock()
+	return syscall.Close(l.fd)
+}
+
+// newXtablesFileLock opens a new lock on the xtables lockfile without
+// acquiring the lock
+func newXtablesFileLock() (*fileLock, error) {
+	fd, err := syscall.Open(xtablesLockFilePath, os.O_CREATE, defaultFilePerm)
+	if err != nil {
+		return nil, err
+	}
+	return &fileLock{fd: fd}, nil
+}