fix: regression on tests as we moved to context

Signed-off-by: Marcel Fest <marcel.fest@telekom.de>

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# Enable GitHub funding
+
+github: [kube-vip]
+

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: gomod
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: /
+    schedule:
+      interval: weekly

.github/suggestion-comment.md

@@ -0,0 +1,13 @@
+I'll help you add a suggestion. Unfortunately, I can't directly add a suggestion to an existing comment through the API. However, here's what I recommend:
+
+**Option 1: Reply with a suggestion**
+Create a new comment with a suggested fix:
+
+```suggestion
+failed to get an IPv6 address after %d attempt(s), giving up, error: %s
+```
+
+**Option 2: Edit your existing comment**
+Update your comment to include the suggestion details pointing out that line 284 in the error message says "IPv4" but should say "IPv6" since this is the DHCPv6Client.
+
+Would you like me to create a new reply comment with the suggestion instead?

.github/workflows/anchore-syft.yml

@@ -0,0 +1,31 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# This workflow checks out code, builds an image, performs a container image
+# scan with Anchore's Syft tool, and uploads the results to the GitHub Dependency
+# submission API.
+
+# For more information on the Anchore sbom-action usage
+# and parameters, see https://github.com/anchore/sbom-action. For more
+# information about the Anchore SBOM tool, Syft, see
+# https://github.com/anchore/syft
+name: Anchore Syft SBOM scan
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  sbom:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+        with:
+          ref: ${{ github.ref_name }}
+      - name: Anchore SBOM Action
+        uses: anchore/sbom-action@v0.24.2
+        with:
+          format: cyclonedx-json

.github/workflows/ci-pull-request.yaml

@@ -0,0 +1,88 @@
+name: For each PR
+on:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  e2e-tests:
+    runs-on: ubuntu-latest
+    name: E2E tests
+    timeout-minutes: 120
+    env:
+      GINKGO_PROCS: ${{ matrix.ginkgo-procs }}
+    strategy:
+      fail-fast: false
+      max-parallel: 3
+      matrix:
+        include:
+          - mode: arp
+            ginkgo-procs: 4
+          - mode: rt
+            ginkgo-procs: 4
+          - mode: bgp
+            ginkgo-procs: 4
+    steps:
+      - name: Get current date
+        id: date
+        run: echo "date=$(date +'%Y-%m-%d-%H-%M')" >> "$GITHUB_OUTPUT"
+      - name: Ensure fs wont cause issues
+        run: sudo sysctl fs.inotify.max_user_instances=8192 &&  sudo sysctl fs.inotify.max_user_watches=524288
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Install Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+      - name: Build image locally
+        run: make dockerx86Local
+      - name: Run Manifest generation tests
+        run: make manifest-test
+        if: matrix.mode == 'arp'
+      - name: Run ARP mode tests
+        run: DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true GINKGO_ARGS="--output-dir=/tmp --json-report=kube-vip-test-report-arp.json" make e2e-tests-arp
+        if: matrix.mode == 'arp'
+      - name: Run RT mode tests
+        run: DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true GINKGO_ARGS="--output-dir=/tmp --json-report=kube-vip-test-report-rt.json" make e2e-tests-rt
+        if: matrix.mode == 'rt'
+      - name: Get GoBGP binaries
+        run: make get-gobgp
+        if: matrix.mode == 'bgp'
+      - name: Run BGP mode tests
+        run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true GINKGO_ARGS="--output-dir=/tmp --json-report=kube-vip-test-report-bgp.json" make e2e-tests-bgp
+        if: matrix.mode == 'bgp'
+      - name: Change log directory permissions
+        run: sudo chmod -R 755 /tmp/kube-vip-test*
+        if: matrix.mode == 'bgp' && always()
+      - name: Save logs
+        uses: actions/upload-artifact@v7
+        with:
+          name: e2e-test-logs-${{ matrix.mode }}-${{ steps.date.outputs.date }}
+          path: /tmp/kube-vip-test*
+        if: always()
+  service-e2e-tests:
+    runs-on: ubuntu-latest
+    name: E2E service tests
+    timeout-minutes: 60
+    steps:
+      - name: Get current date
+        id: date
+        run: echo "date=$(date +'%Y-%m-%d-%H-%M')" >> "$GITHUB_OUTPUT"
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Install Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+      - name: Build image with iptables
+        run: make dockerx86ActionIPTables
+      - name: Run tests
+        run: DOCKER_API_VERSION=1.48 DOCKERTAG=action E2E_KEEP_LOGS=true make service-tests
+      - name: Save logs
+        uses: actions/upload-artifact@v7
+        with:
+          name: services-test-logs-${{ steps.date.outputs.date }}
+          path: /tmp/kube-vip-services*
+        if: always()

.github/workflows/ci.yaml

@@ -1,27 +1,88 @@
-name: For each commit and PR
+name: For each commit
 on:
   push:
+    branches: [main]
+    tags: ['v*']
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   validation:
     runs-on: ubuntu-latest
-    env:
-      CGO_ENABLED: 0
+    name: Checks and linters
+    timeout-minutes: 30
     steps:
-    - name: Init
-      run: sudo apt-get update && sudo apt-get install -y build-essential golint
-    - name: Checkout code
-      uses: actions/checkout@v2
-    - name: Install Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: '1.15'
-    - name: checks
-      run: make check
-    - name: test docker build
-      run: make dockerx86Action
-    - name: Manifest generate
-      run: ./testing/testing.sh
-    - name: e2e tests
-      run: DOCKERTAG=action make e2e-tests
+      - name: Init
+        run: sudo apt-get update && sudo apt-get install -y build-essential &&  sudo sysctl fs.inotify.max_user_instances=8192 &&  sudo sysctl fs.inotify.max_user_watches=524288
+      - name: Install golangci-lint
+        uses: golangci/golangci-lint-action@v9
+        with:
+          version: v2.12
+          install-only: true
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Install Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+      - name: Verify gofmt
+        run: |
+          unformatted=$(gofmt -l .)
+          if [ -n "$unformatted" ]; then
+            echo "The following files are not gofmt-formatted:"
+            echo "$unformatted"
+            exit 1
+          fi
+      - name: All checks
+        run: make check
+  unit-tests:
+    runs-on: ubuntu-latest
+    name: Unit tests
+    timeout-minutes: 30
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Install Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+      - name: Run tests
+        run: make unit-tests
+      - name: Run privileged network tests
+        run: |
+          sudo -E env PATH="$PATH" KUBE_VIP_REQUIRE_NETNS=1 go test -race ./pkg/services ./pkg/vip ./pkg/instance \
+            -run 'TestRecover|TestServiceAddressRetained|TestRetainControlPlaneVIPs|TestAddressProtocol|TestKubeVIPAddressProtocol|TestCleanupKubeVIPAddresses|TestMonitorDefaultInterfaceReturnsErrorWhenTestLinkIsSetDown|TestCleanupLinkAttachmentsOnlyDeletesOwnedVLAN'
+  integration-tests:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Install Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+      - name: Run tests
+        run: make integration-tests
+  image-vul-check:
+    runs-on: ubuntu-latest
+    name: Image vulnerability scan
+    timeout-minutes: 30
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Build image
+        run: make dockerx86Action
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: "plndr/kube-vip:action"
+          format: "table"
+          exit-code: "1"
+          ignore-unfixed: true
+          vuln-type: "os,library"
+          severity: "CRITICAL,HIGH"

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,75 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '17 10 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v7
+      
+    - name: Install Go
+      uses: actions/setup-go@v7
+      with:
+        go-version-file: go.mod
+        
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v4
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v4
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v4

.github/workflows/main.yaml

@@ -1,43 +1,52 @@
-name: Publish the latest dev image
+name: Build and publish main image regularly
 
 on:
-  push:
-    branches:
-      - 'master'
+  schedule:
+    - cron: '25 0 * * *'
+  workflow_dispatch:
+
 
 jobs:
-  docker:
+  nightly_build:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Prepare Names
-        id: prep
-        run: |
-          DOCKER_IMAGE=plndr/kube-vip
-          VERSION=$(echo ${GITHUB_SHA} | cut -c1-8)
-          TAGS="${DOCKER_IMAGE}:${VERSION}"
-          TAGS="$TAGS,${DOCKER_IMAGE}:nightly"
-          echo ::set-output name=tags::${TAGS}
-
+        uses: actions/checkout@v7
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v4
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v4
       - name: Login to DockerHub
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Build and push main branch 
-        uses: docker/build-push-action@v2
+      - name: Login to Github Packages
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}   
+      - name: Build standard version
+        id: docker_build
+        uses: docker/build-push-action@v7
         with:
           context: .
-          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
           push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.prep.outputs.tags }}
-
+          tags: >-
+            plndr/kube-vip:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip:${{ github.ref_name }}
+      - name: Build iptables version
+        id: docker_build_iptables
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: Dockerfile_iptables
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: >-
+            plndr/kube-vip-iptables:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip-iptables:${{ github.ref_name }}
       - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
+        run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/nightly-e2e.yaml

@@ -0,0 +1,111 @@
+name: Nightly e2e
+
+on:
+  schedule:
+    - cron: '30 2 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  unit-coverage:
+    runs-on: ubuntu-latest
+    name: Unit tests with coverage
+    timeout-minutes: 30
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Install Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+      - name: Run tests
+        run: make unit-tests
+      - name: Summarize coverage
+        if: always()
+        run: |
+          if test -f coverage.out; then
+            echo "### Unit coverage" >> "$GITHUB_STEP_SUMMARY"
+            go tool cover -func=coverage.out | tail -1 >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "### Unit coverage: report missing" >> "$GITHUB_STEP_SUMMARY"
+          fi
+      - name: Upload coverage
+        uses: actions/upload-artifact@v7
+        with:
+          name: unit-coverage
+          path: coverage.out
+          if-no-files-found: error
+        if: always()
+  etcd-e2e:
+    runs-on: ubuntu-latest
+    name: Etcd E2E tests
+    timeout-minutes: 60
+    steps:
+      - name: Ensure fs wont cause issues
+        run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Install Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+      - name: Build image locally
+        run: make dockerx86Local
+      - name: Prepare Etcd artifacts
+        if: always()
+        run: |
+          mkdir -p /tmp/kube-vip-etcd-artifacts
+          : > /tmp/kube-vip-etcd-artifacts/suite.log
+          printf '[]\n' > /tmp/kube-vip-etcd-artifacts/report.json
+      - name: Run Etcd tests
+        id: etcd
+        # Scheduled failures are tolerated only during the initial stabilization window.
+        # The enforcement step below makes manual runs and later schedules blocking.
+        continue-on-error: true
+        shell: bash
+        run: |
+          set +e
+          set -o pipefail
+          DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true \
+            GINKGO_ARGS="--json-report=report.json --output-dir=/tmp/kube-vip-etcd-artifacts" \
+            make e2e-tests-etcd 2>&1 | tee /tmp/kube-vip-etcd-artifacts/suite.log
+          exit_code=${PIPESTATUS[0]}
+          echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT"
+          exit "$exit_code"
+      - name: Summarize Etcd suite
+        if: always()
+        env:
+          OUTCOME: ${{ steps.etcd.outcome }}
+          EXIT_CODE: ${{ steps.etcd.outputs.exit_code }}
+        run: |
+          echo "### Etcd E2E result: ${OUTCOME}" >> "$GITHUB_STEP_SUMMARY"
+          printf '{"outcome":"%s","exit_code":%s,"event":"%s","cutoff":"2026-10-01"}\n' \
+            "${OUTCOME:-skipped}" "${EXIT_CODE:-null}" "$GITHUB_EVENT_NAME" \
+            > /tmp/kube-vip-etcd-artifacts/result.json
+      - name: Save logs
+        uses: actions/upload-artifact@v7
+        continue-on-error: true
+        with:
+          name: etcd-e2e-logs
+          path: |
+            /tmp/kube-vip-etcd-artifacts
+            /tmp/kube-vip-test*
+          if-no-files-found: warn
+        if: always()
+      - name: Enforce Etcd result
+        if: always()
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          OUTCOME: ${{ steps.etcd.outcome }}
+        run: |
+          if test "$OUTCOME" = success; then
+            exit 0
+          fi
+          if test "$EVENT_NAME" = schedule && test "$(date -u +%Y-%m-%d)" \< 2026-10-01; then
+            echo "::warning::etcd e2e suite outcome was ${OUTCOME:-skipped} during stabilization through 2026-09-30"
+            exit 0
+          fi
+          echo "::error::etcd e2e suite outcome was ${OUTCOME:-skipped}; see the etcd-e2e-logs artifact"
+          exit 1

.github/workflows/release.yaml

@@ -1,48 +1,64 @@
-name: Publish Releases to Docker Hub
+name: Publish Releases to Docker Hub and GitHub Container Registry
 
 on:
   push:
     tags:
       - '*'
+  workflow_dispatch:
+
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Prepare Names
-        id: prep
-        run: |
-          DOCKER_IMAGE=plndr/kube-vip
-          VERSION=${GITHUB_REF#refs/tags/}
-          TAGS="${DOCKER_IMAGE}:${VERSION},ghcr.io/kube-vip/kube-vip:${VERSION}"
-          TAGS="$TAGS,${DOCKER_IMAGE}:latest,ghcr.io/kube-vip/kube-vip:latest"
-          echo ::set-output name=tags::${TAGS}
-
+        uses: actions/checkout@v7
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v4
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v4
       - name: Login to DockerHub
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Login to Github Packages
-        uses: docker/login-action@v1
+        uses: docker/login-action@v4
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and push main branch 
-        uses: docker/build-push-action@v2
+      - name: Generate Metadata
+        uses: docker/metadata-action@v6.2.0
+        id: metadata
+        with:
+          labels: |
+            org.opencontainers.image.documentation=https://kube-vip.io/docs/
+      - name: Build and push main branch
+        id: docker_build
+        uses: docker/build-push-action@v7
         with:
           context: .
-          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
           push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.prep.outputs.tags }}
-
+          labels: ${{ steps.metadata.outputs.labels }}
+          tags: >-
+            plndr/kube-vip:${{ github.ref_name }},
+            plndr/kube-vip:latest,
+            ghcr.io/kube-vip/kube-vip:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip:latest
+      - name: Build iptables version and push main branch
+        id: docker_build_iptables
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: Dockerfile_iptables
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          labels: ${{ steps.metadata.outputs.labels }}
+          tags: >-
+            plndr/kube-vip-iptables:${{ github.ref_name }},
+            plndr/kube-vip-iptables:latest,
+            ghcr.io/kube-vip/kube-vip-iptables:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip-iptables:latest
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}

.gitignore

@@ -0,0 +1,9 @@
+.idea
+kube-vip
+.vscode
+bin
+testing/e2e/etcd/certs
+coverage.out
+pkg/etcd/etcd.pid
+pkg/etcd/etcd-data
+testing/e2e/e2e.test

.golangci.yml

@@ -0,0 +1,38 @@
+version: "2"
+linters:
+  enable:
+    - bodyclose
+    - gosec
+    - misspell
+    - unconvert
+    - unparam
+  settings:
+    misspell:
+      ignore-rules:
+        - creater
+    staticcheck:
+      checks:
+        - all
+        # Disable QF1008 to retain embedded fields for better readability.
+        - "-QF1008"
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  enable:
+    - gofmt
+    - goimports
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

CHANGELOG.md

@@ -0,0 +1,62 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Fixed
+- Propagate `bgp_attach_ip_to_interface` into per-service config so it attaches BGP-mode Service VIPs to the interface as configured.
+- Add a configurable kube-vip instance name and use it to isolate internal nftables egress tables, persist table ownership on Services, and migrate per-Service chains without affecting other deployments. Fixes #1634.
+- Retry on 403 Forbidden and 401 Unauthorized in `ServicesWatcher` at startup with exponential backoff. Fixes #1464.
+- Reintroduce BGP config via node annotations. Fixes #1488.
+- Fail fast in runtime `manager` and `service` paths when legacy `vip_address` is used without `vip_subnet` in control-plane ARP, BGP, or Routing Table mode.
+- Cancel the mode context on init or configuration failure before waiting on goroutines during shutdown.
+
+
+### Added
+- Configurable control-plane health check for BGP mode without leader election
+  - Polls a configurable HTTP(S) endpoint (e.g. `https://localhost:6443/livez`) to verify the exposed service is healthy (usually the local kube-apiserver)
+  - Withdraws the BGP route after a configurable number of consecutive failures, removing the unhealthy node from the ECMP set
+  - Re-announces the route automatically once the endpoint recovers
+  - Gracefully withdraws the route on shutdown (SIGTERM)
+  - Supports custom CA certificates for TLS verification
+  - Configuration via environment variables or CLI flags:
+    - `control_plane_health_check_address` / `--controlPlaneHealthCheckAddress`: URL to poll
+    - `control_plane_health_check_period_seconds` / `--controlPlaneHealthCheckPeriodSeconds`: interval between checks (default: 5)
+    - `control_plane_health_check_timeout_seconds` / `--controlPlaneHealthCheckTimeoutSeconds`: per-request timeout (default: 3)
+    - `control_plane_health_check_failure_threshold` / `--controlPlaneHealthCheckFailureThreshold`: consecutive failures before withdrawal (default: 3)
+    - `control_plane_health_check_ca_path` / `--controlPlaneHealthCheckCAPath`: CA cert for HTTPS verification
+- SIGUSR1 signal handler for runtime configuration dumps (#1301)
+  - Send SIGUSR1 to kube-vip process to dump current configuration to stdout
+  - Configuration dump includes:
+    - Basic configuration (VIP, interface, port, namespace settings)
+    - BGP configuration (enabled status, AS number, router ID, peers)
+    - ARP/NDP configuration (enabled status, broadcast rate)
+    - Services configuration (enabled status, load balancer settings)
+    - Network interfaces status
+    - Leader election configuration (type, lease details)
+    - Runtime statistics (load balancer, Prometheus, health check settings)
+  - Output format: Human-readable plaintext via fmt.Printf()
+  - Thread-safe implementation using mutex protection
+  - Non-disruptive: Process continues running after configuration dump
+  - Added comprehensive unit tests for all dump methods
+  - Added E2E tests for signal handling
+- Opt-in support for endpointless `LoadBalancer` services with `externalTrafficPolicy: Cluster`
+  - Annotation: `kube-vip.io/allow-reconcile-without-endpoints: "true"`
+  - Starts service handling path for opted-in endpointless Cluster services while preserving default endpoint-gated behavior for non-opt-in services and `Local` policy
+  - Added endpoint behavior tests and README usage documentation
+  - Added support in ipoib interfaces in ARP mode. Fixes #694
+
+### Changed
+- BGP mode now honours `enable_leader_election` for services: a single global services leader advertises the service VIPs instead of every node advertising them. Deployments that enabled `enable_leader_election` for the control plane and relied on ECMP/multipath for services must unset it (or switch to `enable_service_election`) to keep the previous datapath. kube-vip logs a warning on startup when this path is taken.
+- Updated signal handlers in manager_arp.go, manager_bgp.go, manager_wireguard.go, and manager_table.go to use switch statement pattern for handling multiple signals (SIGUSR1, SIGINT, SIGTERM)
+- wireguard.go now manages a complete wireguard interface on the current network namespace
+- manager_wireguard.go uses the new wireguard.go implementation
+
+## [v1.0.1] - Previous Release
+
+### Previous changes
+- See git history for changes prior to CHANGELOG.md introduction

CONTRIBUTING.md

@@ -113,8 +113,8 @@ and *merged* sorts of commits.
 To make it easier for reviewers to review your PR, consider the following:
 
 1. Follow the golang [coding conventions](https://github.com/golang/go/wiki/CodeReviewComments).
-2. Format your code with `make golangci-fix`; if the [linters](ci/README.md) flag an issue that
-   cannot be fixed automatically, an error message will be displayed so you can address the issue.
+2. Format your code with `make simplify` to automatically fix formatting issues.
+2. Lint your code with `make check`; if the linters flag an issue that cannot be fixed automatically, an error message will be displayed so you can address the issue.
 3. Follow [git commit](https://chris.beams.io/posts/git-commit/) guidelines.
 4. Follow [logging](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md) guidelines.
 
@@ -154,7 +154,7 @@ the `$GOPATH`.
 
 To develop locally, you can follow these steps:
 
- 1. [Install Go 1.15](https://golang.org/doc/install)
+ 1. [Install Go 1.19](https://golang.org/doc/install)
  2. Checkout your feature branch and `cd` into it.
  3. To build all Go files and install them under `bin`, run `make bin`
  4. To run all Go unit tests, run `make test-unit`

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:experimental
 
-FROM golang:1.16.5-alpine3.13 as dev
+FROM golang:1.27.1-alpine3.23 as dev
 RUN apk add --no-cache git ca-certificates make
 RUN adduser -D appuser
 COPY . /src/
@@ -16,4 +16,4 @@ FROM scratch
 COPY --from=dev /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 # Add kube-vip binary
 COPY --from=dev /src/kube-vip /
-ENTRYPOINT ["/kube-vip"]
+ENTRYPOINT ["/kube-vip"]

Dockerfile_iptables

@@ -0,0 +1,21 @@
+# syntax=docker/dockerfile:experimental
+
+FROM golang:1.27.1-alpine3.23 as dev
+RUN apk add --no-cache git make
+RUN adduser -D appuser
+COPY . /src/
+WORKDIR /src
+
+ENV GO111MODULE=on
+RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
+    --mount=type=cache,sharing=locked,id=goroot,target=/root/.cache/go-build \
+    CGO_ENABLED=0 GOOS=linux make build
+
+FROM alpine:3.24.1
+# Update pkgs and add iptables
+RUN apk upgrade && \
+    apk add --no-cache iptables iptables-legacy
+
+# Add kube-vip binary
+COPY --from=dev /src/kube-vip /
+ENTRYPOINT ["/kube-vip"]

Makefile

@@ -2,10 +2,11 @@ SHELL := /bin/sh
 
 # The name of the executable (default is current directory name)
 TARGET := kube-vip
-.DEFAULT_GOAL: $(TARGET)
+.DEFAULT_GOAL := $(TARGET)
 
 # These will be provided to the target
-VERSION := v0.3.7
+VERSION := v1.2.3
+
 BUILD := `git rev-parse HEAD`
 
 # Operating System Default (LINUX)
@@ -14,9 +15,15 @@ TARGETOS=linux
 # Use linker flags to provide version/build settings to the target
 LDFLAGS=-ldflags "-s -w -X=main.Version=$(VERSION) -X=main.Build=$(BUILD) -extldflags -static"
 DOCKERTAG ?= $(VERSION)
-REPOSITORY = plndr
+REPOSITORY ?= docker.io/plndr
+GO_VERSION := $(word 2,$(shell grep '^go ' go.mod))
+K8S_VERSION ?= v1.35.0
+GINKGO_ARGS ?=
+GINKGO_PROCS ?=
+GINKGO_PARALLEL := $(if $(GINKGO_PROCS),--procs=$(GINKGO_PROCS),-p)
+BUILDX_CACHE_FLAGS ?=
 
-.PHONY: all build clean install uninstall fmt simplify check run e2e-tests
+.PHONY: all build clean install uninstall simplify check run e2e-tests unit-tests integration-tests unit-tests-docker integration-tests-docker e2e-tests-etcd
 
 all: check install
 
@@ -36,17 +43,30 @@ install:
 uninstall: clean
 	@rm -f $$(which ${TARGET})
 
-fmt:
-	@gofmt -l -w ./...
-
 demo:
 	@cd demo
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 	@cd ..
 
 ## Remote (push of images)
 # This build a local docker image (x86 only) for quick testing
+
+dockerx86Dev:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 --push -t $(REPOSITORY)/$(TARGET):dev .
+	@echo New single x86 Architecture Docker image created
+
+dockerx86Iptables:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 -f ./Dockerfile_iptables --push -t $(REPOSITORY)/$(TARGET):dev .
+	@echo New single x86 Architecture Docker image created
+
+dockerx86IptablesLocal:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 -f ./Dockerfile_iptables -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@echo New single x86 Architecture Docker image created
+
 dockerx86:
 	@-rm ./kube-vip
 	@docker buildx build  --platform linux/amd64 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
@@ -54,48 +74,118 @@ dockerx86:
 
 docker:
 	@-rm ./kube-vip
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 ## Local (docker load of images)
 # This will build a local docker image (x86 only), use make dockerLocal for all architectures
 dockerx86Local:
 	@-rm ./kube-vip
-	@docker buildx build  --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) $(BUILDX_CACHE_FLAGS) .
 	@echo New Multi Architecture Docker image created
 
 dockerx86Action:
 	@-rm ./kube-vip
-	@docker buildx build  --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):action .
+	@docker buildx build --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):action $(BUILDX_CACHE_FLAGS) .
+	@echo New Multi Architecture Docker image created
+
+dockerx86ActionIPTables:
+	@-rm ./kube-vip
+	@docker buildx build --platform linux/amd64 -f ./Dockerfile_iptables --load -t $(REPOSITORY)/$(TARGET):action $(BUILDX_CACHE_FLAGS) .
 	@echo New Multi Architecture Docker image created
 
 dockerLocal:
 	@-rm ./kube-vip
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 simplify:
-	@gofmt -s -l -w ./...
+	@gofmt -s -l -w *.go pkg cmd
 
 check:
 	go mod tidy
 	test -z "$(git status --porcelain)"
-	test -z $(shell gofmt -l main.go | tee /dev/stderr) || echo "[WARN] Fix formatting issues with 'make fmt'"
-	golint ./...
+	test -z $(shell gofmt -l *.go pkg cmd) || echo "[WARN] Fix formatting issues with 'make simplify'"
+	golangci-lint run
 	go vet ./...
-	
+
 run: install
 	@$(TARGET)
 
 manifests:
 	@make build
 	@mkdir -p ./docs/manifests/$(VERSION)/
-	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-arp.yaml
-	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --bgp --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-bgp.yaml
-	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
-	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
-	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster --provider-config /etc/cloud-sa/cloud-sa.json > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
+	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-arp.yaml
+	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-lb.yaml
+	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-bgp.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
 	@-rm ./kube-vip
 
-e2e-tests:
-	E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/ginkgo -v -p testing/e2e
+manifest-test: 
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --enableLoadBalancer
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --controlplane --services
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster
+
+unit-tests:
+	go test -race -coverprofile=coverage.out -covermode=atomic ./...
+
+unit-tests-docker:
+	docker run --rm -w /kube-vip -v $$(pwd):/kube-vip -v kube-vip-gomod-cache:/go/pkg/mod -v kube-vip-gobuild-cache:/root/.cache/go-build golang:$(GO_VERSION) sh -c "make unit-tests; status=$$?; chmod 666 coverage.out 2>/dev/null || true; exit $$status"
+
+integration-tests:
+	go test -tags=integration,e2e -v ./pkg/etcd
+
+e2e-tests-arp: get-whoami
+	GOMAXPROCS=4 TEST_MODE=arp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e
+
+e2e-tests-rt: get-whoami
+	GOMAXPROCS=4 TEST_MODE=rt K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e
+
+e2e-tests-bgp: get-whoami get-gobgp
+	GOMAXPROCS=4 TEST_MODE=bgp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e
+
+e2e-tests-etcd: get-whoami
+	GOMAXPROCS=4 K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e/etcd
+
+e2e-tests: e2e-tests-arp e2e-tests-rt e2e-tests-bgp
+
+service-tests:
+	$(MAKE) -C testing/e2e/e2e dockerLocal
+	E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/services -Services -simple -deployments -leaderActive -leaderFailover -localDeploy -electionFaults -egress -egressIPv6 -dualStack -egressInternal
+
+trivy: dockerx86ActionIPTables
+	docker run -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy:0.47.0 \
+		image  \
+		--format table \
+		--exit-code  1 \
+		--ignore-unfixed \
+		--vuln-type  'os,library' \
+		--severity  'CRITICAL,HIGH'  \
+		$(REPOSITORY)/$(TARGET):action
+
+kind-quick:
+	echo "Standing up your cluster"
+	kind create cluster --config ./testing/kind/kind.yaml --name kube-vip
+	kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
+	kubectl create configmap --namespace kube-system kubevip --from-literal range-global=172.18.100.10-172.18.100.30
+	kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
+	kind load docker-image --name kube-vip $(REPOSITORY)/$(TARGET):$(DOCKERTAG)
+	docker run --network host --rm $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --services --inCluster --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --servicesElection --interface  eth0 | kubectl apply -f -
+
+kind-reload:
+	kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name services
+	kubectl rollout restart -n kube-system daemonset/kube-vip-ds
+
+get-gobgp:
+	mkdir -p bin
+	wget -nc --directory-prefix=bin https://github.com/osrg/gobgp/releases/download/v4.6.0/gobgp_4.6.0_linux_amd64.tar.gz
+	tar -xvzf bin/gobgp_4.6.0_linux_amd64.tar.gz -C bin
+
+get-whoami:
+	docker pull ghcr.io/traefik/whoami:v1.11

README.md

@@ -1,8 +1,10 @@
 # kube-vip
 
-High Availability and Load-Balancing 
+High Availability and Load-Balancing
 
-![](https://kube-vip.io/kube-vip.png)
+![](https://github.com/kube-vip/kube-vip/raw/main/kube-vip.png)
+
+[![Build and publish main image regularly](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml/badge.svg)](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml) [![LFX Health Score](https://img.shields.io/static/v1?label=Health%20Score&message=Healthy&color=A7F3D0&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip) [![LFX Active Contributors](https://img.shields.io/static/v1?label=Active%20contributors%20(1Y)&message=212&color=0094FF&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip)
 
 ## Overview
 Kubernetes Virtual IP and Load-Balancer for both control plane and Kubernetes services
@@ -16,6 +18,8 @@ The idea behind `kube-vip` is a small self-contained Highly-Available option for
 
 **NOTE:** All documentation of both usage and architecture are now available at [https://kube-vip.io](https://kube-vip.io).
 
+For upgrading an existing install in place (static Pod or DaemonSet), see the [upgrade guide](https://kube-vip.io/docs/upgrade/).
+
 ## Features
 
 Kube-Vip was originally created to provide a HA solution for the Kubernetes control plane, over time it has evolved to incorporate that same functionality into Kubernetes service type [load-balancers](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer).
@@ -30,7 +34,8 @@ Kube-Vip was originally created to provide a HA solution for the Kubernetes cont
 - Service LoadBalancer address pools per namespace or global
 - Service LoadBalancer address via (existing network DHCP)
 - Service LoadBalancer address exposure to gateway via UPNP
-- ... manifest generation, vendor API integrations and many nore... 
+- Egress! Kube-vip will utilise a service loadbalancer as both the ingress and **egress** for a pod. 
+- ... manifest generation, vendor API integrations and many more...
 
 ## Why?
 
@@ -55,4 +60,81 @@ All of these would require a separate level of configuration and in some infrast
 
 ## Troubleshooting and Feedback
 
+### SELinux and IPVS kernel modules
+
+When using IPVS load balancing on nodes with SELinux enforcing, kube-vip may be
+blocked from requesting kernel modules from inside the container. Symptoms can
+include the kube-vip pod entering `Error` or `CrashLoopBackOff`, logs that show
+`ensure IPVS kernel modules are loaded`, or audit denials for `module_request`
+from `container_t`.
+
+Load the required IPVS modules on every node that can run kube-vip before
+deploying it:
+
+```shell
+sudo modprobe ip_vs
+sudo modprobe ip_vs_rr
+```
+
+To persist this across reboots, add the modules to a file such as
+`/etc/modules-load.d/kube-vip-ipvs.conf`:
+
+```text
+ip_vs
+ip_vs_rr
+```
+
+Preloading only the required modules is preferred to enabling the SELinux
+`domain_kernel_load_modules` boolean for containers.
+
+### Gateway API `LoadBalancer` services with no endpoints
+
+Some Gateway API controllers create `LoadBalancer` services that intentionally have no Endpoints/EndpointSlices backends.
+
+If you want kube-vip to reconcile such a service, opt in with:
+
+```yaml
+metadata:
+  annotations:
+    kube-vip.io/allow-reconcile-without-endpoints: "true"
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Cluster
+```
+
+Scope:
+- Works only with `externalTrafficPolicy: Cluster`
+- No effect for `Local`
+- Default endpoint-gated behavior remains unchanged for services without this annotation
+
 Please raise issues on the GitHub repository and as mentioned check the documentation at [https://kube-vip.io](https://kube-vip.io/).
+
+## Community Tools
+
+- **[KubeStellar Console — Guided kube-vip Install](https://console.kubestellar.io/missions/install-kube-vip)** — A step-by-step guided installation experience for kube-vip with pre-flight checks, validation, troubleshooting, and rollback support.
+
+## Contributing
+
+Thanks for taking the time to join our community and start contributing! We welcome pull requests. Feel free to dig through the [issues](https://github.com/kube-vip/kube-vip/issues) and jump in.
+
+:warning: This project has issue compiling on MacOS, please compile it on linux distribution
+
+Additionally it is now relatively easy and quick to develop with [skaffold](https://skaffold.dev/), and the `skaffold.yaml` exists within the root folder of the gir repository.
+
+### Set up a kind development environment
+
+1. `kind create cluster --config ./testing/kind.yaml`
+2. `kubectl apply -f https://kube-vip.io/manifests/rbac.yaml`
+3. Create a load balancer range `configMap` from the kind cluster
+4. Apply the CCM manifest
+5. Start skaffold with `skaffold dev`
+6. Start developing and see changes applied in real-time.
+
+## Star History
+
+[![Star History Chart](https://star-history.dera.page/svg?repos=kube-vip/kube-vip&type=Date)](https://star-history.dera.page/#kube-vip/kube-vip&type=date)
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fkube-vip%2Fkube-vip.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fkube-vip%2Fkube-vip?ref=badge_shield)
+
+
+## License
+[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fkube-vip%2Fkube-vip.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fkube-vip%2Fkube-vip?ref=badge_large)

ROADMAP.md

@@ -1,10 +1,10 @@
 # Kube-Vip Roadmap
 
-This document outlines the roadmap for the **kube-vip** project and only covers the technologies within this particular project, other projects that augment or provide additional functionality (such as cloud-providers) may have their own roadmaps in future. The functionality for **kube-vip** has grown either been developed organically or through real-world needs, and this is the first attempt to put into words a plan for the future of **kube-vip** and will additional evolve over time. This means that items listed or detailed here are not neccessarily set in stone and the roadmap can grow/shrink as the project matures. We definitely welcome suggestions and ideas from everyone about the roadmap and **kube-vip** features. Reach us through Issues, Slack or email <catch-all>@kube-vip.io.
-  
+This document outlines the roadmap for the **kube-vip** project and only covers the technologies within this particular project, other projects that augment or provide additional functionality (such as cloud-providers) may have their own roadmaps in future. The functionality for **kube-vip** has grown either been developed organically or through real-world needs, and this is the first attempt to put into words a plan for the future of **kube-vip** and will additional evolve over time. This means that items listed or detailed here are not necessarily set in stone and the roadmap can grow/shrink as the project matures. We definitely welcome suggestions and ideas from everyone about the roadmap and **kube-vip** features. Reach us through Issues, Slack or email <catch-all>@kube-vip.io.
+
 ## Release methodology
 
-The **kube-vip** project attempts to follow a tick-tock release cycle, this typically means that one release will come **packed** with new features where the following release will come with fixes, code sanitation and performane enhancements.
+The **kube-vip** project attempts to follow a tick-tock release cycle, this typically means that one release will come **packed** with new features where the following release will come with fixes, code sanitation and performance enhancements.
 
 ## Roadmap
 
@@ -13,12 +13,12 @@ The **kube-vip** project offers two main areas of functionality:
 - HA Kubernetes clusters through a control-plane VIP
 - Kubernetes `service type:LoadBalancer`
 
-Whilst both of these functions share underlying technologies and code they will have slightly differening roadmaps. 
+Whilst both of these functions share underlying technologies and code they will have slightly differing roadmaps.
 
 ### HA Kubernetes Control Plane
 
-- **Re-implememt LoadBalancing** - due to a previous request the HTTP loadbalancing was removed leaving just HA for the control plane. This functionality will be re-implemented either through the original round-robin HTTP requests or utilising IPVS. 
-- **Utilise the Kubernetes API to determine additional Control Plane members** - Once a single node cluster is running **kube-vip** could use the API to determine the additional members, at this time a Cluster-API provider needs to drop a static manifest per CP node. 
+- **Re-implement LoadBalancing** - due to a previous request the HTTP loadbalancing was removed leaving just HA for the control plane. This functionality will be re-implemented either through the original round-robin HTTP requests or utilising IPVS.
+- **Utilise the Kubernetes API to determine additional Control Plane members** - Once a single node cluster is running **kube-vip** could use the API to determine the additional members, at this time a Cluster-API provider needs to drop a static manifest per CP node.
 - **Re-evaluate raft** - **kube-vip** is mainly designed to run within a Kubernetes cluster, however it's original design was a raft cluster external to Kubernetes. Unfortunately given some of the upgrade paths identified in things like CAPV moving to leaderElection within Kubernetes became a better idea.
 
 ## Kubernetes `service type:LoadBalancer`
@@ -29,7 +29,7 @@ Whilst both of these functions share underlying technologies and code they will
 ## Global **Kube-Vip** items
 
 - **Improved metrics** - At this time the scaffolding for monitoring exists, however this needs drastically extending to provide greater observability to what is happening within **kube-vip**
-- **Windows support** - The Go SDK didn't support the capability for low-levels sockets for ARP originally, this should be revisted. 
+- **Windows support** - The Go SDK didn't support the capability for low-levels sockets for ARP originally, this should be revisited.
 - **Additional BGP features** :
   - Communities
   - BFD

cmd/kube-vip-config.go

@@ -1,146 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-
-	"github.com/ghodss/yaml"
-	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	appv1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-// [sample configuration] - flags
-var cliConfig kubevip.Config
-var cliConfigLB kubevip.LoadBalancer
-var cliLocalPeer string
-var cliRemotePeers, cliBackends []string
-
-func init() {
-	kubeVipSampleConfig.Flags().StringVar(&cliConfig.Interface, "interface", "eth0", "Name of the interface to bind to")
-	kubeVipSampleConfig.Flags().StringVar(&cliConfig.VIP, "vip", "192.168.0.1", "The Virtual IP address")
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfig.SingleNode, "singleNode", false, "Start this instance as a single node")
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfig.StartAsLeader, "startAsLeader", false, "Start this instance as the cluster leader")
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfig.EnableARP, "arp", true, "Use ARP broadcasts to improve VIP re-allocations")
-	kubeVipSampleConfig.Flags().StringVar(&cliLocalPeer, "localPeer", "server1:192.168.0.1:10000", "Settings for this peer, format: id:address:port")
-	kubeVipSampleConfig.Flags().StringSliceVar(&cliRemotePeers, "remotePeers", []string{"server2:192.168.0.2:10000", "server3:192.168.0.3:10000"}, "Comma seperated remotePeers, format: id:address:port")
-	// Load Balancer flags
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfigLB.BindToVip, "lbBindToVip", false, "Bind example load balancer to VIP")
-	kubeVipSampleConfig.Flags().StringVar(&cliConfigLB.Type, "lbType", "tcp", "Type of load balancer instance (TCP/HTTP)")
-	kubeVipSampleConfig.Flags().StringVar(&cliConfigLB.Name, "lbName", "Example Load Balancer", "The name of a load balancer instance")
-	kubeVipSampleConfig.Flags().IntVar(&cliConfigLB.Port, "lbPort", 8080, "Port that load balancer will expose on")
-	kubeVipSampleConfig.Flags().StringSliceVar(&cliBackends, "lbBackends", []string{"192.168.0.1:8080", "192.168.0.2:8080"}, "Comma seperated backends, format: address:port")
-}
-
-var kubeVipSampleConfig = &cobra.Command{
-	Use:   "config",
-	Short: "Generate a Sample configuration",
-	Run: func(cmd *cobra.Command, args []string) {
-
-		// // Parse localPeer
-		// p, err := kubevip.ParsePeerConfig(cliLocalPeer)
-		// if err != nil {
-		// 	cmd.Help()
-		// 	log.Fatalln(err)
-		// }
-		// cliConfig.LocalPeer = *p
-
-		// // Parse remotePeers
-		// //Iterate backends
-		// for i := range cliRemotePeers {
-		// 	p, err := kubevip.ParsePeerConfig(cliRemotePeers[i])
-		// 	if err != nil {
-		// 		cmd.Help()
-		// 		log.Fatalln(err)
-		// 	}
-		// 	cliConfig.RemotePeers = append(cliConfig.RemotePeers, *p)
-		// }
-
-		// //Iterate backends
-		// for i := range cliBackends {
-		// 	b, err := kubevip.ParseBackendConfig(cliBackends[i])
-		// 	if err != nil {
-		// 		cmd.Help()
-		// 		log.Fatalln(err)
-		// 	}
-		// 	cliConfigLB.Backends = append(cliConfigLB.Backends, *b)
-		// }
-
-		// Add the basic Load-Balancer to the configuration
-		cliConfig.LoadBalancers = append(cliConfig.LoadBalancers, cliConfigLB)
-
-		err := cliConfig.ParseFlags(cliLocalPeer, cliRemotePeers, cliBackends)
-		if err != nil {
-			cmd.Help()
-			log.Fatalln(err)
-		}
-
-		err = kubevip.ParseEnvironment(&cliConfig)
-		if err != nil {
-			cmd.Help()
-			log.Fatalln(err)
-		}
-
-		cliConfig.PrintConfig()
-	},
-}
-
-var kubeVipSampleManifest = &cobra.Command{
-	Use:   "manifest",
-	Short: "Generate a Sample kubernetes manifest",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Generate the sample manifest specification
-		p := &appv1.Pod{
-			TypeMeta: metav1.TypeMeta{
-				Kind:       "Pod",
-				APIVersion: "v1",
-			},
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      "kube-vip",
-				Namespace: "kube-system",
-			},
-			Spec: appv1.PodSpec{
-				Containers: []appv1.Container{
-					{
-						Name:  "kube-vip",
-						Image: fmt.Sprintf("docker.io/plndr/kube-vip:%s", Release.Version),
-						SecurityContext: &appv1.SecurityContext{
-							Capabilities: &appv1.Capabilities{
-								Add: []appv1.Capability{
-									"NET_ADMIN",
-									"SYS_TIME",
-								},
-							},
-						},
-						Args: []string{
-							"start",
-							"-c",
-							"/etc/kube-vip/config.yaml",
-						},
-						VolumeMounts: []appv1.VolumeMount{
-							{
-								Name:      "config",
-								MountPath: "/etc/kube-vip/",
-							},
-						},
-					},
-				},
-				Volumes: []appv1.Volume{
-					{
-						Name: "config",
-						VolumeSource: appv1.VolumeSource{
-							HostPath: &appv1.HostPathVolumeSource{
-								Path: "/etc/kube-vip/",
-							},
-						},
-					},
-				},
-				HostNetwork: true,
-			},
-		}
-
-		b, _ := yaml.Marshal(p)
-		fmt.Printf(string(b))
-	},
-}

cmd/kube-vip-kubeadm.go

@@ -1,18 +1,13 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
-	"net"
 	"os"
 
+	log "log/slog"
+
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
 )
 
 // kubeadm adds two subcommands for managing a vip during a kubeadm init/join
@@ -28,134 +23,137 @@ func init() {
 var kubeKubeadm = &cobra.Command{
 	Use:   "kubeadm",
 	Short: "Kubeadm functions",
-	Run: func(cmd *cobra.Command, args []string) {
-		cmd.Help()
-		// TODO - A load of text detailing what's actually happening
+	Long: `This command group provides utilities for generating static Pod manifests specifically tailored for the kubeadm bootstrapping process.
+	It contains two subcommands:
+  		- init:  Generates a manifest to be used during 'kubeadm init' on the first control-plane node.
+  		- join:  Generates a manifest to be used during 'kubeadm join' for additional control-plane nodes.
+
+The generated YAML manifest should be saved to the kubeadm static Pod directory (typically /etc/kubernetes/manifests/) so that kubeadm launches the kube-vip static Pod automatically.`,
+	Run: func(cmd *cobra.Command, _ []string) {
+		_ = cmd.Help()
 	},
 }
 
 var kubeKubeadmInit = &cobra.Command{
 	Use:   "init",
 	Short: "kube-vip init",
-	Long:  "The \"init\" subcommand will generate the Kubernetes manifest that will be started by kubeadm through the kubeadm init process",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Long: `The 'init' subcommand generates a Kubernetes Pod manifest that kubeadm will start as a static Pod during the cluster initialisation phase.
+
+This manifest runs kube-vip on the first control-plane node to advertise the Virtual IP (VIP) for the API server. The VIP is typically configured using ARP (Layer 2) or BGP (dynamic routing).
+
+Required flags for this command:
+  --interface   : The network interface to bind the VIP to (e.g., eth0).
+  --vip or --address : The Virtual IP address or DNS name to use.
+
+Example:
+  kube-vip kubeadm init --interface eth0 --vip 192.168.1.100 --controlplane
+
+The output YAML should be written to the kubeadm manifests directory, e.g.:
+  kube-vip kubeadm init ... > /etc/kubernetes/manifests/kube-vip.yaml`,
+	Run: func(cmd *cobra.Command, _ []string) {
+
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
-		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
+		err := kubevip.ParseEnvironment(&initConfig)
+		if err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+		if err := initConfig.Validate(); err != nil {
+			log.Error("validating configuration", "err", err)
+			return
+		}
+
 		// TODO - check for certain things VIP/interfaces
 		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+			_ = cmd.Help()
+			log.Error("No interface is specified for kube-vip to bind to")
+			return
 		}
 
 		if initConfig.VIP == "" && initConfig.Address == "" {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
-		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
 
-		fmt.Println(cfg)
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("generating VIPSubnet", "err", err)
+				return
+			}
+		}
+
+		cfg, err := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
+		if err != nil {
+			log.Error("unable to create manifest", "err", err)
+			return
+		}
+		fmt.Println(cfg) // output manifest to stdout
 	},
 }
 
 var kubeKubeadmJoin = &cobra.Command{
 	Use:   "join",
 	Short: "kube-vip join",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Long: `The 'join' subcommand generates a Kubernetes Pod manifest for additional control-plane nodes joining an existing cluster via 'kubeadm join'.
+
+It functions identically to the 'init' subcommand, but is intended for secondary control-plane nodes. It validates that the kubeconfig file (specified by --config, defaulting to /etc/kubernetes/admin.conf) exists on the node to ensure the node can authenticate with the cluster.
+
+Required flags for this command:
+  --interface   : The network interface to bind the VIP to.
+  --vip or --address : The Virtual IP address or DNS name (must match the VIP used during 'init').
+
+Example:
+  kube-vip kubeadm join --interface eth0 --vip 192.168.1.100
+
+The output YAML should be saved to the kubeadm manifests directory on the joining node.`,
+	Run: func(cmd *cobra.Command, _ []string) {
 
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
-		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
+		err := kubevip.ParseEnvironment(&initConfig)
+		if err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+		if err := initConfig.Validate(); err != nil {
+			log.Error("validating configuration", "err", err)
+			return
+		}
+
 		// TODO - check for certain things VIP/interfaces
 		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+			_ = cmd.Help()
+			log.Error("No interface is specified for kube-vip to bind to")
+			return
 		}
 
 		if initConfig.VIP == "" && initConfig.Address == "" {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
 
 		if _, err := os.Stat(kubeConfigPath); os.IsNotExist(err) {
-			log.Fatalf("Unable to find file [%s]", kubeConfigPath)
+			log.Error("kubeConfig not found", "Path", kubeConfigPath)
+			return
 		}
 
-		// We will use kubeconfig in order to find all the master nodes
-		// use the current context in kubeconfig
-		config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
-		if err != nil {
-			log.Fatal(err.Error())
-		}
-
-		// create the clientset
-		clientset, err := kubernetes.NewForConfig(config)
-		if err != nil {
-			log.Fatal(err.Error())
-		}
-
-		opts := metav1.ListOptions{}
-		opts.LabelSelector = "node-role.kubernetes.io/master"
-		nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), opts)
-
-		// Iterate over all nodes that are masters and find the details to build a peer list
-		for x := range nodes.Items {
-			// Get hostname and address
-			var nodeAddress, nodeHostname string
-			for y := range nodes.Items[x].Status.Addresses {
-				switch nodes.Items[x].Status.Addresses[y].Type {
-				case corev1.NodeHostName:
-					nodeHostname = nodes.Items[x].Status.Addresses[y].Address
-				case corev1.NodeInternalIP:
-					nodeAddress = nodes.Items[x].Status.Addresses[y].Address
-				}
-			}
-
-			newPeer, err := kubevip.ParsePeerConfig(fmt.Sprintf("%s:%s:%d", nodeHostname, nodeAddress, 10000))
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
 			if err != nil {
-				panic(err.Error())
+				log.Error("generating VIPSubnet", "err", err)
+				return
 			}
-			initConfig.RemotePeers = append(initConfig.RemotePeers, *newPeer)
-
 		}
-		// Generate manifest and print
-		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
-		fmt.Println(cfg)
+
+		cfg, err := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
+		if err != nil {
+			log.Error("unable to create manifest", "err", err)
+			return
+		}
+		fmt.Println(cfg) // output manifest to stdout
 	},
 }
-
-func autoGenLocalPeer() (*kubevip.RaftPeer, error) {
-	// hostname // address // defaultport
-	h, err := os.Hostname()
-	if err != nil {
-		return nil, err
-	}
-
-	var a string
-	addrs, err := net.InterfaceAddrs()
-	if err != nil {
-		return nil, err
-	}
-	for _, address := range addrs {
-		// check the address type and if it is not a loopback the display it
-		if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
-			if ipnet.IP.To16() != nil {
-				a = ipnet.IP.String()
-				break
-			}
-		}
-	}
-	if a == "" {
-		return nil, fmt.Errorf("Unable to find local address")
-	}
-	return &kubevip.RaftPeer{
-		ID:      h,
-		Address: a,
-		Port:    10000,
-	}, nil
-
-}

cmd/kube-vip-manifests.go

@@ -3,8 +3,9 @@ package cmd
 import (
 	"fmt"
 
+	log "log/slog"
+
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
@@ -12,75 +13,200 @@ import (
 // manifests will be used to generate:
 // - Pod spec manifest, mainly used for a static pod (kubeadm)
 // - Daemonset manifest, mainly used to run kube-vip as a deamonset within Kubernetes (k3s/rke)
+// - RBAC manifest, used to generate the RBAC permissions for kube-vip
 
-//var inCluster bool
-var taint bool
+var taint, role, rolebinding bool
 
 func init() {
 	kubeManifest.PersistentFlags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
+	kubeManifest.PersistentFlags().StringVar(&image, "image", "ghcr.io/kube-vip/kube-vip", "Define a hardcoded image with or without tag for the manifest")
 	kubeManifestDaemon.PersistentFlags().BoolVar(&taint, "taint", false, "Taint the manifest for only running on control planes")
+	kubeManifestRbac.PersistentFlags().BoolVar(&role, "role", false, "Generate only a Role inside the serviceNamespace access")
+	kubeManifestRbac.PersistentFlags().BoolVar(&rolebinding, "rolebinding", false, "Generate only a RoleBinding for namespaced access")
 
 	kubeManifest.AddCommand(kubeManifestPod)
 	kubeManifest.AddCommand(kubeManifestDaemon)
+	kubeManifest.AddCommand(kubeManifestRbac)
 }
 
 var kubeManifest = &cobra.Command{
 	Use:   "manifest",
 	Short: "Manifest functions",
-	Run: func(cmd *cobra.Command, args []string) {
-		cmd.Help()
-		// TODO - A load of text detailing what's actually happening
+	Long: `This command group provides flexible manifest generation for deploying kube-vip in various Kubernetes environments.
+
+Unlike the 'kubeadm' subcommands, which are tightly coupled to kubeadm's static Pod requirements, these generators produce standard Kubernetes manifests (Pod, DaemonSet, RBAC) that can be used with any Kubernetes distribution (e.g., k3s, RKE, or vanilla Kubernetes).
+
+Subcommands:
+  pod        : Generates a standalone Pod manifest (similar to a static pod).
+  daemonset  : Generates a DaemonSet manifest to run kube-vip on selected nodes.
+  rbac       : Generates the necessary ServiceAccount, Role/ClusterRole, and Binding manifests.
+
+All output is written to stdout as YAML, typically piped to 'kubectl apply -f -' or saved to a file.`,
+	Run: func(cmd *cobra.Command, _ []string) {
+		_ = cmd.Help()
 	},
 }
 
 var kubeManifestPod = &cobra.Command{
 	Use:   "pod",
 	Short: "Generate a Pod Manifest",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Long: `Generate a standalone Pod manifest for kube-vip.
+
+This is ideal for environments that do not use DaemonSets or where you want to run kube-vip as a static Pod (similar to the 'kubeadm' subcommand, but without kubeadm-specific assumptions). It includes all the necessary container specifications, volumes, and environment variables derived from the provided flags.
+
+Key flags:
+  --interface : Network interface for the VIP.
+  --vip or --address : The Virtual IP address or DNS name.
+  --image     : Override the container image (default: ghcr.io/kube-vip/kube-vip).
+
+The manifest is generated based on the current configuration flags set on the root command.
+
+Example:
+  kube-vip manifest pod --interface eth0 --vip 10.0.0.100 --controlplane | kubectl apply -f -`,
+	Run: func(cmd *cobra.Command, _ []string) {
+		var err error
+
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
-		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
-		// TODO - check for certain things VIP/interfaces
-		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+		if err := initConfig.Validate(); err != nil {
+			log.Error("validating configuration", "err", err)
+			return
 		}
 
 		// The control plane has a requirement for a VIP being specified
-		if initConfig.EnableControlPane && (initConfig.VIP == "" && initConfig.Address == "" && initConfig.DDNS == false) {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+			_ = cmd.Help()
+			log.Error("no address is specified for kube-vip to expose services on")
+			return
 		}
-		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
 
-		fmt.Println(cfg)
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("config parse", "err", err)
+				return
+			}
+		}
+
+		cfg, err := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
+		if err != nil {
+			log.Error("unable to create manifest", "err", err)
+			return
+		}
+		fmt.Println(cfg) // output manifest to stdout
 	},
 }
 
 var kubeManifestDaemon = &cobra.Command{
 	Use:   "daemonset",
 	Short: "Generate a Daemonset Manifest",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Long: `Generate a DaemonSet manifest to run kube-vip across multiple nodes.
+
+This is the recommended deployment method for production clusters running kube-vip as a service. It ensures that kube-vip runs on all control-plane nodes (or selected nodes via tolerations) and can handle both control-plane HA and service load-balancing.
+
+Flags specific to this subcommand:
+  --taint  : Adds a toleration to the DaemonSet so that pods are scheduled only on nodes with the control-plane taint (node-role.kubernetes.io/control-plane:NoSchedule). This is essential for control-plane-only deployments.
+
+All other standard kube-vip flags (--interface, --vip, --enableARP, --enableBGP, etc.) are respected and embedded into the DaemonSet pod template.
+
+Example:
+  kube-vip manifest daemonset --interface eth0 --vip 192.168.1.100 --controlplane --taint | kubectl apply -f -`,
+	Run: func(cmd *cobra.Command, _ []string) {
+		var err error
+
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
-		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
-		// TODO - check for certain things VIP/interfaces
-		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+		if err := initConfig.Validate(); err != nil {
+			log.Error("validating configuration", "err", err)
+			return
+		}
+		// The control plane has a requirement for a VIP being specified
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+			_ = cmd.Help()
+			log.Error("no address is specified for kube-vip to expose services on")
+			return
+		}
+
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("config parse", "err", err)
+				return
+			}
+		}
+
+		cfg, err := kubevip.GenerateDaemonsetManifestFromConfig(&initConfig, image, Release.Version, inCluster, taint)
+		if err != nil {
+			log.Error("unable to create manifest", "err", err)
+			return
+		}
+		fmt.Println(cfg) // output manifest to stdout
+	},
+}
+
+var kubeManifestRbac = &cobra.Command{
+	Use:   "rbac",
+	Short: "Generate an RBAC Manifest",
+	Long: `Generate the RBAC (Role-Based Access Control) manifests required for kube-vip to interact with the Kubernetes API.
+
+kube-vip needs permissions to watch services, endpoints, configmaps, and manage leader election leases. This command outputs the minimum required ServiceAccount, Role (or ClusterRole), and the corresponding binding.
+
+Flags:
+  --role        : If true, generates a namespaced Role instead of a ClusterRole. The namespace is taken from the root --namespace flag (default: kube-system).
+  --rolebinding : If true, generates a RoleBinding (if --role is also true). If --role is false, a ClusterRoleBinding is generated automatically.
+
+The output is a multi-document YAML (separated by '---'). It is safe to apply directly:
+  kube-vip manifest rbac --role --rolebinding | kubectl apply -f -
+
+Without --role, it generates a ClusterRole and ClusterRoleBinding, which is the default behaviour and suitable for most cluster-wide deployments.`,
+	Run: func(cmd *cobra.Command, _ []string) {
+		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
+		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+		if err := initConfig.Validate(); err != nil {
+			log.Error("validating configuration", "err", err)
+			return
 		}
 
 		// The control plane has a requirement for a VIP being specified
-		if initConfig.EnableControlPane && (initConfig.VIP == "" && initConfig.Address == "" && initConfig.DDNS == false) {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+			_ = cmd.Help()
+			log.Error("no address is specified for kube-vip to expose services on")
+			return
 		}
-		cfg := kubevip.GenerateDeamonsetManifestFromConfig(&initConfig, Release.Version, inCluster, taint)
 
-		fmt.Println(cfg)
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			var err error
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("generating VIPSubnet", "err", err)
+				return
+			}
+		}
+		saCfg := kubevip.GenerateSA(&initConfig)
+		roleCfg := kubevip.GenerateRole(&initConfig, role)
+		if role {
+			rolebinding = true
+		}
+		roleBindingCfg := kubevip.GenerateRoleBinding(rolebinding, saCfg, roleCfg)
+
+		// Output the YAML manifests to stdout
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(saCfg))
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(roleCfg))
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(roleBindingCfg))
 	},
 }

cmd/kube-vip-start.go

@@ -1,136 +0,0 @@
-package cmd
-
-import (
-	"os"
-	"os/signal"
-
-	"github.com/kube-vip/kube-vip/pkg/bgp"
-	"github.com/kube-vip/kube-vip/pkg/cluster"
-	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-)
-
-// Start as a single node (no cluster), start as a leader in the cluster
-var startConfig kubevip.Config
-var startConfigLB kubevip.LoadBalancer
-var startLocalPeer, startKubeConfigPath string
-var startRemotePeers, startBackends []string
-var inCluster bool
-
-func init() {
-	// Get the configuration file
-	kubeVipStart.Flags().StringVarP(&configPath, "config", "c", "", "Path to a kube-vip configuration")
-	kubeVipStart.Flags().BoolVarP(&disableVIP, "disableVIP", "d", false, "Disable the VIP functionality")
-
-	// Pointers so we can see if they're nil (and not called)
-	kubeVipStart.Flags().StringVar(&startConfig.Interface, "interface", "eth0", "Name of the interface to bind to")
-	kubeVipStart.Flags().StringVar(&startConfig.VIP, "vip", "192.168.0.1", "The Virtual IP address")
-	kubeVipStart.Flags().StringVar(&startConfig.Address, "address", "", "an address (IP or DNS name) to use as a VIP")
-	kubeVipStart.Flags().IntVar(&startConfig.Port, "port", 6443, "listen port for the VIP")
-	kubeVipStart.Flags().BoolVar(&startConfig.DDNS, "ddns", false, "use Dynamic DNS + DHCP to allocate VIP for address")
-	kubeVipStart.Flags().BoolVar(&startConfig.SingleNode, "singleNode", false, "Start this instance as a single node")
-	kubeVipStart.Flags().BoolVar(&startConfig.StartAsLeader, "startAsLeader", false, "Start this instance as the cluster leader")
-	kubeVipStart.Flags().BoolVar(&startConfig.EnableARP, "arp", false, "Use ARP broadcasts to improve VIP re-allocations")
-	kubeVipStart.Flags().StringVar(&startLocalPeer, "localPeer", "server1:192.168.0.1:10000", "Settings for this peer, format: id:address:port")
-	kubeVipStart.Flags().StringSliceVar(&startRemotePeers, "remotePeers", []string{"server2:192.168.0.2:10000", "server3:192.168.0.3:10000"}, "Comma seperated remotePeers, format: id:address:port")
-	// Load Balancer flags
-	kubeVipStart.Flags().BoolVar(&startConfigLB.BindToVip, "lbBindToVip", false, "Bind example load balancer to VIP")
-	kubeVipStart.Flags().StringVar(&startConfigLB.Type, "lbType", "tcp", "Type of load balancer instance (TCP/HTTP)")
-	kubeVipStart.Flags().StringVar(&startConfigLB.Name, "lbName", "Example Load Balancer", "The name of a load balancer instance")
-	kubeVipStart.Flags().IntVar(&startConfigLB.Port, "lbPort", 8080, "Port that load balancer will expose on")
-	kubeVipStart.Flags().IntVar(&startConfigLB.BackendPort, "lbBackEndPort", 6443, "A port that all backends may be using (optional)")
-	kubeVipStart.Flags().StringSliceVar(&startBackends, "lbBackends", []string{"192.168.0.1:8080", "192.168.0.2:8080"}, "Comma seperated backends, format: address:port")
-
-	// Cluster configuration
-	kubeVipStart.Flags().StringVar(&startKubeConfigPath, "kubeConfig", "/etc/kubernetes/admin.conf", "The path of a kubernetes configuration file")
-	kubeVipStart.Flags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
-	kubeVipStart.Flags().BoolVar(&startConfig.EnableLeaderElection, "leaderElection", false, "Use the Kubernetes leader election mechanism for clustering")
-
-	// This sets the namespace that the lock should exist in
-	kubeVipStart.Flags().StringVarP(&startConfig.Namespace, "namespace", "n", "kube-system", "The configuration map defined within the cluster")
-}
-
-var kubeVipStart = &cobra.Command{
-	Use:   "start",
-	Short: "Start the Virtual IP / Load balancer",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
-		var err error
-
-		// If a configuration file is loaded, then it will overwrite flags
-
-		if configPath != "" {
-			c, err := kubevip.OpenConfig(configPath)
-			if err != nil {
-				log.Fatalf("%v", err)
-			}
-			startConfig = *c
-		}
-
-		// parse environment variables, these will overwrite anything loaded or flags
-		err = kubevip.ParseEnvironment(&startConfig)
-		if err != nil {
-			log.Fatalln(err)
-		}
-
-		newCluster, err := cluster.InitCluster(&startConfig, disableVIP)
-		if err != nil {
-			log.Fatalf("%v", err)
-		}
-		var bgpServer *bgp.Server
-		if startConfig.SingleNode {
-			// If the Virtual IP isn't disabled then create the netlink configuration
-			// Start a single node cluster
-			newCluster.StartSingleNode(&startConfig, disableVIP)
-		} else {
-			if disableVIP {
-				log.Fatalln("Cluster mode requires the Virtual IP to be enabled, use single node with no VIP")
-			}
-
-			if startConfig.EnableLeaderElection {
-				cm, err := cluster.NewManager(startKubeConfigPath, inCluster, startConfig.Port)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-
-				if startConfig.EnableBGP {
-					log.Info("Starting the BGP server to adverise VIP routes to VGP peers")
-					bgpServer, err = bgp.NewBGPServer(&startConfig.BGPConfig)
-					if err != nil {
-						log.Fatalf("%v", err)
-					}
-
-					// Defer a function to check if the bgpServer has been created and if so attempt to close it
-					defer func() {
-						if bgpServer != nil {
-							bgpServer.Close()
-						}
-					}()
-				}
-
-				// Leader Cluster will block
-				err = newCluster.StartLeaderCluster(&startConfig, cm, bgpServer)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-			} else {
-
-				// // Start a multi-node (raft) cluster, this doesn't block so will wait on signal
-				err = newCluster.StartRaftCluster(&startConfig)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-				signalChan := make(chan os.Signal, 1)
-				signal.Notify(signalChan, os.Interrupt)
-
-				<-signalChan
-
-				newCluster.Stop()
-			}
-
-		}
-
-	},
-}

cmd/kube-vip.go

@@ -3,42 +3,41 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"net"
 	"net/http"
 	"os"
+	"slices"
+	"strconv"
+	"strings"
+	"sync"
 	"time"
 
+	log "log/slog"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/spf13/cobra"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+
+	"github.com/kube-vip/kube-vip/pkg/debouncer"
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
 	"github.com/kube-vip/kube-vip/pkg/manager"
-	"github.com/kube-vip/kube-vip/pkg/packet"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
+	"github.com/kube-vip/kube-vip/pkg/metrics"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
 )
 
-// Path to the configuration file
-var configPath string
+// Is an option to set the image
+var image string
 
-// Path to the configuration file
-var namespace string
-
-// Disable the Virtual IP (bind to the existing network stack)
-var disableVIP bool
-
-// Disable the Virtual IP (bind to the existing network stack)
-var controlPlane bool
-
-// Run as a load balancer service (within a pod / kubernetes)
-var serviceArp bool
+// Is kube-vip running within cluster
+var inCluster bool
 
 // ConfigMap name within a Kubernetes cluster
 var configMap string
 
-// Configure the level of loggin
-var logLevel uint32
-
-// Provider Config
-var providerConfig string
+// Points to a kubernetes configuration file
+var kubeConfigPath string
 
 // Release - this struct contains the release information populated when building kube-vip
 var Release struct {
@@ -47,110 +46,164 @@ var Release struct {
 }
 
 // Structs used via the various subcommands
-var initConfig kubevip.Config
-var initLoadBalancer kubevip.LoadBalancer
-
-// Points to a kubernetes configuration file
-var kubeConfigPath string
+var (
+	initConfig       kubevip.Config
+	initLoadBalancer kubevip.LoadBalancer
+)
 
 var kubeVipCmd = &cobra.Command{
-	Use:   "kube-vip",
-	Short: "This is a server for providing a Virtual IP and load-balancer for the Kubernetes control-plane",
+	Use:           "kube-vip",
+	Short:         "This is a server for providing a Virtual IP and load-balancer for the Kubernetes control-plane",
+	SilenceErrors: true,
 }
 
 func init() {
-
-	localpeer, err := autoGenLocalPeer()
-	if err != nil {
-		log.Fatalln(err)
-	}
-	initConfig.LocalPeer = *localpeer
-	//initConfig.Peers = append(initConfig.Peers, *localpeer)
+	// Basic flags
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Interface, "interface", "", "Name of the interface to bind to")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesInterface, "serviceInterface", "", "Name of the interface to bind to (for services)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.AllowInterfaceNotUp, "allowInterfaceNotUp", false, "Allow kube-vip to start even if the interface is not up")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIP, "vip", "", "The Virtual IP address")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPSubnet, "vipSubnet", "", "The Virtual IP address subnet e.g. /32 /24 /8 etc.. (Default to 32 for IPv4 and 128 for IPv6)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.NodeName, "nodeName", "", "Name to be used for lease holder. Must be unique for each node/instance")
+
+	// VIP flags
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Address, "address", "", "an address (IP or DNS name) to use as a VIP")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.Port, "port", 6443, "listen port for the VIP")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPCIDR, "cidr", "32", "The CIDR range for the virtual IP address")
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for Vip changes")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Annotations, "annotations", "", "Set Node annotations prefix for parsing")
+	kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.Port, "port", 6443, "Port for the VIP")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for VIP changes")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableWireguard, "wireguard", false, "Enable Wireguard for services VIPs")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableRoutingTable, "table", false, "Enable Routing Table for services VIPs")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.PreserveVIPOnLeadershipLoss, "preserveVipOnLeadershipLoss", false, "Preserve ARP VIP addresses on interface when leadership is lost (default: false for backward compatibility)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoseLeadership, "loseLeadership", false, "Lose leadership when VIP interface goes down")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.LoseLeadershipTimeoutSeconds, "loseLeadershiptTimeoutSeconds", 30, "Timeout before re-electing a leader when the VIP interface is down")
+
+	// LoadBalancer flags
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "enableLoadBalancer", false, "enable loadbalancing on the VIP with IPVS")
+	kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.LoadBalancerPort, "lbPort", 6443, "loadbalancer port for the VIP")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerForwardingMethod, "lbForwardingMethod", "local", "loadbalancer forwarding method")
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DDNS, "ddns", false, "use Dynamic DNS + DHCP to allocate VIP for address")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MirrorDestInterface, "mirrorDestInterface", "", "network interface where all traffic that traverses the service interface will be mirrored to. Source interface will use default interface is servicesInterface is not set.")
 
 	// Clustering type (leaderElection)
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLeaderElection, "leaderElection", false, "Use the Kubernetes leader election mechanism for clustering")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 5, "Length of time a Kubernetes leader lease can be held for")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time a Kubernetes leader can attempt to renew its lease")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Number of times the host will retry to hold a lease")
-
-	// Clustering type (raft)
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.StartAsLeader, "startAsLeader", false, "Start this instance as the cluster leader")
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.AddPeersAsBackends, "addPeersToLB", true, "Add raft peers to the load-balancer")
-
-	// Packet flags
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableMetal, "metal", false, "This will use the Equinix Metal API (requires the token ENV) to update the EIP <-> VIP")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalAPIKey, "metalKey", "", "The API token for authenticating with the Equinix Metal API")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalProject, "metalProject", "", "The name of project already created within Equinix Metal")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalProjectID, "metalProjectID", "", "The ID of project already created within Equinix Metal")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ProviderConfig, "provider-config", "", "The path to a provider configuration")
-
-	// Load Balancer flags
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "lbEnable", false, "Enable a load-balancer on the VIP")
-	kubeVipCmd.PersistentFlags().BoolVar(&initLoadBalancer.BindToVip, "lbBindToVip", true, "Bind example load balancer to VIP")
-	kubeVipCmd.PersistentFlags().StringVar(&initLoadBalancer.Type, "lbType", "tcp", "Type of load balancer instance (TCP/HTTP)")
-	kubeVipCmd.PersistentFlags().StringVar(&initLoadBalancer.Name, "lbName", "Kubeadm Load Balancer", "The name of a load balancer instance")
-	kubeVipCmd.PersistentFlags().IntVar(&initLoadBalancer.Port, "lbPort", 6443, "Port that load balancer will expose on")
-	kubeVipCmd.PersistentFlags().IntVar(&initLoadBalancer.BackendPort, "lbBackEndPort", 6444, "A port that all backends may be using (optional)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaderElectionType, "leaderElectionType", "kubernetes", "Defines the backend to run the leader election: kubernetes or etcd. Defaults to kubernetes.")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaseName, "leaseName", "plndr-cp-lock", "Name of the lease that is used for leader election")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 15, "Length of time (in seconds) a Kubernetes leader lease can be held for")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 10, "Length of time (in seconds) a Kubernetes leader can attempt to renew its lease")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 2, "Length of time (in seconds) the LeaderElector clients should wait between tries of actions")
 
 	// BGP flags
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableBGP, "bgp", false, "This will enable BGP support within kube-vip")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPAttachIPToInterface, "bgpAttachIPToInterface", false, "Assign BGP service VIPs to the configured interface")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.RouterID, "bgpRouterID", "", "The routerID for the bgp server")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.SourceIF, "sourceIF", "", "The source interface for bgp peering (not to be used with sourceIP)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.SourceIP, "sourceIP", "", "The source address for bgp peering (not to be used with sourceIF)")
 	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.AS, "localAS", 65000, "The local AS number for the bgp server")
+	kubeVipCmd.PersistentFlags().Uint64Var(&initConfig.BGPConfig.HoldTime, "bgpHoldTimer", 30, "The hold timer for all bgp peers (it defines the time a session is held)")
+	kubeVipCmd.PersistentFlags().Uint64Var(&initConfig.BGPConfig.KeepaliveInterval, "bgpKeepAliveInterval", 10, "The keepalive interval for all bgp peers (it defines the heartbeat of keepalive messages)")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPPeerConfig.Address, "peerAddress", "", "The address of a BGP peer")
 	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPPeerConfig.AS, "peerAS", 65000, "The AS number for a BGP peer")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPPeerConfig.Password, "peerPass", "", "The md5 password for a BGP peer")
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPPeerConfig.MultiHop, "multihop", false, "This will enable BGP multihop support")
-	kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.BGPPeers, "bgppeers", []string{}, "Comma seperated BGP Peer, format: address:as:password:multihop")
+	kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.BGPPeers, "bgppeers", []string{}, "Comma separated BGP Peer, format: address:as:password:multihop")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Annotations, "annotations", "", "Set Node annotations prefix for parsing")
 
-	// Control plane specific flags
-	kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The configuration map defined within the cluster")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPConfig.Zebra.Enabled, "zebra", false, "This will enable Zebra support within kube-vip")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.URL, "zebraUrl", "unix:/var/run/frr/zserv.api", "Path to the unix domain socket for connecting to Zebra daemon")
+	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.Zebra.Version, "zebraVersion", 6, "Zebra API Version")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.SoftwareName, "zebraSoftwareName", "frr8.3", "Software Name for Zebra")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ControlPlaneHealthCheck.Address, "controlPlaneHealthCheckAddress", "", "URL to poll for the control-plane health check when using BGP without leader election")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.ControlPlaneHealthCheck.PeriodSeconds, "controlPlaneHealthCheckPeriodSeconds", 5, "Seconds between control-plane health checks")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.ControlPlaneHealthCheck.TimeoutSeconds, "controlPlaneHealthCheckTimeoutSeconds", 3, "Timeout for each control-plane health check request")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.ControlPlaneHealthCheck.FailureThreshold, "controlPlaneHealthCheckFailureThreshold", 3, "Consecutive control-plane health check failures before withdrawing the BGP route")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ControlPlaneHealthCheck.CAPath, "controlPlaneHealthCheckCAPath", "", "Path to CA certificate for TLS verification when the control-plane health check URL is HTTPS")
+
+	// Namespace for kube-vip
+	kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The namespace for the configmap defined within the cluster")
 
 	// Manage logging
-	kubeVipCmd.PersistentFlags().Uint32Var(&logLevel, "log", 4, "Set the level of logging")
+	kubeVipCmd.PersistentFlags().Int32Var(&initConfig.Logging, "log", 0, "Set the level of logging")
 
 	// Service flags
 	kubeVipService.Flags().StringVarP(&configMap, "configMap", "c", "plndr", "The configuration map defined within the cluster")
 
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableControlPane, "controlplane", false, "Enable HA for control plane, hybrid mode")
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServices, "services", false, "Enable Kubernetes services, hybrid mode")
+	// Routing Table flags
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableID, "tableID", 198, "The routing table used for all table entries")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableType, "tableType", unix.RTN_UNICAST, "The type of route that will be added to the routing table")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingProtocol, "routingProtocol", 248, "The routing protocol value used to create routes")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.CleanRoutingTable, "cleanRoutingTable", false, "Clean routing table of redundant routes on start")
+
+	// Behaviour flags
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableControlPlane, "controlplane", false, "Enable HA for control plane")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DetectControlPlane, "autodetectcp", false, "Determine working address for control plane (from loopback)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServices, "services", false, "Enable Kubernetes services")
+
+	// Extended behaviour flags
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServicesElection, "servicesElection", false, "Enable leader election per kubernetes service")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassOnly, "lbClassOnly", false, fmt.Sprintf("Enable load balancing only for services with LoadBalancerClass %q", kubevip.LBClassName))
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerClassName, "lbClassName", kubevip.LBClassName, fmt.Sprintf("Name of load balancer class for kube-VIP, defaults to %q", kubevip.LBClassName))
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassLegacyHandling, "lbClassNameLegacyHandling", true, "Use legacy LoadBalancer class name handling (e.g. accepting services both with empty and non-empty class)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServiceSecurity, "onlyAllowTrafficServicePorts", false, "Only allow traffic to service ports, others will be dropped, defaults to false")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableNodeLabeling, "enableNodeLabeling", false, fmt.Sprintf("Enable leader node labeling with %q, defaults to false", kubevip.HasIP))
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesLeaseName, "servicesLeaseName", "plndr-svcs-lock", "Name of the lease that is used for leader election for services (in arp mode)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.DNSMode, "dnsMode", "first", "Name of the mode that DNS lookup will be performed (first, ipv4, ipv6, dual)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.DHCPMode, "dhcpMode", "", "Mode DHCP resolving will use to obtain IP addresses (ipv4, ipv6, dual)")
+	kubeVipCmd.PersistentFlags().UintVar(&initConfig.DHCPBackoffAttempts, "dhcpBackoffAttempts", kubevip.DefaultDHCPBackoffAttempts,
+		fmt.Sprintf("number of times DHCP client will try to obtain an IP address (defaults to: %d, 0 for unlimited retries)", kubevip.DefaultDHCPBackoffAttempts))
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DisableServiceUpdates, "disableServiceUpdates", false, "If true, kube-vip will process services as usual, but will not update service's Status.LoadBalancer.Ingress slice")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpoints, "enableEndpoints", false, "If enabled, kube-vip will only advertise services, but will use the (deprecated since v1.33) endpoints for IP addresses")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoInterfaceGlobalScope, "loInterfaceGlobalScope", false, "If true, kube-vip will set global scope when using the lo interface, otherwise a host scope will be used by default")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.HealthCheckPort, "healthCheckPort", 0, "If set to non-zero (> 1024), then this is the port that the healthcheck will listen on")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.DebounceTime, "debounceTime", debouncer.DefaultTime,
+		"Configures the time that the event debouncer will wait for the events arrival (default 0s - debouncer disabled, enable with min. 200ms)")
 
 	// Prometheus HTTP Server
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.PrometheusHTTPServer, "promethuesHTTPServer", ":2112", "Host and port used to expose Promethues metrics via an HTTP server")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.PrometheusHTTPServer, "prometheusHTTPServer", ":2112", "Host and port used to expose Prometheus metrics via an HTTP server")
+
+	// Etcd
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.CAFile, "etcdCACert", "", "Verify certificates of TLS-enabled secure servers using this CA bundle file")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.ClientCertFile, "etcdCert", "", "Identify secure client using this TLS certificate file")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.ClientKeyFile, "etcdKey", "", "Identify secure client using this TLS key file")
+	kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.Etcd.Endpoints, "etcdEndpoints", nil, "Etcd member endpoints")
+
+	// Kubernetes client specific flags
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.K8sConfigFile, "k8sConfigPath", "/etc/kubernetes/admin.conf", "Path to the configuration file used with the Kubernetes client")
+
+	// Configuration file flag
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ConfigFile, "config-file", "", "Path to a JSON/YAML configuration file to load settings from")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.InstanceName, "instanceName", "", "Unique name for this kube-vip instance (currently used to isolate nftables egress tables)")
+
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EgressWithNftables, "egressWithNftables", true, "Use nftables-based egress implementation")
+
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.PerServiceElectionOnDemand, "perServiceElectionOnDemand", false, "Allow kube-vip to use per-service election for annotated services")
 
 	kubeVipCmd.AddCommand(kubeKubeadm)
 	kubeVipCmd.AddCommand(kubeManifest)
 	kubeVipCmd.AddCommand(kubeVipManager)
 	kubeVipCmd.AddCommand(kubeVipSample)
 	kubeVipCmd.AddCommand(kubeVipService)
-	kubeVipCmd.AddCommand(kubeVipStart)
 	kubeVipCmd.AddCommand(kubeVipVersion)
 
-	// Sample commands
-	kubeVipSample.AddCommand(kubeVipSampleConfig)
-	kubeVipSample.AddCommand(kubeVipSampleManifest)
-
+	// Set the logging level for all subsequent functions
+	log.SetLogLoggerLevel(log.Level(initConfig.Logging))
 }
 
 // Execute - starts the command parsing process
-func Execute() {
-	if err := kubeVipCmd.Execute(); err != nil {
-		fmt.Println(err)
-		os.Exit(1)
+func Execute() int {
+	cmd, err := kubeVipCmd.ExecuteC()
+	if err != nil {
+		log.Error("command failed", "err", err)
+		if cmd == kubeVipCmd {
+			_ = cmd.Usage()
+		}
+		return 1
 	}
+	return 0
 }
 
 var kubeVipVersion = &cobra.Command{
 	Use:   "version",
 	Short: "Version and Release information about the Kubernetes Virtual IP Server",
-	Run: func(cmd *cobra.Command, args []string) {
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 		fmt.Printf("Kube-VIP Release Information\n")
 		fmt.Printf("Version:  %s\n", Release.Version)
 		fmt.Printf("Build:    %s\n", Release.Build)
@@ -160,22 +213,44 @@ var kubeVipVersion = &cobra.Command{
 var kubeVipSample = &cobra.Command{
 	Use:   "sample",
 	Short: "Generate a Sample configuration",
-	Run: func(cmd *cobra.Command, args []string) {
-		cmd.Help()
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		_ = cmd.Help()
 	},
 }
 
 var kubeVipService = &cobra.Command{
 	Use:   "service",
 	Short: "Start the Virtual IP / Load balancer as a service within a Kubernetes cluster",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	RunE: func(cmd *cobra.Command, args []string) error { //nolint TODO
+		cmd.SilenceUsage = true
 
-		// parse environment variables, these will overwrite anything loaded or flags
+		// Load configuration from file if specified (lowest priority)
+		if initConfig.ConfigFile != "" {
+			err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
+			if err != nil {
+				return fmt.Errorf("loading config file: %w", err)
+			}
+		}
+
+		// parse environment variables, these will overwrite anything loaded from config file
 		err := kubevip.ParseEnvironment(&initConfig)
 		if err != nil {
-			log.Fatalln(err)
+			return fmt.Errorf("parsing environment: %w", err)
+		}
+		if err := initConfig.Validate(); err != nil {
+			return fmt.Errorf("validating configuration: %w", err)
+		}
+
+		// Change RTN_UNSPEC to default type
+		if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
+			initConfig.RoutingProtocol = unix.RTN_UNICAST
+		}
+
+		// Set the logging level for all subsequent functions
+		log.SetLogLoggerLevel(log.Level(initConfig.Logging))
+
+		if err := initConfig.CheckInterface(); err != nil {
+			return fmt.Errorf("checking interface: %w", err)
 		}
 
 		// User Environment variables as an option to make manifest clearer
@@ -184,35 +259,205 @@ var kubeVipService = &cobra.Command{
 			configMap = envConfigMap
 		}
 
+		// Legacy vip_address requires vip_subnet for control-plane ARP, BGP, and Routing Table modes.
+		if initConfig.EnableControlPlane &&
+			(initConfig.EnableARP || initConfig.EnableBGP || initConfig.EnableRoutingTable) {
+			if err := initConfig.CheckSubnetExists(); err != nil {
+				return fmt.Errorf("checking subnet exists if vip_address defined: %w", err)
+			}
+		}
+
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				return fmt.Errorf("generating CIDR: %w", err)
+			}
+		}
+
+		ctx, cancel := context.WithCancel(cmd.Context())
+		defer cancel()
+
 		// Define the new service manager
-		mgr, err := manager.New(configMap, &initConfig)
+		mgr, err := manager.New(ctx, configMap, &initConfig)
 		if err != nil {
-			log.Fatalf("%v", err)
+			return fmt.Errorf("new manager: %w", err)
 		}
 
 		// Start the service manager, this will watch the config Map and construct kube-vip services for it
-		err = mgr.Start()
+		err = mgr.Start(ctx)
 		if err != nil {
-			log.Fatalf("%v", err)
+			return fmt.Errorf("manager start: %w", err)
 		}
+		return nil
 	},
 }
 
 var kubeVipManager = &cobra.Command{
 	Use:   "manager",
 	Short: "Start the kube-vip manager",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	RunE: func(cmd *cobra.Command, args []string) error { //nolint TODO
+		cmd.SilenceUsage = true
 
-		go servePrometheusHTTPServer(cmd.Context(), PrometheusHTTPServerConfig{
-			Addr: initConfig.PrometheusHTTPServer,
-		})
+		// Load configuration from file if specified (lowest priority)
+		if initConfig.ConfigFile != "" {
+			err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
+			if err != nil {
+				return fmt.Errorf("loading config file: %w", err)
+			}
+		}
 
-		// parse environment variables, these will overwrite anything loaded or flags
+		// parse environment variables, these will overwrite anything loaded from config file
 		err := kubevip.ParseEnvironment(&initConfig)
 		if err != nil {
-			log.Fatalln(err)
+			return fmt.Errorf("parsing environment: %w", err)
+		}
+		if err := initConfig.Validate(); err != nil {
+			return fmt.Errorf("validating configuration: %w", err)
+		}
+
+		// Change RTN_UNSPEC to default type
+		if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
+			initConfig.RoutingProtocol = unix.RTN_UNICAST
+		}
+
+		// Set the logging level for all subsequent functions
+		log.SetLogLoggerLevel(log.Level(initConfig.Logging))
+
+		// Legacy vip_address requires vip_subnet for control-plane ARP, BGP, and Routing Table modes.
+		if initConfig.EnableControlPlane &&
+			(initConfig.EnableARP || initConfig.EnableBGP || initConfig.EnableRoutingTable) {
+			if err := initConfig.CheckSubnetExists(); err != nil {
+				return fmt.Errorf("checking subnet exists if vip_address defined: %w", err)
+			}
+		}
+
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				return fmt.Errorf("generating CIDR: %w", err)
+			}
+		}
+
+		// Welome messages
+		log.Info("kube-vip.io", "version", Release.Version, "build", Release.Build)
+
+		wg := sync.WaitGroup{}
+		defer wg.Wait()
+
+		// create main manager context
+		ctx, cancel := context.WithCancelCause(cmd.Context())
+		defer cancel(nil)
+
+		// start prometheus server
+		if initConfig.PrometheusHTTPServer != "" {
+			wg.Go(func() {
+				servePrometheusHTTPServer(ctx, PrometheusHTTPServerConfig{
+					Addr: initConfig.PrometheusHTTPServer,
+				})
+			})
+		}
+
+		// Determine the kube-vip mode
+		var (
+			mode         string
+			modesEnabled int
+		)
+		if initConfig.EnableARP {
+			mode = "ARP"
+			modesEnabled++
+		}
+
+		if initConfig.EnableBGP {
+			mode = "BGP"
+			modesEnabled++
+		}
+
+		if initConfig.EnableWireguard {
+			mode = "Wireguard"
+			modesEnabled++
+		}
+
+		if initConfig.EnableRoutingTable {
+			mode = "Routing Table"
+			modesEnabled++
+		}
+
+		if mode == "" {
+			return fmt.Errorf("no valid kube-vip mode detected, ensure a supported mode is configured")
+		}
+
+		if modesEnabled > 1 {
+			return fmt.Errorf("multiple kube-vip modes detected, ensure only one mode is configured")
+		}
+
+		// Provide configuration to output/logging
+		log.Info("starting", "namespace", initConfig.Namespace, "Mode", mode, "Control Plane", initConfig.EnableControlPlane, "Services", initConfig.EnableServices)
+
+		// End if nothing is enabled
+		if !initConfig.EnableServices && !initConfig.EnableControlPlane {
+			return fmt.Errorf("no features are enabled")
+		}
+
+		if !initConfig.EnableARP && strings.Contains(initConfig.VIPSubnet, kubevip.Auto) {
+			return fmt.Errorf("auto subnet discovery cannot be used outside ARP mode")
+		}
+
+		if strings.Contains(initConfig.VIPSubnet, kubevip.Auto) && initConfig.Address != "" {
+			return fmt.Errorf("auto subnet discovery cannot be used if VIP address was provided")
+		}
+
+		// If we're using wireguard then all traffic goes through the wg0 interface
+		if initConfig.EnableWireguard {
+			if initConfig.Interface == "" {
+				// Set the vip interface to the wireguard interface
+				initConfig.Interface = "wg0"
+			}
+
+			log.Info("configuring Wireguard networking")
+			l, err := netlink.LinkByName(initConfig.Interface)
+			if err != nil {
+				if strings.Contains(err.Error(), "Link not found") {
+					log.Warn("attempting to create wireguard interface", "interface not found", initConfig.Interface)
+					err = netlink.LinkAdd(&netlink.Wireguard{LinkAttrs: netlink.LinkAttrs{Name: initConfig.Interface}})
+					if err != nil {
+						return fmt.Errorf("adding link: %w", err)
+					}
+					l, err = netlink.LinkByName(initConfig.Interface)
+					if err != nil {
+						return fmt.Errorf("finding link: %w", err)
+					}
+				}
+			}
+			err = netlink.LinkSetUp(l)
+			if err != nil {
+				return fmt.Errorf("setting link UP: %w", err)
+			}
+
+		} else { // if we're not using Wireguard then we'll need to use an actual interface
+			// Check if the interface needs auto-detecting
+			if initConfig.Interface == "" {
+				log.Info("No interface is specified for VIP in config, auto-detecting default Interface")
+				defaultIF, err := vip.GetDefaultGatewayInterface()
+				if err != nil {
+					_ = cmd.Help()
+					return fmt.Errorf("detecting interface: %w", err)
+				}
+				initConfig.Interface = defaultIF.Name
+				log.Info("kube-vip bind", "interface", initConfig.Interface)
+
+				wg.Go(func() {
+					if err := vip.MonitorDefaultInterface(ctx, defaultIF); err != nil {
+						log.Error("interface monitor", "err", err)
+						cancel(err)
+					}
+				})
+			}
+		}
+		// Perform a check on the state of the interface
+		if err := initConfig.CheckInterface(); err != nil {
+			return fmt.Errorf("checking interface: %w", err)
 		}
 
 		// User Environment variables as an option to make manifest clearer
@@ -221,34 +466,25 @@ var kubeVipManager = &cobra.Command{
 			configMap = envConfigMap
 		}
 
-		// If Packet is enabled and there is a provider configuration passed
-		if initConfig.EnableMetal {
-			if providerConfig != "" {
-				providerAPI, providerProject, err := packet.GetPacketConfig(providerConfig)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-				initConfig.MetalAPIKey = providerAPI
-				initConfig.MetalProject = providerProject
-			}
-		}
-
 		// Define the new service manager
-		mgr, err := manager.New(configMap, &initConfig)
+		mgr, err := manager.New(ctx, configMap, &initConfig)
 		if err != nil {
-			log.Fatalf("%v", err)
+			return fmt.Errorf("new manager: %w", err)
 		}
 
-		prometheus.MustRegister(mgr.PrometheusCollector()...)
+		metrics.RegisterPrometheusMetrics()
+		metrics.BuildInfo.WithLabelValues(Release.Version, Release.Build, initConfig.NodeName)
 
 		// Start the service manager, this will watch the config Map and construct kube-vip services for it
-		err = mgr.Start()
+		err = mgr.Start(ctx)
 		if err != nil {
-			log.Fatalf("%v", err)
+			return fmt.Errorf("start manager: %w", err)
 		}
+		return nil
 	},
 }
 
+// PrometheusHTTPServerConfig defines the Prometheus server configuration.
 type PrometheusHTTPServerConfig struct {
 	// Addr sets the http server address used to expose the metric endpoint
 	Addr string
@@ -258,35 +494,86 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
 	var err error
 	mux := http.NewServeMux()
 	mux.Handle("/metrics", promhttp.Handler())
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { //nolint TODO
+		_, _ = w.Write([]byte(`<html>
+			<head><title>kube-vip</title></head>
+			<body>
+			<h1>kube-vip Metrics</h1>
+			<p><a href="` + "/metrics" + `">Metrics</a></p>
+			</body>
+			</html>`))
+	})
 
 	srv := &http.Server{
-		Addr:    config.Addr,
-		Handler: mux,
+		Addr:              config.Addr,
+		Handler:           mux,
+		ReadHeaderTimeout: 2 * time.Second,
 	}
 
-	go func() {
-		if err = srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
-			log.Fatalf("listen:%+s\n", err)
-		}
-	}()
+	wg := sync.WaitGroup{}
 
-	log.Printf("server started")
+	wg.Go(func() {
+		if err = srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			log.Error("prometheus HTTP server", "err", err)
+			return
+		}
+	})
+
+	log.Info("prometheus HTTP server started")
 
 	<-ctx.Done()
 
-	log.Printf("server stopped")
-
+	// create prometheus shutdown context (independent of other contexts)
 	ctxShutDown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer func() {
 		cancel()
 	}()
 
 	if err = srv.Shutdown(ctxShutDown); err != nil {
-		log.Fatalf("server Shutdown Failed:%+s", err)
+		log.Error("shutting down prometheus HTTP server", "err", err)
+		return
 	}
 
 	if err == http.ErrServerClosed {
 		err = nil
 	}
 
+	log.Info("prometheus HTTP server stopped")
+
+	wg.Wait()
+}
+
+func GenerateCidrRange(address string, dnsMode string) (string, error) {
+	var cidrs []string
+
+	addresses := strings.Split(address, ",")
+	for _, a := range addresses {
+		ip := net.ParseIP(a)
+		if ip == nil {
+			// we probably are a DNS name
+			ips, err := utils.LookupHost(a, dnsMode, true)
+			if len(ips) == 0 || err != nil {
+				return "", fmt.Errorf("invalid IP address: %s from [%s], %v", a, address, err)
+			}
+			for _, addr := range ips {
+				ip = net.ParseIP(addr)
+				if ip.To4() != nil {
+					cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv4))
+				} else {
+					cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv6))
+				}
+			}
+		} else {
+			if ip.To4() != nil {
+				cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv4))
+			} else {
+				cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv6))
+			}
+		}
+	}
+	// compact as DNS could have a lot of addresses
+	slices.Sort(cidrs)
+	cidrs = slices.Compact(cidrs)
+	slices.Reverse(cidrs)
+	return strings.Join(cidrs, ","), nil
 }

demo/README.md

@@ -0,0 +1,46 @@
+# Demo client-server
+
+This contains some example code to determine how long "failovers" are taking within kube-vip, the server component should live within the cluster and the client should be externally.
+
+## Deploy the server
+
+Simply apply the manifest to a working cluster that has kube-vip deployed:
+
+```
+kubectl apply -f ./demo/server/deploy.yaml
+```
+
+Retrieve the loadBalancer IP that is fronting the service:
+
+```
+kubectl get svc demo-service
+NAME           TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)           AGE
+demo-service   LoadBalancer   10.104.18.147   192.168.0.217   10002:32529/UDP   117m
+```
+
+## Connect the client
+
+From elsewhere, clone the kube-vip repository and connect the client to the server endpoint (loadBalancer IP) with the following command:
+
+```
+go run ./demo/client/main.go -address=<vip>
+```
+
+You will only see output when the client has reconcilled the connection to a pod beneath the service, where it will print the timestamp to reconnection along with the time in milliseconds it took:
+
+```
+15:58:35.916952 3008
+15:58:45.947506 2005
+15:58:57.983151 3007
+15:59:08.013450 2005
+15:59:20.046491 3008
+15:59:30.076341 2507
+15:59:42.110747 3008
+```
+
+## Kill some pods to test
+
+On a machine or control plane that has `kubectl` and has the credentials to speak to the cluster we will run a command to find the demo pod and kill it every 10 seconds:
+
+`while true ; do kubectl delete pod $(kubectl get pods | grep -v NAME | grep vip| awk '{ print $1 }'); sleep 10; done`
+

demo/client/main.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"net"
+	"time"
+)
+
+const udpdata = "a3ViZS12aXAK=kube-vip"
+
+func main() {
+	address := flag.String("address", "127.0.0.1", "The address of the server")
+	port := flag.Int("port", 10002, "the port of the server")
+	interval := flag.Float64("interval", 1000, "Interval in milliseconds")
+	flag.Parse()
+	var errorTime time.Time
+	var errorOccurred bool
+	for {
+		p := make([]byte, 2048)
+		conn, err := net.Dial("udp", net.JoinHostPort(*address, fmt.Sprint(port)))
+		if err != nil {
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			continue
+		}
+
+		err = conn.SetDeadline(time.Now().Add(time.Duration(*interval) * time.Millisecond))
+		if err != nil {
+			//fmt.Printf("Connectivity error [%v]", err)
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			if err = conn.Close(); err != nil {
+				fmt.Printf("Error closing connection [%v]", err)
+			}
+			continue
+		}
+
+		_, err = fmt.Fprint(conn, udpdata)
+		if err != nil {
+			fmt.Printf("Error writing data [%v]", err)
+		}
+
+		_, err = bufio.NewReader(conn).Read(p)
+		if err != nil {
+			//fmt.Printf("read error %v\n", err)
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			if err = conn.Close(); err != nil {
+				fmt.Printf("Error closing connection [%v]", err)
+			}
+			continue
+		}
+		time.Sleep(time.Duration(*interval) * time.Millisecond)
+		if errorOccurred {
+			finishTime := time.Since(errorTime)
+			//fmt.Printf("connectivity reconciled in %dms\n", finishTime.Milliseconds())
+			//t :=time.Now().Format("15:04:05.000000")
+			fmt.Printf("%s %d\n", time.Now().Format("15:04:05.000000"), finishTime.Milliseconds())
+
+			errorOccurred = false
+		}
+	}
+}

demo/server/Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:experimental
 
-FROM golang:1.13-alpine as dev
+FROM golang:1.19-alpine as dev
 RUN apk add --no-cache git ca-certificates
 RUN adduser -D appuser
 COPY main.go /src/
@@ -13,4 +13,4 @@ RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
 
 FROM scratch
 COPY --from=dev /src/demo /
-CMD ["/demo"]
+CMD ["/demo"]

demo/server/Makefile

@@ -45,7 +45,7 @@ fmt:
 	@gofmt -l -w $(SRC)
 
 docker:
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 simplify:

demo/server/deploy.yaml

@@ -6,7 +6,7 @@ metadata:
     app: kube-vip-demo
   name: kube-vip-demo
 spec:
-  replicas: 3
+  replicas: 1
   selector:
     matchLabels:
       app: kube-vip-demo
@@ -30,4 +30,25 @@ spec:
         ports:
             - containerPort: 10001
             - containerPort: 10002
-status: {}
+status: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: demo-service
+  namespace: default
+  labels:
+    app: demo-service
+  annotations:
+    kube-vip.io/egress: "true"
+spec:
+  type: LoadBalancer
+  # "Local" preserves the client source IP and avoids a second hop for
+  # LoadBalancer and NodePort
+  externalTrafficPolicy: Local
+  ports:
+  - name: demo-udp
+    port: 10002
+    protocol: UDP
+  selector:
+    app: kube-vip-demo

demo/server/main.go

@@ -32,7 +32,7 @@ func main() {
 		}
 	}
 
-	if serverType == strings.ToLower("udp") {
+	if serverType == "udp" {
 		// Start the UDP echo server
 
 		ServerAddr, err := net.ResolveUDPAddr("udp", ":10002")

docs/Dockerfile

@@ -1,5 +0,0 @@
-FROM nginx:latest
-RUN apt-get update; apt-get install -y nodejs npm; npm install -g markdown-styles;
-COPY . /docs
-WORKDIR /docs
-RUN generate-md --layout github --input ./ --output /usr/share/nginx/html/

docs/architecture/index.md

@@ -1,135 +0,0 @@
-# **kube-vip** architecture
-
-This section covers two parts of the architecture: 
-
-1. The technical capabilities of `kube-vip`
-2. The components to build a load-balancing service within [Kubernetes](https://kubernetes.io)
-
-The `kube-vip` project is designed to provide both a highly available networking endpoint and load-balancing functionality for underlying networking services. The project was originally designed for the purpose of providing a resilient control-plane for Kubernetes, it has since expanded to provide the same functionality for applications within a Kubernetes cluster.
-
-Additionally `kube-vip` is designed to be lightweight and **multi-architecture**, all of the components are built for Linux but are also built for both `x86` and `armv7`,`armhvf`,`ppc64le`. This means that `kube-vip` will run fine in **bare-metal**, **virtual** and **edge** (raspberry pi or small arm SoC devices). 
-
-## Technologies
-
-There are a number of technologies or functional design choices that provide high-availability or networking functions as part of a VIP/Load-balancing solution.
-
-### Cluster
-
-The `kube-vip` service builds a multi-node or multi-pod cluster to provide High-Availability. In ARP mode a leader is elected, this node will inherit the Virtual IP and become the leader of the load-balancing within the cluster, whereas with BGP all nodes will advertise the VIP address.
-
-When using ARP or layer2 it will use [leader election](https://godoc.org/k8s.io/client-go/tools/leaderelection)
-
-It is also possible to use [raft](https://en.wikipedia.org/wiki/Raft_(computer_science) clustering technology, but this approach has largely been superseded by leader election especially when running in cluster.
-
-### Virtual IP
-
-The leader within the cluster will assume the **vip** and will have it bound to the selected interface that is declared within the configuration. When the leader changes it will evacuate the **vip** first or in failure scenarios the **vip** will be directly assumed by the next elected leader.
-
-When the **vip** moves from one host to another any host that has been using the **vip** will retain the previous `vip <-> MAC address` mapping until the ARP (Address resolution protocol) expires the old entry (typically 30 seconds) and retrieves a new `vip <-> MAC` mapping. This can be improved using Gratuitous ARP broadcasts (when enabled), this is detailed below.
-
-### ARP
-
-(Optional) The `kube-vip` can be configured to broadcast a [gratuitous arp](https://wiki.wireshark.org/Gratuitous_ARP) that will typically immediately notify all local hosts that the `vip <-> MAC` has changed.
-
-**Below** we can see that the failover is typically done within a few seconds as the ARP broadcast is recieved.
-
-```
-64 bytes from 192.168.0.75: icmp_seq=146 ttl=64 time=0.258 ms
-64 bytes from 192.168.0.75: icmp_seq=147 ttl=64 time=0.240 ms
-92 bytes from 192.168.0.70: Redirect Host(New addr: 192.168.0.75)
-Vr HL TOS  Len   ID Flg  off TTL Pro  cks      Src      Dst
- 4  5  00 0054 bc98   0 0000  3f  01 3d16 192.168.0.95  192.168.0.75 
-
-Request timeout for icmp_seq 148
-92 bytes from 192.168.0.70: Redirect Host(New addr: 192.168.0.75)
-Vr HL TOS  Len   ID Flg  off TTL Pro  cks      Src      Dst
- 4  5  00 0054 75ff   0 0000  3f  01 83af 192.168.0.95  192.168.0.75 
-
-Request timeout for icmp_seq 149
-92 bytes from 192.168.0.70: Redirect Host(New addr: 192.168.0.75)
-Vr HL TOS  Len   ID Flg  off TTL Pro  cks      Src      Dst
- 4  5  00 0054 2890   0 0000  3f  01 d11e 192.168.0.95  192.168.0.75 
-
-Request timeout for icmp_seq 150
-64 bytes from 192.168.0.75: icmp_seq=151 ttl=64 time=0.245 ms
-```
-
-### Load Balancing
-
-Within a Kubernetes cluster, the load-balancing is managed by the `plndr-cloud-provider` which watches all service that are created, and for those of `type=LoadBalancer` will create the configuration for `kube-vip` to consume.
-
-#### Load Balancing (Inside a cluster)
-
-When using `type=LoadBalancer` within a Kubernetes cluster `kube-vip` will assign the VIP to the leader (when using ARP) or to all running Pods (when using BGP). When traffic is directed to a node with the VIP then the rules configured by `kube-proxy` will redirect the traffic to one of the pods running in the service.
-
-#### Load Balancing (Outside a cluster)
-
-Within the configuration of `kube-vip` multiple load-balancers can be created, below is the example load-balancer for a Kubernetes Control-plane:
-
-```
-loadBalancers:
-- name: Kubernetes Control Plane
-  type: tcp
-  port: 6443
-  bindToVip: true
-  backends:
-  - port: 6444
-    address: 192.168.0.70
-  - port: 6444
-    address: 192.168.0.71
-  - port: 6444
-    address: 192.168.0.72
-```
-
-The above load balancer will create an instance that listens on port `6443` and will forward traffic to the array of backend addresses. If the load-balancer type is `tcp` then the backends will be IP addresses, however if the backend is set to `http` then the backends should be URLs:
-
-```
-  type: http
-  port: 6443
-  bindToVip: true
-  backends:
-  - port: 6444
-    address: https://192.168.0.70
-```
-
-Additionally the load-balancing within `kibe-vip` has two modes of operation:
-
-`bindToVip: false` - will result in every node in the cluster binding all load-balancer port(s) to all interfaces on the host itself
-
-`bindToVip: true` - The load-balancer will only **bind** to the VIP address.
-
-
-## Components within a Kubernetes Cluster
-
-The `kube-vip` kubernetes load-balancer requires a number of components in order to function:
-
-- The Plunder Cloud Provider -> [https://github.com/kube-vip/plndr-cloud-provider](https://github.com/kube-vip/plndr-cloud-provider)
-- The Kube-Vip Deployment -> [https://github.com/kube-vip/kube-vip](https://github.com/kube-vip/kube-vip)
-
-### Architecture overview
-
-![kubernetes-vip-architecture.png](kubernetes-vip-architecture.png)
-
-### Plunder Cloud Provider
-
-The cloud provider works like all Kubernetes cloud providers and is built using the Kubernetes cloud-provider SDK. It's role is to provide the same cloud "like" services one would expect from services such as AWS / Azure / GCP etc.. in that when a user requests functionality then the cloud provider can speak natively to the underlying vendor and provision the required service 
-
-e.g. _In AWS when requesting a Kubernetes LoadBalancer, the cloud provider will provision an **ELB**_ 
-
-The `Plunder cloud Provider` is *currently* only designed to intercept the creation of LoadBalancers and translate that into a `kube-vip` load balancer. 
-
-It is configured by a `configMap` within the `kube-system` namespace that contains the ranges of addresses that the other `kube-vip` load-balancers can use, it will also manage the allocation of addresses and then build the configMap configurations in these namespaces for consumption by `kube-vip`. The IP addresses for each namespace should be in the structure `cidr-<namespace>` followed by the cidr range for the address pool.
-
-**Example `ConfigMap`**
-
-```
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: plndr
-  namespace: kube-system
-data:
-  cidr-default: 192.168.0.200/29
-  cidr-plunder: 192.168.0.210/29
-  cidr-testing: 192.168.0.220/29
-``` 

docs/control-plane/0.1.4/index.md

@@ -1,147 +0,0 @@
-# Load Balancing a Kubernetes Cluster (Control-Plane) (pre 0.1.5)
-
-This document covers all of the details for using `kube-vip` to build a HA Kubernetes cluster
-
-`tl;dr version`
-- Generate/modify first node `kube-vip` config/manifest
-- `init` first node
-- `join` remaining nodes
-- Add remaining config/manifests
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.75 |
-| controlPlane01 | 10.0.0.70 |
-| controlPlane02 | 10.0.0.71 |
-| controlPlane03 | 10.0.0.72 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.17.0.
-
-### Generate the `kube-vip` configuration
-
-Make sure that the config directory exists: `sudo mkdir -p /etc/kube-vip/`, this directory can be any directory however the `hostPath` in the manifest will need modifying to point to the correct path.
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1 /kube-vip sample config | sudo tee /etc/kube-vip/config.yaml
-```
-
-### Modify the configuration
-
-**Cluster Configuration**
-Modify the `remotePeers` to point to the correct addresses of the other two nodes, ensure that their `id` is unique otherwise this will confuse the raft algorithm. The `localPeer` should be the configuration of the current node (`controlPlane01`), which is where this instance of the cluster will run. 
-
-As this node will be the first node, it will need to elect itself leader as until this occurs the VIP won’t be activated!
-
-`startAsLeader: true`
-
-**VIP Config**
-We will need to set our VIP address to `192.168.0.75` and to ensure all hosts are updated when the VIP moves we will enable ARP broadcasts `gratuitousARP: true`
-
-**Load Balancer**
-We will configure the load balancer to sit on the standard API-Server port `6443` and we will configure the backends to point to the API-servers that will be configured to run on port `6444`. Also for the Kubernetes Control Plane we will configure the load balancer to be of `type: tcp`.
-
-We can also use `6443` for both the VIP and the API-Servers, in order to do this we need to specify that the api-server is bound to it's local IP. To do this we use the `--apiserver-advertise-address` flag as part of the `init`, this means that we can then bind the same port to the VIP and we wont have a port conflict.
-
-**config.yaml**
-
-`user@controlPlane01:/etc/kube-vip$ cat config.yaml`
-
-...
-
-``` 
-remotePeers:
-- id: server2
-  address: 192.168.0.71
-  port: 10000
-- id: server3
-  address: 192.168.0.72
-  port: 10000
-localPeer:
-  id: server1
-  address: 192.168.0.70
-  port: 10000
-vip: 192.168.0.75
-gratuitousARP: true
-singleNode: false
-startAsLeader: true
-interface: ens192
-loadBalancers:
-- name: Kubernetes Control Plane
-  type: tcp
-  port: 6443
-  bindToVip: true
-  backends:
-  - port: 6444
-    address: 192.168.0.70
-  - port: 6444
-    address: 192.168.0.71
-  - port: 6444
-    address: 192.168.0.72
-```
-
-### First Node
-
-To generate the basic Kubernetes static pod `yaml` configuration:
-
-Make sure that the manifest directory exists: `sudo mkdir -p /etc/kubernetes/manifests/`
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1 /kube-vip sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Ensure that `image: plndr/kube-vip:<x>` is modified to point to a specific version (`0.1` at the time of writing), refer to [docker hub](https://hub.docker.com/r/plndr/kube-vip/tags) for details. Also ensure that the `hostPath` points to the correct `kube-vip` configuration, if it isn’t the above path. 
-
-The **vip** is set to `192.168.0.75` and this first node will elect itself as leader, and as part of the `kubeadm init` it will use the VIP in order to speak back to the initialising api-server.
-
-`sudo kubeadm init --control-plane-endpoint “192.168.0.75:6443” --apiserver-bind-port 6444 --upload-certs --kubernetes-version “v1.17.0”`
-
-Once this node is up and running we will be able to see the control-plane pods, including the `kube-vip` pod:
-
-```
-$ kubectl get pods -A
-NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
-<...>
-kube-system   kube-vip-controlplane01                  1/1     Running   0          10m
-```
-
-### Remaining Nodes
-
-We first will need to create the `kube-vip` configuration that resides in `/etc/kube-vip/config.yaml` or we can regenerate it from scratch using the above example. Ensure that the configuration is almost identical with the `localPeer` and `remotePeers` sections are updated for each node. Finally, ensure that the remaining nodes will behave as standard cluster nodes by setting `startAsLeader: false`.
-
-At this point **DON’T** generate the manifests, this is due to some bizarre `kubeadm/kubelet` behaviour.
-
-```
-  kubeadm join 192.168.0.75:6443 --token <tkn> \
-    --discovery-token-ca-cert-hash sha256:<hash> \
-    --control-plane --certificate-key <key> 
-
-```
-
-**After** this node has been added to the cluster, we can add the manifest to also add this node as a `kube-vip` member. (Adding the manifest afterwards doesn’t interfere with `kubeadm`). 
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1 /kube-vip sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Once this node is added we will be able to see that the `kube-vip` pod is up and running as expected:
-
-```
-user@controlPlane01:~$ kubectl get pods -A | grep vip
-kube-system   kube-vip-controlplane01                  1/1     Running             1          16m
-kube-system   kube-vip-controlplane02                  1/1     Running             0          18m
-kube-system   kube-vip-controlplane03                  1/1     Running             0          20m
-
-```
-
-If we look at the logs, we can see that the VIP is running on the second node and we’re waiting for our third node to join the cluster:
-
-```
-$ kubectl logs kube-vip-controlplane02  -n kube-system
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-
-```

docs/control-plane/0.1.5/index.md

@@ -1,195 +0,0 @@
-# Load Balancing a Kubernetes Cluster (Control-Plane)
-
-This document covers the newer (post `0.1.5`) method for using `kube-vip` to provide HA for a Kubernetes Cluster. The documentation for older releases can be found [here](./0.1.4/)
-
-This document covers all of the details for using `kube-vip` to build a HA Kubernetes cluster
-
-`tl;dr version`
-- Generate/modify first node `kube-vip` config/manifest
-- `init` first node
-- `join` remaining nodes
-- Add remaining config/manifests
-
-Below are examples of the steps required:
-
-```
-# First Node
-sudo docker run --network host --rm plndr/kube-vip:0.1.5 kubeadm init --interface ens192 --vip 192.168.0.81 --startAsLeader=true | sudo tee /etc/kubernetes/manifests/vip.yaml
-
-sudo kubeadm init --kubernetes-version 1.17.0 --control-plane-endpoint 192.168.0.81 --upload-certs
-
-# Additional Node(s)
-
-sudo kubeadm join 192.168.0.81:6443 --token w5atsr.blahblahblah --control-plane --certificate-key abc123
-
-sudo docker run -v /etc/kubernetes/admin.conf:/etc/kubernetes/admin.conf --network host --rm plndr/kube-vip:0.1.5 kubeadm join --interface ens192 --vip 192.168.0.81 --startAsLeader=false | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.75 |
-| controlPlane01 | 10.0.0.70 |
-| controlPlane02 | 10.0.0.71 |
-| controlPlane03 | 10.0.0.72 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.17.0.
-
-### Generate the `kube-vip` configuration
-
-Kube-Vip no longer requires storing it's configuration in a seperate directory and will now store its configuration in the actual manifest that defines the static pods. 
-
-```
-sudo docker run --network host \
-	--rm plndr/kube-vip:0.1.5 \
-	kubeadm init \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--startAsLeader=true | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-The above command will "initialise" the manifest within the `/etc/kubernetes/manifests` directory, that will be started when we actually initialise our Kubernetes cluster with `kubeadm init`
-
-### Modify the configuration
-
-**Cluster Configuration**
-As this node will be the first node, it will need to elect itself leader as until this occurs the VIP won’t be activated!
-
-`--startAsLeader=true`
-
-**VIP Config**
-We will need to set our VIP address to `192.168.0.75` with `--vip 192.168.0.75` and to ensure all hosts are updated when the VIP moves we will enable ARP broadcasts `--arp` (defaults to `true`)
-
-**Load Balancer**
-We will configure the load balancer to sit on the standard API-Server port `6443` and we will configure the backends to point to the API-servers that will be configured to run on port `6444`. Also for the Kubernetes Control Plane we will configure the load balancer to be of `type: tcp`.
-
-We can also use `6443` for both the VIP and the API-Servers, in order to do this we need to specify that the api-server is bound to it's local IP. To do this we use the `--apiserver-advertise-address` flag as part of the `init`, this means that we can then bind the same port to the VIP and we wont have a port conflict.
-
-**vip.yaml** Static-pod Manifest
-
-`$ sudo cat /etc/kubernetes/manifests/vip.yaml`
-
-
-``` 
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - start
-    env:
-    - name: vip_arp
-      value: "true"
-    - name: vip_interface
-      value: ens192
-    - name: vip_address
-      value: 192.168.0.81
-    - name: vip_startleader
-      value: "true"
-    - name: vip_addpeerstolb
-      value: "true"
-    - name: vip_localpeer
-      value: controlPlane01:192.168.0.70:10000
-    - name: lb_backendport
-      value: "6443"
-    - name: lb_name
-      value: Kubeadm Load Balancer
-    - name: lb_type
-      value: tcp
-    - name: lb_bindtovip
-      value: "true"
-    image: plndr/kube-vip:0.1.5
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - SYS_TIME
-  hostNetwork: true
-status: {}
-```
-
-### First Node
-
-To generate the basic Kubernetes static pod `yaml` configuration:
-
-Make sure that the manifest directory exists: `sudo mkdir -p /etc/kubernetes/manifests/`
-
-```
-sudo docker run --network host \
-	--rm plndr/kube-vip:0.1.5 \
-	kubeadm init \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--startAsLeader=true | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-Ensure that `image: plndr/kube-vip:<x>` is modified to point to a specific version (`0.1.5` at the time of writing), refer to [docker hub](https://hub.docker.com/r/plndr/kube-vip/tags) for details. 
-
-The **vip** is set to `192.168.0.75` and this first node will elect itself as leader, and as part of the `kubeadm init` it will use the VIP in order to speak back to the initialising api-server.
-
-`sudo kubeadm init --control-plane-endpoint “192.168.0.75:6443” --apiserver-bind-port 6444 --upload-certs --kubernetes-version “v1.17.0”`
-
-Once this node is up and running we will be able to see the control-plane pods, including the `kube-vip` pod:
-
-```
-$ kubectl get pods -A
-NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
-<...>
-kube-system   kube-vip-controlplane01                  1/1     Running   0          10m
-```
-
-### Remaining Nodes
-
-
-At this point **DON’T** generate the manifests, this is due to some bizarre `kubeadm/kubelet` behaviour.
-
-```
-  kubeadm join 192.168.0.75:6443 --token <tkn> \
-    --discovery-token-ca-cert-hash sha256:<hash> \
-    --control-plane --certificate-key <key> 
-
-```
-
-**After** this node has been added to the cluster, we can add the manifest to also add this node as a `kube-vip` member. (Adding the manifest afterwards doesn’t interfere with `kubeadm`). 
-
-```
-sudo docker run \
-	-v /etc/kubernetes/admin.conf:/etc/kubernetes/admin.conf \
-	--network host \
-	--rm plndr/kube-vip:0.1.5 \
-	kubeadm join \
-	--interface ens192 \
-	--vip 192.168.0.81 \
-	--startAsLeader=false | sudo tee /etc/kubernetes/manifests/vip.yaml
-
-```
-
-Once this node is added we will be able to see that the `kube-vip` pod is up and running as expected:
-
-```
-user@controlPlane01:~$ kubectl get pods -A | grep vip
-kube-system   kube-vip-controlplane01                  1/1     Running             1          16m
-kube-system   kube-vip-controlplane02                  1/1     Running             0          18m
-kube-system   kube-vip-controlplane03                  1/1     Running             0          20m
-
-```
-
-If we look at the logs, we can see that the VIP is running on the second node and we’re waiting for our third node to join the cluster:
-
-```
-$ kubectl logs kube-vip-controlplane02  -n kube-system
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-
-```

docs/control-plane/index.md

@@ -1,421 +0,0 @@
-# Load Balancing a Kubernetes Cluster (Control-Plane)
-
-**Note**: The most common deployment currently for HA Kubernetes clusters w/`kube-vip` involved `kubeadm`, however recently we've worked to bring a method of bringing `kube-vip` to other types of Kubernetes cluster. Typically this deployment method makes use of a daemonset that is usually brought up during the cluster instantiation.. So for those wanting to deploy [k3s](https://k3s.io), we now have installation steps available [here](https://kube-vip.io/control-plane/#k3s),
-
-This document covers the newer (post `0.1.6`) method for using `kube-vip` to provide HA for a Kubernetes Cluster. The documentation for older releases can be found [here](./0.1.5/)
-
-From version `0.1.6` we've moved `kube-vip` from raft to leaderElection within the Kubernetes cluster. After a lot of testing it became clear that the leaderElection gave quicker reconciliation when removing nodes etc.. during upgrades and failures.
-
-For **more** configuration around LeaderElection click [here](https://kube-vip.io/control-plane/#leaderelection-configuration).
-
-This document covers all of the details for using `kube-vip` to build a HA Kubernetes cluster
-
-`tl;dr version`
-- Generate/modify first node `kube-vip` config/manifest
-- `init` first node
-- `join` remaining nodes
-- Add remaining config/manifests
-
-Below are examples of the steps required:
-
-```
-# First Node
-sudo docker run --network host --rm plndr/kube-vip:0.2.1 manifest pod \
---interface ens192 \
---vip 192.168.0.75 \
---arp \
---leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-
-sudo kubeadm init --kubernetes-version 1.17.0 --control-plane-endpoint 192.168.0.75 --upload-certs
-
-# Additional Node(s)
-
-sudo kubeadm join 192.168.0.75:6443 --token w5atsr.blahblahblah --control-plane --certificate-key abc123
-
-sudo docker run --network host --rm plndr/kube-vip:0.2.1 manifest pod \
---interface ens192 \
---vip 192.168.0.75 \
---arp \
---leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.75 |
-| controlPlane01 | 10.0.0.70 |
-| controlPlane02 | 10.0.0.71 |
-| controlPlane03 | 10.0.0.72 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.17.0.
-
-### Generate the `kube-vip` configuration
-
-`kube-vip` no longer requires storing its configuration in a separate directory and will now store its configuration in the actual manifest that defines the static pods. 
-
-```
-sudo docker run --network host \
-	--rm plndr/kube-vip:0.2.1 \
-	manifest pod \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--arp \
-	--leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-The above command will "initialise" the manifest within the `/etc/kubernetes/manifests` directory, that will be started when we actually initialise our Kubernetes cluster with `kubeadm init`
-
-### Modify the configuration
-
-**Cluster Configuration**
-To enable Kubernetes leader Election passing the `--leaderElection` flag will enable `kube-vip` to use the Kubernetes leaderElection functionality to work out which member is the leader.
-
-**VIP Config**
-We will need to set our VIP address to `192.168.0.75` with `--vip 192.168.0.75` and to ensure all hosts are updated when the VIP moves we will enable ARP broadcasts `--arp` (defaults to `true`)
-
-**vip.yaml** Static-pod Manifest
-
-`$ sudo cat /etc/kubernetes/manifests/vip.yaml`
-
-
-``` 
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - start
-    env:
-    - name: vip_arp
-      value: "true"
-    - name: vip_interface
-      value: ens160
-    - name: vip_leaderelection
-      value: "true"
-    - name: vip_leaseduration
-      value: "5"
-    - name: vip_renewdeadline
-      value: "3"
-    - name: vip_retryperiod
-      value: "1"
-    - name: vip_address
-      value: 192.168.0.75
-    image: plndr/kube-vip:0.2.1
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - SYS_TIME
-  hostNetwork: true
-status: {}
-```
-
-### First Node
-
-To generate the basic Kubernetes static pod `yaml` configuration:
-
-Make sure that the manifest directory exists: `sudo mkdir -p /etc/kubernetes/manifests/`
-
-```
-sudo docker run --network host \
-	--rm plndr/kube-vip:0.2.1 \
-	manifest pod \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--arp \
-	--leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-Ensure that `image: plndr/kube-vip:<x>` is modified to point to a specific version (`0.1.8` at the time of writing), refer to [docker hub](https://hub.docker.com/r/plndr/kube-vip/tags) for details. 
-
-The **vip** is set to `192.168.0.75` and this first node will elect itself as leader, and as part of the `kubeadm init` it will use the VIP in order to speak back to the initialising api-server.
-
-`sudo kubeadm init --control-plane-endpoint “192.168.0.75:6443” --upload-certs --kubernetes-version “v1.17.0”`
-
-Once this node is up and running we will be able to see the control-plane pods, including the `kube-vip` pod:
-
-```
-$ kubectl get pods -A
-NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
-<...>
-kube-system   kube-vip-controlplane01                  1/1     Running   0          10m
-```
-
-### Remaining Nodes
-
-
-At this point **DON’T** generate the manifests, this is due to some bizarre `kubeadm/kubelet` behaviour.
-
-```
-  kubeadm join 192.168.0.75:6443 --token <tkn> \
-    --discovery-token-ca-cert-hash sha256:<hash> \
-    --control-plane --certificate-key <key> 
-
-```
-
-**After** this node has been added to the cluster, we can add the manifest to also add this node as a `kube-vip` member. (Adding the manifest afterwards doesn’t interfere with `kubeadm`). 
-
-```
-sudo docker run --network host \
-	--rm plndr/kube-vip:0.2.1 \
-	manifest pod \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--arp \
-	--leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-
-```
-
-Once this node is added we will be able to see that the `kube-vip` pod is up and running as expected:
-
-```
-user@controlPlane01:~$ kubectl get pods -A | grep vip
-kube-system   kube-vip-controlplane01                  1/1     Running             1          16m
-kube-system   kube-vip-controlplane02                  1/1     Running             0          18m
-kube-system   kube-vip-controlplane03                  1/1     Running             0          20m
-
-```
-
-## DNS Support
-
-### Static DNS Support (added in 0.2.0)
-
-A new flag `--address` is introduced to support using a DNS record as the control plane endpoint. `kube-vip` will do a dns lookup to retrieve the IP for the DNS record, and use that IP as the VIP. An `dnsUpdater` periodically checks and updates the system if IP changes for the DNS record.
-
-### Dynamic DNS Support (added in 0.2.1)
-
-`kube-vip` was also updated to support DHCP + [Dynamic DNS](https://en.wikipedia.org/wiki/Dynamic_DNS), for the use case where it's not able to reserve a static IP for the control plane endpoint.
-
-A new flag `--ddns` is introduced. Once enabled, `kube-vip` expects the input `--address` will be a FQDN without binding to an IP. Then `kube-vip` will start a dhcp client to allocate an IP for the hostname of FQDN, and maintain the lease for it.
-
-Once DHCP returns an IP for the FQDN, the same `dnsUpdater` runs to periodically checks and updates if IP got changed.
-
-## BGP Support (added in 0.1.8)
-
-In version `0.1.8`+ `kube-vip` was updated to support [BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol) as a VIP failover mechanism. When a node is elected as a leader then it will update it's peers so that they are aware to route traffic to that node in order to access the VIP. 
-
-The following new flags are used:
-
-- `--bgp` This will enable BGP support within kube-vip
-- `--localAS` The local AS number
-- `--bgpRouterID` The local router address
-- `--peerAS` The AS number for a BGP peer
-- `--peerAddress` The address of a BGP peer
-
-### BGP Packet support
-
-If the `--bgp` flag is passed alone with the Packet flags `packet, packetKey and packetProject`, then the Packet API will be used in order to determine the BGP configuration for the nodes being used in the cluster. This automates a lot of the process and makes using BGP within Packet much simpler.
-
-## Packet Support (added in 0.1.7)
-
-Recently in version `0.1.7` of `kube-vip` we added the functionality to use a Packet Elastic IP as the virtual IP fronting the Kubernetes Control plane cluster. In order to first get out virtual IP we will need to use our Packet account and create a EIP (either public (eek) or private). We will only need a single address so a `/32` will suffice, once this is created as part of a Packet project we can now apply this address to the servers that live in the same project. 
-
-In this example we've logged into the UI can created a new EIP of `147.75.1.2`, and we've deployed three small server instances with Ubuntu.
-
-The following new flags are used:
-
-- `--packet` which enables the use of the Packet API
-- `--packetKey` which is our API key
-- `--packetProject`which is the name of our Packet project where our servers and EIP are located.
-
-*Also* the `--arp` flag should NOT be used as it wont work within the Packet network.
-
-### Variables
-
-```
-export EIP=1.1.1.1
-export PACKET_AUTH_TOKEN=XYZ
-```
-
-### First node
-
-```
-# Generate the manifest
-
-sudo docker run --network host --rm plndr/kube-vip:0.2.1 manifest pod \
---arp=false \
---interface lo \
---vip $EIP \
---leaderElection \
---packet \
---packetKey $PACKET_AUTH_TOKEN \
---packetProject vipTest | sudo tee /etc/kubernetes/manifests/vip.yaml\
-
-# Init Kubernetes
-
-sudo kubeadm init --kubernetes-version 1.18.5 --control-plane-endpoint $EIP --upload-certs
-```
-
-### Other nodes
-
-```
-# Join
-kubeadm join $EIP:6443 --token BLAH --control-plane --certificate-key BLAH --discovery-token-ca-cert-hash sha:blah
-
-# Generate Manifest
-
-sudo docker run --network host --rm plndr/kube-vip:0.2.1 manifest pod \
---arp=false \
---interface lo \
---vip $EIP \
---leaderElection \
---packet \
---packetKey $PACKET_AUTH_TOKEN \
---packetProject vipTest | sudo tee /etc/kubernetes/manifests/vip.yaml\
-```
-
-The Elastic IP failover takes some time (30+ seconds) to move from a failed host to a new leader, so in this release it is mainly for testing. 
-
-
-## Upgrades
-
-From above we have a 3 node cluster and the controlPlane01 is leader:
-
-```
-$ kubectl logs -n kube-system kube-vip-controlplane01 -f
-time="2020-07-04T15:12:52Z" level=info msg="Beginning cluster membership, namespace [kube-system], lock name [plunder-lock], id [controlPlane01]"
-I0704 15:12:52.290420       1 leaderelection.go:242] attempting to acquire leader lease  kube-system/plunder-lock...
-I0704 15:12:56.373113       1 leaderelection.go:252] successfully acquired lease kube-system/plunder-lock
-time="2020-07-04T15:12:56Z" level=info msg="This node is assuming leadership of the cluster"
-time="2020-07-04T15:12:56Z" level=error msg="This node is leader and is adopting the virtual IP"
-time="2020-07-04T15:12:56Z" level=info msg="Starting TCP Load Balancer for service [192.168.0.81:0]"
-time="2020-07-04T15:12:56Z" level=info msg="Load Balancer [Kubeadm Load Balancer] started"
-time="2020-07-04T15:12:56Z" level=info msg="Broadcasting ARP update for 192.168.0.81 (00:50:56:a5:69:a1) via ens192"
-time="2020-07-04T15:12:56Z" level=info msg="Starting TCP Load Balancer for service [192.168.0.81:0]"
-time="2020-07-04T15:12:56Z" level=info msg="Load Balancer [Kubeadm Load Balancer] started"
-time="2020-07-04T15:12:56Z" level=info msg="Broadcasting ARP update for 192.168.0.81 (00:50:56:a5:69:a1) via ens192"
-time="2020-07-04T15:12:56Z" level=info msg="new leader elected: controlPlane01"
-```
-
-We will kill this node and watch `kube-vip` logs from another node:
-
-#### Pinging VIP
-
-```
-64 bytes from 192.168.0.81: icmp_seq=667 ttl=64 time=0.387 ms
-Request timeout for icmp_seq 668
-Request timeout for icmp_seq 669
-Request timeout for icmp_seq 670
-Request timeout for icmp_seq 671
-Request timeout for icmp_seq 672
-64 bytes from 192.168.0.81: icmp_seq=673 ttl=64 time=0.453 ms
-```
-
-#### Logs 
-```
-$ kubectl logs -n kube-system kube-vip-controlplane03 -f
-time="2020-07-04T15:17:53Z" level=info msg="Beginning cluster membership, namespace [kube-system], lock name [plunder-lock], id [controlPlane03]"
-I0704 15:17:53.484698       1 leaderelection.go:242] attempting to acquire leader lease  kube-system/plunder-lock...
-time="2020-07-04T15:17:53Z" level=info msg="new leader elected: controlPlane01"
-E0704 15:20:18.864141       1 leaderelection.go:331] error retrieving resource lock kube-system/plunder-lock: etcdserver: request timed out
-time="2020-07-04T15:20:20Z" level=info msg="new leader elected: controlPlane02"
-```
-
-#### Adding `controlPlane04`
-
-A kubeadm join will fail as the `controlPlane01` still exists as an endpoint, so we have two options (manual steps and configmap edit to remove all mention of this node, or we can bring this node up and `kubeadm reset` the node (which we will do)). 
-
-```
-$ kubectl get nodes
-NAME             STATUS     ROLES    AGE   VERSION
-controlplane01   NotReady   master   14m   v1.17.0
-controlplane02   Ready      master   13m   v1.17.2
-controlplane03   Ready      master   13m   v1.17.0
-controlplane04   NotReady   master   9s    v1.17.0
-```
- After this we can add this node into `kube-vip` with the same manifest created by `docker run`.
-
-## LeaderElection configuration
-
-The Kubernetes LeaderElection that is used to manage the election of a new leader now supports having it's settings managed through flags. 
-
- - `--leaseDuration` Length of time a Kubernetes leader lease can be held for
- - `--leaseRenewDuration` Length of time a Kubernetes leader can attempt to renew its lease
- - `--leaseRetry` Number of times the host will retry to hold a lease
-
-For larger clusters the `--leaseDuration` and `--leaseRenewDuration` may need extending due to slower `etcd` performance. (Tested with 2000 nodes)
-
-## k3s
-
-This section details the steps required to deploye `k3s` in a Highly available manner, using kube-vip deployed within k3s as a daemonset on the control plane nodes. As of `k3s` v1 the persistent datastore is back to etcd, however this guide will also include the steps for using `mysql`.
-
-### Example MySQL deployment (optional)
-
-To quickly validate this we can use docker on a host to quickly spin up a mysql database to store the persistent Kubernetes data.
-
-#### Create local directory for BD storage
-`mkdir mysql`
-
-#### Start Docker MySQL container
-`sudo docker run --cap-add SYS_NICE -p 3306:3306 --name k3s-mysql -v /home/dan/mysql:/var/lib/mysql -e MYSQL_ROOT_PASSWORD=k3s-password -d mysql:8`
-
-### Create `kube-vip` manifest
-
-The `kube-vip` manifest contains all the configuration for starting up `kube-vip` within the `k3s` cluster, it runs as a daemonset with affinity/taints for the control-plane nodes. As `k3s` starts it will parse all manifests in the manifests folder and start the highly available VIP across all control plane nodes in the cluster.
-
-#### Create the `k3` manifests directory
-
-Create the manifests directory, this directory is used by `k3s` for all of it's other deployments once it's up and running.
-`sudo mkdir -p /var/lib/rancher/k3s/server/manifests/`
-
-#### Generate the manifest
-
-Modify the `vipAddress` and `vipInterface` to match the floating IP address you'd like to use and the interface it should bind to.
-
-`curl -sL kube-vip.io/k3s | vipAddress=192.168.0.10 vipInterface=ens192 sh | sudo tee /var/lib/rancher/k3s/server/manifests/vip.yaml`
-
-### Start `k3s`
-
-Set the VIP **first**
-
-`export VIP=192.168.0.10`
-
-
-From online `-->`
-
-```
-curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--write-kubeconfig-mode 644 \
--t agent-secret --tls-san $VIP" sh -
-```
-
-From local `-->`
-
-```
-sudo ./k3s server --tls-san $VIP
-```
-
-#### With MySQL
-
-From online `-->`
-
-```
-curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--write-kubeconfig-mode 644 \
---datastore-endpoint mysql://root:k3s-password@tcp(192.168.0.43:3306)/kubernetes \
--t agent-secret --tls-san $VIP" sh -
-```
-
-From local `-->`
-
-```
-sudo ./k3s server --tls-san $VIP \
---datastore-endpoint="mysql://root:k3s-password@tcp(192.168.0.43:3306)/kubernetes"
-```
-
-### Get a `kubeconfig` that uses the vip
-
-````
-mkdir -p $HOME/.kube
-sudo cat /etc/rancher/k3s/k3s.yaml | sed 's/127.0.0.1/'$VIP'/g' > $HOME/.kube/config
-sudo chown $(id -u):$(id -g) $HOME/.kube/config
-```

docs/flags/index.md

@@ -1,84 +0,0 @@
-# Kube-Vip Flag / Environment Variable reference
-
-## Flags
-
-These flags are typically used in manifest generation.
-
-| Category     | Flag | Usage | Notes |
-|--------------|------|-------|-------|
-|**Mode**          ||||
-|              |`--controlPlane`|Enables `kube-vip` control-plane functionality||
-|              |`--services`|Enables `kube-vip` to watch services of type:LoadBalancer||
-|**Vip Config**   ||||
-|              |`--arp`|Enables ARP brodcasts from Leader||
-|              |`--bgp`|Enables BGP peering from `kube-vip`||
-|              |`--vip`|`<IP Address>`|(deprecated)|
-|              |`--address`|`<IP Address>` or `<DNS name>`||
-|              |`--interface`|`<linux interface>`||
-|              |`--leaderElection`|Enables Kubernetes LeaderElection|Used by ARP, as only the leader can broadcast|
-|**Services**||||
-|              |`--cidr`|Defaults "32"|Used when advertising BGP addresses (typically as `x.x.x.x/32`)|
-|**Kubernetes**||||
-|              |`--inCluster`|Defaults to looking inside the Pod for the token||
-|              |`--taint`|Enables a taint, stopping control plane daemonset being on workers||
-|**LeaderElection**||||
-|              |`--leaseDuration`|default 5|Seconds a lease is held for|
-|              |`--leaseRenewDuration`|default 3|Seconds a leader can attempt to renew the lease|
-|              |`--leaseRetry`|default 1|Number of times the leader will hold the lease for|
-|              |`--namespace`|"kube-vip"|The namespace where the lease will reside|
-|**BGP**||||
-|              |`--bgpRouterID`|`<IP Address>`|Typically the address of the local node|
-|              |`--localAS`|default 65000|The AS we peer from|
-|              |`--bgppeers`|`<address:AS:password:mutlihop>`|Comma seperate list of BGP peers|
-|              |`--peerAddress`|`<IP Address>`|Address of a single BGP Peer|
-|              |`--peerAS`|default 65000|AS of a single BGP Peer|
-|              |`--peerPass`|""| Password to work with a single BGP Peer|
-|              |`--multiHop`|Enables eBGP MultiHop| Enable multiHop with a single BGP Peer|
-|              |`--annotaions`|`<provider string>`|Startup will be paused until the node annotaions contain the BGP configuration|
-|**Equinix Metal**|||(May be deprecated)|
-|              |`--metal`|Enables Equinix Metal API calls||
-|              |`--metalKey`|Equinix Metal API token||
-|              |`--metalProject`|Equinix Metal Project (Name)||
-|              |`--metalProjectID`|Equinix Metal Project (UUID)||
-|              |`--provider-config`|Path to the Equinix Metal provider configuration|Requires the Equinix Metal CCM|
-
-## Environment Variables
-
-These environment variables are usually part of a kube-vip manifest.
-
-More environment variables can be read through the `pkg/kubevip/config_envvar.go` file.
-
-| Category     | Environment Variable | Usage | Notes |
-|--------------|------|-------|-------|
-|**Mode**          ||||
-|              |`cp_enable`|Enables `kube-vip` control-plane functionality||
-|              |`svc_enable`|Enables `kube-vip` to watch services of `type:LoadBalancer`||
-|**Vip Config**   ||||
-|              |`vip_arp`|Enables ARP brodcasts from Leader||
-|              |`bgp_enable`|Enables BGP peering from `kube-vip`||
-|              |`vip_address`|`<IP Address>`|(deprecated)|
-|              |`address`|`<IP Address>` or `<DNS name>`||
-|              |`vip_interface`|`<linux interface>`||
-|              |`vip_leaderelection`|Enables Kubernetes LeaderElection|Used by ARP, as only the leader can broadcast|
-|**Services**||||
-|              |`vip_cidr`|Defaults "32"|Used when advertising BGP addresses (typically as `x.x.x.x/32`)|
-|**LeaderElection**||||
-|              |`vip_leaseduration`|default 5|Seconds a lease is held for|
-|              |`vip_renewdeadline`|default 3|Seconds a leader can attempt to renew the lease|
-|              |`vip_retryperiod`|default 1|Number of times the leader will hold the lease for|
-|              |`cp_namespace`|"kube-vip"|The namespace where the lease will reside|
-|**BGP**||||
-|              |`bgp_routerid`|`<IP Address>`|Typically the address of the local node|
-|              |`bgp_as`|default 65000|The AS we peer from|
-|              |`bgp_peers`|`<address:AS:password:mutlihop>`|Comma seperate list of BGP peers|
-|              |`bgp_peeraddress`|`<IP Address>`|Address of a single BGP Peer|
-|              |`bgp_peeras`|default 65000|AS of a single BGP Peer|
-|              |`bgp_peerpass`|""| Password to work with a single BGP Peer|
-|              |`bgp_multihop`|Enables eBGP MultiHop| Enable multiHop with a single BGP Peer|
-|              |`annotaions`|`<provider string>`|Startup will be paused until the node annotaions contain the BGP configuration|
-|**Equinix Metal**|||(May be deprecated)|
-|              |`vip_packet`|Enables Equinix Metal API calls||
-|              |`PACKET_AUTH_TOKEN`|Equinix Metal API token||
-|              |`vip_packetproject`|Equinix Metal Project (Name)||
-|              |`vip_packetprojectid`|Equinix Metal Project (UUID)||
-|              |`provider_config`|Path to the Equinix Metal provider configuration|Requires the Equinix Metal CCM|

docs/hybrid/daemonset/index.md

@@ -1,226 +0,0 @@
-# Kube-Vip as a daemonset
-
-In Hybrid mode `kube-vip` will manage a virtual IP address that is passed through it's configuration for a Highly Available Kubernetes cluster, it will also "watch" services of `type:LoadBalancer` and once their `spec.LoadBalancerIP` is updated (typically by a cloud controller) it will advertise this address using BGP/ARP.
-
-
-**Note about Daemonsets**
-
-The "hybrid" mode is now the default mode in `kube-vip` from `0.2.3` onwards, and allows both modes to be enabled at the same time. 
-
-If the Kubernetes installer allows for adding a Virtual IP as an additional [SAN](https://en.wikipedia.org/wiki/Subject_Alternative_Name) to the API server certificate then we can apply `kube-vip` to the cluster once the first node has been brought up. 
-
-Unlike generating the static manifest there are a few more things that may need configuring, this page will cover most scenarios.
-
-## Create the RBAC settings
-
-As a daemonSet runs within the Kubernetes cluster it needs the correct access to be able to watch Kubernetes services and other objects. In order to do this we create a User, Role, and a binding.. we can apply this with the command:
-
-```
-kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
-```
-
-## Generating a Manifest
-
-This section only covers generating a simple *BGP* configuration, as the main focus is will be on additional changes to the manifest. For more examples we can look at [here](/hybrid/static/).
-
-**Note:** Pay attention if using the "static" examples, as the `manifest` subcommand should use `daemonset` and NOT `pod`.
-
-### Set configuration details
-
-`export VIP=192.168.0.40`
-
-`export INTERFACE=<interface>`
-
-### Configure to use a container runtime
-
-The easiest method to generate a manifest is using the container itself, below will create an alias for different container runtimes.
-
-#### containerd
-`alias kube-vip="ctr run --rm --net-host docker.io/plndr/kube-vip:0.3.1 vip"`
-
-#### Docker
-`alias kube-vip="docker run --network host --rm plndr/kube-vip:0.3.1"`
-
-### BGP Example
-
-This configuration will create a manifest that will start `kube-vip` providing **controlplane** and **services** management. **Unlike** ARP, all nodes in the BGP configuration will advertise virtual IP addresses. 
-
-**Note** we bind the address to `lo` as we don't want multiple devices that have the same address on public interfaces. We can specify all the peers in a comma seperate list in the format of `address:AS:password:multihop`.
-
-**Note 2** we pass the `--inCluster` flag as this is running as a daemonSet within the Kubernetes cluster and therefore will have access to the token inside the running pod.
-
-**Note 2** we pass the `--taint` flag as we're deploying `kube-vip` as both a daemonset and as advertising controlplane, we want to taint this daemonset to only run on the worker nodes.
-
-`export INTERFACE=lo`
-
-```
-kube-vip manifest daemonset \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --inCluster \
-    --taint \
-    --bgp \
-    --bgppeers 192.168.0.10:65000::false,192.168.0.11:65000::false
-```
-
-### Generated Manifest
-
-```
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: lo
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: "true"
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_peers
-          value: "192.168.0.10:65000::false,192.168.0.11:65000::false"
-        - name: vip_address
-          value: 192.168.0.40
-        image: plndr/kube-vip:0.2.3
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      nodeSelector:
-        node-role.kubernetes.io/master: "true"
-      tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
-  updateStrategy: {}
-```
-
-### Manifest Overview
-
-- `nodeSelector` - Ensures that this particular daemonset only runs on control plane nodes
-- `serviceAccountName: kube-vip` - this specifies the user in the `rbac` that will give us the permissions to get/update services.
-- `hostNetwork: true` - This pod will need to modify interfaces (for VIPs)
-- `env {...}` - We pass the configuration into the kube-vip pod through environment variables.
-
-## Equinix Metal Overview (using the [Equinix Metal CCM](https://github.com/packethost/packet-ccm))
-
-The below example is for running `type:LoadBalancer` services on worker nodes only and will create a daemonset that will run `kube-vip`. 
-
-**NOTE** This use-case requires the [Equinix Metal CCM](https://github.com/packethost/packet-ccm) to be installed and that the cluster/kubelet is configured to use an "external" cloud provider.
-
-This is important as the CCM will apply the BGP configuration to the [node annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) making it easy for `kube-vip` to find the networking configuration it needs to expose load balancer addresses. The `--annotations metal.equinix.com` will cause kube-vip to "watch" the annotations of the worker node that it is running on, once all of the configuarion has been applied by the CCM then the `kube-vip` pod is ready to advertise BGP addresses for the service.
-
-```
-kube-vip manifest daemonset \
-  --interface $INTERFACE \
-  --services \
-  --bgp \
-  --annotations metal.equinix.com \
-  --inCluster | k apply -f -
-```
-
-### Troubleshooting
-
-If `kube-vip` has been sat waiting for a long time then you may need to investigate that the annotations have been applied correctly by doing running the `describe` on the node:
-
-```
-kubectl describe node k8s.bgp02
-...
-Annotations:        kubeadm.alpha.kubernetes.io/cri-socket: /var/run/dockershim.sock
-                    node.alpha.kubernetes.io/ttl: 0
-                    metal.equinix.com/node-asn: 65000
-                    metal.equinix.com/peer-asn: 65530
-                    metal.equinix.com/peer-ip: x.x.x.x
-                    metal.equinix.com/src-ip: x.x.x.x
-```
-
-If there are errors regarding `169.254.255.1` or `169.254.255.2` in the `kube-vip` logs then the routes to the ToR switches that provide BGP peering may by missing from the nodes. They can be replaced with the below command:
-
-```
-GATEWAY_IP=$(curl https://metadata.platformequinix.com/metadata | jq -r ".network.addresses[] | select(.public == false) | .gateway")
-ip route add 169.254.255.1 via $GATEWAY_IP
-ip route add 169.254.255.2 via $GATEWAY_IP
-```
-
-Additionally examining the logs of the Packet CCM may reveal why the node is not yet ready.
-
-## K3s overview (on Equinix Metal)
-
-### Step 1: TIDY (best if something was running before)
-`rm -rf /var/lib/rancher /etc/rancher ~/.kube/*; ip addr flush dev lo; ip addr add 127.0.0.1/8 dev lo; mkdir -p /var/lib/rancher/k3s/server/manifests/`
-
-### Step 2: Get rbac
-`curl https://kube-vip.io/manifests/rbac.yaml > /var/lib/rancher/k3s/server/manifests/rbac.yaml`
-
-### Step 3: Generate kube-vip (get EIP from CLI or UI)
-
-```
-export EIP=x.x.x.x
-export INTERFACE=lo
-```
-
-```
-kube-vip manifest daemonset \
-  --interface $INTERFACE \
-  --vip $EIP \
-  --controlplane \
-  --services \
-  --inCluster \
-  --taint \
-  --bgp \
-  --metal \
-  --provider-config /etc/cloud-sa/cloud-sa.json | tee /var/lib/rancher/k3s/server/manifests/vip.yaml
-```
-
-NOTE: the `—provider-config` actually comes from the secret we apply in step 5 (this will leave kube-vip waiting to start)
-
-### Step 4: Up Cluster
-`K3S_TOKEN=SECRET k3s server --cluster-init --tls-san $EIP --no-deploy servicelb --disable-cloud-controller`
-
-### Step 5: Add CCM
-
-`alias k="k3s kubectl"`
-`k apply -f ./secret.yaml`
-
-(^ https://github.com/packethost/packet-ccm/blob/master/deploy/template/secret.yaml)
-
-`k apply -f https://gist.githubusercontent.com/thebsdbox/c86dd970549638105af8d96439175a59/raw/4abf90fb7929ded3f7a201818efbb6164b7081f0/ccm.yaml`
-
-### Step 6: Demo !
-`k apply -f https://k8s.io/examples/application/deployment.yaml`
-`k expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx`
-
-### Step 7 watch and test:
-`k get svc --watch`

docs/hybrid/index.md

@@ -1,141 +0,0 @@
-# Using `kube-vip` in Hybrid Mode
-
-We can deploy kube-vip in two different methods, which completely depends on your use-case and method for installing Kubernetes:
-
-- Static Pods (hybrid)
-- Daemonset (hybrid, requires taint)
-
-## **Prerequisites**
-
-In order for `kube-vip` to be able to speak with the Kubernetes API server, we need to be able to resolve the hostname within the pod. In order to ensure this will work as expected the `/etc/hosts` file should have the `hostname` of the server within it. The `/etc/hosts` file is passed into the running container and will ensure that the pod isn't "confused" by any Kubernetes networking.
-
-## Kubernetes Services (`type:LoadBalancer`)
-
-To learn more about how `kube-vip` in hybrid works with the LoadBalancer services within a kubernetes cluster the documentation is [here](./services/). To get `kube-vip` deployed read on !
-
-## Static Pods
-
-Static pods are a Kubernetes pod that is ran by the `kubelet` on a single node, and is **not** managed by the Kubernetes cluster itself. This means that whilst the pod can appear within Kubernetes it can't make use of a variety of kubernetes functionality (such as the kubernetes token or `configMaps`). The static pod approach is primarily required for [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/), this is due to the sequence of actions performed by `kubeadm`. Ideally we want `kube-vip` to be part of the kubernetes cluster, for various bits of functionality we also need `kube-vip` to provide a HA virtual IP as part of the installation. 
-
-The sequence of events for this to work follows:
-1. Generate a `kube-vip` manifest in the static pods manifest folder
-2. Run `kubeadm init`, this generates the manifests for the control plane and wait to connect to the VIP
-3. The `kubelet` will parse and execute all manifest, including the `kube-vip` manifest
-4. `kube-vip` starts and advertises our VIP
-5. The `kubeadm init` finishes succesfully.
-
-## Daemonset
-
-Other Kubernetes distributions can bring up a Kubernetes cluster, without depending on a VIP (BUT they are configured to support one). A prime example of this would be k3s, that can be configured to start and also sign the certificates to allow incoming traffic to a virtual IP. Given we don't need the VIP to exist **before** the cluster, we can bring up the k3s node(s) and then add `kube-vip` as a daemonset for all control plane nodes.
-
-# Deploying `kube-vip`
-
-The simplest method for generating the Kubernetes manifests is with `kube-vip` itself.. The subcommand `manifest pod|daemonset` can be used to generate specific types of Kubernetes manifests for use in a cluster. These subcommands can be configured with additional flags to enable/disable BGP/ARP/LeaderElection and a host of other options.
-
-Both Examples will use the same Architecture:
-
-#### Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.40 |
-| controlPlane01 | 10.0.0.41 |
-| controlPlane02 | 10.0.0.42 |
-| controlPlane03 | 10.0.0.43 |
-| worker01       | 10.0.0.44 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.19.0, we only have one worker as we're going to use our controlPlanes in "hybrid" mode.
-
-## As a static Pod (for kubeadm)
-
-The details for creating a static pod are available [here](./static/)
-
-## As a daemonset
-
-When using `kube-vip` as a daemonset the details are available [here](./daemonset/)
-
-# Kube-Vip flag reference
-
-| Category     | Flag | Usage | Notes |
-|--------------|------|-------|-------|
-|**Mode**          ||||
-|              |`--controlPlane`|Enables `kube-vip` control-plane functionality||
-|              |`--services`|Enables `kube-vip` to watch services of type:LoadBalancer||
-|**Vip Config**   ||||
-|              |`--arp`|Enables ARP brodcasts from Leader||
-|              |`--bgp`|Enables BGP peering from `kube-vip`||
-|              |`--vip`|`<IP Address>`|(deprecated)|
-|              |`--address`|`<IP Address>` or `<DNS name>`||
-|              |`--interface`|`<linux interface>`||
-|              |`--leaderElection`|Enables Kubernetes LeaderElection|Used by ARP, as only the leader can broadcast|
-|**Services**||||
-|              |`--cidr`|Defaults "32"|Used when advertising BGP addresses (typically as `x.x.x.x/32`)|
-|**Kubernetes**||||
-|              |`--inCluster`|Defaults to looking inside the Pod for the token||
-|              |`--taint`|Enables a taint, stopping control plane daemonset being on workers||
-|**LeaderElection**||||
-|              |`--leaseDuration`|default 5|Seconds a lease is held for|
-|              |`--leaseRenewDuration`|default 3|Seconds a leader can attempt to renew the lease|
-|              |`--leaseRetry`|default 1|Number of times the leader will hold the lease for|
-|              |`--namespace`|"kube-vip"|The namespace where the lease will reside|
-|**BGP**||||
-|              |`--bgpRouterID`|`<IP Address>`|Typically the address of the local node|
-|              |`--localAS`|default 65000|The AS we peer from|
-|              |`--bgppeers`|`<address:AS:password:mutlihop>`|Comma seperate list of BGP peers|
-|              |`--peerAddress`|`<IP Address>`|Address of a single BGP Peer|
-|              |`--peerAS`|default 65000|AS of a single BGP Peer|
-|              |`--peerPass`|""| Password to work with a single BGP Peer|
-|              |`--multiHop`|Enables eBGP MultiHop| Enable multiHop with a single BGP Peer|
-|              |`--annotaions`|`<provider string>`|Startup will be paused until the node annotaions contain the BGP configuration|
-|**Equinix Metal**|||(May be deprecated)|
-|              |`--metal`|Enables Equinix Metal API calls||
-|              |`--metalKey`|Equinix Metal API token||
-|              |`--metalProject`|Equinix Metal Project (Name)||
-|              |`--metalProjectID`|Equinix Metal Project (UUID)||
-|              |`--provider-config`|Path to the Equinix Metal provider configuration|Requires the Equinix Metal CCM|
-
-## Changelog
-
-### Static DNS Support (added in 0.2.0)
-
-A new flag `--address` is introduced to support using a DNS record as the control plane endpoint. `kube-vip` will do a dns lookup to retrieve the IP for the DNS record, and use that IP as the VIP. An `dnsUpdater` periodically checks and updates the system if IP changes for the DNS record.
-
-### Dynamic DNS Support (added in 0.2.1)
-
-`kube-vip` was also updated to support DHCP + [Dynamic DNS](https://en.wikipedia.org/wiki/Dynamic_DNS), for the use case where it's not able to reserve a static IP for the control plane endpoint.
-
-A new flag `--ddns` is introduced. Once enabled, `kube-vip` expects the input `--address` will be a FQDN without binding to an IP. Then `kube-vip` will start a dhcp client to allocate an IP for the hostname of FQDN, and maintain the lease for it.
-
-Once DHCP returns an IP for the FQDN, the same `dnsUpdater` runs to periodically checks and updates if IP got changed.
-
-## BGP Support (added in 0.1.8)
-
-In version `0.1.8` `kube-vip` was updated to support [BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol) as a VIP failover mechanism. When a node is elected as a leader then it will update it's peers so that they are aware to route traffic to that node in order to access the VIP. 
-
-The following new flags are used:
-
-- `--bgp` This will enable BGP support within kube-vip
-- `--localAS` The local AS number
-- `--bgpRouterID` The local router address
-- `--peerAS` The AS number for a BGP peer
-- `--peerAddress` The address of a BGP peer
-
-### Equinix Metal BGP support
-
-If the `--bgp` flag is passed along with the Equinix Metal flags `metal, metalKey and metalProject`, then Equinix Metal  API will be used in order to determine the BGP configuration for the nodes being used in the cluster. This automates a lot of the process and makes using BGP within Equinix Metal much simpler.
-
-## Equinix Metal Control Plane Support (added in 0.1.8)
-
-Recently in version `0.1.7` of `kube-vip` we added the functionality to use a Equinix Metal Elastic IP as the virtual IP fronting the Kubernetes Control plane cluster. In order to first get out virtual IP we will need to use our Equinix Metal account and create a EIP (either public or private). We will only need a single address so a `/32` will suffice, once this is created as part of a Equinix Metal project we can now apply this address to the servers that live in the same project. 
-
-In this example we've logged into the UI can created a new EIP of `147.75.1.2`, and we've deployed three small server instances with Ubuntu.
-
-The following new flags are used:
-
-- `--metal` which enables the use of the Equinix Metal API
-- `--metalKey` which is our API key
-- `--metalProject`which is the name of our Equinix Metal project where our servers and EIP are located.
-
-*Also* the `--arp` flag should NOT be used as it wont work within the Equinix Metal network.

docs/hybrid/services/index.md

@@ -1,269 +0,0 @@
-# Kube-vip services 
-
-We've designed `kube-vip` to be as de-coupled or agnostic from other components that may exist within a Kubernetes cluster as possible. This has lead to `kube-vip` having a very simplistic but robust approach to advertising Kubernetes services to the outside world and marking these services as ready to use.
-
-## Flow
-
-This section details the flow of events in order for `kube-vip` to advertise a Kubernetes service:
-
-1. An end user exposes a application through Kubernetes as a LoadBalancer => `kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx`
-2. Within the Kubernetes cluster a service object is created with the `svc.Spec.Type = ServiceTypeLoadBalancer`
-3. A controller (typically a Cloud Controller) has a loop that "watches" for services of the type `LoadBalancer`.
-4. The controller now has the responsibility of providing an IP address for this service along with doing anything that is network specific for the environment where the cluster is running.
-5. Once the controller has an IP address it will update the service `svc.Spec.LoadBalancerIP` with it's new IP address.
-6. The `kube-vip` pods also implement a "watcher" for services that have a `svc.Spec.LoadBalancerIP` address attached.
-7. When a new service appears `kube-vip` will start advertising this address to the wider network (through BGP/ARP) which will allow traffic to come into the cluster and hit the service network.
-8. Finally `kube-vip` will update the service status so that the API reflects that this LoadBalancer is ready. This is done by updating the `svc.Status.LoadBalancer.Ingress` with the VIP address.
-
-## CCM
-
-We can see from the [flow](#Flow) above that `kube-vip` isn't coupled to anything other than the Kubernetes API, and will only act upon an existing Kubernetes primative (in this case the object of type `Service`). This makes it easy for existing CCMs to simply apply their logic to services of type LoadBalancer and leave `kube-vip` to take the next steps to advertise these load-balancers to the outside world. 
-
-
-## Using the Kube-vip Cloud Provider
-
-The below instructions *should just work* on Kubernetes regardless of architecture (Linux Operating System is the only requirement) - you can quickly install the "latest" components:
-
-**Install the `kube-vip-cloud-provider`**
-
-```
-$ kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
-```
-
-It uses a `statefulSet` and can always be viewed with the following command:
-
-```
-kubectl describe pods -n kube-system kube-vip-cloud-provider-0
-```
-
-**Create a global CIDR or IP Range**
-
-Any `service` in any `namespace` can use an address from the global CIDR `cidr-global` or range `range-global`
-
-```
-kubectl create configmap --namespace kube-system kubevip --from-literal cidr-global=192.168.0.220/29
-```
-or
-```
-kubectl create configmap --namespace kube-system kubevip --from-literal range-global=192.168.1.220-192.168.1.230
-```
-
-Creating services of `type: LoadBalancer` in *any namespace* will now take addresses from the **global** cidr defined in the `configmap` unless a specific 
-
-
-## The Detailed guide
-
-### Deploy the Kube-vip Cloud Provider
-
-**Install the `kube-vip-cloud-provider`**
-
-```
-$ kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
-```
-
-The following output should appear when the manifest is applied: 
-
-```
-serviceaccount/kube-vip-cloud-controller created
-clusterrole.rbac.authorization.k8s.io/system:kube-vip-cloud-controller-role created
-clusterrolebinding.rbac.authorization.k8s.io/system:kube-vip-cloud-controller-binding created
-statefulset.apps/kube-vip-cloud-provider created
-```
-
-We can validate the cloud provider by examining the pods and following the logs:
-
-```
-kubectl describe pods -n kube-system kube-vip-cloud-provider-0
-kubectl logs -n kube-system kube-vip-cloud-provider-0 -f
-```
-
-### The Kube-vip Cloud Provider `configmap`
-
-To manage the IP address ranges for the load balancer instances the `kube-vip-cloud-provider` uses a `configmap` held in the `kube-system` namespace. IP address ranges can be configured using:
-- IP address pools by CIDR
-- IP ranges [start address - end address]
-- Multiple pools by CIDR per namespace
-- Multiple IP ranges per namespace (handles overlapping ranges)
-- Setting of static addresses through --load-balancer-ip=x.x.x.x
-
-To control which IP address range is used for which service the following rules are applied:
-- Global address pools (`cidr-global` or `range-global`) are available for use by *any* `service` in *any* `namespace`
-- Namespace specific address pools (`cidr-<namespace>` or `range-<namespace>`) are *only* available for use by `service` in the *specific* `namespace`
-- Static IP addresses can be applied to a load balancer `service` using the `loadbalancerIP` setting, even outside of the assigned ranges
-
-Example Configmap:
-
-```
-$ kubectl get configmap -n kube-system kubevip -o yaml
-
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kubevip
-  namespace: kube-system
-data:
-  cidr-default: 192.168.0.200/29                      # CIDR-based IP range for use in the default namespace
-  range-development: 192.168.0.210-192.168.0.219      # Range-based IP range for use in the development namespace
-  cidr-finance: 192.168.0.220/29,192.168.0.230/29     # Multiple CIDR-based ranges for use in the finance namespace
-  cidr-global: 192.168.0.240/29                       # CIDR-based range which can be used in any namespace
-```
-
-### Expose a service
-
-We can now expose a service and once the cloud provider has provided an address `kube-vip` will start to advertise that address to the outside world as shown below!
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx
-```
-
-or via a `service` YAML definition
-
-```
-apiVersion: v1
-kind: Service
-metadata:
-  name: nginx
-spec:
-  ports:
-  - name: http
-    port: 80
-    protocol: TCP
-  selector:
-    app: nginx
-  type: LoadBalancer
-  ```
-
-
-We can also expose a specific address by specifying it on the command line:
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```
-
-or including it in the `service` definition:
-
-```
-apiVersion: v1
-kind: Service
-metadata:
-  name: nginx
-spec:
-  ports:
-  - name: http
-    port: 80
-    protocol: TCP
-  selector:
-    app: nginx
-  type: LoadBalancer
-  loadBalancerIP: "1.1.1.1"
-```
-
-### Using DHCP for Load Balancers (experimental)
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to use the local network DHCP server to provide `kube-vip` with a load-balancer address that can be used to access a
- Kubernetes service on the network. 
-
-In order to do this we need to signify to `kube-vip` and the cloud-provider that we don't need one of their managed addresses. We do this by explicitly exposing a service on the 
-address `0.0.0.0`. When `kube-vip` sees a service on this address it will create a `macvlan` interface on the host and request a DHCP address, once this address is provided it will assign it as the VIP and update the Kubernetes service!
-
-```
-$ k expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx-dhcp --load-balancer-ip=0.0.0.0; k get svc
-service/nginx-dhcp exposed
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   0.0.0.0         80:31184/TCP   0s
-
-{ ... a second or so later ... }
-
-$ k get svc
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   192.168.0.155   80:31184/TCP   3s
-```
-
-### Using UPNP to expose a service to the outside world
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to expose a load-balancer on a specific port and using UPNP (on a supported gateway) expose this service to the inte
-rnet.
-
-Most simple networks look something like the following:
-
-`<----- <internal network 192.168.0.0/24> <Gateway / router> <external network address> ----> Internet`
-
-Using UPNP we can create a matching port on the `<external network address>` allowing your service to be exposed to the internet.
-
-#### Enable UPNP
-
-Add the following to the `kube-vip` `env:` section, and the rest should be completely automated. 
-
-**Note** some environments may require (Unifi) will require `Secure mode` being `disabled` (this allows a host with a different address to register a port)
-
-```
-- name: enableUPNP
-  value: "true"
-```
-
-#### Exposing a service
-
-To expose a port successfully we'll need to change the command slightly:
-
-`--target-port=80` the port of the application in the pods (HTT/NGINX)
-`--port=32380` the port the service will be exposed on (and what you should connect to in order to receive traffic from the service)
-
-`kubectl expose deployment plunder-nginx --port=32380 --target-port=80 --type=LoadBalancer --namespace plunder`
-
-The above example should expose a port on your external (internet facing address), that can be tested externally with:
-
-```
-$ curl externalIP:32380
-<!DOCTYPE html>
-<html>
-...
-```
-
-### Expose with Equinix Metal (using the `kube-vip-cloud-provider`)
-
-Either through the CLI or through the UI, create a public IPv4 EIP address.. and this is the address you can expose through BGP!
-
-```
-# packet ip request -p xxx-bbb-ccc -f ams1 -q 1 -t public_ipv4                                                                   
-+-------+---------------+--------+----------------------+
-|   ID  |    ADDRESS    | PUBLIC |       CREATED        |
-+-------+---------------+--------+----------------------+
-| xxxxx |     1.1.1.1   | true   | 2020-11-10T15:57:39Z |
-+-------+---------------+--------+----------------------+
-
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```
-
-## Equinix Metal Overview (using the [Equinix Metal CCM](https://github.com/packethost/packet-ccm))
-
-Below are two examples for running `type:LoadBalancer` services on worker nodes only and will create a daemonset that will run `kube-vip`. 
-
-**NOTE** This use-case requires the [Equinix Metal CCM](https://github.com/packethost/packet-ccm) to be installed and that the cluster/kubelet is configured to use an "external" cloud provider.
-
-### Using Annotations
-
-This is important as the CCM will apply the BGP configuration to the [node annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) making it easy for `kube-vip` to find the networking configuration it needs to expose load balancer addresses. The `--annotations metal.equinix.com` will cause kube-vip to "watch" the annotations of the worker node that it is running on, once all of the configuarion has been applied by the CCM then the `kube-vip` pod is ready to advertise BGP addresses for the service.
-
-```
-kube-vip manifest daemonset \
-  --interface $INTERFACE \
-  --services \
-  --bgp \
-  --annotations metal.equinix.com \
-  --inCluster | k apply -f -
-```
-
-### Using the existing CCM secret 
-
-Alternatively it is possible to create a daemonset that will use the existing CCM secret to do an API lookup, this will allow for discovering the networking configuration needed to advertise loadbalancer addresses through BGP.
-
-```
-kube-vip manifest daemonset --interface $INTERFACE \
---services \
---inCluster \
---bgp \
---metal \
---provider-config /etc/cloud-sa/cloud-sa.json | kubectl apply -f -
-```

docs/hybrid/static/index.md

@@ -1,129 +0,0 @@
-# Kube-vip as a Static Pod
-
-In Hybrid mode `kube-vip` will manage a virtual IP address that is passed through it's configuration for a Highly Available Kubernetes cluster, it will also "watch" services of `type:LoadBalancer` and once their `spec.LoadBalancerIP` is updated (typically by a cloud controller) it will advertise this address using BGP/ARP.
-
-The "hybrid" mode is now the default mode in `kube-vip` from `0.2.3` onwards, and allows both modes to be enabled at the same time. 
-
-## Generating a Manifest
-
-This section details creating a number of manifests for various use cases
-
-### Set configuration details
-
-`export VIP=192.168.0.40`
-
-`export INTERFACE=<interface>`
-
-### Configure to use a container runtime
-
-The easiest method to generate a manifest is using the container itself, below will create an alias for different container runtimes.
-
-#### containerd
-`alias kube-vip="ctr run --rm --net-host docker.io/plndr/kube-vip:0.3.1 vip /kube-vip"`
-
-#### Docker
-`alias kube-vip="docker run --network host --rm plndr/kube-vip:0.3.1"`
-
-
-### ARP
-
-This configuration will create a manifest that starts `kube-vip` providing **controlplane** and **services** management, using **leaderElection**. When this instance is elected as the leader it will bind the `vip` to the specified `interface`, this is also the same for services of `type:LoadBalancer`.
-
-`export INTERFACE=eth0`
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --arp \
-    --leaderElection | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-### BGP
-
-This configuration will create a manifest that will start `kube-vip` providing **controlplane** and **services** management. **Unlike** ARP, all nodes in the BGP configuration will advertise virtual IP addresses. 
-
-**Note** we bind the address to `lo` as we don't want multiple devices that have the same address on public interfaces. We can specify all the peers in a comma seperate list in the format of `address:AS:password:multihop`.
-
-`export INTERFACE=lo`
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --localAS 65000 \
-    --bgpRouterID 192.168.0.2 \
-    --bgppeers 192.168.0.10:65000::false,192.168.0.11:65000::false | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-### BGP with Equinix Metal
-
-When deploying Kubernetes with Equinix Metal with the `--controlplane` functionality we need to pre-populate the BGP configuration in order for the control plane to be advertised and work in a HA scenario. Luckily Equinix Metal provides the capability to "look up" the configuration details (for BGP) that we need in order to advertise our virtual IP for HA functionality. We can either make use of the [Equinix Metal API](https://metal.equinix.com/developers/api/) or we can parse the [Equinix Metal Metadata service](https://metal.equinix.com/developers/docs/servers/metadata/).
-
-**Note** If this cluster will be making use of Equinix Metal for `type:LoadBalancer` (by using the [Equinix Metal CCM](https://github.com/packethost/packet-ccm)) then we will need to ensure that nodes are set to use an external cloud-provider. Before doing a `kubeadm init|join` ensure the kubelet has the correct flags by using the following command `echo KUBELET_EXTRA_ARGS=\"--cloud-provider=external\" > /etc/default/kubelet`.
-
-#### Creating a manifest using the API
-
-We can enable `kube-vip` with the capability to discover the required configuration for BGP by passing the `--metal` flag and the API Key and our project ID.
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE\
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --metal \
-    --metalKey xxxxxxx \
-    --metalProjectID xxxxx | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-#### Creating a manifest using the metadata
-
-We can parse the metadata, *however* it requires that the tools `curl` and `jq` are installed. 
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE\
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --peerAS $(curl https://metadata.platformequinix.com/metadata | jq '.bgp_neighbors[0].peer_as') \
-    --peerAddress $(curl https://metadata.platformequinix.com/metadata | jq -r '.bgp_neighbors[0].peer_ips[0]') \
-    --localAS $(curl https://metadata.platformequinix.com/metadata | jq '.bgp_neighbors[0].customer_as') \
-    --bgpRouterID $(curl https://metadata.platformequinix.com/metadata | jq -r '.bgp_neighbors[0].customer_ip') | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-## Deploy your Kubernetes Cluster
-
-### First node
-
-```
-sudo kubeadm init \
-    --kubernetes-version 1.19.0 \
-    --control-plane-endpoint $VIP \
-    --upload-certs
-```
-
-### Additional Node(s)
-
-Due to an oddity with `kubeadm` we can't have our `kube-vip` manifest present **before** joining our additional nodes. So on these control plane nodes we will add them first to the cluster.
-
-```
-sudo kubeadm join $VIP:6443 \
-    --token w5atsr.blahblahblah 
-    --control-plane \
-    --certificate-key abc123
-```
-
-**Once**, joined these nodes can have the same command that we ran on the first node to populate the `/etc/kubernetes/manifests/` folder with the `kube-vip` manifest.
-
-## Services
-
-At this point your `kube-vip` static pods will be up and running and where used with the `--services` flag will also be watching for Kubernetes services that they can advertise. In order for `kube-vip` to advertise a service it needs a CCM or other controller to apply an IP address to the `spec.LoadBalancerIP`, which marks the loadbalancer as defined. 

docs/index.md

@@ -1,80 +0,0 @@
-![kube-vip.png](kube-vip.png)
-
-## Overview
-Kubernetes Virtual IP and Load-Balancer for both control plane and Kubernetes services
-
-The idea behind `kube-vip` is a small self-contained Highly-Available option for all environments, especially:
-
-- Bare-Metal
-- On-Prem
-- Edge (ARM / Raspberry PI)
-- Virtualisation
-- Pretty much anywhere else :)
-
-## Features
-
-Kube-Vip was originally created to provide a HA solution for the Kubernetes control plane, over time it has evolved to incorporate that same functionality into Kubernetes service type [load-balancers](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer).
-
-- VIP addresses can be both IPv4 or IPv6
-- Control Plane with ARP (Layer 2) or BGP (Layer 3)
-- Control Plane using either [leader election](https://godoc.org/k8s.io/client-go/tools/leaderelection) or [raft](https://en.wikipedia.org/wiki/Raft_(computer_science))
-- Control Plane HA with kubeadm (static Pods)
-- Control Plane HA with K3s/and others (daemonsets)
-- Service LoadBalancer using [leader election](https://godoc.org/k8s.io/client-go/tools/leaderelection) for ARP (Layer 2)
-- Service LoadBalancer using multiple nodes with BGP
-- Service LoadBalancer address pools per namespace or global
-- Service LoadBalancer address via (existing network DHCP)
-- Service LoadBalancer address exposure to gateway via UPNP
-- ... manifest generation, vendor API integrations and many nore... 
-
-## Why?
-
-The "original" purpose of `kube-vip` was to simplify the building of HA Kubernetes clusters, which at this time can involve a few components and configurations that all need to be managed. This was blogged about in detail by [thebsdbox](https://twitter.com/thebsdbox/) here -> [https://thebsdbox.co.uk/2020/01/02/Designing-Building-HA-bare-metal-Kubernetes-cluster/#Networking-load-balancing](https://thebsdbox.co.uk/2020/01/02/Designing-Building-HA-bare-metal-Kubernetes-cluster/#Networking-load-balancing). As the project evolved it now can use those same technologies to provide load-balancing capabilities within a Kubernetes Cluster.
-
-
-## Architecture
-
-The architecture for `kube-vip` (and associated kubernetes components) is covered in detail [here](/architecture/)
-
-## Installation
-
-There are two main routes for deploying `kube-vip`, either through a [static pod](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/) when bringing up a Kubernetes cluster with [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/) or as a [daemon set](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) (typically with distributions like [k3s](https://k3s.io)). 
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.40 |
-| controlPlane01 | 10.0.0.41 |
-| controlPlane02 | 10.0.0.42 |
-| controlPlane03 | 10.0.0.43 |
-| worker01       | 10.0.0.44 |
-
-All nodes are running Ubuntu 20.04, Docker CE and will use Kubernetes 1.21.0, we only have one worker as we're going to use our controlPlanes in "hybrid" mode.
-
-- [Static Pod](/install_static)
-- [Daemon Set](/install_daemonset)
-
-## Usage
-
-- [On-Prem with the kube-vip cloud controller](/usage/on-prem)
-- [KIND](/usage/kind)
-- [Equinix Metal](/usage/EquinixMetal)
-- [k3s](/usage/k3s)
-
-## Flags/Environment Variables
-
-- [Flags and Environment variables](/flags/)
-
-## Links
-
-- The Kube-Vip Cloud Provider Repository -> [https://github.com/kube-vip/kube-vip-cloud-provider](https://github.com/kube-vip/kube-vip-cloud-provider)
-- The Kube-Vip Repository -> [https://github.com/kube-vip/kube-vip](https://github.com/kube-vip/kube-vip)
-
-## Copyright
-
-© 2021 [The Linux Foundation](https://www.linuxfoundation.org/). All right reserved
-
-The Linux Foundation has registered trademarks and uses trademarks.
-
-For a list trademarks of The Linux Foundation, please see our [Trademark Usage page](https://www.linuxfoundation.org/en/trademark-usage).

docs/install_daemonset/index.md

@@ -1,167 +0,0 @@
-# Kube-Vip as a daemonset
-
-## Daemonset
-
-Other Kubernetes distributions can bring up a Kubernetes cluster, without depending on a VIP (BUT they are configured to support one). A prime example of this would be k3s, that can be configured to start and also sign the certificates to allow incoming traffic to a virtual IP. Given we don't need the VIP to exist **before** the cluster, we can bring up the k3s node(s) and then add `kube-vip` as a daemonset for all control plane nodes.
-
-If the Kubernetes installer allows for adding a Virtual IP as an additional [SAN](https://en.wikipedia.org/wiki/Subject_Alternative_Name) to the API server certificate then we can apply `kube-vip` to the cluster once the first node has been brought up. 
-
-## Kube-Vip as **HA**, **Load-Balancer** or both ` ¯\_(ツ)_/¯`
-
-When generating a manifest for `kube-vip` we will pass in the flags `--controlplane` / `--services` these will enable the various types of functionality within `kube-vip`. 
-
-With both enabled `kube-vip` will manage a virtual IP address that is passed through it's configuration for a Highly Available Kubernetes cluster, it will also "watch" services of `type:LoadBalancer` and once their `spec.LoadBalancerIP` is updated (typically by a cloud controller) it will advertise this address using BGP/ARP.
-
-**Note about Daemonsets**
-
-Unlike generating the static manifest there are a few more things that may need configuring, this page will cover most scenarios.
-
-## Create the RBAC settings
-
-As a daemonSet runs within the Kubernetes cluster it needs the correct access to be able to watch Kubernetes services and other objects. In order to do this we create a User, Role, and a binding.. we can apply this with the command:
-
-```
-kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
-```
-
-## Generating a Manifest
-
-This section only covers generating a simple *BGP* configuration, as the main focus is will be on additional changes to the manifest. For more examples we can look at [here](/hybrid/static/).
-
-**Note:** Pay attention if using the "static" examples, as the `manifest` subcommand should use `daemonset` and NOT `pod`.
-
-### Set configuration details
-
-`export VIP=192.168.0.40`
-
-`export INTERFACE=<interface>`
-
-### Configure to use a container runtime
-
-The easiest method to generate a manifest is using the container itself, below will create an alias for different container runtimes.
-
-#### containerd
-`alias kube-vip="ctr run --rm --net-host docker.io/plndr/kube-vip:0.3.1 vip"`
-
-#### Docker
-`alias kube-vip="docker run --network host --rm plndr/kube-vip:0.3.1"`
-
-### BGP Example
-
-This configuration will create a manifest that will start `kube-vip` providing **controlplane** and **services** management. **Unlike** ARP, all nodes in the BGP configuration will advertise virtual IP addresses. 
-
-**Note** we bind the address to `lo` as we don't want multiple devices that have the same address on public interfaces. We can specify all the peers in a comma seperate list in the format of `address:AS:password:multihop`.
-
-**Note 2** we pass the `--inCluster` flag as this is running as a daemonSet within the Kubernetes cluster and therefore will have access to the token inside the running pod.
-
-**Note 2** we pass the `--taint` flag as we're deploying `kube-vip` as both a daemonset and as advertising controlplane, we want to taint this daemonset to only run on the worker nodes.
-
-`export INTERFACE=lo`
-
-```
-kube-vip manifest daemonset \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --inCluster \
-    --taint \
-    --bgp \
-    --bgppeers 192.168.0.10:65000::false,192.168.0.11:65000::false
-```
-
-### Generated Manifest
-
-```
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: lo
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: bgp_peers
-          value: 192.168.0.10:65000::false,192.168.0.11:65000::false
-        - name: vip_address
-          value: 192.168.0.40
-        image: 'plndr/kube-vip:'
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      nodeSelector:
-        node-role.kubernetes.io/master: "true"
-      serviceAccountName: kube-vip
-      tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
-        operator: Exists
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-```
-
-### Managing a `routerID` as a daemonset
-
-The routerID needs to be unique on each node that participates in BGP advertisements. In order to do this we can modify the manifest so that when `kube-vip` starts it will look up its local address and use that as the routerID.
-
-```
-          - name: bgp_routerinterface
-            value: "ens160"
-```
-
-This will instruct each instance of `kube-vip` as part of the daemonset to look up the IP address on that interface and use it as the routerID.
-
-### Manifest Overview
-
-- `nodeSelector` - Ensures that this particular daemonset only runs on control plane nodes
-- `serviceAccountName: kube-vip` - this specifies the user in the `rbac` that will give us the permissions to get/update services.
-- `hostNetwork: true` - This pod will need to modify interfaces (for VIPs)
-- `env {...}` - We pass the configuration into the kube-vip pod through environment variables.

docs/install_static/index.md

@@ -1,75 +0,0 @@
-# Kube-vip as a Static Pod
-
-## Static Pods
-
-Static pods are a Kubernetes pod that is ran by the `kubelet` on a single node, and is **not** managed by the Kubernetes cluster itself. This means that whilst the pod can appear within Kubernetes it can't make use of a variety of kubernetes functionality (such as the kubernetes token or `configMaps`). The static pod approach is primarily required for [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/), this is due to the sequence of actions performed by `kubeadm`. Ideally we want `kube-vip` to be part of the kubernetes cluster, for various bits of functionality we also need `kube-vip` to provide a HA virtual IP as part of the installation. 
-
-The sequence of events for this to work follows:
-1. Generate a `kube-vip` manifest in the static pods manifest folder
-2. Run `kubeadm init`, this generates the manifests for the control plane and wait to connect to the VIP
-3. The `kubelet` will parse and execute all manifest, including the `kube-vip` manifest
-4. `kube-vip` starts and advertises our VIP
-5. The `kubeadm init` finishes succesfully.
-
-## Kube-Vip as **HA**, **Load-Balancer** or both ` ¯\_(ツ)_/¯`
-
-When generating a manifest for `kube-vip` we will pass in the flags `--controlplane` / `--services` these will enable the various types of functionality within `kube-vip`. 
-
-With both enabled `kube-vip` will manage a virtual IP address that is passed through it's configuration for a Highly Available Kubernetes cluster, it will also "watch" services of `type:LoadBalancer` and once their `spec.LoadBalancerIP` is updated (typically by a cloud controller) it will advertise this address using BGP/ARP.
-
-## Generating a Manifest
-
-This section details creating a number of manifests for various use cases
-
-### Set configuration details
-
-`export VIP=192.168.0.40`
-
-`export INTERFACE=<interface>`
-
-## Configure to use a container runtime
-
-The easiest method to generate a manifest is using the container itself, below will create an alias for different container runtimes.
-
-### containerd
-`alias kube-vip="ctr run --rm --net-host docker.io/plndr/kube-vip:0.3.1 vip /kube-vip"`
-
-### Docker
-`alias kube-vip="docker run --network host --rm plndr/kube-vip:0.3.1"`
-
-
-## ARP
-
-This configuration will create a manifest that starts `kube-vip` providing **controlplane** and **services** management, using **leaderElection**. When this instance is elected as the leader it will bind the `vip` to the specified `interface`, this is also the same for services of `type:LoadBalancer`.
-
-`export INTERFACE=eth0`
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --arp \
-    --leaderElection | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-## BGP
-
-This configuration will create a manifest that will start `kube-vip` providing **controlplane** and **services** management. **Unlike** ARP, all nodes in the BGP configuration will advertise virtual IP addresses. 
-
-**Note** we bind the address to `lo` as we don't want multiple devices that have the same address on public interfaces. We can specify all the peers in a comma seperate list in the format of `address:AS:password:multihop`.
-
-`export INTERFACE=lo`
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --localAS 65000 \
-    --bgpRouterID 192.168.0.2 \
-    --bgppeers 192.168.0.10:65000::false,192.168.0.11:65000::false | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```

docs/k3s

@@ -1,61 +0,0 @@
-#!/bin/bash
-
-echo "apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: \"true\"
-        - name: vip_interface
-          value: $vipInterface
-        - name: port
-          value: \"6443\"
-        - name: vip_cidr
-          value: \"32\"
-        - name: cp_enable
-          value: \"true\"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: \"false\"
-        - name: vip_address
-          value: $vipAddress
-        image: plndr/kube-vip:v0.3.5
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      nodeSelector:
-        node-role.kubernetes.io/master: \"true\"
-      serviceAccountName: kube-vip
-      tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0"

docs/kubernetes/arp/index.md

@@ -1,181 +0,0 @@
-# Kube-vip (Layer 2 / ARP)
-
-**BEFORE** we begin we should ensure that ipvs has `strict` ARP enabled:
-
-```
-$ kubectl describe configmap -n kube-system kube-proxy | grep ARP
-  strictARP: false
-```
-
-If this is false we can enable it with the command:
-
-```
-$ kubectl get configmap kube-proxy -n kube-system -o yaml | \
- sed -e "s/strictARP: false/strictARP: true/" | \
- kubectl apply -f - -n kube-system
-```
-
-and confirm with:
-
-```
-$ kubectl describe configmap -n kube-system kube-proxy | grep ARP
-  strictARP: true
-```
-
-## Deploy `kube-vip`
-
-To deploy the [latest] then `kubectl apply -f https://kube-vip.io/manifests/kube-vip.yaml`, specific versions should be found in the repository as detailed below:
-
-From the GitHub repository [https://github.com/kube-vip/kube-vip/tree/master/example/deploy](https://github.com/kube-vip/kube-vip/tree/master/example/deploy) find the version of the `kube-vip` to deploy (although typically the highest version number will provider more functionality/stability). The [raw] option in Github will provide the url that can be applied directly with a `kubectl apply -f <url>`.
-
-The following output should appear when the manifest is applied: 
-```
-serviceaccount/vip created
-role.rbac.authorization.k8s.io/vip-role created
-rolebinding.rbac.authorization.k8s.io/vip-role-bind created
-deployment.apps/kube-vip-cluster created
-```
-
-*NOTE* The manifest for the `kube-vip` deployment has rules to ensure affinity (pods are always distributed to different nodes for HA). By default the replicas are set to `3` in the event you have less than `3` worker nodes then those replicas will sit as `pending`. This in itself isn't an issue, it means when new workers are added then they will be scheduled. *However*, tooling such as `kapps` will inspect the manifest before it's applied an error because of issues such as this.
-
-### Editing `kube-vip` configuration
-
-Either download and edit the manifest locally or apply as above and edit the deployment with `kubectl edit deploy/kube-vip-cluster` (change namespace where appropriate `-n`)
-
-```
-		- name: vip_interface
-		  value: ens192
-		- name: vip_configmap
-		  value: plndr
-		- name: vip_arp
-		  value: "true"
-		- name: vip_loglevel
-		  value: "5"
-```
-
-- `vip_interface` - defines the interface that the VIP will bind to
-- `vip_configmap` - defines the `configmap` that `kube-vip` will watch for service configuration
-- `vip_arp` - determines if ARP broadcasts are enabled
-- `vip_loglevel` - determines the verbosity of logging
-
-## Using other namespaces
-
-In this example we'll deploy and load-balance within the namespace `plunder`
-
-### Create the namespace
-
-`kubectl create namespace plunder`
-
-### Add a network range/cidr for this namespace
-
-`kubectl edit -n kube-system configmap/plndr`
-
-We will add the range 192.168.0.210/29 for the namespace plunder underneath the existing range for the namespace default:
-
-```
-apiVersion: v1
-data:
-  cidr-default: 192.168.0.200/29
-  cidr-global: 192.168.0.210/29
-  cidr-plunder: 192.168.0.220/29
-<...>
-```
-
-### Deploy `kube-vip` in the namespace **plunder**
-
-In the same way we deployed `kube-vip` into the default namespace we can deploy the same manifest into a different namespace using `-n namespace` e.g.
-
-**Note** change the version of manifest when actually deploying!
-
-```
-kubectl apply -f https://kube-vip.io/manifests/kube-vip.yaml -n plunder
-```
-
-## Usage
-
-This example will deploy into the namespace `plunder` as mention in the [Using other namespaces](Using other namespaces) example. Remove the `-n plunder` to deploy within the `default` namespace.
-
-### Deploy nginx
-
-```
-kubectl create deployment --image nginx plunder-nginx --namespace plunder
-```
-
-### Create a load balancer
-
-```
-kubectl expose deployment plunder-nginx --port=80 --type=LoadBalancer --namespace plunder
-```
-
-## Using DHCP for Load Balancers (experimental)
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to use the local network DHCP server to provide `kube-vip` with a load-balancer address that can be used to access a Kubernetes service on the network. 
-
-In order to do this we need to signify to `kube-vip` and the cloud-provider that we don't need one of their managed addresses. We do this by explicitly exposing a service on the address `0.0.0.0`. When `kube-vip` sees a service on this address it will create a `macvlan` interface on the host and request a DHCP address, once this address is provided it will assign it as the VIP and update the Kubernetes service!
-
-```
-$ k expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx-dhcp --load-balancer-ip=0.0.0.0; k get svc
-service/nginx-dhcp exposed
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   0.0.0.0         80:31184/TCP   0s
-
-{ ... a second or so later ... }
-
-$ k get svc
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   192.168.0.155   80:31184/TCP   3s
-```
-
-## Using UPNP to expose a service to the outside world
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to expose a load-balancer on a specific port and using UPNP (on a supported gateway) expose this service to the internet.
-
-Most simple networks look something like the following:
-
-`<----- <internal network 192.168.0.0/24> <Gateway / router> <external network address> ----> Internet`
-
-Using UPNP we can create a matching port on the `<external network address>` allowing your service to be exposed to the internet.
-
-### Enable UPNP
-
-Add the following to the `kube-vip` `env:` section, and the rest should be completely automated. 
-
-**Note** some environments may require (Unifi) will require `Secure mode` being `disabled` (this allows a host with a different address to register a port)
-
-```
-- name: enableUPNP
-  value: "true"
-```
-
-### Exposing a service
-
-To expose a port successfully we'll need to change the command slightly:
-
-`--target-port=80` the port of the application in the pods (HTT/NGINX)
-`--port=32380` the port the service will be exposed on (and what you should connect to in order to receive traffic from the service)
-
-`kubectl expose deployment plunder-nginx --port=32380 --target-port=80 --type=LoadBalancer --namespace plunder`
-
-The above example should expose a port on your external (internet facing address), that can be tested externally with:
-
-```
-$ curl externalIP:32380
-<!DOCTYPE html>
-<html>
-...
-```
-
-## Troubleshooting
-
-Typically the logs from the `kube-vip` controller will reveal the most clues as to where a problem may lie.
-
-The `ClusterRoleBinding` is missing will result in the following:
-
-```
-E0229 17:36:38.014351       1 retrywatcher.go:129] Watch failed: unknown (get endpoints)
-E0229 17:36:38.014352       1 retrywatcher.go:129] Watch failed: unknown (get endpoints)
-```
-
-Additionally ensure that the vip_interface matches the correct interface from `ip addr`

docs/kubernetes/bgp/index.md

@@ -1,87 +0,0 @@
-# Kube-vip (Layer 3 / BGP)
-
-## Deploy `kube-vip`
-
-To deploy the [latest] then `kubectl apply -f https://kube-vip.io/manifests/kube-vip.yaml`, specific versions should be found in the repository as detailed below:
-
-From the GitHub repository [https://github.com/kube-vip/kube-vip/tree/master/example/deploy](https://github.com/kube-vip/kube-vip/tree/master/example/deploy) find the version of the `kube-vip` to deploy (although typically the highest version number will provider more functionality/stability). The [raw] option in Github will provide the url that can be applied directly with a `kubectl apply -f <url>`.
-
-The following output should appear when the manifest is applied: 
-```
-serviceaccount/vip created
-role.rbac.authorization.k8s.io/vip-role created
-rolebinding.rbac.authorization.k8s.io/vip-role-bind created
-deployment.apps/kube-vip-cluster created
-```
-
-*NOTE* The manifest for the `kube-vip` deployment has rules to ensure affinity (pods are always distributed to different nodes for HA). By default the replicas are set to `3` in the event you have less than `3` worker nodes then those replicas will sit as `pending`. This in itself isn't an issue, it means when new workers are added then they will be scheduled. *However*, tooling such as `kapps` will inspect the manifest before it's applied an error because of issues such as this.
-
-### Editing `kube-vip` configuration
-
-Either download and edit the manifest locally or apply as above and edit the deployment with `kubectl edit deploy/kube-vip-cluster` (change namespace where appropriate `-n`)
-
-Ensure the `vip_arp` isn't enabled as ARP and BGP can't be used at the same time (today), also that the `vip_interface` is set to localhost (`lo`).
-
-```
-        - name: vip_interface
-		  value: "lo"
-		- name: vip_configmap
-		  value: "plndr" 
-		- name: bgp_enable
-		  value: "true"
-		- name: vip_loglevel
-		  value: "5"
-```
-
-### BGP Specific configuration
-
-Additionally for BGP we'll need some configuration details, your local friendly network admin should be able to help here:
-
-```
-		- name: bgp_routerid
-		  value: "192.168.0.45"
-		- name: bgp_as
-          value: "65000" 
-		- name: bgp_peeraddress
-    	  value: "10.0.0.1"
-		- name: bgp_peeras
-    	  value: "65522"
-```
-
-### BGP on Packet
-
-If you're lucky enough to be running services on Packet then The above BGP information can be found from the API, instead of specifying the above we need to use the following:
-
-```
-		- name: vip_packet
-		  value: "true"
-		- name: vip_packetproject
-		  value: "My Project" 
-		- name: PACKET_AUTH_TOKEN
-		  value: "XXYZZYVVY"
-```
-
-With the above configuration in place, all `kube-vip` pods will start in active mode and when a service is exposed then all nodes will advertise the VIP to the routers.
-
-## Expose a service
-
-Given that `kube-vip` doesn't know your network (at this point) ask your local friendly network OPs for an address you can advertise. That is the address you can expose to the outside world as shown below!
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```
-
-## Expose with packet 
-
-Either through the CLI or through the UI, create a public IPv4 EIP address.. and this is the address you can expose through BGP!
-
-```
-# packet ip request -p xxx-bbb-ccc -f ams1 -q 1 -t public_ipv4                                                                   
-+-------+---------------+--------+----------------------+
-|   ID  |    ADDRESS    | PUBLIC |       CREATED        |
-+-------+---------------+--------+----------------------+
-| xxxxx |     1.1.1.1   | true   | 2020-11-10T15:57:39Z |
-+-------+---------------+--------+----------------------+
-
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```

docs/kubernetes/index.md

@@ -1,75 +0,0 @@
-# Usage
-
-The below instructions *should just work* on Kubernetes regardless of architecture, Linux as the Operating System is the only requirement.
-
-## The `tl;dr` guide
-
-If you just want things to "work", then you can quickly install the "latest" components:
-
-**NOTE** the `kube-vip.yaml` may need customising to set ARP/BGP OR to configure which interface to bind VIPs too.
-
-**Install the `plndr-cloud-provider`, and `kube-vip`**
-
-```
-kubectl apply -f https://kube-vip.io/manifests/controller.yaml
-kubectl apply -f https://kube-vip.io/manifests/kube-vip.yaml
-```
-
-**Create the `cidr` for the `global` namespace**
-
-```
-kubectl create configmap --namespace kube-system plndr --from-literal cidr-global=192.168.0.200/29
-```
-
-Creating services of `type: LoadBalancer` in the default namespace will now take addresses from the **global** cidr defined in the `configmap`.
-
-**Additional namespaces**
-
-Edit the `configmap` and add in the cidr ranges for those namespaces, the key in the cidr should be `cidr-<namespace>`, then ensure that `kube-vip` is deployed into that namespace with the above `apply` command with the `-n namespace` flag.
-
-## The Detailed guide
-
-### Deploy the `plndr-cloud-provider`
-
-To deploy the [latest] then `kubectl apply -f https://kube-vip.io/manifests/controller.yaml`, specific versions should be found in the repository as detailed below:
-
-From the GitHub repository [https://github.com/kube-vip/plndr-cloud-provider/tree/master/example/pod](https://github.com/kube-vip/plndr-cloud-provider/tree/master/example/pod), find the version of the plunder cloud provider manifest (although typically the highest version number will provider more functionality/stability). The [raw] option in Github will provide the url that can be applied directly with a `kubectl apply -f <url>`.
-
-The following output should appear when the manifest is applied: 
-
-```
-serviceaccount/plunder-cloud-controller created
-clusterrole.rbac.authorization.k8s.io/system:plunder-cloud-controller-role created
-clusterrolebinding.rbac.authorization.k8s.io/system:plunder-cloud-controller-binding created
-pod/plndr-cloud-provider created
-```
-
-We can validate the cloud-provider by examining the pods:
-
-`kubectl logs -n kube-system plndr-cloud-provider-0 -f`
-
-#### The `plndr-cloud-provider` `configmap`
-
-The `configmap` details a CIDR range *per* namespace, however as of (`kube-vip 0.2.1` and `plnder-cloud-provider 0.1.4`), there is now the option of having a **global** CIDR range (`cidr-global)`.  
-
-To manage the ranges for the load-balancer instances, the `plndr-cloud-provider` has a `configmap` held in the `kube-system` namespace. The structure for the key/values within the `configmap` should be that the key is in the format `cidr-<namespace>` and the value should be the cidr range.
-
-Example Configmap:
-
-```
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: plndr
-  namespace: kube-system
-data:
-  cidr-default: 192.168.0.200/29
-  cidr-global: 192.168.0.210/29
-```
-
-### Deploying `kube-vip`
-
-To use `kube-vip` in Layer2/ARP the follow this [guide](/kubernetes/arp/)
-
-To use `kube-vip` in Layer3/BGP the follow this [guide](/kubernetes/bgp/)
-

docs/manifests/0.3.1/kube-vip-arp-ds.yaml

@@ -1,63 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "true"
-        - name: vip_interface
-          value: eth0
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: vip_address
-          value: 192.168.0.1
-        image: plndr/kube-vip:0.3.1
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/0.3.1/kube-vip-bgp-ds.yaml

@@ -1,72 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: eth0
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: vip_address
-          value: 192.168.0.1
-        image: plndr/kube-vip:0.3.1
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/0.3.1/kube-vip-bgp-em-ds.yaml

@@ -1,82 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: lo
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "false"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: provider_config
-          value: /etc/cloud-sa/cloud-sa.json
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: vip_address
-          value: 192.168.0.1
-        image: plndr/kube-vip:0.3.1
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-        volumeMounts:
-        - mountPath: /etc/cloud-sa
-          name: cloud-sa-volume
-          readOnly: true
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      volumes:
-      - name: cloud-sa-volume
-        secret:
-          secretName: packet-cloud-config
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/0.3.1/kube-vip-bgp.yaml

@@ -1,62 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - manager
-    env:
-    - name: vip_arp
-      value: "false"
-    - name: vip_interface
-      value: eth0
-    - name: port
-      value: "6443"
-    - name: vip_cidr
-      value: "32"
-    - name: cp_enable
-      value: "true"
-    - name: cp_namespace
-      value: kube-system
-    - name: svc_enable
-      value: "true"
-    - name: vip_startleader
-      value: "false"
-    - name: vip_addpeerstolb
-      value: "true"
-    - name: vip_localpeer
-      value: code:192.168.0.22:10000
-    - name: bgp_enable
-      value: "true"
-    - name: bgp_routerid
-    - name: bgp_as
-      value: "65000"
-    - name: bgp_peeraddress
-    - name: bgp_peerpass
-    - name: bgp_peeras
-      value: "65000"
-    - name: vip_address
-      value: 192.168.0.1
-    image: plndr/kube-vip:0.3.1
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_RAW
-        - SYS_TIME
-    volumeMounts:
-    - mountPath: /etc/kubernetes/admin.conf
-      name: kubeconfig
-  hostNetwork: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/admin.conf
-    name: kubeconfig
-status: {}
-

docs/manifests/controller.yaml

@@ -1,70 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip-cloud-controller
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-cloud-controller-role
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "create", "update", "list", "put"]
-  - apiGroups: [""]
-    resources: ["configmaps", "endpoints","events","services/status", "leases"]
-    verbs: ["*"]
-  - apiGroups: [""]
-    resources: ["nodes", "services"]
-    verbs: ["list","get","watch","update"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-cloud-controller-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-cloud-controller-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip-cloud-controller
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: kube-vip-cloud-provider
-  namespace: kube-system
-spec:
-  serviceName: kube-vip-cloud-provider
-  podManagementPolicy: OrderedReady
-  replicas: 1
-  revisionHistoryLimit: 10
-  selector:
-    matchLabels:
-      app: kube-vip
-      component: kube-vip-cloud-provider
-  template:
-    metadata:
-      labels:
-        app: kube-vip
-        component: kube-vip-cloud-provider
-    spec:
-      containers:
-      - command:
-        - /kube-vip-cloud-provider
-        - --leader-elect-resource-name=kube-vip-cloud-controller
-        image: kubevip/kube-vip-cloud-provider:latest
-        name: kube-vip-cloud-provider
-        imagePullPolicy: Always
-        resources: {}
-      dnsPolicy: ClusterFirst
-      restartPolicy: Always
-      schedulerName: default-scheduler
-      securityContext: {}
-      terminationGracePeriodSeconds: 30
-      serviceAccountName: kube-vip-cloud-controller

docs/manifests/kube-vip-arp.yaml

@@ -1,85 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "create", "update", "list", "put"]
-  - apiGroups: [""]
-    resources: ["services"]
-    verbs: ["list","get","watch", "update"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-workers
-  name: kube-vip-workers
-  namespace: kube-system
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-workers
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-workers
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-workers
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: plndr/kube-vip:0.2.2
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        env:
-          - name: vip_interface
-            value: "ens160"
-          - name: vip_arp
-            value: "true"
-          - name: vip_loglevel
-            value: "5"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-      serviceAccountName: kube-vip
-status: {}

docs/manifests/kube-vip-bgp.yaml

@@ -1,90 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: [""]
-    resources: ["services"]
-    verbs: ["list","get","watch", "update"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-workers
-  name: kube-vip-workers
-  namespace: kube-system
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-workers
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-workers
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-workers
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: plndr/kube-vip:0.2.2
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        env:
-          - name: vip_interface
-            value: "lo"
-          - name: bgp_enable
-            value: "true"
-          - name: vip_loglevel
-            value: "5"
-          - name: bgp_routerinterface
-            value: "ens160"
-          - name: bgp_as
-            value: "64512"
-          - name: bgp_peeraddress
-            value: "192.168.0.1"
-          - name: bgp_peeras
-            value: "64512"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-      serviceAccountName: kube-vip
-status: {}

docs/manifests/kube-vip-em.yaml

@@ -1,91 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: [""]
-    resources: ["services", "services/status"]
-    verbs: ["list","get","watch", "update"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: lo
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: svc_enable
-          value: "true"
-        - name: provider_config
-          value: /etc/cloud-sa/cloud-sa.json
-        - name: vip_packet
-          value: "true"
-        - name: bgp_enable
-          value: "true"
-        image: plndr/kube-vip:0.2.3
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - SYS_TIME
-        volumeMounts:
-        - mountPath: /etc/cloud-sa
-          name: cloud-sa-volume
-          readOnly: true
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      volumes:
-      - name: cloud-sa-volume
-        secret:
-          secretName: packet-cloud-config
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0

docs/manifests/kube-vip.yaml

@@ -1,83 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: vip
----
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: vip-role
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "create", "update", "list", "put"]
-  - apiGroups: [""]
-    resources: ["configmaps", "endpoints"]
-    verbs: ["watch", "get"]
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: vip-role-bind
-subjects:
-  - kind: ServiceAccount
-    name: vip 
-    apiGroup: ""
-roleRef:
-  kind: Role
-  name: vip-role
-  apiGroup: "" 
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-cluster
-  name: kube-vip-cluster
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-cluster
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-cluster
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-cluster
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: plndr/kube-vip:0.1.3
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        env:
-          - name: vip_interface
-            value: "ens192"
-          - name: vip_configmap
-            value: "plndr" 
-          - name: vip_arp
-            value: "true"
-          - name: vip_loglevel
-            value: "5"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-      serviceAccountName: vip
-status: {}

docs/manifests/rbac.yaml

@@ -1,32 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: [""]
-    resources: ["services", "services/status", "nodes"]
-    verbs: ["list","get","watch", "update"]
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["list", "get", "watch", "update", "create"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system

docs/publish.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-echo Deploying updated documentation
-
-# Sleep for dramatic purposes!
-
-sleep 5
-
-generate-md --layout github --input ./ --output /var/www/kube-vip/

docs/usage/EquinixMetal/index.md

@@ -1,114 +0,0 @@
-# Equinix Metal Overview (using the [Equinix Metal CCM](https://github.com/packethost/packet-ccm))
-
-## BGP with Equinix Metal
-
-When deploying Kubernetes with Equinix Metal with the `--controlplane` functionality we need to pre-populate the BGP configuration in order for the control plane to be advertised and work in a HA scenario. Luckily Equinix Metal provides the capability to "look up" the configuration details (for BGP) that we need in order to advertise our virtual IP for HA functionality. We can either make use of the [Equinix Metal API](https://metal.equinix.com/developers/api/) or we can parse the [Equinix Metal Metadata service](https://metal.equinix.com/developers/docs/servers/metadata/).
-
-**Note** If this cluster will be making use of Equinix Metal for `type:LoadBalancer` (by using the [Equinix Metal CCM](https://github.com/packethost/packet-ccm)) then we will need to ensure that nodes are set to use an external cloud-provider. Before doing a `kubeadm init|join` ensure the kubelet has the correct flags by using the following command `echo KUBELET_EXTRA_ARGS=\"--cloud-provider=external\" > /etc/default/kubelet`.
-
-## Creating HA clusters in Equinix Metal
-
-### Creating a manifest using the API
-
-We can enable `kube-vip` with the capability to discover the required configuration for BGP by passing the `--metal` flag and the API Key and our project ID.
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE\
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --metal \
-    --metalKey xxxxxxx \
-    --metalProjectID xxxxx | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-### Creating a manifest using the metadata
-
-We can parse the metadata, *however* it requires that the tools `curl` and `jq` are installed. 
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE\
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --peerAS $(curl https://metadata.platformequinix.com/metadata | jq '.bgp_neighbors[0].peer_as') \
-    --peerAddress $(curl https://metadata.platformequinix.com/metadata | jq -r '.bgp_neighbors[0].peer_ips[0]') \
-    --localAS $(curl https://metadata.platformequinix.com/metadata | jq '.bgp_neighbors[0].customer_as') \
-    --bgpRouterID $(curl https://metadata.platformequinix.com/metadata | jq -r '.bgp_neighbors[0].customer_ip') | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-## Load Balancing servies on Equinix Metal
-
-Below are two examples for running `type:LoadBalancer` services on worker nodes only and will create a daemonset that will run `kube-vip`. 
-
-**NOTE** This use-case requires the [Equinix Metal CCM](https://github.com/packethost/packet-ccm) to be installed and that the cluster/kubelet is configured to use an "external" cloud provider.
-
-### Using Annotations
-
-This is important as the CCM will apply the BGP configuration to the [node annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) making it easy for `kube-vip` to find the networking configuration it needs to expose load balancer addresses. The `--annotations metal.equinix.com` will cause kube-vip to "watch" the annotations of the worker node that it is running on, once all of the configuarion has been applied by the CCM then the `kube-vip` pod is ready to advertise BGP addresses for the service.
-
-```
-kube-vip manifest daemonset \
-  --interface $INTERFACE \
-  --services \
-  --bgp \
-  --annotations metal.equinix.com \
-  --inCluster | k apply -f -
-```
-
-### Using the existing CCM secret 
-
-Alternatively it is possible to create a daemonset that will use the existing CCM secret to do an API lookup, this will allow for discovering the networking configuration needed to advertise loadbalancer addresses through BGP.
-
-```
-kube-vip manifest daemonset --interface $INTERFACE \
---services \
---inCluster \
---bgp \
---metal \
---provider-config /etc/cloud-sa/cloud-sa.json | kubectl apply -f -
-```
-
-### Expose with Equinix Metal (using the `kube-vip-cloud-provider`)
-
-Either through the CLI or through the UI, create a public IPv4 EIP address.. and this is the address you can expose through BGP!
-
-```
-# packet ip request -p xxx-bbb-ccc -f ams1 -q 1 -t public_ipv4                                                                   
-+-------+---------------+--------+----------------------+
-|   ID  |    ADDRESS    | PUBLIC |       CREATED        |
-+-------+---------------+--------+----------------------+
-| xxxxx |     1.1.1.1   | true   | 2020-11-10T15:57:39Z |
-+-------+---------------+--------+----------------------+
-
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```
-
-## Troubleshooting
-
-If `kube-vip` has been sat waiting for a long time then you may need to investigate that the annotations have been applied correctly by doing running the `describe` on the node:
-
-```
-kubectl describe node k8s.bgp02
-...
-Annotations:        kubeadm.alpha.kubernetes.io/cri-socket: /var/run/dockershim.sock
-                    node.alpha.kubernetes.io/ttl: 0
-                    metal.equinix.com/node-asn: 65000
-                    metal.equinix.com/peer-asn: 65530
-                    metal.equinix.com/peer-ip: x.x.x.x
-                    metal.equinix.com/src-ip: x.x.x.x
-```
-
-If there are errors regarding `169.254.255.1` or `169.254.255.2` in the `kube-vip` logs then the routes to the ToR switches that provide BGP peering may by missing from the nodes. They can be replaced with the below command:
-
-```
-GATEWAY_IP=$(curl https://metadata.platformequinix.com/metadata | jq -r ".network.addresses[] | select(.public == false) | .gateway")
-ip route add 169.254.255.1 via $GATEWAY_IP
-ip route add 169.254.255.2 via $GATEWAY_IP
-```
-
-Additionally examining the logs of the Packet CCM may reveal why the node is not yet ready.

docs/usage/k3s/index.md

@@ -1,82 +0,0 @@
-# K3s overview (on Equinix Metal)
-
-## Optional Tidy environment (best if something was running before)
-```
-rm -rf /var/lib/rancher /etc/rancher ~/.kube/*; \ 
-ip addr flush dev lo; \
-ip addr add 127.0.0.1/8 dev lo; 
-```
-
-## Step 1: Create Manifests folder
-
-This is required, this folder will contain all of the generated manifests that `k3s` will execute as it starts. We will create it before `k3s` and place our `kube-vip` manifests within it.
-
-```
-mkdir -p /var/lib/rancher/k3s/server/manifests/
-```
-
-## Step 2: Get rbac for `Kube-Vip`
-
-As `kube-vip` runs inside of the Kubernetes cluster, we will need to ensure that the required permissions exist.
-
-```
-curl https://kube-vip.io/manifests/rbac.yaml > /var/lib/rancher/k3s/server/manifests/rbac.yaml
-```
-
-## Step 3: Generate kube-vip (A VIP address for the network will be required)
-
-Configure your virtual IP (for the control plane) and interface that will expose this VIP first.
-
-```
-export VIP=x.x.x.x
-export INTERFACE=ethx
-```
-
-Modify the `VIP` and `INTERFACE` to match the floating IP address you'd like to use and the interface it should bind to.
-
-To generate the manifest we have two options! We can generate the manifest from [kube-vip.io](kube-vip.io) or use a kube-vip image to generate the manifest!
-
-## Step 3.1: Generate from kube-vip.io
- 
-```
-curl -sL kube-vip.io/k3s | vipAddress=$VIP vipInterface=$INTERFACE sh | sudo tee /var/lib/rancher/k3s/server/manifests/vip.yaml
-```
-
-## Step 3.2 Genereate from container image
-
-### containerd
-`alias kube-vip="ctr run --rm --net-host docker.io/plndr/kube-vip:0.3.1 vip /kube-vip"`
-
-### Docker
-`alias kube-vip="docker run --network host --rm plndr/kube-vip:0.3.1"`
-
-
-```
-kube-vip manifest daemonset \
-  --interface $INTERFACE \
-  --vip $VIP \
-  --controlplane \
-  --services \
-  --inCluster \
-  --taint \
-  --arp 
-```
-
-## Step 4: Up Cluster
-
-From online `-->`
-
-```
-curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--write-kubeconfig-mode 644 \
--t agent-secret --tls-san $VIP" sh -
-```
-
-From local `-->`
-
-```
-sudo ./k3s server --tls-san $VIP
-```
-
-## Step 5: Service Load-Balancing
-
-For this refer to the [on-prem](../on-prem) documentation 

docs/usage/kind/index.md

@@ -1,57 +0,0 @@
-# Kube-vip on KIND
-
-## Deploying KIND
-
-The documentation for KIND is fantastic and it's quickstart guide will have you up and running in no time -> [https://kind.sigs.k8s.io/docs/user/quick-start/](https://kind.sigs.k8s.io/docs/user/quick-start/)
-
-## Find Address Pool for Kube-Vip
-
-We will need to find addresses that can be used by Kube-Vip:
-
-```
-docker network inspect kind -f '{{ range $i, $a := .IPAM.Config }}{{ println .Subnet }}{{ end }}'
-```
-
-This will return a cidr range such as `172.18.0.0/16` and from here we can select a range.
-
-## Deploy the Kube-Vip Cloud Controller
-
-```
-$ kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
-```
-
-## Add our Address range
-
-```
-kubectl create configmap --namespace kube-system kubevip --from-literal range-global=172.18.100.10-172.18.100.30
-```
-
-## Install kube-vip
-
-Set the correct `alias` for `kube-vip`
-```
-alias kube-vip="docker run --network host --rm plndr/kube-vip:v0.3.5"
-```
-
-Install Kube-vip deamonset inside of KIND
-
-```
-kube-vip manifest daemonset --services --inCluster --arp --interface eth0 | ./kubectl apply -f -
-```
-
-## Test
-
-```
-kubectl apply -f https://k8s.io/examples/application/deployment.yaml
-```
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx
-```
-
-```
-kubectl get svc
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        74m
-nginx        LoadBalancer   10.96.196.235   172.18.100.11   80:31236/TCP   6s
-```

docs/usage/on-prem/index.md

@@ -1,222 +0,0 @@
-# Kube-vip on-prem 
-
-We've designed `kube-vip` to be as de-coupled or agnostic from other components that may exist within a Kubernetes cluster as possible. This has lead to `kube-vip` having a very simplistic but robust approach to advertising Kubernetes services to the outside world and marking these services as ready to use.
-
-## Flow
-
-This section details the flow of events in order for `kube-vip` to advertise a Kubernetes service:
-
-1. An end user exposes a application through Kubernetes as a LoadBalancer => `kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx`
-2. Within the Kubernetes cluster a service object is created with the `svc.Spec.Type = ServiceTypeLoadBalancer`
-3. A controller (typically a Cloud Controller) has a loop that "watches" for services of the type `LoadBalancer`.
-4. The controller now has the responsibility of providing an IP address for this service along with doing anything that is network specific for the environment where the cluster is running.
-5. Once the controller has an IP address it will update the service `svc.Spec.LoadBalancerIP` with it's new IP address.
-6. The `kube-vip` pods also implement a "watcher" for services that have a `svc.Spec.LoadBalancerIP` address attached.
-7. When a new service appears `kube-vip` will start advertising this address to the wider network (through BGP/ARP) which will allow traffic to come into the cluster and hit the service network.
-8. Finally `kube-vip` will update the service status so that the API reflects that this LoadBalancer is ready. This is done by updating the `svc.Status.LoadBalancer.Ingress` with the VIP address.
-
-## CCM
-
-We can see from the [flow](#Flow) above that `kube-vip` isn't coupled to anything other than the Kubernetes API, and will only act upon an existing Kubernetes primative (in this case the object of type `Service`). This makes it easy for existing CCMs to simply apply their logic to services of type LoadBalancer and leave `kube-vip` to take the next steps to advertise these load-balancers to the outside world. 
-
-
-## Using the Kube-vip Cloud Provider
-
-The below instructions *should just work* on Kubernetes regardless of architecture (Linux Operating System is the only requirement) - you can quickly install the "latest" components:
-
-**Install the `kube-vip-cloud-provider`**
-
-```
-$ kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
-```
-
-It uses a `statefulSet` and can always be viewed with the following command:
-
-```
-kubectl describe pods -n kube-system kube-vip-cloud-provider-0
-```
-
-**Create a global CIDR or IP Range**
-
-Any `service` in any `namespace` can use an address from the global CIDR `cidr-global` or range `range-global`
-
-```
-kubectl create configmap --namespace kube-system kubevip --from-literal cidr-global=192.168.0.220/29
-```
-or
-```
-kubectl create configmap --namespace kube-system kubevip --from-literal range-global=192.168.1.220-192.168.1.230
-```
-
-Creating services of `type: LoadBalancer` in *any namespace* will now take addresses from the **global** cidr defined in the `configmap` unless a specific 
-
-
-## The Detailed guide
-
-### Deploy the Kube-vip Cloud Provider
-
-**Install the `kube-vip-cloud-provider`**
-
-```
-$ kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
-```
-
-The following output should appear when the manifest is applied: 
-
-```
-serviceaccount/kube-vip-cloud-controller created
-clusterrole.rbac.authorization.k8s.io/system:kube-vip-cloud-controller-role created
-clusterrolebinding.rbac.authorization.k8s.io/system:kube-vip-cloud-controller-binding created
-statefulset.apps/kube-vip-cloud-provider created
-```
-
-We can validate the cloud provider by examining the pods and following the logs:
-
-```
-kubectl describe pods -n kube-system kube-vip-cloud-provider-0
-kubectl logs -n kube-system kube-vip-cloud-provider-0 -f
-```
-
-### The Kube-vip Cloud Provider `configmap`
-
-To manage the IP address ranges for the load balancer instances the `kube-vip-cloud-provider` uses a `configmap` held in the `kube-system` namespace. IP address ranges can be configured using:
-- IP address pools by CIDR
-- IP ranges [start address - end address]
-- Multiple pools by CIDR per namespace
-- Multiple IP ranges per namespace (handles overlapping ranges)
-- Setting of static addresses through --load-balancer-ip=x.x.x.x
-
-To control which IP address range is used for which service the following rules are applied:
-- Global address pools (`cidr-global` or `range-global`) are available for use by *any* `service` in *any* `namespace`
-- Namespace specific address pools (`cidr-<namespace>` or `range-<namespace>`) are *only* available for use by `service` in the *specific* `namespace`
-- Static IP addresses can be applied to a load balancer `service` using the `loadbalancerIP` setting, even outside of the assigned ranges
-
-Example Configmap:
-
-```
-$ kubectl get configmap -n kube-system kubevip -o yaml
-
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kubevip
-  namespace: kube-system
-data:
-  cidr-default: 192.168.0.200/29                      # CIDR-based IP range for use in the default namespace
-  range-development: 192.168.0.210-192.168.0.219      # Range-based IP range for use in the development namespace
-  cidr-finance: 192.168.0.220/29,192.168.0.230/29     # Multiple CIDR-based ranges for use in the finance namespace
-  cidr-global: 192.168.0.240/29                       # CIDR-based range which can be used in any namespace
-```
-
-### Expose a service
-
-We can now expose a service and once the cloud provider has provided an address `kube-vip` will start to advertise that address to the outside world as shown below!
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx
-```
-
-or via a `service` YAML definition
-
-```
-apiVersion: v1
-kind: Service
-metadata:
-  name: nginx
-spec:
-  ports:
-  - name: http
-    port: 80
-    protocol: TCP
-  selector:
-    app: nginx
-  type: LoadBalancer
-  ```
-
-
-We can also expose a specific address by specifying it on the command line:
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```
-
-or including it in the `service` definition:
-
-```
-apiVersion: v1
-kind: Service
-metadata:
-  name: nginx
-spec:
-  ports:
-  - name: http
-    port: 80
-    protocol: TCP
-  selector:
-    app: nginx
-  type: LoadBalancer
-  loadBalancerIP: "1.1.1.1"
-```
-
-### Using DHCP for Load Balancers (experimental)
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to use the local network DHCP server to provide `kube-vip` with a load-balancer address that can be used to access a
- Kubernetes service on the network. 
-
-In order to do this we need to signify to `kube-vip` and the cloud-provider that we don't need one of their managed addresses. We do this by explicitly exposing a service on the 
-address `0.0.0.0`. When `kube-vip` sees a service on this address it will create a `macvlan` interface on the host and request a DHCP address, once this address is provided it will assign it as the VIP and update the Kubernetes service!
-
-```
-$ k expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx-dhcp --load-balancer-ip=0.0.0.0; k get svc
-service/nginx-dhcp exposed
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   0.0.0.0         80:31184/TCP   0s
-
-{ ... a second or so later ... }
-
-$ k get svc
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   192.168.0.155   80:31184/TCP   3s
-```
-
-### Using UPNP to expose a service to the outside world
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to expose a load-balancer on a specific port and using UPNP (on a supported gateway) expose this service to the inte
-rnet.
-
-Most simple networks look something like the following:
-
-`<----- <internal network 192.168.0.0/24> <Gateway / router> <external network address> ----> Internet`
-
-Using UPNP we can create a matching port on the `<external network address>` allowing your service to be exposed to the internet.
-
-#### Enable UPNP
-
-Add the following to the `kube-vip` `env:` section, and the rest should be completely automated. 
-
-**Note** some environments may require (Unifi) will require `Secure mode` being `disabled` (this allows a host with a different address to register a port)
-
-```
-- name: enableUPNP
-  value: "true"
-```
-
-#### Exposing a service
-
-To expose a port successfully we'll need to change the command slightly:
-
-`--target-port=80` the port of the application in the pods (HTT/NGINX)
-`--port=32380` the port the service will be exposed on (and what you should connect to in order to receive traffic from the service)
-
-`kubectl expose deployment plunder-nginx --port=32380 --target-port=80 --type=LoadBalancer --namespace plunder`
-
-The above example should expose a port on your external (internet facing address), that can be tested externally with:
-
-```
-$ curl externalIP:32380
-<!DOCTYPE html>
-<html>
-...
-```

example/deploy/0.1.2.yaml

@@ -1,74 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-cluster
-  name: kube-vip-cluster
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-cluster
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-cluster
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-cluster
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: plndr/kube-vip:0.1.2
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        - --configMap
-        - plndr-configmap
-        - --arp
-        - --interface
-        - ens192
-        - --log
-        - "5"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-status: {}
----
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: lease-access
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "create", "update", "list", "put"]
-  - apiGroups: [""]
-    resources: ["configMap"]
-    verbs: ["get"]
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: lease-access
-subjects:
-  - kind: User
-    name: system:serviceaccount:default:default
-roleRef:
-  kind: Role
-  name: lease-access
-  apiGroup: rbac.authorization.k8s.io

example/deploy/0.1.3.yaml

@@ -1,83 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: vip
----
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: vip-role
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "create", "update", "list", "put"]
-  - apiGroups: [""]
-    resources: ["configmaps", "endpoints"]
-    verbs: ["watch", "get"]
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: vip-role-bind
-subjects:
-  - kind: ServiceAccount
-    name: vip 
-    apiGroup: ""
-roleRef:
-  kind: Role
-  name: vip-role
-  apiGroup: "" 
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-cluster
-  name: kube-vip-cluster
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-cluster
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-cluster
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-cluster
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: plndr/kube-vip:0.1.3
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        env:
-          - name: vip_interface
-            value: "ens192"
-          - name: vip_configmap
-            value: "plndr" 
-          - name: vip_arp
-            value: "true"
-          - name: vip_loglevel
-            value: "5"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-      serviceAccountName: vip
-status: {}

example/deploy/0.1.4.yaml

@@ -1,83 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: vip
----
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: vip-role
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "create", "update", "list", "put"]
-  - apiGroups: [""]
-    resources: ["configmaps", "endpoints"]
-    verbs: ["watch", "get"]
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: vip-role-bind
-subjects:
-  - kind: ServiceAccount
-    name: vip 
-    apiGroup: ""
-roleRef:
-  kind: Role
-  name: vip-role
-  apiGroup: "" 
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-cluster
-  name: kube-vip-cluster
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-cluster
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-cluster
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-cluster
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: plndr/kube-vip:0.1.4
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        env:
-          - name: vip_interface
-            value: "ens192"
-          - name: vip_configmap
-            value: "plndr" 
-          - name: vip_arp
-            value: "true"
-          - name: vip_loglevel
-            value: "5"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-      serviceAccountName: vip
-status: {}

example/deploy/0.3.5.yaml

@@ -1,55 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "true"
-        - name: vip_interface
-          value: eth0
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: svc_enable
-          value: "true"
-        - name: vip_startleader
-          value: "false"
-        - name: vip_addpeerstolb
-          value: "true"
-        - name: vip_localpeer
-          value: ip-172-20-40-207:172.20.40.207:10000
-        - name: vip_address
-        image: plndr/kube-vip:v0.3.5
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0

example/deployment.yaml

@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:latest
+        ports:
+        - containerPort: 80

example/service-interface-annotation.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-interface-ens192-service
+  annotations:
+    kube-vip.io/serviceInterface: ens192
+spec:
+  selector:
+    app: nginx
+  type: LoadBalancer
+  ports:
+  - name: http
+    port: 80
+    targetPort: 80
+    protocol: TCP

example/service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-service
+spec:
+  selector:
+    app: nginx
+  type: LoadBalancer
+  ports:
+  - name: http
+    port: 80
+    targetPort: 80
+    protocol: TCP

go.mod

@@ -1,66 +1,154 @@
 module github.com/kube-vip/kube-vip
 
-go 1.14
+go 1.26.4
 
 require (
-	github.com/armon/go-metrics v0.3.8 // indirect
-	github.com/davecgh/go-spew v1.1.1
-	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
-	github.com/fatih/color v1.10.0 // indirect
-	github.com/ghodss/yaml v1.0.0
-	github.com/golang/protobuf v1.5.2
-	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/uuid v1.2.0 // indirect
-	github.com/googleapis/gnostic v0.5.5 // indirect
-	github.com/hashicorp/go-hclog v0.16.0 // indirect
-	github.com/hashicorp/go-immutable-radix v1.3.0 // indirect
-	github.com/hashicorp/go-msgpack v1.1.5 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
-	github.com/hashicorp/raft v1.3.1
-	github.com/imdario/mergo v0.3.12 // indirect
-	github.com/insomniacslk/dhcp v0.0.0-20210621130208-1cac67f12b1e
+	github.com/cloudflare/ipvs v0.12.0
+	github.com/containernetworking/plugins v1.9.1
+	github.com/docker/docker v28.5.2+incompatible
+	github.com/florianl/go-conntrack v0.7.0
+	github.com/google/go-cmp v0.7.0
+	github.com/google/go-containerregistry v0.22.1
+	github.com/google/nftables v0.3.0
+	github.com/gookit/slog v0.7.1
+	github.com/huin/goupnp v1.3.0
+	github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82
 	github.com/jpillora/backoff v1.0.0
-	github.com/json-iterator/go v1.1.11 // indirect
-	github.com/k-sone/critbitgo v1.4.0 // indirect
-	github.com/kamhlos/upnp v0.0.0-20210324072331-5661950dff08
-	github.com/magiconair/properties v1.8.5 // indirect
-	github.com/mdlayher/ndp v0.0.0-20200602162440-17ab9e3e5567
-	github.com/mdlayher/raw v0.0.0-20210412142147-51b895745faf // indirect
-	github.com/mitchellh/mapstructure v1.4.1 // indirect
-	github.com/onsi/ginkgo v1.11.0
-	github.com/onsi/gomega v1.7.0
-	github.com/osrg/gobgp v2.0.0+incompatible
-	github.com/packethost/packngo v0.13.0
-	github.com/pelletier/go-toml v1.9.0 // indirect
+	github.com/mdlayher/ndp v1.1.0
+	github.com/onsi/ginkgo/v2 v2.32.2
+	github.com/onsi/gomega v1.43.0
+	github.com/osrg/gobgp/v4 v4.9.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.10.0
-	github.com/prometheus/common v0.23.0 // indirect
-	github.com/sirupsen/logrus v1.8.1
-	github.com/spf13/afero v1.6.0 // indirect
-	github.com/spf13/cast v1.3.1 // indirect
-	github.com/spf13/cobra v1.1.3
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/viper v1.7.1 // indirect
-	github.com/vishvananda/netlink v1.1.1-0.20200221165523-c79a4b7b4066
-	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
-	golang.org/x/crypto v0.0.0-20210503195802-e9a32991a82e // indirect
-	golang.org/x/net v0.0.0-20210505024714-0287a6fb4125
-	golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c // indirect
-	golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea
-	golang.org/x/term v0.0.0-20210503060354-a79de5458b56 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20210505142820-a42aa055cf76 // indirect
-	google.golang.org/grpc v1.37.0 // indirect
-	gopkg.in/ini.v1 v1.62.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/api v0.21.0
-	k8s.io/apimachinery v0.21.0
-	k8s.io/client-go v0.21.0
-	k8s.io/klog v1.0.0
-	k8s.io/klog/v2 v2.8.0
-	k8s.io/utils v0.0.0-20210305010621-2afb4311ab10 // indirect
-	sigs.k8s.io/kind v0.10.0
-	sigs.k8s.io/structured-merge-diff/v4 v4.1.1 // indirect
+	github.com/prometheus/client_golang v1.24.1
+	github.com/spf13/cobra v1.10.2
+	github.com/stretchr/testify v1.12.1
+	github.com/vishvananda/netlink v1.3.2-0.20260830232854-cf01b55a4a4b
+	github.com/vishvananda/netns v0.0.5
+	go.etcd.io/etcd/api/v3 v3.7.1
+	go.etcd.io/etcd/client/pkg/v3 v3.7.1
+	go.etcd.io/etcd/client/v3 v3.7.1
+	go.uber.org/zap v1.28.0
+	golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
+	golang.org/x/sync v0.23.0
+	golang.org/x/sys v0.48.0
+	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
+	google.golang.org/grpc v1.83.2
+	gopkg.in/yaml.v3 v3.0.1
+	k8s.io/api v0.36.4
+	k8s.io/apimachinery v0.36.4
+	k8s.io/client-go v0.36.4
+	k8s.io/klog/v2 v2.140.0
+	sigs.k8s.io/kind v0.33.0
+	sigs.k8s.io/yaml v1.6.0
 )
 
-replace github.com/osrg/gobgp v2.0.0+incompatible => github.com/osrg/gobgp v0.0.0-20191101114856-a42a1a5f6bf0
+require (
+	al.essio.dev/pkg/shellescape v1.5.1 // indirect
+	github.com/BurntSushi/toml v1.5.0 // indirect
+	github.com/Masterminds/semver/v3 v3.4.0 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/coreos/go-semver v0.3.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.7.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/go-connections v0.7.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/eapache/channels v1.1.0 // indirect
+	github.com/eapache/queue v1.1.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.13.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
+	github.com/gaissmai/bart v0.26.1 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/gnostic-models v0.7.0 // indirect
+	github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gookit/color v1.6.1 // indirect
+	github.com/gookit/goutil v0.7.6 // indirect
+	github.com/gookit/gsr v0.1.1 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/josharian/native v1.1.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/k-sone/critbitgo v1.4.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/mailru/easyjson v0.9.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mdlayher/genetlink v1.3.2 // indirect
+	github.com/mdlayher/netlink v1.8.0 // indirect
+	github.com/mdlayher/packet v1.1.2 // indirect
+	github.com/mdlayher/socket v0.5.1 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/sys/atomicwriter v0.1.0 // indirect
+	github.com/moby/term v0.5.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
+	github.com/morikuni/aec v1.1.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect
+	github.com/pelletier/go-toml v1.9.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.70.1 // indirect
+	github.com/prometheus/procfs v0.21.1 // indirect
+	github.com/sagikazarmark/locafero v0.7.0 // indirect
+	github.com/segmentio/fasthash v1.0.3 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.12.0 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/spf13/pflag v1.0.10 // indirect
+	github.com/spf13/viper v1.20.1 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
+	go.opentelemetry.io/otel v1.44.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect
+	go.opentelemetry.io/otel/metric v1.44.0 // indirect
+	go.opentelemetry.io/otel/trace v1.44.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.4 // indirect
+	go.yaml.in/yaml/v3 v3.0.5 // indirect
+	golang.org/x/crypto v0.55.0 // indirect
+	golang.org/x/mod v0.39.0 // indirect
+	golang.org/x/net v0.58.0 // indirect
+	golang.org/x/oauth2 v0.36.0 // indirect
+	golang.org/x/term v0.45.0 // indirect
+	golang.org/x/text v0.41.0 // indirect
+	golang.org/x/time v0.14.0 // indirect
+	golang.org/x/tools v0.49.0 // indirect
+	golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
+	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect
+	k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect
+	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect
+)

kubernetes-control-plane.md

@@ -1,147 +0,0 @@
-# Load Balancing a Kubernetes Cluster (Control-Plane)
-
-This document covers all of the details for using `kube-vip` to build a HA Kubernetes cluster
-
-`tl;dr version`
-- Generate/modify first node `kube-vip` config/manifest
-- `init` first node
-- `join` remaining nodes
-- Add remaining config/manifests
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.75 |
-| controlPlane01 | 10.0.0.70 |
-| controlPlane02 | 10.0.0.71 |
-| controlPlane03 | 10.0.0.72 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.17.0.
-
-### Generate the `kube-vip` configuration
-
-Make sure that the config directory exists: `sudo mkdir -p /etc/kube-vip/`, this directory can be any directory however the `hostPath` in the manifest will need modifying to point to the correct path.
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1.5 sample config | sudo tee /etc/kube-vip/config.yaml
-```
-
-### Modify the configuration
-
-**Cluster Configuration**
-Modify the `remotePeers` to point to the correct addresses of the other two nodes, ensure that their `id` is unique otherwise this will confuse the raft algorithm. The `localPeer` should be the configuration of the current node (`controlPlane01`), which is where this instance of the cluster will run. 
-
-As this node will be the first node, it will need to elect itself leader as until this occurs the VIP won’t be activated!
-
-`startAsLeader: true`
-
-**VIP Config**
-We will need to set our VIP address to `192.168.0.75` and to ensure all hosts are updated when the VIP moves we will enable ARP broadcasts `gratuitousARP: true`
-
-**Load Balancer**
-We will configure the load balancer to sit on the standard API-Server port `6443` and we will configure the backends to point to the API-servers that will be configured to run on port `6444`. Also for the Kubernetes Control Plane we will configure the load balancer to be of `type: tcp`.
-
-We can also use `6443` for both the VIP and the API-Servers, in order to do this we need to specify that the api-server is bound to it's local IP. To do this we use the `--apiserver-advertise-address` flag as part of the `init`, this means that we can then bind the same port to the VIP and we wont have a port conflict.
-
-**config.yaml**
-
-`user@controlPlane01:/etc/kube-vip$ cat config.yaml`
-
-...
-
-``` 
-remotePeers:
-- id: server2
-  address: 192.168.0.71
-  port: 10000
-- id: server3
-  address: 192.168.0.72
-  port: 10000
-localPeer:
-  id: server1
-  address: 192.168.0.70
-  port: 10000
-vip: 192.168.0.75
-gratuitousARP: true
-singleNode: false
-startAsLeader: true
-interface: ens192
-loadBalancers:
-- name: Kubernetes Control Plane
-  type: tcp
-  port: 6443
-  bindToVip: true
-  backends:
-  - port: 6444
-    address: 192.168.0.70
-  - port: 6444
-    address: 192.168.0.71
-  - port: 6444
-    address: 192.168.0.72
-```
-
-### First Node
-
-To generate the basic Kubernetes static pod `yaml` configuration:
-
-Make sure that the manifest directory exists: `sudo mkdir -p /etc/kubernetes/manifests/`
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1.5 sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Ensure that `image: plndr/kube-vip:<x>` is modified to point to a specific version (`0.1.5` at the time of writing), refer to [docker hub](https://hub.docker.com/r/plndr/kube-vip/tags) for details. Also ensure that the `hostPath` points to the correct `kube-vip` configuration, if it isn’t the above path.
-
-The **vip** is set to `192.168.0.75` and this first node will elect itself as leader, and as part of the `kubeadm init` it will use the VIP in order to speak back to the initialising api-server.
-
-`sudo kubeadm init --control-plane-endpoint “192.168.0.75:6443” --apiserver-bind-port 6444 --upload-certs --kubernetes-version “v1.17.0”`
-
-Once this node is up and running we will be able to see the control-plane pods, including the `kube-vip` pod:
-
-```
-$ kubectl get pods -A
-NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
-<...>
-kube-system   kube-vip-controlplane01                  1/1     Running   0          10m
-```
-
-### Remaining Nodes
-
-We first will need to create the `kube-vip` configuration that resides in `/etc/kube-vip/config.yaml` or we can regenerate it from scratch using the above example. Ensure that the configuration is almost identical with the `localPeer` and `remotePeers` sections are updated for each node. Finally, ensure that the remaining nodes will behave as standard cluster nodes by setting `startAsLeader: false`.
-
-At this point **DON’T** generate the manifests, this is due to some bizarre `kubeadm/kubelet` behaviour.
-
-```
-  kubeadm join 192.168.0.75:6443 --token <tkn> \
-    --discovery-token-ca-cert-hash sha256:<hash> \
-    --control-plane --certificate-key <key> 
-
-```
-
-**After** this node has been added to the cluster, we can add the manifest to also add this node as a `kube-vip` member. (Adding the manifest afterwards doesn’t interfere with `kubeadm`). 
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1.5 sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Once this node is added we will be able to see that the `kube-vip` pod is up and running as expected:
-
-```
-user@controlPlane01:~$ kubectl get pods -A | grep vip
-kube-system   kube-vip-controlplane01                  1/1     Running             1          16m
-kube-system   kube-vip-controlplane02                  1/1     Running             0          18m
-kube-system   kube-vip-controlplane03                  1/1     Running             0          20m
-
-```
-
-If we look at the logs, we can see that the VIP is running on the second node and we’re waiting for our third node to join the cluster:
-
-```
-$ kubectl logs kube-vip-controlplane02  -n kube-system
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-
-```

main.go

@@ -1,6 +1,10 @@
 package main
 
-import "github.com/kube-vip/kube-vip/cmd"
+import (
+	"os"
+
+	"github.com/kube-vip/kube-vip/cmd"
+)
 
 // Version is populated from the Makefile and is tied to the release TAG
 var Version string
@@ -9,7 +13,8 @@ var Version string
 var Build string
 
 func main() {
+
 	cmd.Release.Version = Version
 	cmd.Release.Build = Build
-	cmd.Execute()
+	os.Exit(cmd.Execute())
 }

pkg/arp/arp.go

@@ -0,0 +1,296 @@
+package arp
+
+import (
+	"context"
+	"fmt"
+	log "log/slog"
+	"sync"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	"github.com/vishvananda/netlink"
+)
+
+const linkSubscriptionBuffer = 64
+
+type Manager struct {
+	mu        sync.Mutex
+	instances map[string]*Instance
+	config    *kubevip.Config
+}
+
+type Instance struct {
+	network vip.Network
+	ndp     *vip.NdpResponder
+	counter int
+}
+
+func NewManager(config *kubevip.Config) *Manager {
+	if config.ArpBroadcastRate < 500 {
+		log.Warn("[ARP manager] arp broadcast rate is too low", "rate (ms)", config.ArpBroadcastRate, "setting to (ms)", "3000")
+		config.ArpBroadcastRate = 3000
+	}
+	return &Manager{
+		instances: make(map[string]*Instance),
+		config:    config,
+	}
+}
+
+func NewInstance(network vip.Network, ndp *vip.NdpResponder) *Instance {
+	return &Instance{
+		ndp:     ndp,
+		network: network,
+		counter: 1,
+	}
+}
+
+func (i *Instance) Name() string {
+	return i.network.ARPName()
+}
+
+func (m *Manager) Insert(instance *Instance) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	existing := m.instances[instance.Name()]
+	if existing == nil {
+		m.instances[instance.Name()] = instance
+		log.Info("[ARP manager] inserting ARP/NDP instance", "name", instance.Name())
+		return
+	}
+	existing.counter++
+}
+
+func (m *Manager) Remove(instance *Instance) {
+	m.RemoveWithIPDelete(instance, true)
+}
+
+// RemoveOnLeadershipLoss removes an ARP instance when leadership is lost
+func (m *Manager) RemoveOnLeadershipLoss(instance *Instance) {
+	// Use the inverse of PreserveVIPOnLeadershipLoss to decide whether to delete the IP
+	// If preserve is true, don't delete IP (deleteIP = false)
+	// If preserve is false, delete IP (deleteIP = true), This is the legacy behavior
+	deleteIP := !m.config.PreserveVIPOnLeadershipLoss
+	m.RemoveWithIPDelete(instance, deleteIP)
+}
+
+func (m *Manager) RemoveWithIPDelete(instance *Instance, deleteIP bool) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	i := m.instances[instance.Name()]
+	if i != nil {
+		i.counter--
+		if i.counter == 0 {
+			log.Info("[ARP manager] removing ARP/NDP instance", "name", instance.Name())
+			if deleteIP {
+				if _, err := instance.network.DeleteIP(); err != nil {
+					log.Error("failed to delete IP", "address", instance.network.IP(), "err", err)
+				}
+			}
+			delete(m.instances, instance.Name())
+		}
+	} else {
+		log.Warn("[ARP manager] unable to remove the instance - instance not found", "name", instance.Name())
+	}
+}
+
+func (m *Manager) Count(name string) int {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	i := m.instances[name]
+	if i != nil {
+		return i.counter
+	}
+	return 0
+}
+
+func (m *Manager) StartAdvertisement(ctx context.Context, killFunc func()) {
+	if m.config.LoseLeadership {
+		var wg sync.WaitGroup
+		defer wg.Wait()
+
+		log.Info("[ARP manager] starting watching network device", "interface", m.config.Interface)
+
+		duration := time.Duration(m.config.LoseLeadershipTimeoutSeconds) * time.Second
+		timeout := time.NewTimer(duration)
+		timeout.Stop()
+
+		wg.Go(func() {
+			select {
+			case <-timeout.C:
+				killFunc()
+			case <-ctx.Done():
+				return
+			}
+		})
+
+		wg.Go(func() {
+			if err := watch(ctx, m.config.Interface, func(s netlink.LinkOperState) {
+				if isUp(s) {
+					timeout.Stop()
+					return
+				}
+				timeout.Reset(duration)
+			}); err != nil {
+				log.Error("[ARP manager] stopped watching interface", "err", err)
+				killFunc()
+			}
+		})
+	}
+
+	log.Info("[ARP manager] starting ARP/NDP advertisement")
+
+	ticker := time.NewTicker(time.Duration(m.config.ArpBroadcastRate) * time.Millisecond)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done(): // if cancel() execute
+			return
+		case <-ticker.C: // send gratuitous ARP/NDP on each tick
+			m.advertiseAll()
+		}
+	}
+}
+
+func (m *Manager) advertiseAll() {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	for _, instance := range m.instances {
+		if instance.counter > 0 {
+			ensureIPAndSendGratuitous(instance)
+		} else if _, err := instance.network.DeleteIP(); err != nil {
+			log.Error("[ARP manager] failed to delete IP", "address", instance.network.IP(), "err", err)
+		}
+	}
+}
+
+// ensureIPAndSendGratuitous - adds IP to the interface if missing, and send
+// either a gratuitous ARP or gratuitous NDP. Re-adds the interface if it is IPv6
+// and in a dadfailed state.
+func ensureIPAndSendGratuitous(instance *Instance) {
+	iface := instance.network.Interface()
+	ipString := instance.network.IP()
+
+	// Check if IP is dadfailed
+	if instance.network.IsDADFAIL() {
+		log.Warn("IP address is in dadfailed state, removing config", "ip", ipString, "interface", iface)
+		deleted, err := instance.network.DeleteIP()
+		if err != nil {
+			log.Warn(err.Error())
+		}
+		if deleted {
+			log.Info("deleted and recreating address with NODAD flag to skip DAD", "IP", ipString, "interface", iface)
+			// Re-add immediately without DAD check since we're recovering from DADFAILED
+			// The AddIP function will set IFA_F_NODAD flag for IPv6 addresses when skipDAD=true
+			if _, err := instance.network.AddIP(false, true); err != nil {
+				log.Error("failed to recreate address after DADFAILED", "IP", ipString, "interface", iface, "err", err)
+			} else {
+				log.Info("successfully recreated address after DADFAILED recovery", "IP", ipString, "interface", iface)
+			}
+		}
+		// Return early after DADFAILED recovery to avoid double IP addition
+		return
+	}
+
+	// Normal case: add IP with precheck and normal DAD process
+	if added, err := instance.network.AddIP(true, false); err != nil {
+		log.Warn(err.Error())
+	} else if added {
+		log.Warn("Re-applied the VIP configuration", "ip", ipString, "interface", iface)
+	}
+
+	if utils.IsIPv6(ipString) {
+		// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
+		if instance.ndp == nil {
+			log.Error("NDP responder was not created")
+		} else {
+			err := instance.ndp.SendGratuitous(ipString)
+			if err != nil {
+				log.Warn(err.Error())
+			}
+		}
+	} else {
+		// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
+		err := vip.ARPSendGratuitous(ipString, iface)
+		if err != nil {
+			log.Warn(err.Error())
+		}
+	}
+}
+
+// watch subscribing to the network interface events and calls handler
+func watch(ctx context.Context, interfaceName string, operStateHandler func(netlink.LinkOperState)) error {
+	ifname, err := netlink.LinkByName(interfaceName)
+	if err != nil {
+		return fmt.Errorf("failed to watch interface %q: %w", interfaceName, err)
+	}
+
+	// verify if this interface is physical device
+	if _, ok := ifname.(*netlink.Device); !ok {
+		return fmt.Errorf("interface %s is not physical, ignoring", interfaceName)
+	}
+
+	// The subscription is buffered and drained on exit: netlink parks its reader
+	// goroutine on an unread send, which closing done alone does not release.
+	events := make(chan netlink.LinkUpdate, linkSubscriptionBuffer)
+	done := make(chan struct{})
+
+	if err := netlink.LinkSubscribe(events, done); err != nil {
+		return fmt.Errorf("failed to subscribe to the interface events: %w", err)
+	}
+	defer func() {
+		close(done)
+		drainLinkUpdates(events)
+	}()
+
+	//  handle initial state
+	operStateHandler(ifname.Attrs().OperState)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case event, ok := <-events:
+			if !ok {
+				return fmt.Errorf("interface events channel closed")
+			}
+
+			attrs := event.Attrs()
+			// LinkSubscribe captures events for all network devices found
+			// so we only care about vip interface
+			if ifname.Attrs().Name != attrs.Name {
+				continue
+			}
+			log.Debug("handling device change", "state", attrs.OperState)
+			operStateHandler(attrs.OperState)
+		}
+	}
+}
+
+func isUp(operState netlink.LinkOperState) bool {
+	return operState == netlink.OperUp
+}
+
+// drainLinkUpdates releases a netlink sender that is parked on an unread update
+// so its goroutine can observe the closed subscription and exit.
+func drainLinkUpdates(events <-chan netlink.LinkUpdate) {
+	timer := time.NewTimer(100 * time.Millisecond)
+	defer timer.Stop()
+
+	for {
+		select {
+		case _, ok := <-events:
+			if !ok {
+				return
+			}
+		case <-timer.C:
+			return
+		}
+	}
+}

pkg/arp/arp_test.go

@@ -0,0 +1,119 @@
+package arp
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/vishvananda/netlink"
+	v1 "k8s.io/api/core/v1"
+)
+
+func TestDrainLinkUpdatesReleasesParkedSender(t *testing.T) {
+	events := make(chan netlink.LinkUpdate)
+	sent := make(chan struct{})
+	go func() {
+		events <- netlink.LinkUpdate{}
+		close(sent)
+	}()
+
+	drainLinkUpdates(events)
+
+	select {
+	case <-sent:
+	case <-time.After(time.Second):
+		t.Fatal("netlink sender is still parked on an unread link update")
+	}
+}
+
+// stubNetwork is a minimal vip.Network implementation; only ARPName matters here.
+type stubNetwork struct {
+	name          string
+	deleteStarted chan struct{}
+	releaseDelete chan struct{}
+}
+
+func (s *stubNetwork) AddIP(bool, bool, ...int) (bool, error) { return false, nil }
+func (s *stubNetwork) AddRoute(bool) (bool, error)            { return false, nil }
+func (s *stubNetwork) ReplaceRoute() error                    { return nil }
+func (s *stubNetwork) DeleteIP() (bool, error) {
+	if s.deleteStarted != nil {
+		close(s.deleteStarted)
+		<-s.releaseDelete
+	}
+	return true, nil
+}
+func (s *stubNetwork) DeleteRoute() error            { return nil }
+func (s *stubNetwork) UpdateRoutes() (bool, error)   { return false, nil }
+func (s *stubNetwork) IsSet() (*netlink.Addr, error) { return nil, nil }
+func (s *stubNetwork) IP() string                    { return "" }
+func (s *stubNetwork) CIDR() string                  { return "" }
+func (s *stubNetwork) IPisLinkLocal() bool           { return false }
+func (s *stubNetwork) PrepareRoute() *netlink.Route  { return nil }
+func (s *stubNetwork) RouteHash() string             { return "" }
+func (s *stubNetwork) SetIP(string) error            { return nil }
+func (s *stubNetwork) SetServicePorts(*v1.Service)   {}
+func (s *stubNetwork) Interface() string             { return "eth0" }
+func (s *stubNetwork) IsDADFAIL() bool               { return false }
+func (s *stubNetwork) IsDNS() bool                   { return false }
+func (s *stubNetwork) IsDDNS() bool                  { return false }
+func (s *stubNetwork) DDNSHostName() string          { return "" }
+func (s *stubNetwork) DNSName() string               { return "" }
+func (s *stubNetwork) SetMask(string) error          { return nil }
+func (s *stubNetwork) SetHasEndpoints(bool)          {}
+func (s *stubNetwork) HasEndpoints() bool            { return false }
+func (s *stubNetwork) ARPName() string               { return s.name }
+func (s *stubNetwork) GetPossibleSubnets() string    { return "" }
+func (s *stubNetwork) DHCPFamily() string            { return "" }
+func (s *stubNetwork) IPVSMark() uint32              { return 0 }
+
+// TestManagerInsertConcurrentFirstRegistrationsDoNotLoseClaims guards the
+// get-then-store race: two never-before-seen instances for the same ARP name
+// registering concurrently must both be counted, not just the last writer.
+func TestManagerInsertConcurrentFirstRegistrationsDoNotLoseClaims(t *testing.T) {
+	m := NewManager(&kubevip.Config{ArpBroadcastRate: 3000})
+	const concurrent = 8
+
+	var wg sync.WaitGroup
+	for range concurrent {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			m.Insert(NewInstance(&stubNetwork{name: "shared"}, nil))
+		}()
+	}
+	wg.Wait()
+
+	if got := m.Count("shared"); got != concurrent {
+		t.Fatalf("Count() = %d, want %d claims registered", got, concurrent)
+	}
+}
+
+func TestManagerInsertDoesNotJoinEntryBeingRemoved(t *testing.T) {
+	m := NewManager(&kubevip.Config{ArpBroadcastRate: 3000})
+	deleteStarted := make(chan struct{})
+	releaseDelete := make(chan struct{})
+	first := NewInstance(&stubNetwork{name: "shared", deleteStarted: deleteStarted, releaseDelete: releaseDelete}, nil)
+	m.Insert(first)
+
+	removeDone := make(chan struct{})
+	go func() {
+		m.Remove(first)
+		close(removeDone)
+	}()
+	<-deleteStarted
+
+	insertDone := make(chan struct{})
+	go func() {
+		m.Insert(NewInstance(&stubNetwork{name: "shared"}, nil))
+		close(insertDone)
+	}()
+	close(releaseDelete)
+	<-removeDone
+	<-insertDone
+
+	if got := m.Count("shared"); got != 1 {
+		t.Fatalf("Count() = %d, want replacement claim registered", got)
+	}
+}

pkg/backend/backend.go

@@ -0,0 +1,107 @@
+package backend
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/k8s"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+type Entry struct {
+	Addr    string
+	Port    uint16
+	IsLocal bool
+}
+
+type Map map[Entry]bool
+
+// kubeConfigPath is an explicitly configured kubeconfig used by Check when
+// set; static pod deployments configure it since neither admin.conf nor
+// in-cluster config are available there.
+var (
+	kubeConfigPath string
+	pathMtx        sync.Mutex
+)
+
+// SetKubeConfigPath configures the kubeconfig used by backend health checks.
+func SetKubeConfigPath(path string) {
+	pathMtx.Lock()
+	defer pathMtx.Unlock()
+	kubeConfigPath = path
+}
+
+func (e *Entry) Check() bool {
+	var client *kubernetes.Clientset
+	var err error
+	var config *rest.Config
+
+	adminConfigPath := "/etc/kubernetes/admin.conf"
+	// TODO: add one more switch case of homeConfigPath if there is such scenario in future
+	// homeConfigPath := filepath.Join(os.Getenv("HOME"), ".kube", "config")
+
+	var k8sAddr string
+	if utils.IsIPv6(e.Addr) {
+		k8sAddr = fmt.Sprintf("[%s]:%v", e.Addr, e.Port)
+	} else {
+		k8sAddr = fmt.Sprintf("%s:%v", e.Addr, e.Port)
+	}
+
+	switch {
+	case kubeConfigPath != "" && utils.FileExists(kubeConfigPath):
+		config, err = k8s.NewRestConfig(kubeConfigPath, false, k8sAddr)
+		if err != nil {
+			log.Error("create k8s REST config", "path", kubeConfigPath, "err", err)
+			return false
+		}
+	case utils.FileExists(adminConfigPath):
+		config, err = k8s.NewRestConfig(adminConfigPath, false, k8sAddr)
+		if err != nil {
+			log.Error("create k8s REST config", "path", adminConfigPath, "err", err)
+			return false
+		}
+	default:
+		config, err = k8s.NewRestConfig("", true, k8sAddr)
+		if err != nil {
+			log.Error("create k8s REST config", "err", err)
+			return false
+		}
+	}
+
+	client, err = k8s.NewClientset(config)
+	if err != nil {
+		log.Error("create k8s client", "err", err)
+		return false
+	}
+
+	_, err = client.DiscoveryClient.ServerVersion()
+	if err != nil {
+		log.Error("discover k8s version", "err", err)
+		return false
+	}
+	return true
+}
+
+func Watch(ctx context.Context, interval int, tickAction func()) {
+	if interval <= 0 {
+		interval = 5
+	}
+
+	ticker := time.NewTicker(time.Second * time.Duration(interval))
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			tickAction()
+		}
+	}
+}

pkg/bgp/hosts.go

@@ -3,46 +3,80 @@ package bgp
 import (
 	"context"
 	"fmt"
+	log "log/slog"
 	"net"
 
-	api "github.com/osrg/gobgp/api"
+	"github.com/osrg/gobgp/v4/pkg/apiutil"
 )
 
 // AddHost will update peers of a host
-func (b *Server) AddHost(addr string) (err error) {
-	ip, _, err := net.ParseCIDR(addr)
-	if err != nil {
-		return err
+func (b *Server) AddHost(ctx context.Context, addr string, object string) error {
+	b.mtx.Lock()
+	defer b.mtx.Unlock()
+
+	objects, exists := b.tracker[addr]
+
+	if !exists {
+		b.tracker[addr] = make(map[string]bool)
+		objects = b.tracker[addr]
+
+		ip, _, err := net.ParseCIDR(addr)
+		if err != nil {
+			return err
+		}
+
+		p := b.getPath(ip)
+		if p == nil {
+			return fmt.Errorf("failed to get path for %v", ip)
+		}
+
+		if _, err := b.s.AddPath(apiutil.AddPathRequest{
+			Paths: []*apiutil.Path{p},
+		}); err != nil {
+			return err
+		}
+		log.Debug("[BGP] added host", "addr", addr, "cnt", len(objects)+1, "object", object)
 	}
 
-	p := b.getPath(ip)
-	if p == nil {
-		return fmt.Errorf("failed to get path for %v", ip)
-	}
+	objects[object] = true
 
-	_, err = b.s.AddPath(context.Background(), &api.AddPathRequest{
-		Path: p,
-	})
-
-	if err != nil {
-		return err
-	}
-
-	return
+	return nil
 }
 
 // DelHost will inform peers to remove a host
-func (b *Server) DelHost(addr string) (err error) {
+func (b *Server) DelHost(ctx context.Context, addr string, object string) error {
+	b.mtx.Lock()
+	defer b.mtx.Unlock()
+
+	objects, exists := b.tracker[addr]
+	if !exists {
+		log.Debug("[BGP] deleting host - nothing to delete", "addr", addr, "object", object)
+		return nil
+	}
+
 	ip, _, err := net.ParseCIDR(addr)
 	if err != nil {
 		return err
 	}
-	p := b.getPath(ip)
-	if p == nil {
-		return
+
+	delete(objects, object)
+
+	if len(objects) == 0 {
+		p := b.getPath(ip)
+		if p == nil {
+			return nil
+		}
+
+		if err := b.s.DeletePath(apiutil.DeletePathRequest{
+			Paths: []*apiutil.Path{p},
+		}); err != nil {
+			return err
+		}
+		delete(b.tracker, addr)
+		log.Debug("[BGP] deleted host", "addr", addr, "cnt", len(objects), "object", object)
+	} else {
+		log.Debug("[BGP] deleting from tracker only", "addr", addr, "object", object)
 	}
 
-	return b.s.DeletePath(context.Background(), &api.DeletePathRequest{
-		Path: p,
-	})
+	return nil
 }

pkg/bgp/peers.go

@@ -3,37 +3,46 @@ package bgp
 import (
 	"context"
 	"fmt"
+	log "log/slog"
 	"net"
+	"net/netip"
 	"strconv"
-	"strings"
+	"time"
 
-	"github.com/golang/protobuf/ptypes"
-	"github.com/golang/protobuf/ptypes/any"
-	api "github.com/osrg/gobgp/api"
+	"github.com/jpillora/backoff"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	api "github.com/osrg/gobgp/v4/api"
+
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/osrg/gobgp/v4/pkg/apiutil"
+	"github.com/osrg/gobgp/v4/pkg/config/oc"
+	bgp "github.com/osrg/gobgp/v4/pkg/packet/bgp"
+	"github.com/osrg/gobgp/v4/pkg/server"
 )
 
-//AddPeer will add peers to the BGP configuration
-func (b *Server) AddPeer(peer Peer) (err error) {
-	port := 179
+const defaultBGPPort uint32 = 179
 
-	if t := strings.SplitN(peer.Address, ":", 2); len(t) == 2 {
-		peer.Address = t[0]
-
-		if port, err = strconv.Atoi(t[1]); err != nil {
-			return fmt.Errorf("Unable to parse port '%s' as int: %s", t[1], err)
-		}
+// AddPeer will add peers to the BGP configuration
+func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error) {
+	remotePort := defaultBGPPort
+	if peer.Port != 0 {
+		remotePort = uint32(peer.Port)
 	}
 
 	p := &api.Peer{
 		Conf: &api.PeerConf{
-			NeighborAddress: peer.Address,
-			PeerAs:          peer.AS,
-			AuthPassword:    peer.Password,
+			NeighborAddress:   peer.Address,
+			PeerAsn:           peer.AS,
+			NeighborInterface: peer.Interface,
+			AuthPassword:      peer.Password,
 		},
 
 		Timers: &api.Timers{
 			Config: &api.TimersConfig{
-				ConnectRetry: 10,
+				ConnectRetry:      10,
+				HoldTime:          b.c.HoldTime,
+				KeepaliveInterval: b.c.KeepaliveInterval,
 			},
 		},
 
@@ -46,95 +55,273 @@ func (b *Server) AddPeer(peer Peer) (err error) {
 		Transport: &api.Transport{
 			MtuDiscovery:  true,
 			RemoteAddress: peer.Address,
-			RemotePort:    uint32(port),
+			RemotePort:    remotePort,
 		},
 	}
 
-	// if b.c.SourceIP != "" {
-	// 	p.Transport.LocalAddress = b.c.SourceIP
-	// }
+	if peer.BFDEnabled {
+		p.Bfd = &api.BfdPeerConfig{
+			Enabled:                  true,
+			DesiredMinimumTxInterval: peer.BFDTransmitInterval,
+			RequiredMinimumReceive:   peer.BFDReceiveInterval,
+			DetectionMultiplier:      peer.BFDDetectMultiplier,
+			Port:                     3784, // TODO: Should this be configurable??
+		}
+	}
 
-	// if b.c.SourceIF != "" {
-	// 	p.Transport.BindInterface = b.c.SourceIF
-	// }
+	if peer.Interface != "" {
+		neighborAddress, err := getIPv6LinkLocalNeighborAddress(ctx, peer.Interface)
+		if err != nil {
+			return fmt.Errorf("failed to get link-local address of interface %s: %w", peer.Interface, err)
+		}
 
-	return b.s.AddPeer(context.Background(), &api.AddPeerRequest{
-		Peer: p,
-	})
+		p.State = &api.PeerState{
+			NeighborAddress: neighborAddress,
+		}
+	}
+
+	mpBGP := b.c.MpbgpNexthop
+
+	if peer.MpbgpNexthop != "" {
+		mpBGP = peer.MpbgpNexthop
+	}
+
+	if mpBGP != "" {
+		ipv4Address, ipv6Address, err := peer.FindMpbgpAddresses(p, b.c)
+		if err != nil {
+			log.Error("failed to get MP-BGP addresses, will not us MP-BGP for this host", "error", err)
+			b.setPeerSource(p)
+		} else {
+			p.AfiSafis = []*api.AfiSafi{
+				{
+					Config: &api.AfiSafiConfig{
+						Family: &api.Family{
+							Afi:  api.Family_AFI_IP,
+							Safi: api.Family_SAFI_UNICAST,
+						},
+						Enabled: true,
+					},
+				},
+				{
+					Config: &api.AfiSafiConfig{
+						Family: &api.Family{
+							Afi:  api.Family_AFI_IP6,
+							Safi: api.Family_SAFI_UNICAST,
+						},
+						Enabled: true,
+					},
+				},
+			}
+
+			peer.SetMpbgpOptions(b.c)
+
+			mask := strconv.Itoa(vip.DefaultMaskIPv6)
+			address := ipv4Address
+			family := api.Family_AFI_IP
+			if utils.IsIPv4(p.Conf.NeighborAddress) {
+				mask = strconv.Itoa(vip.DefaultMaskIPv4)
+				address = ipv6Address
+				family = api.Family_AFI_IP6
+			}
+
+			err = b.s.AddDefinedSet(ctx, &api.AddDefinedSetRequest{
+				DefinedSet: &api.DefinedSet{
+					DefinedType: api.DefinedType_DEFINED_TYPE_NEIGHBOR,
+					Name:        fmt.Sprintf("peer-%s", p.Conf.NeighborAddress),
+					List:        []string{fmt.Sprintf("%s/%s", p.Conf.NeighborAddress, mask)},
+				},
+			})
+			if err != nil {
+				return fmt.Errorf("failed to add defined set: %v", err)
+			}
+
+			if address != "" {
+				if err := insertPolicy(ctx, b.s, address, p, family); err != nil {
+					return fmt.Errorf("failed to add policy: %w", err)
+				}
+			}
+		}
+	} else {
+		b.setPeerSource(p)
+	}
+
+	if err := b.s.AddPeer(ctx, &api.AddPeerRequest{Peer: p}); err != nil {
+		return fmt.Errorf("failed to add peer: %v", err)
+	}
+	log.Info("[BGP]", "peer", p.Conf.NeighborAddress, "AS", p.Conf.PeerAsn, "BFD", p.Bfd)
+	return nil
 }
 
-func (b *Server) getPath(ip net.IP) *api.Path {
-	var pfxLen uint32 = 32
-	if ip.To4() == nil {
-		if !b.c.IPv6 {
+func (b *Server) setPeerSource(p *api.Peer) {
+	if b.c.SourceIP != "" {
+		p.Transport.LocalAddress = b.c.SourceIP
+	}
+
+	if b.c.SourceIF != "" {
+		p.Transport.BindInterface = b.c.SourceIF
+	}
+}
+
+func (b *Server) getPath(ip net.IP) *apiutil.Path {
+	isV6 := ip.To4() == nil
+
+	if !isV6 {
+		prefix, err := bgp.NewIPAddrPrefix(netip.MustParsePrefix(
+			fmt.Sprintf("%s/%d", ip.String(), vip.DefaultMaskIPv4),
+		))
+		if err != nil {
 			return nil
 		}
 
-		pfxLen = 128
+		nh, err := bgp.NewPathAttributeNextHop(netip.MustParseAddr("0.0.0.0"))
+		if err != nil {
+			return nil
+		}
+
+		return &apiutil.Path{
+			Family: bgp.RF_IPv4_UC,
+			Nlri:   prefix,
+			Attrs: []bgp.PathAttributeInterface{
+				bgp.NewPathAttributeOrigin(0),
+				nh,
+			},
+		}
 	}
 
-	nlri, _ := ptypes.MarshalAny(&api.IPAddressPrefix{
-		Prefix:    ip.String(),
-		PrefixLen: pfxLen,
-	})
-
-	a1, _ := ptypes.MarshalAny(&api.OriginAttribute{
-		Origin: 0,
-	})
-
-	var nh string
-	if b.c.NextHop != "" {
-		nh = b.c.NextHop
-	} else if b.c.SourceIP != "" {
-		nh = b.c.SourceIP
-	} else {
-		nh = b.c.RouterID
+	prefix, err := bgp.NewIPAddrPrefix(netip.MustParsePrefix(
+		fmt.Sprintf("%s/%d", ip.String(), vip.DefaultMaskIPv6),
+	))
+	if err != nil {
+		return nil
 	}
 
-	a2, _ := ptypes.MarshalAny(&api.NextHopAttribute{
-		NextHop: nh,
-	})
+	mpReach, err := bgp.NewPathAttributeMpReachNLRI(
+		bgp.RF_IPv6_UC,
+		[]bgp.PathNLRI{{NLRI: prefix}},
+		netip.MustParseAddr("::"),
+	)
+	if err != nil {
+		return nil
+	}
 
-	return &api.Path{
-		Family: &api.Family{
-			Afi:  api.Family_AFI_IP,
-			Safi: api.Family_SAFI_UNICAST,
+	return &apiutil.Path{
+		Family: bgp.RF_IPv6_UC,
+		Nlri:   prefix,
+		Attrs: []bgp.PathAttributeInterface{
+			bgp.NewPathAttributeOrigin(0),
+			mpReach,
 		},
-		Nlri:   nlri,
-		Pattrs: []*any.Any{a1, a2},
 	}
 }
-
-// ParseBGPPeerConfig - take a string and parses it into an array of peers
-func ParseBGPPeerConfig(config string) (bgpPeers []Peer, err error) {
-	peers := strings.Split(config, ",")
-	if len(peers) == 0 {
-		return nil, fmt.Errorf("No BGP Peer configurations found")
+func insertPolicy(ctx context.Context, s *server.BgpServer, address string, p *api.Peer, family api.Family_Afi) error {
+	familyType := "v4"
+	if family == api.Family_AFI_IP6 {
+		familyType = "v6"
 	}
 
-	for x := range peers {
-		peer := strings.Split(peers[x], ":")
-		if len(peer) != 4 {
-			return nil, fmt.Errorf("BGP Peer configuration format error <host>:<AS>:<password>:<multihop>")
-		}
-		ASNumber, err := strconv.Atoi(peer[1])
-		if err != nil {
-			return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
+	setName := fmt.Sprintf("peer-%s", p.Conf.NeighborAddress)
+	policyName := fmt.Sprintf("%s-%s", setName, familyType)
+
+	policy := &api.Policy{
+		Name: policyName,
+		Statements: []*api.Statement{
+			{
+				Conditions: &api.Conditions{
+					AfiSafiIn: []*api.Family{
+						{
+							Afi:  family,
+							Safi: api.Family_SAFI_UNICAST,
+						},
+					},
+					NeighborSet: &api.MatchSet{
+						Type: api.MatchSet_TYPE_ANY,
+						Name: setName,
+					},
+				},
+				Actions: &api.Actions{
+					RouteAction: api.RouteAction_ROUTE_ACTION_ACCEPT,
+					Nexthop: &api.NexthopAction{
+						Address: address,
+					},
+				},
+			},
+			{
+				Conditions: &api.Conditions{
+					NeighborSet: &api.MatchSet{
+						Type: api.MatchSet_TYPE_ANY,
+						Name: setName,
+					},
+				},
+				Actions: &api.Actions{
+					RouteAction: api.RouteAction_ROUTE_ACTION_ACCEPT,
+				},
+			},
+		},
+	}
+
+	err := s.AddPolicy(ctx, &api.AddPolicyRequest{
+		Policy: policy,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to add policy: %w", err)
+	}
+
+	err = s.AddPolicyAssignment(ctx, &api.AddPolicyAssignmentRequest{
+		Assignment: &api.PolicyAssignment{
+			Name:      "global",
+			Direction: api.PolicyDirection_POLICY_DIRECTION_EXPORT,
+			Policies: []*api.Policy{
+				{
+					Name: policy.Name,
+				},
+			},
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("failed to add policy assignment: %v", err)
+	}
+
+	return nil
+}
+
+func getIPv6LinkLocalNeighborAddress(ctx context.Context, peerInterface string) (string, error) {
+	neighCtx, neighCancel := context.WithTimeout(ctx, time.Minute)
+	defer neighCancel()
+
+	bo := backoff.Backoff{
+		Factor: 2,
+		Jitter: true,
+		Min:    1 * time.Second,
+		Max:    5 * time.Second,
+	}
+
+	maxAttempts := 20.0
+
+	var err error
+	for {
+		select {
+		case <-neighCtx.Done():
+			if err != nil {
+				return "", fmt.Errorf("failed to get link-local address of interface %s: %w", peerInterface, err)
+			}
+			return "", fmt.Errorf("failed to get link-local address of interface %s: %w", peerInterface, neighCtx.Err())
+		default:
+			dur := bo.Duration()
+			var neighborAddress string
+			neighborAddress, err = oc.GetIPv6LinkLocalNeighborAddress(peerInterface)
+			if err != nil && bo.Attempt() >= maxAttempts {
+				return "", fmt.Errorf("failed to get link-local address of interface %s: %w", peerInterface, err)
+			}
+			if neighborAddress != "" {
+				return neighborAddress, nil
+			}
+			t := time.NewTimer(dur)
+			select {
+			case <-neighCtx.Done():
+				t.Stop()
+			case <-t.C:
+			}
+		}
 
-		}
-		multiHop, err := strconv.ParseBool(peer[3])
-		if err != nil {
-			return nil, fmt.Errorf("BGP MultiHop format error (true/false) [%s]", peer[1])
-		}
-
-		peerConfig := Peer{
-			Address:  peer[0],
-			AS:       uint32(ASNumber),
-			Password: peer[2],
-			MultiHop: multiHop,
-		}
-
-		bgpPeers = append(bgpPeers, peerConfig)
 	}
-	return
 }

pkg/bgp/peers_config_test.go

@@ -0,0 +1,146 @@
+package bgp
+
+import (
+	"context"
+	"sync"
+	"testing"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	api "github.com/osrg/gobgp/v4/api"
+	gobgp "github.com/osrg/gobgp/v4/pkg/server"
+)
+
+func TestAddPeerConfiguresTransportOptions(t *testing.T) {
+	tests := []struct {
+		name          string
+		newServer     func(*testing.T) *Server
+		peer          kubevip.BGPPeer
+		wantPort      uint32
+		wantLocalAddr string
+		wantInterface string
+	}{
+		{
+			name: "configured remote port",
+			newServer: func(t *testing.T) *Server {
+				return newStartedTestBGPServer(t, kubevip.BGPConfig{
+					AS:       65000,
+					RouterID: "192.0.2.1",
+					Peers:    []kubevip.BGPPeer{{Address: "192.0.2.10", AS: 65001}},
+				})
+			},
+			peer:     kubevip.BGPPeer{Address: "192.0.2.10", AS: 65001, Port: 180},
+			wantPort: 180,
+		},
+		{
+			name: "configured source interface after MP-BGP fallback",
+			newServer: func(t *testing.T) *Server {
+				return newPeerTestServer(t, kubevip.BGPConfig{
+					AS:           65000,
+					RouterID:     "192.0.2.1",
+					SourceIF:     "lo",
+					MpbgpNexthop: "fixed",
+					Peers:        []kubevip.BGPPeer{{Address: "192.0.2.20", AS: 65001}},
+					MpbgpIPv4:    "",
+					MpbgpIPv6:    "",
+				})
+			},
+			peer:          kubevip.BGPPeer{Address: "192.0.2.20", AS: 65001},
+			wantInterface: "lo",
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			server := tt.newServer(t)
+			if err := server.AddPeer(context.Background(), tt.peer); err != nil {
+				t.Fatalf("AddPeer() error = %v", err)
+			}
+
+			peer := listTestPeer(t, server, tt.peer.Address)
+			if peer.GetTransport() == nil {
+				t.Fatal("configured peer has no transport")
+			}
+			if tt.wantPort != 0 && peer.GetTransport().GetRemotePort() != tt.wantPort {
+				t.Fatalf("remote port = %d, want %d", peer.GetTransport().GetRemotePort(), tt.wantPort)
+			}
+			if tt.wantLocalAddr != "" && peer.GetTransport().GetLocalAddress() != tt.wantLocalAddr {
+				t.Fatalf("local address = %q, want %q", peer.GetTransport().GetLocalAddress(), tt.wantLocalAddr)
+			}
+			if tt.wantInterface != "" && peer.GetTransport().GetBindInterface() != tt.wantInterface {
+				t.Fatalf("bind interface = %q, want %q", peer.GetTransport().GetBindInterface(), tt.wantInterface)
+			}
+		})
+	}
+}
+
+func newStartedTestBGPServer(t *testing.T, config kubevip.BGPConfig) *Server {
+	t.Helper()
+
+	server, err := NewBGPServer(config, log.LevelError)
+	if err != nil {
+		t.Fatalf("NewBGPServer() error = %v", err)
+	}
+
+	go server.s.Serve()
+	if err := server.s.StartBgp(context.Background(), &api.StartBgpRequest{
+		Global: &api.Global{
+			Asn:        config.AS,
+			RouterId:   config.RouterID,
+			ListenPort: -1,
+		},
+	}); err != nil {
+		server.s.Stop()
+		t.Fatalf("StartBgp() error = %v", err)
+	}
+	t.Cleanup(server.s.Stop)
+
+	return server
+}
+
+func listTestPeer(t *testing.T, server *Server, address string) *api.Peer {
+	t.Helper()
+
+	var got *api.Peer
+	if err := server.s.ListPeer(context.Background(), &api.ListPeerRequest{Address: address}, func(peer *api.Peer) {
+		got = peer
+	}); err != nil {
+		t.Fatalf("ListPeer() error = %v", err)
+	}
+	if got == nil {
+		t.Fatalf("ListPeer() returned no peer for %s", address)
+	}
+	return got
+}
+
+func newPeerTestServer(t *testing.T, cfg kubevip.BGPConfig) *Server {
+	t.Helper()
+	raw := startEmbeddedRawBGP(t)
+	return &Server{s: raw, c: &cfg, tracker: make(map[string]map[string]bool)}
+}
+
+func startEmbeddedRawBGP(t *testing.T) *gobgp.BgpServer {
+	t.Helper()
+	raw := gobgp.NewBgpServer()
+	go raw.Serve()
+	if err := raw.StartBgp(context.Background(), &api.StartBgpRequest{
+		Global: &api.Global{
+			Asn:        65000,
+			RouterId:   "192.0.2.1",
+			ListenPort: -1,
+		},
+	}); err != nil {
+		t.Fatalf("starting embedded BGP server: %v", err)
+	}
+	var stopOnce sync.Once
+	t.Cleanup(func() {
+		stopOnce.Do(func() {
+			if err := raw.StopBgp(context.Background(), &api.StopBgpRequest{}); err != nil {
+				t.Logf("stopping embedded BGP server: %v", err)
+			}
+		})
+	})
+	return raw
+}

pkg/bgp/server.go

@@ -3,49 +3,94 @@ package bgp
 import (
 	"context"
 	"fmt"
-	"log"
+	"sync"
 	"time"
 
-	api "github.com/osrg/gobgp/api"
-	gobgp "github.com/osrg/gobgp/pkg/server"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	api "github.com/osrg/gobgp/v4/api"
+	"github.com/osrg/gobgp/v4/pkg/apiutil"
+	bgp "github.com/osrg/gobgp/v4/pkg/packet/bgp"
+	gobgp "github.com/osrg/gobgp/v4/pkg/server"
 )
 
+type BGPManager interface {
+	AddHost(ctx context.Context, addr string, object string) error
+	DelHost(ctx context.Context, addr string, object string) error
+}
+
+// Server manages a server object
+type Server struct {
+	s       *gobgp.BgpServer
+	c       *kubevip.BGPConfig
+	mtx     sync.Mutex
+	tracker map[string]map[string]bool
+}
+
 // NewBGPServer takes a configuration and returns a running BGP server instance
-func NewBGPServer(c *Config) (b *Server, err error) {
+func NewBGPServer(c kubevip.BGPConfig, logLevel log.Level) (b *Server, err error) {
 	if c.AS == 0 {
-		return nil, fmt.Errorf("You need to provide AS")
+		return nil, fmt.Errorf("you need to provide AS")
 	}
 
-	// if c.SourceIP != "" && c.SourceIF != "" {
-	// 	return nil, fmt.Errorf("SourceIP and SourceIF are mutually exclusive")
-	// }
+	if c.SourceIP != "" && c.SourceIF != "" {
+		return nil, fmt.Errorf("sourceIP and SourceIF are mutually exclusive")
+	}
 
 	if len(c.Peers) == 0 {
-		return nil, fmt.Errorf("You need to provide at least one peer")
+		return nil, fmt.Errorf("you need to provide at least one peer")
 	}
+	bgpLogger := log.Default()
+	lvl := &log.LevelVar{}
+	lvl.Set(logLevel)
 
 	b = &Server{
-		s: gobgp.NewBgpServer(),
-		c: c,
+		s:       gobgp.NewBgpServer(gobgp.LoggerOption(bgpLogger, lvl)),
+		c:       &c,
+		tracker: make(map[string]map[string]bool),
 	}
+	return
+}
+
+// Start starts the BGP server
+func (b *Server) Start(ctx context.Context, peerStateChangeCallback func(*apiutil.WatchEventMessage_PeerEvent)) (err error) {
 	go b.s.Serve()
 
-	if err = b.s.StartBgp(context.Background(), &api.StartBgpRequest{
+	if err = b.s.StartBgp(ctx, &api.StartBgpRequest{
 		Global: &api.Global{
-			As:         c.AS,
-			RouterId:   c.RouterID,
+			Asn:        b.c.AS,
+			RouterId:   b.c.RouterID,
 			ListenPort: -1,
 		},
 	}); err != nil {
 		return
 	}
 
-	if err = b.s.MonitorPeer(context.Background(), &api.MonitorPeerRequest{}, func(p *api.Peer) { log.Println(p) }); err != nil {
+	if err = b.s.WatchEvent(ctx, gobgp.WatchEventMessageCallbacks{
+		OnPeerUpdate: func(p *apiutil.WatchEventMessage_PeerEvent, _ time.Time) {
+			log.Info("[BGP]", "peer", fmt.Sprintf("%+v", p))
+			if peerStateChangeCallback != nil {
+				peerStateChangeCallback(p)
+			}
+		},
+	}, gobgp.WatchPeer()); err != nil {
 		return
 	}
 
-	for _, p := range c.Peers {
-		if err = b.AddPeer(p); err != nil {
+	for _, p := range b.c.Peers {
+		if err = b.AddPeer(ctx, p); err != nil {
+			return
+		}
+	}
+
+	if b.c.Zebra.Enabled {
+		if err = b.s.EnableZebra(ctx, &api.EnableZebraRequest{
+			Url:          b.c.Zebra.URL,
+			Version:      b.c.Zebra.Version,
+			SoftwareName: b.c.Zebra.SoftwareName,
+		}); err != nil {
+			log.Error(err.Error())
 			return
 		}
 	}
@@ -55,7 +100,51 @@ func NewBGPServer(c *Config) (b *Server, err error) {
 
 // Close will stop a running BGP Server
 func (b *Server) Close() error {
-	ctx, cf := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cf()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
 	return b.s.StopBgp(ctx, &api.StopBgpRequest{})
 }
+
+// ListAdvertisedRoutes retrieves all active routes inside GoBGP's local RIB.
+// It queries the GLOBAL table type to find routes that kube-vip has requested GoBGP to advertise.
+func (b *Server) ListAdvertisedRoutes(ctx context.Context, isIPv6 bool) ([]*api.Destination, error) {
+	afi := bgp.AFI_IP
+
+	if isIPv6 {
+		afi = bgp.AFI_IP6
+	}
+
+	family := bgp.NewFamily(uint16(afi), bgp.SAFI_UNICAST)
+
+	var destinations []*api.Destination
+
+	req := apiutil.ListPathRequest{
+		TableType: api.TableType_TABLE_TYPE_GLOBAL,
+		Family:    family,
+	}
+
+	// GoBGP's embedded server API uses a callback function to stream results
+	// locally without requiring a gRPC client stream setup.
+	err := b.s.ListPath(req, func(prefix bgp.NLRI, paths []*apiutil.Path) {
+		var newPaths []*api.Path
+		for _, p := range paths {
+			np, err := apiutil.NewPath(p.Family, p.Nlri, p.Withdrawal, p.Attrs, time.Unix(p.Age, 0))
+			if err != nil {
+				log.Error("failed to create BGP path details", "err", err)
+				continue
+			}
+			newPaths = append(newPaths, np)
+		}
+		d := &api.Destination{
+			Prefix: prefix.String(),
+			Paths:  newPaths,
+		}
+		destinations = append(destinations, d)
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to extract local RIB: %w", err)
+	}
+
+	return destinations, nil
+}

pkg/bgp/types.go

@@ -1,29 +0,0 @@
-package bgp
-
-import gobgp "github.com/osrg/gobgp/pkg/server"
-
-// Peer defines a BGP Peer
-type Peer struct {
-	Address  string
-	AS       uint32
-	Password string
-	MultiHop bool
-}
-
-// Config defines the BGP server configuration
-type Config struct {
-	AS       uint32
-	RouterID string
-	NextHop  string
-	SourceIP string
-	SourceIF string
-
-	Peers []Peer
-	IPv6  bool
-}
-
-// Server manages a server object
-type Server struct {
-	s *gobgp.BgpServer
-	c *Config
-}

pkg/cluster/cluster.go

@@ -1,52 +1,288 @@
 package cluster
 
 import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/networkinterface"
+	"github.com/kube-vip/kube-vip/pkg/node"
+	"github.com/kube-vip/kube-vip/pkg/route"
 	"github.com/kube-vip/kube-vip/pkg/vip"
 )
 
-const leaderLogcount = 5
-
 // Cluster - The Cluster object manages the state of the cluster for a particular node
 type Cluster struct {
-	stateMachine FSM
-	stop         chan bool
-	completed    chan bool
-	Network      vip.Network
+	stop                  chan struct{}
+	stopMu                sync.Mutex
+	service               *servicesWorker
+	Network               []vip.Network
+	arpMgr                *arp.Manager
+	routeMgr              *route.Manager
+	nodeLabelMgr          node.Labeler
+	labelAdded            bool
+	healthCheckHTTPClient *http.Client
+}
+
+type servicesWorker struct {
+	stop         chan struct{}
+	done         chan struct{}
+	stopping     bool
+	preserveVIPs map[string]struct{}
 }
 
 // InitCluster - Will attempt to initialise all of the required settings for the cluster
-func InitCluster(c *kubevip.Config, disableVIP bool) (*Cluster, error) {
-
-	// TODO - Check for root (needed to netlink)
-	var network vip.Network
+func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.Manager, arpMgr *arp.Manager,
+	routeMgr *route.Manager, nodeLabelMgr node.Labeler) (*Cluster, error) {
+	var networks []vip.Network
+	var healthCheckHTTPClient *http.Client
 	var err error
 
 	if !disableVIP {
 		// Start the Virtual IP Networking configuration
-		network, err = startNetworking(c)
+		networks, err = startNetworking(c, intfMgr)
 		if err != nil {
 			return nil, err
 		}
 	}
+
+	if c.ControlPlaneHealthCheck.Address != "" {
+		healthCheckHTTPClient, err = newHealthCheckHTTPClient(c)
+		if err != nil {
+			return nil, fmt.Errorf("initializing BGP health check client: %w", err)
+		}
+	}
+
 	// Initialise the Cluster structure
 	newCluster := &Cluster{
-		Network: network,
+		Network:               networks,
+		arpMgr:                arpMgr,
+		stop:                  make(chan struct{}),
+		routeMgr:              routeMgr,
+		nodeLabelMgr:          nodeLabelMgr,
+		healthCheckHTTPClient: healthCheckHTTPClient,
 	}
 
+	log.Debug("service security", "enabled", c.EnableServiceSecurity)
+
 	return newCluster, nil
 }
 
-func startNetworking(c *kubevip.Config) (vip.Network, error) {
+func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vip.Network, error) {
 	address := c.VIP
 
 	if c.Address != "" {
 		address = c.Address
 	}
 
-	network, err := vip.NewConfig(address, c.Interface, c.DDNS)
-	if err != nil {
-		return nil, err
+	addresses := vip.Split(address)
+
+	networks := []vip.Network{}
+	for _, addr := range addresses {
+		network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.DHCPMode,
+			c.RequireDualStack, c.IsDualStack, c.RoutingTableID, c.RoutingTableType, c.RoutingProtocol, c.DNSMode,
+			c.LoadBalancerForwardingMethod, c.IptablesBackend, c.EnableLoadBalancer, c.LoadBalancerPort,
+			c.EnableServiceSecurity, intfMgr, c.EgressWithNftables, c.SkipDAD)
+		if err != nil {
+			return nil, err
+		}
+		networks = append(networks, network...)
+	}
+
+	return networks, nil
+}
+
+// Stop - Will stop the Cluster and release VIP if needed
+func (cluster *Cluster) Stop() {
+	cluster.stopMu.Lock()
+	defer cluster.stopMu.Unlock()
+	if cluster.service != nil {
+		workers := cluster.service
+		if workers.stopping {
+			return
+		}
+		workers.stopping = true
+		cluster.stop = make(chan struct{})
+		close(workers.stop)
+		return
+	}
+	stop := cluster.stop
+	cluster.stop = make(chan struct{})
+	close(stop)
+}
+
+// StopAndWait signals the current Service worker generation and waits until it
+// has finished its datapath cleanup.
+func (cluster *Cluster) StopAndWait() {
+	cluster.stopAndWait(nil)
+}
+
+// StopAndWaitPreserving stops the current Service worker generation while
+// preserving the supplied VIPs for another Service that shares the same lease.
+func (cluster *Cluster) StopAndWaitPreserving(addresses ...string) {
+	preserve := make(map[string]struct{}, len(addresses))
+	for _, address := range addresses {
+		preserve[address] = struct{}{}
+	}
+	cluster.stopAndWait(preserve)
+}
+
+func (cluster *Cluster) stopAndWait(preserveVIPs map[string]struct{}) {
+	workers, signal := cluster.prepareServiceStop(preserveVIPs)
+	if workers == nil {
+		return
+	}
+	if signal {
+		close(workers.stop)
+	}
+	<-workers.done
+}
+
+func (cluster *Cluster) prepareServiceStop(preserveVIPs map[string]struct{}) (*servicesWorker, bool) {
+	cluster.stopMu.Lock()
+	defer cluster.stopMu.Unlock()
+	workers := cluster.service
+	if workers == nil {
+		return nil, false
+	}
+	if workers.stopping {
+		workers.preserveVIPs = mergeVIPs(workers.preserveVIPs, preserveVIPs)
+		return workers, false
+	}
+	workers.stopping = true
+	workers.preserveVIPs = preserveVIPs
+	cluster.stop = make(chan struct{})
+	return workers, true
+}
+
+func (cluster *Cluster) startServicesWorker() (<-chan struct{}, chan struct{}, error) {
+	cluster.stopMu.Lock()
+	defer cluster.stopMu.Unlock()
+	if cluster.service != nil {
+		return nil, nil, fmt.Errorf("load balancer workers already running")
+	}
+	workers := &servicesWorker{stop: cluster.stop, done: make(chan struct{})}
+	cluster.service = workers
+	return workers.stop, workers.done, nil
+}
+
+func (cluster *Cluster) preserveServiceVIP(done chan struct{}, address string) bool {
+	cluster.stopMu.Lock()
+	defer cluster.stopMu.Unlock()
+	if cluster.service == nil || cluster.service.done != done {
+		return false
+	}
+	_, preserve := cluster.service.preserveVIPs[address]
+	return preserve
+}
+
+func mergeVIPs(existing, addresses map[string]struct{}) map[string]struct{} {
+	if len(addresses) == 0 {
+		return existing
+	}
+	if existing == nil {
+		existing = make(map[string]struct{}, len(addresses))
+	}
+	for address := range addresses {
+		existing[address] = struct{}{}
+	}
+	return existing
+}
+
+func (cluster *Cluster) finishServicesWorker(done chan struct{}) {
+	cluster.stopMu.Lock()
+	defer cluster.stopMu.Unlock()
+	if cluster.service == nil || cluster.service.done != done {
+		return
+	}
+	cluster.service = nil
+	close(done)
+}
+
+func (cluster *Cluster) StopChannel() <-chan struct{} {
+	cluster.stopMu.Lock()
+	defer cluster.stopMu.Unlock()
+	return cluster.stop
+}
+
+func newHealthCheckHTTPClient(c *kubevip.Config) (*http.Client, error) {
+	defaultTransport, ok := http.DefaultTransport.(*http.Transport)
+	if !ok {
+		return nil, fmt.Errorf("unexpected default HTTP transport type %T", http.DefaultTransport)
+	}
+
+	transport := defaultTransport.Clone()
+	if c.ControlPlaneHealthCheck.CAPath != "" {
+		caCert, err := os.ReadFile(c.ControlPlaneHealthCheck.CAPath)
+		if err != nil {
+			return nil, fmt.Errorf("reading health check CA cert %q: %w", c.ControlPlaneHealthCheck.CAPath, err)
+		}
+
+		rootCAs, err := x509.SystemCertPool()
+		if err != nil || rootCAs == nil {
+			rootCAs = x509.NewCertPool()
+		}
+		if !rootCAs.AppendCertsFromPEM(caCert) {
+			return nil, fmt.Errorf("health check CA cert %q contains no valid certificates", c.ControlPlaneHealthCheck.CAPath)
+		}
+
+		tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12}
+		if transport.TLSClientConfig != nil {
+			tlsConfig = transport.TLSClientConfig.Clone()
+		}
+		tlsConfig.RootCAs = rootCAs
+		transport.TLSClientConfig = tlsConfig
+	}
+
+	return &http.Client{
+		Timeout:   time.Duration(c.ControlPlaneHealthCheck.TimeoutSeconds) * time.Second,
+		Transport: transport,
+	}, nil
+}
+
+// cleanupVIPs releases the control plane VIPs after leadership was lost.
+// Nothing waits for the control plane layer2Update goroutine to observe the
+// cancelled context, so this caller usually still holds its own ARP claim and
+// has to delete the address itself.
+func (cluster *Cluster) cleanupVIPs(c *kubevip.Config) {
+	for i := range cluster.Network {
+		cluster.cleanupVIP(c, cluster.Network[i], 1)
+	}
+}
+
+// cleanupServiceVIPs releases the service VIPs once the services worker has
+// drained. layer2Update already removed this instance's own claim by then, so
+// any remaining claim belongs to another service sharing the VIP.
+func (cluster *Cluster) cleanupServiceVIPs(c *kubevip.Config, done chan struct{}) {
+	for i := range cluster.Network {
+		if cluster.preserveServiceVIP(done, cluster.Network[i].IP()) {
+			continue
+		}
+		cluster.cleanupVIP(c, cluster.Network[i], 0)
+	}
+}
+
+// cleanupVIP deletes the VIP unless somebody else still advertises it.
+// ownClaims is the number of ARP claims the caller may still hold itself.
+func (cluster *Cluster) cleanupVIP(c *kubevip.Config, network vip.Network, ownClaims int) {
+	if c.EnableARP && cluster.arpMgr.Count(network.ARPName()) > ownClaims {
+		return
+	}
+
+	log.Info("[VIP] Deleting VIP", "ip", network.IP())
+	deleted, err := network.DeleteIP()
+	if err != nil {
+		log.Warn(err.Error())
+	}
+	if deleted {
+		log.Info("deleted address", "IP", network.IP(), "interface", network.Interface())
 	}
-	return network, nil
 }

pkg/cluster/clusterDDNS.go

@@ -2,6 +2,7 @@ package cluster
 
 import (
 	"context"
+	"sync"
 
 	"github.com/kube-vip/kube-vip/pkg/vip"
 )
@@ -12,14 +13,13 @@ import (
 // during runtime if IP changes, startDDNS don't have to do reconfigure because
 // dnsUpdater already have the functionality to keep trying resolve the IP
 // and update the VIP configuration if it changes
-func (cluster *Cluster) StartDDNS(ctx context.Context) error {
-	ddnsMgr := vip.NewDDNSManager(ctx, cluster.Network)
-	ip, err := ddnsMgr.Start()
+func (cluster *Cluster) StartDDNS(ctx context.Context, network vip.Network, backoffAttempts uint, wg *sync.WaitGroup) error {
+	ddnsMgr := vip.NewDDNSManager(network, backoffAttempts)
+	ip, err := ddnsMgr.Start(ctx, wg)
 	if err != nil {
 		return err
 	}
-
-	if err = cluster.Network.SetIP(ip); err != nil {
+	if err = network.SetIP(ip); err != nil {
 		return err
 	}
 

pkg/cluster/clusterLeader.go

@@ -1,469 +0,0 @@
-package cluster
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/signal"
-	"path/filepath"
-	"syscall"
-	"time"
-
-	"github.com/kube-vip/kube-vip/pkg/bgp"
-	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	leaderelection "github.com/kube-vip/kube-vip/pkg/leaderElection"
-	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
-	"github.com/kube-vip/kube-vip/pkg/packet"
-
-	"github.com/kube-vip/kube-vip/pkg/vip"
-
-	"github.com/packethost/packngo"
-
-	log "github.com/sirupsen/logrus"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/tools/leaderelection/resourcelock"
-)
-
-const plunderLock = "plndr-cp-lock"
-
-// Manager degines the manager of the load-balancing services
-type Manager struct {
-	KubernetesClient *kubernetes.Clientset
-}
-
-// NewManager will create a new managing object
-func NewManager(path string, inCluster bool, port int) (*Manager, error) {
-	var clientset *kubernetes.Clientset
-	if inCluster {
-		// This will attempt to load the configuration when running within a POD
-		cfg, err := rest.InClusterConfig()
-		if err != nil {
-			return nil, fmt.Errorf("error creating kubernetes client config: %s", err.Error())
-		}
-		clientset, err = kubernetes.NewForConfig(cfg)
-
-		if err != nil {
-			return nil, fmt.Errorf("error creating kubernetes client: %s", err.Error())
-		}
-		// use the current context in kubeconfig
-	} else {
-		if path == "" {
-			path = filepath.Join(os.Getenv("HOME"), ".kube", "config")
-		}
-		config, err := clientcmd.BuildConfigFromFlags("", path)
-		if err != nil {
-			panic(err.Error())
-		}
-
-		// We modify the config so that we can always speak to the correct host
-		id, err := os.Hostname()
-		if err != nil {
-			return nil, err
-		}
-
-		config.Host = fmt.Sprintf("%s:%v", id, port)
-		clientset, err = kubernetes.NewForConfig(config)
-
-		if err != nil {
-			return nil, fmt.Errorf("error creating kubernetes client: %s", err.Error())
-		}
-	}
-
-	return &Manager{
-		KubernetesClient: clientset,
-	}, nil
-}
-
-// StartLeaderCluster - Begins a running instance of the Raft cluster
-func (cluster *Cluster) StartLeaderCluster(c *kubevip.Config, sm *Manager, bgpServer *bgp.Server) error {
-
-	id, err := os.Hostname()
-	if err != nil {
-		return err
-	}
-
-	log.Infof("Beginning cluster membership, namespace [%s], lock name [%s], id [%s]", c.Namespace, plunderLock, id)
-
-	// we use the Lease lock type since edits to Leases are less common
-	// and fewer objects in the cluster watch "all Leases".
-	lock := &resourcelock.LeaseLock{
-		LeaseMeta: metav1.ObjectMeta{
-			Name:      plunderLock,
-			Namespace: c.Namespace,
-		},
-		Client: sm.KubernetesClient.CoordinationV1(),
-		LockConfig: resourcelock.ResourceLockConfig{
-			Identity: id,
-		},
-	}
-
-	// use a Go context so we can tell the leaderelection code when we
-	// want to step down
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	// use a Go context so we can tell the arp loop code when we
-	// want to step down
-	ctxArp, cancelArp := context.WithCancel(context.Background())
-	defer cancelArp()
-
-	// use a Go context so we can tell the dns loop code when we
-	// want to step down
-	ctxDNS, cancelDNS := context.WithCancel(context.Background())
-	defer cancelDNS()
-
-	// listen for interrupts or the Linux SIGTERM signal and cancel
-	// our context, which the leader election code will observe and
-	// step down
-	signalChan := make(chan os.Signal, 1)
-	// Add Notification for Userland interrupt
-	signal.Notify(signalChan, syscall.SIGINT)
-
-	// Add Notification for SIGTERM (sent from Kubernetes)
-	signal.Notify(signalChan, syscall.SIGTERM)
-
-	// Add Notification for SIGKILL (sent from Kubernetes)
-	signal.Notify(signalChan, syscall.SIGKILL)
-
-	go func() {
-		<-signalChan
-		log.Info("Received termination, signaling shutdown")
-		// Cancel the context, which will in turn cancel the leadership
-		cancel()
-		// Cancel the arp context, which will in turn stop any broadcasts
-	}()
-
-	// (attempt to) Remove the virtual IP, incase it already exists
-	cluster.Network.DeleteIP()
-
-	// Managers for Vip load balancers and none-vip loadbalancers
-	nonVipLB := loadbalancer.LBManager{}
-	VipLB := loadbalancer.LBManager{}
-
-	// Defer a function to check if the bgpServer has been created and if so attempt to close it
-	defer func() {
-		if bgpServer != nil {
-			bgpServer.Close()
-		}
-	}()
-
-	// If Packet is enabled then we can begin our preperation work
-	var packetClient *packngo.Client
-	if c.EnableMetal {
-		packetClient, err = packngo.NewClient()
-		if err != nil {
-			log.Error(err)
-		}
-
-		// We're using Packet with BGP, popuplate the Peer information from the API
-		if c.EnableBGP {
-			log.Infoln("Looking up the BGP configuration from packet")
-			err = packet.BGPLookup(packetClient, c)
-			if err != nil {
-				log.Error(err)
-			}
-		}
-	}
-
-	if c.EnableBGP {
-		// Lets start BGP
-		log.Info("Starting the BGP server to adverise VIP routes to VGP peers")
-		bgpServer, err = bgp.NewBGPServer(&c.BGPConfig)
-		if err != nil {
-			log.Error(err)
-		}
-	}
-
-	if c.EnableLoadBalancer {
-
-		// Iterate through all Configurations
-		if len(c.LoadBalancers) != 0 {
-			for x := range c.LoadBalancers {
-				// If the load balancer doesn't bind to the VIP
-				if c.LoadBalancers[x].BindToVip == false {
-					err = nonVipLB.Add("", &c.LoadBalancers[x])
-					if err != nil {
-						log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-					}
-				}
-			}
-		}
-	}
-	// start the leader election code loop
-	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
-		Lock: lock,
-		// IMPORTANT: you MUST ensure that any code you have that
-		// is protected by the lease must terminate **before**
-		// you call cancel. Otherwise, you could have a background
-		// loop still running and another process could
-		// get elected before your background loop finished, violating
-		// the stated goal of the lease.
-		ReleaseOnCancel: true,
-		LeaseDuration:   time.Duration(c.LeaseDuration) * time.Second,
-		RenewDeadline:   time.Duration(c.RenewDeadline) * time.Second,
-		RetryPeriod:     time.Duration(c.RetryPeriod) * time.Second,
-		Callbacks: leaderelection.LeaderCallbacks{
-			OnStartedLeading: func(ctx context.Context) {
-				// we're notified when we start
-				log.Info("This node is starting with leadership of the cluster")
-				// setup ddns first
-				// for first time, need to wait until IP is allocated from DHCP
-				if cluster.Network.IsDDNS() {
-					if err := cluster.StartDDNS(ctxDNS); err != nil {
-						log.Error(err)
-					}
-				}
-
-				// start the dns updater if address is dns
-				if cluster.Network.IsDNS() {
-					log.Infof("starting the DNS updater for the address %s", cluster.Network.DNSName())
-					ipUpdater := vip.NewIPUpdater(cluster.Network)
-					ipUpdater.Run(ctxDNS)
-				}
-
-				err = cluster.Network.AddIP()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				if c.EnableMetal {
-					// We're not using Packet with BGP
-					if !c.EnableBGP {
-						// Attempt to attach the EIP in the standard manner
-						log.Debugf("Attaching the Packet EIP through the API to this host")
-						err = packet.AttachEIP(packetClient, c, id)
-						if err != nil {
-							log.Error(err)
-						}
-					}
-				}
-
-				if c.EnableBGP {
-					// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
-					cidrVip := fmt.Sprintf("%s/%s", cluster.Network.IP(), c.VIPCIDR)
-					log.Debugf("Attempting to advertise the address [%s] over BGP", cidrVip)
-
-					err = bgpServer.AddHost(cidrVip)
-					if err != nil {
-						log.Error(err)
-					}
-				}
-
-				if c.EnableLoadBalancer {
-					// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-					for x := range c.LoadBalancers {
-
-						if c.LoadBalancers[x].BindToVip == true {
-							err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-							if err != nil {
-								log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-
-								// Stop all load balancers associated with the VIP
-								err = VipLB.StopAll()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-
-								err = cluster.Network.DeleteIP()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-							}
-						}
-					}
-				}
-
-				if c.EnableARP == true {
-					ctxArp, cancelArp = context.WithCancel(context.Background())
-
-					ipString := cluster.Network.IP()
-
-					var ndp *vip.NdpResponder
-					if vip.IsIPv6(ipString) {
-						ndp, err = vip.NewNDPResponder(c.Interface)
-						if err != nil {
-							log.Fatalf("failed to create new NDP Responder")
-						}
-					}
-
-					go func(ctx context.Context) {
-						if ndp != nil {
-							defer ndp.Close()
-						}
-
-						for {
-							select {
-							case <-ctx.Done(): // if cancel() execute
-								return
-							default:
-								// Ensure the address exists on the interface before attempting to ARP
-								set, err := cluster.Network.IsSet()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-								if !set {
-									log.Warnf("Re-applying the VIP configuration [%s] to the interface [%s]", ipString, c.Interface)
-									err = cluster.Network.AddIP()
-									if err != nil {
-										log.Warnf("%v", err)
-									}
-								}
-
-								if vip.IsIPv4(ipString) {
-									// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
-									err := vip.ARPSendGratuitous(ipString, c.Interface)
-									if err != nil {
-										log.Warnf("%v", err)
-									}
-								} else {
-									// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
-									err := ndp.SendGratuitous(ipString)
-									if err != nil {
-										log.Warnf("%v", err)
-									}
-								}
-							}
-							time.Sleep(3 * time.Second)
-						}
-					}(ctxArp)
-				}
-			},
-			OnStoppedLeading: func() {
-				// we can do cleanup here
-				log.Info("This node is becoming a follower within the cluster")
-
-				// Stop the dns context
-				cancelDNS()
-				// Stop the Arp context if it is running
-				cancelArp()
-
-				// Stop the BGP server
-				if bgpServer != nil {
-					err = bgpServer.Close()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-
-				// Stop all load balancers associated with the VIP
-				err = VipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				err = cluster.Network.DeleteIP()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				log.Fatal("lost leadership, restarting kube-vip")
-			},
-			OnNewLeader: func(identity string) {
-				// we're notified when new leader elected
-				log.Infof("Node [%s] is assuming leadership of the cluster", identity)
-
-				if identity == id {
-					// We have the lock
-				}
-			},
-		},
-	})
-
-	return nil
-}
-
-// TODO - refactor an active machine func(), this will replace the singleNode code and have a single code block
-
-// func (cluster *Cluster) active(c *kubevip.Config) error {
-// 	// we're notified when we start
-// 	log.Info("This node is starting with leadership of the cluster")
-// 	// setup ddns first
-// 	// for first time, need to wait until IP is allocated from DHCP
-// 	if cluster.Network.IsDDNS() {
-// 		if err := cluster.StartDDNS(ctxDns); err != nil {
-// 			log.Error(err)
-// 		}
-// 	}
-
-// 	// start the dns updater if address is dns
-// 	if cluster.Network.IsDNS() {
-// 		log.Infof("starting the DNS updater for the address %s", cluster.Network.DNSName())
-// 		ipUpdater := vip.NewIPUpdater(cluster.Network)
-// 		ipUpdater.Run(ctxDns)
-// 	}
-
-// 	err := cluster.Network.AddIP()
-// 	if err != nil {
-// 		log.Warnf("%v", err)
-// 	}
-
-// 	if c.EnablePacket {
-// 		// We're not using Packet with BGP
-// 		if !c.EnableBGP {
-// 			// Attempt to attach the EIP in the standard manner
-// 			log.Debugf("Attaching the Packet EIP through the API to this host")
-// 			err = packet.AttachEIP(packetClient, c, id)
-// 			if err != nil {
-// 				log.Error(err)
-// 			}
-// 		}
-// 	}
-
-// 	if c.EnableBGP {
-// 		// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
-// 		cidrVip := fmt.Sprintf("%s/%s", cluster.Network.IP(), c.VIPCIDR)
-// 		log.Debugf("Attempting to advertise the address [%s] over BGP", cidrVip)
-
-// 		err = bgpServer.AddHost(cidrVip)
-// 		if err != nil {
-// 			log.Error(err)
-// 		}
-// 	}
-
-// 	if c.EnableLoadBalancer {
-// 		// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-// 		for x := range c.LoadBalancers {
-
-// 			if c.LoadBalancers[x].BindToVip == true {
-// 				err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-// 				if err != nil {
-// 					log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-
-// 					// Stop all load balancers associated with the VIP
-// 					err = VipLB.StopAll()
-// 					if err != nil {
-// 						log.Warnf("%v", err)
-// 					}
-
-// 					err = cluster.Network.DeleteIP()
-// 					if err != nil {
-// 						log.Warnf("%v", err)
-// 					}
-// 				}
-// 			}
-// 		}
-// 	}
-
-// 	if c.EnableARP == true {
-// 		ctxArp, cancelArp = context.WithCancel(context.Background())
-
-// 		go func(ctx context.Context) {
-// 			for {
-// 				select {
-// 				case <-ctx.Done(): // if cancel() execute
-// 					return
-// 				default:
-// 					// Gratuitous ARP, will broadcast to new MAC <-> IP
-// 					err = vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-// 					if err != nil {
-// 						log.Warnf("%v", err)
-// 					}
-// 				}
-// 				time.Sleep(3 * time.Second)
-// 			}
-// 		}(ctxArp)
-// 	}
-// }

pkg/cluster/clusterLeaderElection.go

@@ -0,0 +1,169 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/election"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/lease"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+
+	log "log/slog"
+)
+
+// StartCluster - Begins a running instance of the Leader Election cluster
+func (cluster *Cluster) StartCluster(ctx context.Context, c *kubevip.Config,
+	em *election.Manager, bgpServer *bgp.Server, leaseMgr *lease.Manager, killFunc func()) error {
+
+	ns, leaseName := lease.NamespaceName(c.LeaseName, c)
+
+	leaseID := lease.NewID(c.LeaderElectionType, ns, leaseName)
+
+	log.Info("cluster membership", "namespace", leaseID.Namespace(), "lock", leaseID.Name(), "id", c.NodeName)
+
+	objectName := lease.ObjectName(leaseID, "cp")
+	objLease, _ := leaseMgr.Acquire(context.Background(), leaseID, objectName)
+	defer leaseMgr.Delete(leaseID, objectName, objLease)
+
+	wg := sync.WaitGroup{}
+	defer wg.Wait()
+
+	electionCtx, cancelElection := objLease.NewElectionContext(ctx)
+	defer cancelElection()
+
+	stop := cluster.StopChannel()
+	wg.Go(func() {
+		select {
+		case <-stop:
+			cancelElection()
+		case <-electionCtx.Done():
+		}
+	})
+
+	// (attempt to) Remove the virtual IP, in case it already exists
+
+	for i := range cluster.Network {
+		deleted, err := cluster.Network[i].DeleteIP()
+		if err != nil {
+			log.Error("could not delete virtualIP", "err", err)
+		}
+		if deleted {
+			log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+		}
+	}
+
+	for {
+		if !objLease.BeginElection() {
+			log.Debug("this election was already done, shared lease", "lease", leaseName)
+			leaderGeneration, elected := objLease.WaitForLeaderGeneration(electionCtx)
+			if !elected {
+				if electionCtx.Err() != nil {
+					return nil
+				}
+				// The runner that owned this shared lease's election ended it
+				// without ever being elected; take over the campaign ourselves
+				// instead of leaving the lease without an active runner.
+				continue
+			}
+
+			leaderCtx, cancelLeader := context.WithCancel(electionCtx)
+			leaderWG := sync.WaitGroup{}
+			leaderWG.Go(func() {
+				cluster.OnStartedLeading(leaderCtx, c, em, bgpServer, killFunc, true)
+			})
+
+			log.Debug("cluster waiting for shared election to finish", "lease", leaseName)
+			objLease.WaitForElectionEndAfter(electionCtx, leaderGeneration)
+			cancelLeader()
+			leaderWG.Wait()
+
+			cluster.OnStoppedLeading(c, bgpServer)
+
+			return nil
+		}
+		break
+	}
+	defer objLease.ElectionStopped()
+
+	run := &election.RunConfig{
+		Config:           c,
+		LeaseID:          leaseID,
+		LeaseAnnotations: c.LeaseAnnotations,
+		VIPs:             controlPlaneElectionVIPs(c),
+		Mgr:              em,
+		OnStartedLeading: func(ctx context.Context) {
+			objLease.ElectionStarted()
+			cluster.OnStartedLeading(ctx, c, em, bgpServer, killFunc, false)
+		},
+		OnStoppedLeading: func() {
+			objLease.ElectionStopped()
+			cluster.OnStoppedLeading(c, bgpServer)
+		},
+		OnNewLeader: func(identity string) {
+			cluster.OnNewLeader(identity, c)
+		},
+	}
+
+	if err := election.RunOrDie(electionCtx, run, c); err != nil {
+		cluster.Stop()
+		return fmt.Errorf("leaderelection failed: %w", err)
+	}
+
+	return nil
+}
+
+func controlPlaneElectionVIPs(config *kubevip.Config) []string {
+	configured := config.VIP
+	if config.Address != "" {
+		configured = config.Address
+	}
+	return vip.Split(configured)
+}
+
+func (cluster *Cluster) OnStartedLeading(ctx context.Context, c *kubevip.Config,
+	em *election.Manager, bgpServer *bgp.Server, killFunc func(), _ bool) {
+	labels := generateLabelsFromConfig(c.Address, kubevip.HasIP)
+	if err := cluster.nodeLabelMgr.AddLabel(labels); err != nil {
+		log.Error("error adding label to node", "err", err)
+	}
+	cluster.labelAdded = true
+
+	// As we're leading lets start the vip service
+	err := cluster.StartVipService(ctx, c, em, bgpServer, killFunc)
+	if err != nil {
+		log.Error("starting VIP service on leader", "err", err)
+		killFunc()
+	}
+}
+
+func (cluster *Cluster) OnStoppedLeading(c *kubevip.Config, bgpServer *bgp.Server) {
+	// we can do cleanup here
+	log.Info("This node is becoming a follower within the cluster")
+
+	if cluster.labelAdded {
+		labels := generateLabelsFromConfig(c.Address, kubevip.HasIP)
+		if err := cluster.nodeLabelMgr.RemoveLabel(labels); err != nil {
+			log.Error("error removing label from node", "err", err)
+		}
+		cluster.labelAdded = false
+	}
+
+	cluster.cleanupVIPs(c)
+
+	log.Error("lost leadership, restarting kube-vip")
+}
+
+func (cluster *Cluster) OnNewLeader(identity string, c *kubevip.Config) {
+	// we're notified when new leader elected
+	log.Info("New leader", "leader", identity)
+}
+
+func generateLabelsFromConfig(addr, labelKey string) map[string]string {
+	return map[string]string{
+		labelKey: utils.SanitizeIPForLabel(addr),
+	}
+}

pkg/cluster/clusterRaft.go

@@ -1,296 +0,0 @@
-package cluster
-
-import (
-	"fmt"
-	"net"
-	"time"
-
-	"github.com/hashicorp/raft"
-	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
-	"github.com/kube-vip/kube-vip/pkg/vip"
-	log "github.com/sirupsen/logrus"
-)
-
-// StartRaftCluster - Begins a running instance of the Raft cluster
-func (cluster *Cluster) StartRaftCluster(c *kubevip.Config) error {
-
-	// Create local configuration address
-	localAddress := fmt.Sprintf("%s:%d", c.LocalPeer.Address, c.LocalPeer.Port)
-
-	// Begin the Raft configuration
-	config := raft.DefaultConfig()
-	config.LocalID = raft.ServerID(c.LocalPeer.ID)
-	logger := log.StandardLogger().Writer()
-	config.LogOutput = logger
-
-	// Initialize communication
-	address, err := net.ResolveTCPAddr("tcp", localAddress)
-	if err != nil {
-		return err
-	}
-
-	// Create transport
-	transport, err := raft.NewTCPTransport(localAddress, address, 3, 10*time.Second, logger)
-	if err != nil {
-		return err
-	}
-
-	// Create Raft structures
-	snapshots := raft.NewInmemSnapshotStore()
-	logStore := raft.NewInmemStore()
-	stableStore := raft.NewInmemStore()
-
-	// Cluster configuration
-	configuration := raft.Configuration{}
-
-	// Add Local Peer
-	configuration.Servers = append(configuration.Servers, raft.Server{
-		ID:      raft.ServerID(c.LocalPeer.ID),
-		Address: raft.ServerAddress(fmt.Sprintf("%s:%d", c.LocalPeer.Address, c.LocalPeer.Port))})
-
-	// If we want to start a node as leader then we will not add any remote peers, this will leave this as a cluster of one
-	// The remotePeers will add themselves to the cluster as they're added
-	if c.StartAsLeader != true {
-		for x := range c.RemotePeers {
-			// Make sure that we don't add in this server twice
-			if c.LocalPeer.Address != c.RemotePeers[x].Address {
-
-				// Build the address from the peer configuration
-				peerAddress := fmt.Sprintf("%s:%d", c.RemotePeers[x].Address, c.RemotePeers[x].Port)
-
-				// Set this peer into the raft configuration
-				configuration.Servers = append(configuration.Servers, raft.Server{
-					ID:      raft.ServerID(c.RemotePeers[x].ID),
-					Address: raft.ServerAddress(peerAddress)})
-			}
-		}
-		log.Info("This node will attempt to start as Follower")
-	} else {
-		log.Info("This node will attempt to start as Leader")
-	}
-
-	// Bootstrap cluster
-	if err := raft.BootstrapCluster(config, logStore, stableStore, snapshots, transport, configuration); err != nil {
-		return err
-	}
-
-	// Create RAFT instance
-	raftServer, err := raft.NewRaft(config, cluster.stateMachine, logStore, stableStore, snapshots, transport)
-	if err != nil {
-		return err
-	}
-
-	cluster.stop = make(chan bool, 1)
-	cluster.completed = make(chan bool, 1)
-	ticker := time.NewTicker(time.Second)
-	isLeader := c.StartAsLeader
-
-	// (attempt to) Remove the virtual IP, incase it already exists
-	cluster.Network.DeleteIP()
-
-	// leader log broadcast - this counter is used to stop flooding STDOUT with leader log entries
-	var leaderbroadcast int
-	// Managers for Vip load balancers and none-vip loadbalancers
-	nonVipLB := loadbalancer.LBManager{}
-	VipLB := loadbalancer.LBManager{}
-
-	// Iterate through all Configurations
-	for x := range c.LoadBalancers {
-		// If the load balancer doesn't bind to the VIP
-		if c.LoadBalancers[x].BindToVip == false {
-			err = nonVipLB.Add("", &c.LoadBalancers[x])
-			if err != nil {
-				log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-			}
-		}
-	}
-
-	// On a cold start the node will sleep for 5 seconds to ensure that leader elections are complete
-	log.Infoln("This instance will wait approximately 5 seconds, from cold start to ensure cluster elections are complete")
-	time.Sleep(time.Second * 5)
-
-	go func() {
-		for {
-			if c.AddPeersAsBackends == true {
-				// Get addresses and change backends
-
-				// c.LoadBalancers[0].Backends
-				// for x := range raftServer.GetConfiguration().Configuration().Servers {
-				// 	raftServer.GetConfiguration().Configuration().Servers[x].Address
-				// }
-
-			}
-			// Broadcast the current leader on this node if it's the correct time (every leaderLogcount * time.Second)
-			if leaderbroadcast == leaderLogcount {
-				log.Infof("The Node [%s] is leading", raftServer.Leader())
-				// Reset the timer
-				leaderbroadcast = 0
-
-				// ensure that if this node is the leader, it is set as the leader
-				if localAddress == string(raftServer.Leader()) {
-					// Re-broadcast arp to ensure network stays up to date
-					if c.EnableARP == true {
-						// Gratuitous ARP, will broadcast to new MAC <-> IP
-						err = vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-					}
-					if !isLeader {
-						log.Infoln("This node is leading, but isnt the leader (correcting)")
-						isLeader = true
-					}
-				} else {
-					// (attempt to) Remove the virtual IP, incase it already exists to keep nodes clean
-					cluster.Network.DeleteIP()
-					isLeader = false
-				}
-
-			}
-			leaderbroadcast++
-
-			select {
-			case leader := <-raftServer.LeaderCh():
-				log.Infoln("New Election event")
-				if leader {
-					isLeader = true
-
-					log.Info("This node is assuming leadership of the cluster")
-					err = cluster.Network.AddIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-
-					// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-
-					for x := range c.LoadBalancers {
-
-						if c.LoadBalancers[x].BindToVip == true {
-							err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-							if err != nil {
-								log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-								log.Errorf("Dropping Leadership to another node in the cluster")
-								raftServer.LeadershipTransfer()
-
-								// Stop all load balancers associated with the VIP
-								err = VipLB.StopAll()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-
-								err = cluster.Network.DeleteIP()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-							}
-						}
-					}
-
-					if c.EnableARP == true {
-						// Gratuitous ARP, will broadcast to new MAC <-> IP
-						err = vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-					}
-				} else {
-					isLeader = false
-
-					log.Info("This node is becoming a follower within the cluster")
-
-					// Stop all load balancers associated with the VIP
-					err = VipLB.StopAll()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-
-					err = cluster.Network.DeleteIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-
-			case <-ticker.C:
-
-				if isLeader {
-
-					result, err := cluster.Network.IsSet()
-					if err != nil {
-						log.WithFields(log.Fields{"error": err, "ip": cluster.Network.IP(), "interface": cluster.Network.Interface()}).Error("Could not check ip")
-					}
-
-					if result == false {
-						log.Error("This node is leader and is adopting the virtual IP")
-
-						err = cluster.Network.AddIP()
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-						// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-
-						for x := range c.LoadBalancers {
-
-							if c.LoadBalancers[x].BindToVip == true {
-								err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-								if err != nil {
-									log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-								}
-							}
-						}
-						if c.EnableARP == true {
-							// Gratuitous ARP, will broadcast to new MAC <-> IP
-							err = vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-							if err != nil {
-								log.Warnf("%v", err)
-							}
-						}
-					}
-				}
-
-			case <-cluster.stop:
-				log.Info("[RAFT] Stopping this node")
-				log.Info("[LOADBALANCER] Stopping load balancers")
-
-				// Stop all load balancers associated with the VIP
-				err = VipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				// Stop all load balancers associated with the Host
-				err = nonVipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				if isLeader {
-					log.Info("[VIP] Releasing the Virtual IP")
-					err = cluster.Network.DeleteIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-
-				close(cluster.completed)
-
-				return
-			}
-		}
-	}()
-
-	log.Info("Started")
-
-	return nil
-}
-
-// Stop - Will stop the Cluster and release VIP if needed
-func (cluster *Cluster) Stop() {
-	// Close the stop chanel, which will shut down the VIP (if needed)
-	close(cluster.stop)
-
-	// Wait until the completed channel is closed, signallign all shutdown tasks completed
-	<-cluster.completed
-
-	log.Info("Stopped")
-}

pkg/cluster/cluster_internal_test.go

@@ -0,0 +1,196 @@
+package cluster
+
+import (
+	"context"
+	"slices"
+	"testing"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/lease"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	"github.com/vishvananda/netlink"
+	v1 "k8s.io/api/core/v1"
+)
+
+func TestControlPlaneElectionVIPsPreservesConfigOrder(t *testing.T) {
+	config := &kubevip.Config{Address: "2001:db8::10,192.0.2.10"}
+	want := []string{"2001:db8::10", "192.0.2.10"}
+	if got := controlPlaneElectionVIPs(config); !slices.Equal(got, want) {
+		t.Fatalf("controlPlaneElectionVIPs() = %v, want %v", got, want)
+	}
+}
+
+type recordingLabeler struct {
+	added   chan struct{}
+	removed chan struct{}
+}
+
+func (l *recordingLabeler) AddLabel(map[string]string) error {
+	l.added <- struct{}{}
+	return nil
+}
+
+func (l *recordingLabeler) RemoveLabel(map[string]string) error {
+	l.removed <- struct{}{}
+	return nil
+}
+
+// stubNetwork is a minimal vip.Network implementation for exercising
+// cleanupVIP without a real interface.
+type stubNetwork struct {
+	ip            string
+	deleteIPCalls int
+}
+
+func (s *stubNetwork) AddIP(bool, bool, ...int) (bool, error) { return false, nil }
+func (s *stubNetwork) AddRoute(bool) (bool, error)            { return false, nil }
+func (s *stubNetwork) ReplaceRoute() error                    { return nil }
+func (s *stubNetwork) DeleteIP() (bool, error)                { s.deleteIPCalls++; return true, nil }
+func (s *stubNetwork) DeleteRoute() error                     { return nil }
+func (s *stubNetwork) UpdateRoutes() (bool, error)            { return false, nil }
+func (s *stubNetwork) IsSet() (*netlink.Addr, error)          { return nil, nil }
+func (s *stubNetwork) IP() string                             { return s.ip }
+func (s *stubNetwork) CIDR() string                           { return s.ip + "/32" }
+func (s *stubNetwork) IPisLinkLocal() bool                    { return false }
+func (s *stubNetwork) PrepareRoute() *netlink.Route           { return nil }
+func (s *stubNetwork) RouteHash() string                      { return "" }
+func (s *stubNetwork) SetIP(string) error                     { return nil }
+func (s *stubNetwork) SetServicePorts(*v1.Service)            {}
+func (s *stubNetwork) Interface() string                      { return "eth0" }
+func (s *stubNetwork) IsDADFAIL() bool                        { return false }
+func (s *stubNetwork) IsDNS() bool                            { return false }
+func (s *stubNetwork) IsDDNS() bool                           { return false }
+func (s *stubNetwork) DDNSHostName() string                   { return "" }
+func (s *stubNetwork) DNSName() string                        { return "" }
+func (s *stubNetwork) SetMask(string) error                   { return nil }
+func (s *stubNetwork) SetHasEndpoints(bool)                   {}
+func (s *stubNetwork) HasEndpoints() bool                     { return false }
+func (s *stubNetwork) ARPName() string                        { return "shared-vip" }
+func (s *stubNetwork) GetPossibleSubnets() string             { return "" }
+func (s *stubNetwork) DHCPFamily() string                     { return "" }
+func (s *stubNetwork) IPVSMark() uint32                       { return 0 }
+
+// TestCleanupVIPRetainsSharedVIPWithOneSiblingLeft reproduces the off-by-one:
+// layer2Update already removes its own ARP claim before cleanupVIP runs, so a
+// single remaining sibling must still block deletion.
+func TestCleanupVIPRetainsSharedVIPWithOneSiblingLeft(t *testing.T) {
+	arpMgr := arp.NewManager(&kubevip.Config{ArpBroadcastRate: 3000})
+	netA := &stubNetwork{ip: "192.0.2.10"}
+	netB := &stubNetwork{ip: "192.0.2.10"}
+
+	instA := arp.NewInstance(netA, nil)
+	instB := arp.NewInstance(netB, nil)
+	arpMgr.Insert(instA)
+	arpMgr.Insert(instB)
+
+	// Cluster A's layer2Update goroutine ends first and drops its own claim,
+	// leaving only sibling B registered.
+	arpMgr.Remove(instA)
+
+	c := &Cluster{arpMgr: arpMgr}
+	c.cleanupVIP(&kubevip.Config{EnableARP: true}, netA, 0)
+
+	if netA.deleteIPCalls != 0 {
+		t.Fatalf("cleanupVIP deleted the shared VIP while a sibling was still registered")
+	}
+}
+
+// TestCleanupVIPsDeletesControlPlaneVIPHoldingItsOwnARPClaim covers the
+// leadership loss path: OnStoppedLeading runs concurrently with the control
+// plane layer2Update goroutine, so the VIP's only ARP claim is still the
+// caller's own and the address must still be removed before the process exits.
+func TestCleanupVIPsDeletesControlPlaneVIPHoldingItsOwnARPClaim(t *testing.T) {
+	arpMgr := arp.NewManager(&kubevip.Config{ArpBroadcastRate: 3000})
+	network := &stubNetwork{ip: "2001:db8::10"}
+	arpMgr.Insert(arp.NewInstance(network, nil))
+
+	c := &Cluster{arpMgr: arpMgr, Network: []vip.Network{network}}
+	c.cleanupVIPs(&kubevip.Config{EnableARP: true})
+
+	if network.deleteIPCalls != 1 {
+		t.Fatalf("cleanupVIPs made %d DeleteIP calls, want 1", network.deleteIPCalls)
+	}
+}
+
+func TestControlPlaneFollowsSharedServiceElection(t *testing.T) {
+	config := &kubevip.Config{KubernetesLeaderElection: kubevip.KubernetesLeaderElection{LeaseName: "default/shared"}}
+	leaseID := lease.NewID(config.LeaderElectionType, "default", "shared")
+	leaseMgr := lease.NewManager()
+	sharedLease, _ := leaseMgr.Acquire(context.Background(), leaseID, "service")
+	if !sharedLease.BeginElection() {
+		t.Fatal("Service election did not start")
+	}
+	sharedLease.ElectionStarted()
+
+	labels := &recordingLabeler{added: make(chan struct{}, 1), removed: make(chan struct{}, 1)}
+	cluster := &Cluster{stop: make(chan struct{}), nodeLabelMgr: labels}
+	done := make(chan error, 1)
+	go func() {
+		done <- cluster.StartCluster(context.Background(), config, nil, nil, leaseMgr, func() {})
+	}()
+	select {
+	case <-labels.added:
+	case <-time.After(time.Second):
+		t.Fatal("control plane did not activate under the shared Service election")
+	}
+
+	sharedLease.ElectionStopped()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Fatalf("shared control-plane follower returned an error: %v", err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("control plane did not stop after shared Service leadership ended")
+	}
+	select {
+	case <-labels.removed:
+	default:
+		t.Fatal("control-plane label was not removed after shared leadership ended")
+	}
+	if sharedLease.Ctx.Err() != nil || leaseMgr.Get(leaseID) != sharedLease {
+		t.Fatal("control-plane cleanup cancelled the surviving Service lease")
+	}
+	leaseMgr.Delete(leaseID, "service", sharedLease)
+}
+
+func TestStopAndWaitPreservingUpgradesInProgressStop(t *testing.T) {
+	done := make(chan struct{})
+	service := &Cluster{
+		stop: make(chan struct{}),
+		service: &servicesWorker{
+			stop:     make(chan struct{}),
+			done:     done,
+			stopping: true,
+		},
+	}
+
+	returned := make(chan struct{})
+	go func() {
+		service.StopAndWaitPreserving("192.0.2.10")
+		close(returned)
+	}()
+
+	deadline := time.Now().Add(time.Second)
+	for {
+		service.stopMu.Lock()
+		_, preserving := service.service.preserveVIPs["192.0.2.10"]
+		service.stopMu.Unlock()
+		if preserving {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("preserving stop did not update the in-progress worker shutdown")
+		}
+		time.Sleep(time.Millisecond)
+	}
+
+	service.finishServicesWorker(done)
+	select {
+	case <-returned:
+	case <-time.After(time.Second):
+		t.Fatal("preserving stop did not return after worker cleanup completed")
+	}
+}

pkg/cluster/cluster_stop_test.go

@@ -0,0 +1,32 @@
+package cluster
+
+import (
+	"sync"
+	"sync/atomic"
+	"testing"
+)
+
+func TestStopConcurrentDoesNotRaceOrPanic(t *testing.T) {
+	c := &Cluster{stop: make(chan struct{})}
+	start := make(chan struct{})
+	var wg sync.WaitGroup
+	var panics atomic.Int64
+
+	for range 128 {
+		wg.Go(func() {
+			<-start
+			defer func() {
+				if recover() != nil {
+					panics.Add(1)
+				}
+			}()
+			c.Stop()
+		})
+	}
+
+	close(start)
+	wg.Wait()
+	if got := panics.Load(); got != 0 {
+		t.Fatalf("concurrent Stop panicked %d time(s)", got)
+	}
+}

pkg/cluster/cluster_test.go

@@ -0,0 +1,125 @@
+package cluster_test
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"encoding/pem"
+	"math/big"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/cluster"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+)
+
+func TestInitCluster_HealthCheckClientNoCA(t *testing.T) {
+	t.Parallel()
+	cfg := &kubevip.Config{
+		EnableBGP: true,
+		ControlPlaneHealthCheck: kubevip.HealthCheck{
+			Address:        "http://localhost:6443/livez",
+			TimeoutSeconds: 5,
+		},
+	}
+
+	_, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
+func TestInitCluster_HealthCheckClientValidCA(t *testing.T) {
+	t.Parallel()
+	caPEM := generateTestCACert(t)
+	caFile := filepath.Join(t.TempDir(), "ca.crt")
+	if err := os.WriteFile(caFile, caPEM, 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	cfg := &kubevip.Config{
+		EnableBGP: true,
+		ControlPlaneHealthCheck: kubevip.HealthCheck{
+			Address:        "https://localhost:6443/livez",
+			TimeoutSeconds: 3,
+			CAPath:         caFile,
+		},
+	}
+
+	_, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
+func TestInitCluster_HealthCheckClientInvalidCAPath(t *testing.T) {
+	t.Parallel()
+	cfg := &kubevip.Config{
+		EnableBGP: true,
+		ControlPlaneHealthCheck: kubevip.HealthCheck{
+			Address: "https://localhost:6443/livez",
+			CAPath:  "/nonexistent/ca.crt",
+		},
+	}
+
+	_, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
+	if err == nil {
+		t.Fatal("expected error for invalid CA path")
+	}
+	if !strings.Contains(err.Error(), "reading health check CA cert") {
+		t.Errorf("expected error about reading CA cert, got: %v", err)
+	}
+}
+
+func TestInitCluster_HealthCheckClientInvalidCAContent(t *testing.T) {
+	t.Parallel()
+	caFile := filepath.Join(t.TempDir(), "bad-ca.crt")
+	if err := os.WriteFile(caFile, []byte("not a certificate"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	cfg := &kubevip.Config{
+		EnableBGP: true,
+		ControlPlaneHealthCheck: kubevip.HealthCheck{
+			Address: "https://localhost:6443/livez",
+			CAPath:  caFile,
+		},
+	}
+
+	_, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
+	if err == nil {
+		t.Fatal("expected error for invalid CA content")
+	}
+	if !strings.Contains(err.Error(), "contains no valid certificates") {
+		t.Errorf("expected error about invalid certificates, got: %v", err)
+	}
+}
+
+// generateTestCACert creates a self-signed CA certificate in PEM format for testing.
+func generateTestCACert(t *testing.T) []byte {
+	t.Helper()
+
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	template := &x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		NotBefore:             time.Now(),
+		NotAfter:              time.Now().Add(time.Hour),
+		IsCA:                  true,
+		BasicConstraintsValid: true,
+	}
+
+	certDER, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
+}

pkg/cluster/service.go

@@ -0,0 +1,626 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
+	"github.com/kube-vip/kube-vip/pkg/backend"
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/election"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config, em *election.Manager, bgpServer bgp.BGPManager, killFunc func()) error {
+
+	var err error
+
+	var wg sync.WaitGroup
+	defer wg.Wait()
+
+	wg.Go(func() {
+		<-ctx.Done()
+		killFunc()
+	})
+
+	loadbalancers := []*loadbalancer.IPVSLoadBalancer{}
+
+	for i := range cluster.Network {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		network := cluster.Network[i]
+
+		if network.IsDDNS() {
+			if err := cluster.StartDDNS(ctx, cluster.Network[i], c.DHCPBackoffAttempts, &wg); err != nil {
+				log.Error("failed to start DDNS", "err", err)
+			}
+		}
+
+		if err := network.SetMask(c.VIPSubnet); err != nil {
+			killFunc()
+			return fmt.Errorf("failed to set mask for subnet %q: %w", c.VIPSubnet, err)
+		}
+
+		// start the dns updater if address is dns
+		if network.IsDNS() {
+			log.Info("starting the DNS updater", "address", network.DNSName())
+			ipUpdater := vip.NewIPUpdater(network)
+			wg.Go(func() {
+				ipUpdater.Run(ctx)
+			})
+		}
+
+		if !c.EnableRoutingTable {
+			// Normal VIP addition, use skipDAD=false for normal DAD process
+			if _, err = network.AddIP(false, false); err != nil {
+				log.Error("failed to add IP", "address", network.IP(), "error", err)
+			}
+		}
+
+		if c.EnableBGP {
+			if c.ControlPlaneHealthCheck.Address != "" {
+				// The health check loop owns route advertisement/withdrawal when configured.
+				wg.Go(func() {
+					cluster.bgpHealthCheckLoop(ctx, c, bgpServer, network.CIDR())
+				})
+			} else {
+				// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation.
+				log.Debug("Attempting to advertise over BGP", "address", network.CIDR())
+				err = bgpServer.AddHost(ctx, network.CIDR(), c.NodeName)
+				if err != nil {
+					log.Error(err.Error())
+				}
+			}
+		}
+
+		if c.EnableLoadBalancer {
+			lb, err := loadbalancer.NewIPVSLB(ctx, network, c.LoadBalancerPort, c.LoadBalancerForwardingMethod,
+				c.BackendHealthCheckInterval, c.EgressWithNftables, killFunc, &wg)
+			if err != nil {
+				killFunc()
+				return fmt.Errorf("creating IPVS LoadBalancer: %w", err)
+			}
+
+			wg.Go(func() {
+				for {
+					select {
+					case <-ctx.Done():
+						return
+					default:
+						err = em.NodeWatcher(ctx, lb, c.Port)
+						if err != nil {
+							log.Error("Error watching node labels", "err", err)
+							if utils.IsPanicError(err) {
+								killFunc()
+								return
+							}
+						}
+					}
+				}
+			})
+
+			loadbalancers = append(loadbalancers, lb)
+		}
+
+		if c.EnableARP {
+			wg.Go(func() {
+				cluster.layer2Update(ctx, network, c)
+			})
+		}
+	}
+
+	if c.EnableLoadBalancer {
+		// Shutdown function that will wait on this signal, unless we call it ourselves
+		<-ctx.Done()
+		for _, lb := range loadbalancers {
+			err = lb.RemoveIPVSLB()
+			if err != nil {
+				log.Error("Error stopping IPVS LoadBalancer", "err", err)
+			}
+		}
+	}
+
+	if c.EnableRoutingTable {
+		backendMapV4 := backend.Map{}
+		backendMapV6 := backend.Map{}
+		// only check localhost
+
+		// An explicitly configured Kubernetes API address (static-pod
+		// deployments point it at the local API server, whose loopback
+		// listener is often the only certificate-valid local endpoint)
+		// takes precedence over the Node object's addresses: the check
+		// answers "is the local API server healthy" for every VIP family,
+		// regardless of the transport family of the override itself.
+		if entry := kubernetesAddrBackendEntry(c.KubernetesAddr, c.Port); entry != nil {
+			log.Info("using configured Kubernetes address for backend health checks", "address", c.KubernetesAddr)
+			backendMapV4[*entry] = false
+			backendMapV6[*entry] = false
+		} else {
+			ips := []string{}
+			if c.NodeName != "" {
+				if ips, err = getNodeIPs(ctx, c.NodeName, em.KubernetesClient); err != nil && !apierrors.IsNotFound(err) {
+					log.Error("failed to get IP of control-plane node", "err", err)
+				}
+			}
+
+			if len(ips) == 0 {
+				if !utils.IsIPv6(cluster.Network[0].IP()) {
+					ips = append(ips, "127.0.0.1")
+				} else {
+					ips = append(ips, "::1")
+				}
+
+				log.Info("no IP address found for node - will fallback to use localhost address", "addresses", ips)
+			}
+
+			for _, ip := range ips {
+				entry := backend.Entry{Addr: ip, Port: c.Port}
+				if !utils.IsIPv6(ip) {
+					backendMapV4[entry] = false
+				} else {
+					backendMapV6[entry] = false
+				}
+			}
+		}
+
+		backend.SetKubeConfigPath(c.K8sConfigFile)
+		backend.Watch(ctx, c.BackendHealthCheckInterval, func() {
+			for i := range cluster.Network {
+				network := cluster.Network[i]
+				networkIP := network.IP()
+				isNetworkV6 := utils.IsIPv6(networkIP)
+				log.Debug("current ip to process", "ip", networkIP)
+
+				backendMap := &backendMapV4
+				if isNetworkV6 {
+					backendMap = &backendMapV6
+				}
+
+				for entry := range *backendMap {
+					log.Debug("entry.Check() for entry", "entry", entry)
+					var healthy bool
+					if c.ControlPlaneHealthCheck.Address != "" {
+						req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, c.ControlPlaneHealthCheck.Address, nil)
+						if reqErr != nil {
+							log.Error("create health check request", "err", reqErr)
+						} else if resp, doErr := cluster.healthCheckHTTPClient.Do(req); doErr != nil {
+							log.Error("health check request failed", "url", c.ControlPlaneHealthCheck.Address, "err", doErr)
+						} else {
+							resp.Body.Close()
+							healthy = resp.StatusCode == http.StatusOK
+							if !healthy {
+								log.Warn("health check returned non-200 status", "url", c.ControlPlaneHealthCheck.Address, "status", resp.StatusCode)
+							}
+						}
+					} else {
+						healthy = entry.Check()
+					}
+					if healthy {
+						log.Debug("entry.Check() true")
+						// Normal VIP addition with precheck, use skipDAD=false for normal DAD process
+						_, err = network.AddIP(true, false)
+						if err != nil {
+							log.Error("error adding address", "err", err)
+						}
+						if !(*backendMap)[entry] {
+							log.Info("added backend", "ip", network.IP())
+						}
+
+						err = cluster.routeMgr.Add(c.NodeName, network, true, false)
+						if err != nil && !errors.Is(err, fs.ErrExist) && !errors.Is(err, syscall.ESRCH) {
+							log.Warn(err.Error())
+						} else if err == nil && !(*backendMap)[entry] {
+							log.Info("added route", "route", network.PrepareRoute())
+						} else if err == nil || errors.Is(err, fs.ErrExist) {
+							// Re-assert the route on every healthy cycle: routing daemons
+							// (e.g. zebra) can miss the single netlink event for the route,
+							// leaving it unadvertised even though it exists in the kernel.
+							// RouteReplace is idempotent and regenerates that event.
+							if replaceErr := network.ReplaceRoute(); replaceErr != nil {
+								log.Warn("re-asserting route", "err", replaceErr)
+							} else {
+								log.Debug("re-asserted route", "route", network.PrepareRoute())
+							}
+						}
+
+						(*backendMap)[entry] = true
+						break
+					}
+					(*backendMap)[entry] = false
+				}
+
+				deleteAddress := true
+				for entry := range *backendMap {
+					if (*backendMap)[entry] {
+						deleteAddress = false
+						break
+					}
+				}
+
+				if deleteAddress {
+					err = cluster.routeMgr.Delete(c.NodeName, network)
+					if err != nil {
+						log.Warn("deleting route", "err", err)
+					}
+
+					deleted, err := network.DeleteIP()
+					if err != nil {
+						log.Error("error deleting IP", "err", err)
+						killFunc()
+						return
+					}
+					if deleted {
+						log.Info("deleted address", "IP", network.IP(), "interface", network.Interface())
+					}
+				}
+			}
+		})
+	}
+
+	if c.EnableBGP {
+		<-ctx.Done()
+	}
+
+	return nil
+}
+
+func (cluster *Cluster) bgpHealthCheck(ctx context.Context, c *kubevip.Config) (bool, error) {
+	statusCode := 0
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.ControlPlaneHealthCheck.Address, nil)
+	if err != nil {
+		return false, fmt.Errorf("building request %v: %w", req, err)
+	} else {
+		resp, err := cluster.healthCheckHTTPClient.Do(req)
+		if err != nil {
+			return false, fmt.Errorf("checking control-plane: %w", err)
+		}
+		defer resp.Body.Close()
+		statusCode = resp.StatusCode
+	}
+	healthy := statusCode == http.StatusOK
+	if !healthy {
+		return healthy, fmt.Errorf("wrong status code: %d", statusCode)
+	}
+	return healthy, nil
+}
+
+func (cluster *Cluster) bgpHealthCheckLoop(ctx context.Context, c *kubevip.Config, bgpServer bgp.BGPManager, vipCIDR string) {
+	period := time.Duration(c.ControlPlaneHealthCheck.PeriodSeconds) * time.Second
+
+	consecutiveFailures := 0
+	routeAnnounced := false
+	ticker := time.NewTicker(period)
+	defer ticker.Stop()
+
+	log.Info("Starting BGP health check",
+		"address", c.ControlPlaneHealthCheck.Address,
+		"cidr", vipCIDR,
+		"period", period,
+		"timeout", cluster.healthCheckHTTPClient.Timeout,
+		"threshold", c.ControlPlaneHealthCheck.FailureThreshold,
+	)
+
+	for {
+		healthy, healthErr := cluster.bgpHealthCheck(ctx, c)
+		if healthy {
+			consecutiveFailures = 0
+			if !routeAnnounced {
+				log.Info("BGP health check passed, announcing route", "cidr", vipCIDR)
+				if err := bgpServer.AddHost(ctx, vipCIDR, c.NodeName); err != nil {
+					log.Error("BGP health check: failed to announce route", "cidr", vipCIDR, "err", err)
+				} else {
+					routeAnnounced = true
+				}
+			}
+		} else {
+			consecutiveFailures++
+			if healthErr != nil {
+				log.Warn("BGP health check failed", "address", c.ControlPlaneHealthCheck.Address, "consecutive", consecutiveFailures, "err", healthErr)
+			}
+			if consecutiveFailures >= c.ControlPlaneHealthCheck.FailureThreshold && routeAnnounced {
+				log.Warn("BGP health check threshold reached, withdrawing route", "failureThreshold", c.ControlPlaneHealthCheck.FailureThreshold, "cidr", vipCIDR)
+				if err := bgpServer.DelHost(ctx, vipCIDR, c.NodeName); err != nil {
+					log.Error("BGP health check: failed to withdraw route", "cidr", vipCIDR, "err", err)
+				} else {
+					routeAnnounced = false
+				}
+			}
+		}
+
+		select {
+		case <-ctx.Done():
+			if routeAnnounced {
+				if err := bgpServer.DelHost(ctx, vipCIDR, c.NodeName); err != nil {
+					log.Error("BGP health check: failed to withdraw route", "cidr", vipCIDR, "err", err)
+				}
+			}
+			return
+		case <-ticker.C:
+		}
+	}
+}
+
+// kubernetesAddrBackendEntry converts an explicitly configured Kubernetes
+// API address override (config.KubernetesAddr, e.g. "https://127.0.0.1:6443"
+// on static-pod deployments) into a backend health-check entry. Returns nil
+// when no usable override is configured.
+func kubernetesAddrBackendEntry(kubernetesAddr string, defaultPort uint16) *backend.Entry {
+	if kubernetesAddr == "" {
+		return nil
+	}
+	u, err := url.Parse(kubernetesAddr)
+	if err != nil || u.Hostname() == "" {
+		return nil
+	}
+	port := defaultPort
+	if p := u.Port(); p != "" {
+		if parsed, err := strconv.ParseUint(p, 10, 16); err == nil {
+			port = uint16(parsed)
+		}
+	}
+	return &backend.Entry{Addr: u.Hostname(), Port: port}
+}
+
+func getNodeIPs(ctx context.Context, nodename string, client *kubernetes.Clientset) ([]string, error) {
+	node, err := client.CoreV1().Nodes().Get(ctx, nodename, metav1.GetOptions{})
+	if err != nil && !apierrors.IsNotFound(err) {
+		return []string{}, fmt.Errorf("failed to get data about '%s' node: %w", nodename, err)
+	}
+	ips := []string{}
+	for _, addr := range node.Status.Addresses {
+		if addr.Type == corev1.NodeInternalIP {
+			ips = append(ips, addr.Address)
+		}
+	}
+	return ips, nil
+}
+
+// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
+func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp bgp.BGPManager, name string, wg *sync.WaitGroup) error {
+	// use a Go context so we can tell the arp loop code when we
+	// want to step down
+	//nolint
+	lbCtx, lbCancel := context.WithCancel(ctx)
+	var lbWg sync.WaitGroup
+	stop, done, err := cluster.startServicesWorker()
+	if err != nil {
+		lbCancel()
+		return err
+	}
+	type startedNetwork struct {
+		network    vip.Network
+		routeAdded bool
+		ipAdded    bool
+		bgpAdded   bool
+	}
+	startedNetworks := make([]startedNetwork, 0, len(cluster.Network))
+	servicesWorkerStarted := false
+	defer func() {
+		if !servicesWorkerStarted {
+			lbCancel()
+			lbWg.Wait()
+			cleanupCtx := context.WithoutCancel(ctx)
+			for index := len(startedNetworks) - 1; index >= 0; index-- {
+				started := startedNetworks[index]
+				if started.bgpAdded && bgp != nil {
+					if err := bgp.DelHost(cleanupCtx, started.network.CIDR(), name); err != nil {
+						log.Warn("failed to withdraw BGP host after startup failure", "address", started.network.CIDR(), "err", err)
+					}
+				}
+				if started.routeAdded && cluster.routeMgr != nil {
+					if err := cluster.routeMgr.Delete(name, started.network); err != nil {
+						log.Warn("failed to delete route after startup failure", "address", started.network.CIDR(), "err", err)
+					}
+				}
+				if started.ipAdded {
+					if _, err := started.network.DeleteIP(); err != nil {
+						log.Warn("failed to delete VIP after startup failure", "address", started.network.IP(), "err", err)
+					}
+				}
+			}
+			cluster.finishServicesWorker(done)
+		}
+	}()
+
+	for i := range cluster.Network {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		network := cluster.Network[i]
+		startedNetworks = append(startedNetworks, startedNetwork{network: network})
+		started := &startedNetworks[len(startedNetworks)-1]
+
+		if network.IsDDNS() {
+			ddnsReady := make(chan struct{})
+			lbWg.Go(func() {
+				// start the DDNS if requested
+				log.Debug("(svcs) start DDNS", "name", network.DNSName())
+				if err := cluster.StartDDNS(lbCtx, cluster.Network[i], c.DHCPBackoffAttempts, &lbWg); err != nil {
+					log.Error("failed to start DDNS", "err", err)
+				}
+
+				close(ddnsReady)
+				<-lbCtx.Done()
+			})
+			<-ddnsReady
+		}
+
+		log.Debug("current ip to process", "ip", network.IP(), "mask", c.VIPSubnet)
+		if err := network.SetMask(c.VIPSubnet); err != nil {
+			log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
+			lbCancel()
+			return utils.WrapPanicError(err, "failed to set mask for subnet %q", c.VIPSubnet)
+		}
+		existing, err := network.IsSet()
+		if err != nil {
+			lbCancel()
+			return fmt.Errorf("check existing VIP %q: %w", network.IP(), err)
+		}
+		log.Debug("config flags", "enable_routing_table", c.EnableRoutingTable, "enable_leader_election", c.EnableLeaderElection, "enable_services_election", c.EnableServicesElection)
+
+		if c.EnableRoutingTable && (c.EnableLeaderElection || c.EnableServicesElection) {
+			err = cluster.routeMgr.Add(name, network, false, false)
+			if err != nil {
+				log.Warn(err.Error())
+			} else {
+				started.routeAdded = true
+				log.Info("successful add Route")
+			}
+		}
+
+		if shouldAddServiceIP(c) {
+			// Normal VIP addition, use skipDAD=false for normal DAD process
+			// Note: When WireGuard is enabled, the VIP is added to the tunnel interface
+			// instead of lo, so we skip adding it here.
+			added, addErr := network.AddIP(false, false)
+			started.ipAdded = existing == nil && added
+			if addErr != nil {
+				log.Warn(addErr.Error())
+			} else {
+				log.Info("successful add IP", "address", network.IP())
+			}
+		}
+
+		if c.EnableARP {
+			lbWg.Go(func() {
+				cluster.layer2Update(lbCtx, network, c)
+			})
+		}
+
+		if c.EnableBGP && (c.EnableLeaderElection || c.EnableServicesElection) {
+			// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
+			log.Debug("(svcs) attempting to advertise over BGP", "address", network.CIDR())
+			err = bgp.AddHost(lbCtx, network.CIDR(), name)
+			if err != nil {
+				log.Error(err.Error())
+			} else {
+				started.bgpAdded = true
+			}
+		}
+	}
+
+	wg.Go(func() {
+		defer cluster.finishServicesWorker(done)
+		for i := range cluster.Network {
+			network := cluster.Network[i]
+
+			// start the dns updater if address is dns
+			if network.IsDNS() {
+				log.Info("(svcs) starting the DNS updater", "address", network.DNSName(), "ip", network.IP())
+				ipUpdater := vip.NewIPUpdater(network)
+				wg.Go(func() {
+					ipUpdater.Run(lbCtx)
+				})
+			}
+		}
+
+		select {
+		case <-stop:
+		case <-ctx.Done():
+		}
+
+		// Stop the loadbalancer context if it is running
+		lbCancel()
+
+		lbWg.Wait() // wait for all cluster ARP/NDP to be finished
+
+		log.Info("[LOADBALANCER] Stopping load balancers", "name", name)
+
+		if c.EnableRoutingTable {
+			for i := range cluster.Network {
+				if err := cluster.routeMgr.Delete(name, cluster.Network[i]); err != nil {
+					log.Warn(err.Error())
+				}
+			}
+
+			return
+		}
+
+		cluster.cleanupServiceVIPs(c, done)
+	})
+	servicesWorkerStarted = true
+
+	return nil
+}
+
+func shouldAddServiceIP(c *kubevip.Config) bool {
+	return !c.EnableRoutingTable && (!c.EnableBGP || c.BGPAttachIPToInterface) && !c.EnableWireguard
+}
+
+// Layer2Update, handles the creation of the
+func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config) {
+	var ndp *vip.NdpResponder
+	var err error
+	ipString := network.IP()
+	if utils.IsIPv6(ipString) {
+		if network.IPisLinkLocal() {
+			log.Error("layer2 is link-local can't use NDP", "address", ipString)
+		} else {
+			ndp, err = waitNDPResponder(ctx, network.Interface())
+			if err != nil {
+				log.Error("failed to create new NDP Responder", "error", err)
+			} else {
+				if ndp != nil {
+					defer ndp.Close()
+				}
+			}
+		}
+	}
+
+	log.Info("layer 2 broadcaster starting", "IP", network.IP(), "device", network.Interface())
+	log.Debug("layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
+
+	arpInstance := arp.NewInstance(network, ndp)
+	cluster.arpMgr.Insert(arpInstance)
+
+	<-ctx.Done() // if cancel() execute
+	log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
+	cluster.arpMgr.RemoveOnLeadershipLoss(arpInstance)
+}
+
+func waitNDPResponder(ctx context.Context, ifaceName string) (*vip.NdpResponder, error) {
+	ndp, err := vip.NewNDPResponder(ifaceName)
+	if err != nil && strings.Contains(err.Error(), "no such device") {
+		log.Warn("unable to create NDP responder at first try", "interface", ifaceName, "err", err)
+		ndpCreateCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
+		defer cancel()
+		ticker := time.NewTicker(time.Second)
+
+		for {
+			select {
+			case <-ndpCreateCtx.Done():
+				return nil, fmt.Errorf("failed to create NDP responder for interface %q: %w", ifaceName, ndpCreateCtx.Err())
+			case <-ticker.C:
+				ndp, err = vip.NewNDPResponder(ifaceName)
+				if err != nil {
+					log.Warn("unable to create NDP responder on retry", "interface", ifaceName, "err", err)
+				} else {
+					return ndp, nil
+				}
+			}
+		}
+	} else if err != nil {
+		return nil, fmt.Errorf("unable to create NDP responder for interface %q: %w", ifaceName, err)
+	}
+	return ndp, nil
+}

pkg/cluster/service_config_test.go

@@ -0,0 +1,55 @@
+package cluster
+
+import (
+	"testing"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+)
+
+func TestShouldAddServiceIP(t *testing.T) {
+	tests := []struct {
+		name   string
+		config *kubevip.Config
+		want   bool
+	}{
+		{
+			name:   "BGP default does not attach IP",
+			config: &kubevip.Config{EnableBGP: true},
+			want:   false,
+		},
+		{
+			name: "BGP opt-in attaches IP",
+			config: &kubevip.Config{
+				EnableBGP:              true,
+				BGPAttachIPToInterface: true,
+			},
+			want: true,
+		},
+		{
+			name: "routing table takes precedence",
+			config: &kubevip.Config{
+				EnableBGP:              true,
+				BGPAttachIPToInterface: true,
+				EnableRoutingTable:     true,
+			},
+			want: false,
+		},
+		{
+			name: "WireGuard takes precedence",
+			config: &kubevip.Config{
+				EnableBGP:              true,
+				BGPAttachIPToInterface: true,
+				EnableWireguard:        true,
+			},
+			want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := shouldAddServiceIP(tt.config); got != tt.want {
+				t.Fatalf("shouldAddServiceIP() = %t, want %t", got, tt.want)
+			}
+		})
+	}
+}

pkg/cluster/service_internal_test.go

@@ -0,0 +1,61 @@
+package cluster
+
+import (
+	"testing"
+)
+
+func TestKubernetesAddrBackendEntry(t *testing.T) {
+	cases := []struct {
+		name     string
+		addr     string
+		port     uint16
+		wantAddr string
+		wantPort uint16
+		wantNil  bool
+	}{
+		{
+			name:     "explicit v4 loopback with port",
+			addr:     "https://127.0.0.1:6443",
+			port:     9999,
+			wantAddr: "127.0.0.1",
+			wantPort: 6443,
+		},
+		{
+			name:     "hostname without port falls back to config port",
+			addr:     "https://localhost",
+			port:     6443,
+			wantAddr: "localhost",
+			wantPort: 6443,
+		},
+		{
+			name:    "empty override",
+			addr:    "",
+			port:    6443,
+			wantNil: true,
+		},
+		{
+			name:    "garbage override",
+			addr:    "://not-a-url",
+			port:    6443,
+			wantNil: true,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			entry := kubernetesAddrBackendEntry(tc.addr, tc.port)
+			if tc.wantNil {
+				if entry != nil {
+					t.Fatalf("expected nil entry, got %+v", entry)
+				}
+				return
+			}
+			if entry == nil {
+				t.Fatal("expected an entry, got nil")
+			}
+			if entry.Addr != tc.wantAddr || entry.Port != tc.wantPort {
+				t.Fatalf("got %+v, want addr %q port %d", entry, tc.wantAddr, tc.wantPort)
+			}
+		})
+	}
+}

pkg/cluster/service_test.go

@@ -0,0 +1,548 @@
+package cluster_test
+
+import (
+	"context"
+	"encoding/pem"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/cluster"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/route"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	"github.com/vishvananda/netlink"
+	corev1 "k8s.io/api/core/v1"
+)
+
+const testCIDR = "10.0.0.34/32"
+
+func TestBGPHealthCheckLoop_AnnouncesOnHealthy(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusOK)
+	t.Cleanup(healthcheck.server.Close)
+
+	bgpManager := newMockBGPRouteManager()
+	startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
+
+	expectEventually(t, func() bool { return bgpManager.isAnnounced() },
+		"route should be announced")
+}
+
+func TestServicesWorkerStopAndWaitDrainsBeforeRestart(t *testing.T) {
+	config := &kubevip.Config{}
+	serviceCluster, err := cluster.InitCluster(config, true, nil, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("InitCluster() error = %v", err)
+	}
+	var workers sync.WaitGroup
+	if err := serviceCluster.StartLoadBalancerService(context.Background(), config, nil, "service", &workers); err != nil {
+		t.Fatalf("first StartLoadBalancerService() error = %v", err)
+	}
+	if err := serviceCluster.StartLoadBalancerService(context.Background(), config, nil, "service", &workers); err == nil {
+		t.Fatal("second StartLoadBalancerService() started while the first workers were active")
+	}
+
+	serviceCluster.StopAndWait()
+	if err := serviceCluster.StartLoadBalancerService(context.Background(), config, nil, "service", &workers); err != nil {
+		t.Fatalf("StartLoadBalancerService() after StopAndWait error = %v", err)
+	}
+	serviceCluster.StopAndWait()
+	workers.Wait()
+}
+
+func TestServicesWorkerStopAndWaitPreservingDrainsBeforeRestart(t *testing.T) {
+	config := &kubevip.Config{}
+	serviceCluster, err := cluster.InitCluster(config, true, nil, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("InitCluster() error = %v", err)
+	}
+	var workers sync.WaitGroup
+	if err := serviceCluster.StartLoadBalancerService(context.Background(), config, nil, "service", &workers); err != nil {
+		t.Fatalf("StartLoadBalancerService() error = %v", err)
+	}
+	serviceCluster.StopAndWait()
+	if err := serviceCluster.StartLoadBalancerService(context.Background(), config, nil, "service", &workers); err != nil {
+		t.Fatalf("StartLoadBalancerService() after preserving stop error = %v", err)
+	}
+	serviceCluster.StopAndWait()
+	workers.Wait()
+}
+
+func TestStartLoadBalancerServiceRollsBackEarlierNetwork(t *testing.T) {
+	first := &mockNetwork{ip: "192.0.2.10", cidr: "192.0.2.10/32"}
+	second := &mockNetwork{ip: "192.0.2.11", cidr: "192.0.2.11/32", setMaskErr: errors.New("set mask")}
+	serviceCluster, err := cluster.InitCluster(&kubevip.Config{}, true, nil, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("InitCluster() error = %v", err)
+	}
+	serviceCluster.Network = []vip.Network{first, second}
+
+	if err := serviceCluster.StartLoadBalancerService(context.Background(), &kubevip.Config{VIPSubnet: "32"}, nil, "service", &sync.WaitGroup{}); err == nil {
+		t.Fatal("StartLoadBalancerService() error = nil, want second-network failure")
+	}
+	first.mu.Lock()
+	addCalls, deleteCalls, present := first.addIPCalls, first.deleteIPCalls, first.present
+	first.mu.Unlock()
+	if addCalls != 1 || deleteCalls != 1 || present {
+		t.Fatalf("first network rollback = add %d, delete %d, present %t; want 1, 1, false", addCalls, deleteCalls, present)
+	}
+	serviceCluster.StopAndWait()
+}
+
+func TestStartLoadBalancerServiceRollbackPreservesExistingVIP(t *testing.T) {
+	first := &mockNetwork{ip: "192.0.2.10", cidr: "192.0.2.10/32", present: true}
+	second := &mockNetwork{ip: "192.0.2.11", cidr: "192.0.2.11/32", setMaskErr: errors.New("set mask")}
+	serviceCluster, err := cluster.InitCluster(&kubevip.Config{}, true, nil, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("InitCluster() error = %v", err)
+	}
+	serviceCluster.Network = []vip.Network{first, second}
+
+	if err := serviceCluster.StartLoadBalancerService(context.Background(), &kubevip.Config{VIPSubnet: "32"}, nil, "service", &sync.WaitGroup{}); err == nil {
+		t.Fatal("StartLoadBalancerService() error = nil, want second-network failure")
+	}
+	first.mu.Lock()
+	addCalls, deleteCalls, present := first.addIPCalls, first.deleteIPCalls, first.present
+	first.mu.Unlock()
+	if addCalls != 1 || deleteCalls != 0 || !present {
+		t.Fatalf("existing VIP rollback = add %d, delete %d, present %t; want 1, 0, true", addCalls, deleteCalls, present)
+	}
+	serviceCluster.StopAndWait()
+}
+
+func TestStartLoadBalancerServiceCancelledContextDoesNotConfigureVIP(t *testing.T) {
+	network := &mockNetwork{ip: "192.0.2.10", cidr: "192.0.2.10/32"}
+	serviceCluster, err := cluster.InitCluster(&kubevip.Config{}, true, nil, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("InitCluster() error = %v", err)
+	}
+	serviceCluster.Network = []vip.Network{network}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	err = serviceCluster.StartLoadBalancerService(ctx, &kubevip.Config{VIPSubnet: "32"}, nil, "service", &sync.WaitGroup{})
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("StartLoadBalancerService() error = %v, want context.Canceled", err)
+	}
+	network.mu.Lock()
+	addCalls := network.addIPCalls
+	network.mu.Unlock()
+	if addCalls != 0 {
+		t.Fatalf("AddIP calls = %d, want 0 after context cancellation", addCalls)
+	}
+}
+
+func TestBGPHealthCheckLoop_NoAnnouncementUntilHealthy(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusInternalServerError)
+	t.Cleanup(healthcheck.server.Close)
+
+	bgpManager := newMockBGPRouteManager()
+	startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
+
+	expectConsistently(t, func() bool { return !bgpManager.isAnnounced() },
+		2*time.Second, "route should not be announced while unhealthy")
+
+	healthcheck.setStatus(http.StatusOK)
+	expectEventually(t, func() bool { return bgpManager.isAnnounced() },
+		"route should be announced after recovery")
+}
+
+func TestBGPHealthCheckLoop_WithdrawsAfterThreshold(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusOK)
+	t.Cleanup(healthcheck.server.Close)
+
+	bgpManager := newMockBGPRouteManager()
+	cfg := newBGPConfig(healthcheck.server.URL, healthcheck.caPath)
+	cfg.ControlPlaneHealthCheck.FailureThreshold = 3
+	startVipService(t, cfg, bgpManager)
+
+	expectEventually(t, func() bool { return bgpManager.isAnnounced() },
+		"route should be announced")
+
+	healthcheck.setStatus(http.StatusServiceUnavailable)
+
+	expectConsistently(t, func() bool { return bgpManager.isAnnounced() },
+		1500*time.Millisecond, "route should stay announced before threshold is reached")
+
+	expectEventually(t, func() bool { return !bgpManager.isAnnounced() },
+		"route should be withdrawn after threshold")
+}
+
+func TestBGPHealthCheckLoop_ReAnnouncesOnRecovery(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusOK)
+	t.Cleanup(healthcheck.server.Close)
+
+	bgpManager := newMockBGPRouteManager()
+	cfg := newBGPConfig(healthcheck.server.URL, healthcheck.caPath)
+	cfg.ControlPlaneHealthCheck.FailureThreshold = 1
+	startVipService(t, cfg, bgpManager)
+
+	expectEventually(t, func() bool { return bgpManager.isAnnounced() },
+		"route should be announced")
+
+	healthcheck.setStatus(http.StatusServiceUnavailable)
+	expectEventually(t, func() bool { return !bgpManager.isAnnounced() },
+		"route should be withdrawn")
+
+	healthcheck.setStatus(http.StatusOK)
+	expectEventually(t, func() bool { return bgpManager.isAnnounced() },
+		"route should be re-announced")
+}
+
+func TestBGPHealthCheckLoop_StopsOnContextCancel(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusOK)
+	t.Cleanup(healthcheck.server.Close)
+
+	bgpManager := newMockBGPRouteManager()
+	cancelContext, vipServiceDone := startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
+
+	expectEventually(t, func() bool { return bgpManager.isAnnounced() },
+		"route should be announced")
+
+	cancelContext()
+
+	select {
+	case <-vipServiceDone:
+	case <-time.After(5 * time.Second):
+		t.Fatal("vipService did not stop after context cancellation")
+	}
+}
+
+func TestBGPHealthCheckLoop_RetriesAddHostOnFailure(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusOK)
+	t.Cleanup(healthcheck.server.Close)
+
+	bgpManager := newMockBGPRouteManager()
+	bgpManager.setAddErr(errTestAddHost)
+	startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
+
+	expectConsistently(t, func() bool { return !bgpManager.isAnnounced() },
+		2*time.Second, "route should not be announced while AddHost errors")
+
+	bgpManager.setAddErr(nil)
+	expectEventually(t, func() bool { return bgpManager.isAnnounced() },
+		"route should be announced after clearing AddHost error")
+}
+
+func TestBGPHealthCheckLoop_RetriesDelHostOnFailure(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusOK)
+	t.Cleanup(healthcheck.server.Close)
+
+	bgpManager := newMockBGPRouteManager()
+	cfg := newBGPConfig(healthcheck.server.URL, healthcheck.caPath)
+	cfg.ControlPlaneHealthCheck.FailureThreshold = 1
+	startVipService(t, cfg, bgpManager)
+
+	expectEventually(t, func() bool { return bgpManager.isAnnounced() },
+		"route should be announced")
+
+	bgpManager.setDelErr(errTestDelHost)
+	healthcheck.setStatus(http.StatusServiceUnavailable)
+
+	expectConsistently(t, func() bool { return bgpManager.isAnnounced() },
+		1500*time.Millisecond, "route should stay announced while DelHost errors")
+
+	bgpManager.setDelErr(nil)
+	expectEventually(t, func() bool { return !bgpManager.isAnnounced() },
+		"route should be withdrawn after clearing DelHost error")
+}
+
+func TestRoutingTableHealthCheck_AddsVIPWhenHealthy(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusOK)
+	t.Cleanup(healthcheck.server.Close)
+
+	network := &mockNetwork{ip: "10.0.0.1", cidr: testCIDR}
+	startRoutingTableVipService(t, newRoutingTableConfig(healthcheck.server.URL, healthcheck.caPath), network)
+
+	expectEventually(t, network.isPresent,
+		"VIP should be added while health check is healthy")
+}
+
+func TestRoutingTableHealthCheck_RemovesVIPWhenUnhealthy(t *testing.T) {
+	t.Parallel()
+	healthcheck := newTestHealthServer(t, http.StatusOK)
+	t.Cleanup(healthcheck.server.Close)
+
+	network := &mockNetwork{ip: "10.0.0.1", cidr: testCIDR}
+	startRoutingTableVipService(t, newRoutingTableConfig(healthcheck.server.URL, healthcheck.caPath), network)
+
+	expectEventually(t, network.isPresent,
+		"VIP should be added while health check is healthy")
+
+	healthcheck.setStatus(http.StatusServiceUnavailable)
+	expectEventually(t, func() bool { return !network.isPresent() },
+		"VIP should be removed once health check becomes unhealthy")
+}
+
+var (
+	errTestAddHost = &testError{msg: "mock AddHost error"}
+	errTestDelHost = &testError{msg: "mock DelHost error"}
+)
+
+type testError struct{ msg string }
+
+func (e *testError) Error() string { return e.msg }
+
+// startVipService launches vipService in a goroutine with a mock network and
+// registers a cleanup to cancel the context and wait for it to finish.
+// Uses InitCluster so the real code parses certs for the BGP health check client.
+func startVipService(t *testing.T, cfg *kubevip.Config, bgpServer bgp.BGPManager) (context.CancelFunc, <-chan struct{}) {
+	t.Helper()
+	c, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
+	if err != nil {
+		t.Fatalf("InitCluster: %v", err)
+	}
+	c.Network = []vip.Network{&mockNetwork{ip: "10.0.0.1", cidr: testCIDR}}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+
+	go func() {
+		_ = c.StartVipService(ctx, cfg, nil, bgpServer, func() {})
+		close(done)
+	}()
+
+	t.Cleanup(func() {
+		cancel()
+		<-done
+	})
+
+	return cancel, done
+}
+
+// startRoutingTableVipService launches vipService in routing-table mode with a
+// mock network and a real route.Manager (which only drives the mock network's
+// route methods, so no netlink calls happen). Registers cleanup to stop it.
+func startRoutingTableVipService(t *testing.T, cfg *kubevip.Config, network *mockNetwork) {
+	t.Helper()
+
+	c, err := cluster.InitCluster(cfg, true, nil, nil, route.NewManager(), nil)
+	if err != nil {
+		t.Fatalf("InitCluster: %v", err)
+	}
+	c.Network = []vip.Network{network}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+
+	go func() {
+		_ = c.StartVipService(ctx, cfg, nil, nil, func() {})
+		close(done)
+	}()
+
+	t.Cleanup(func() {
+		cancel()
+		<-done
+	})
+}
+
+func newRoutingTableConfig(url, caPath string) *kubevip.Config {
+	cfg := newBGPConfig(url, caPath)
+	cfg.EnableBGP = false
+	cfg.EnableRoutingTable = true
+	cfg.BackendHealthCheckInterval = 1
+	return cfg
+}
+
+func newBGPConfig(url, caPath string) *kubevip.Config {
+	return &kubevip.Config{
+		EnableBGP: true,
+		ControlPlaneHealthCheck: kubevip.HealthCheck{
+			Address:          url,
+			CAPath:           caPath,
+			PeriodSeconds:    1,
+			TimeoutSeconds:   2,
+			FailureThreshold: 1,
+		},
+	}
+}
+
+// mockBGPRouteManager tracks announced addresses as a set.
+// AddHost adds, DelHost removes. Errors prevent state changes.
+type mockBGPRouteManager struct {
+	mu        sync.Mutex
+	announced map[string]bool
+	addErr    error
+	delErr    error
+}
+
+func newMockBGPRouteManager() *mockBGPRouteManager {
+	return &mockBGPRouteManager{announced: make(map[string]bool)}
+}
+
+func (m *mockBGPRouteManager) AddHost(_ context.Context, addr string, _ string) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.addErr != nil {
+		return m.addErr
+	}
+	m.announced[addr] = true
+	return nil
+}
+
+func (m *mockBGPRouteManager) DelHost(_ context.Context, addr string, _ string) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.delErr != nil {
+		return m.delErr
+	}
+	delete(m.announced, addr)
+	return nil
+}
+
+func (m *mockBGPRouteManager) isAnnounced() bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.announced[testCIDR]
+}
+
+func (m *mockBGPRouteManager) setAddErr(err error) {
+	m.mu.Lock()
+	m.addErr = err
+	m.mu.Unlock()
+}
+
+func (m *mockBGPRouteManager) setDelErr(err error) {
+	m.mu.Lock()
+	m.delErr = err
+	m.mu.Unlock()
+}
+
+// mockNetwork implements vip.Network with no-op operations.
+type mockNetwork struct {
+	ip   string
+	cidr string
+
+	mu            sync.Mutex
+	present       bool
+	setMaskErr    error
+	addIPCalls    int
+	deleteIPCalls int
+}
+
+func (m *mockNetwork) AddIP(bool, bool, ...int) (bool, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.addIPCalls++
+	m.present = true
+	return true, nil
+}
+func (m *mockNetwork) DeleteIP() (bool, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.deleteIPCalls++
+	deleted := m.present
+	m.present = false
+	return deleted, nil
+}
+func (m *mockNetwork) isPresent() bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.present
+}
+func (m *mockNetwork) AddRoute(bool) (bool, error) { return false, nil }
+func (m *mockNetwork) ReplaceRoute() error         { return nil }
+func (m *mockNetwork) DeleteRoute() error          { return nil }
+func (m *mockNetwork) UpdateRoutes() (bool, error) { return false, nil }
+func (m *mockNetwork) IsSet() (*netlink.Addr, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.present {
+		return &netlink.Addr{}, nil
+	}
+	return nil, nil
+}
+func (m *mockNetwork) IP() string                      { return m.ip }
+func (m *mockNetwork) CIDR() string                    { return m.cidr }
+func (m *mockNetwork) IPisLinkLocal() bool             { return false }
+func (m *mockNetwork) PrepareRoute() *netlink.Route    { return nil }
+func (m *mockNetwork) RouteHash() string               { return "" }
+func (m *mockNetwork) SetIP(string) error              { return nil }
+func (m *mockNetwork) SetServicePorts(*corev1.Service) {}
+func (m *mockNetwork) Interface() string               { return "eth0" }
+func (m *mockNetwork) IsDADFAIL() bool                 { return false }
+func (m *mockNetwork) IsDNS() bool                     { return false }
+func (m *mockNetwork) IsDDNS() bool                    { return false }
+func (m *mockNetwork) DDNSHostName() string            { return "" }
+func (m *mockNetwork) DNSName() string                 { return "" }
+func (m *mockNetwork) SetMask(string) error            { return m.setMaskErr }
+func (m *mockNetwork) SetHasEndpoints(bool)            {}
+func (m *mockNetwork) HasEndpoints() bool              { return false }
+func (m *mockNetwork) ARPName() string                 { return "" }
+func (m *mockNetwork) GetPossibleSubnets() string      { return "" }
+func (m *mockNetwork) DHCPFamily() string              { return "" }
+func (m *mockNetwork) IPVSMark() uint32                { return 0 }
+
+// testHealthServer wraps an HTTPS httptest.Server with an atomic status code.
+// caPath is the path to the server's CA cert for client verification.
+type testHealthServer struct {
+	server     *httptest.Server
+	statusCode atomic.Int64
+	caPath     string
+}
+
+func newTestHealthServer(t *testing.T, status int) *testHealthServer {
+	t.Helper()
+	healthcheck := &testHealthServer{}
+	healthcheck.statusCode.Store(int64(status))
+	healthcheck.server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(int(healthcheck.statusCode.Load()))
+	}))
+
+	cert := healthcheck.server.Certificate()
+	if cert == nil {
+		t.Fatal("TLS server has no certificate")
+	}
+	caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})
+	caFile := filepath.Join(t.TempDir(), "ca.crt")
+	if err := os.WriteFile(caFile, caPEM, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	healthcheck.caPath = caFile
+	return healthcheck
+}
+
+func (ths *testHealthServer) setStatus(code int) {
+	ths.statusCode.Store(int64(code))
+}
+
+// expectConsistently continuously checks that condition remains true for the given duration.
+// Fails immediately if the condition becomes false at any point.
+func expectConsistently(t *testing.T, condition func() bool, duration time.Duration, msg string) {
+	t.Helper()
+	deadline := time.Now().Add(duration)
+	for time.Now().Before(deadline) {
+		if !condition() {
+			t.Fatalf("condition violated: %s", msg)
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+}
+
+// expectEventually polls condition until it returns true or 5s timeout is reached.
+func expectEventually(t *testing.T, condition func() bool, msg string) {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		if condition() {
+			return
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	t.Fatalf("timeout: %s", msg)
+}

pkg/cluster/singleNode.go

@@ -1,219 +0,0 @@
-package cluster
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-
-	"github.com/kube-vip/kube-vip/pkg/bgp"
-	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
-	"github.com/kube-vip/kube-vip/pkg/vip"
-)
-
-// StartSingleNode will start a single node cluster
-func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) error {
-	// Start kube-vip as a single node server
-
-	// TODO - Split all this code out as a seperate function
-	log.Infoln("Starting kube-vip as a single node cluster")
-
-	log.Info("This node is assuming leadership of the cluster")
-
-	cluster.stop = make(chan bool, 1)
-	cluster.completed = make(chan bool, 1)
-
-	// Managers for Vip load balancers and none-vip loadbalancers
-	nonVipLB := loadbalancer.LBManager{}
-	VipLB := loadbalancer.LBManager{}
-
-	// Iterate through all Configurations
-	for x := range c.LoadBalancers {
-		// If the load balancer doesn't bind to the VIP
-		if c.LoadBalancers[x].BindToVip == false {
-			err := nonVipLB.Add("", &c.LoadBalancers[x])
-			if err != nil {
-				log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-			}
-
-		}
-	}
-
-	if !disableVIP {
-		err := cluster.Network.DeleteIP()
-		if err != nil {
-			log.Warnf("Attempted to clean existing VIP => %v", err)
-		}
-
-		err = cluster.Network.AddIP()
-		if err != nil {
-			log.Warnf("%v", err)
-		}
-
-		// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-		for x := range c.LoadBalancers {
-
-			if c.LoadBalancers[x].BindToVip == true {
-				err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-				if err != nil {
-					log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-				}
-			}
-		}
-	}
-
-	if c.EnableARP == true {
-		// Gratuitous ARP, will broadcast to new MAC <-> IP
-		err := vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-		if err != nil {
-			log.Warnf("%v", err)
-		}
-	}
-
-	go func() {
-		for {
-			select {
-			case <-cluster.stop:
-				log.Info("[LOADBALANCER] Stopping load balancers")
-
-				// Stop all load balancers associated with the VIP
-				err := VipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				// Stop all load balancers associated with the Host
-				err = nonVipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				if !disableVIP {
-
-					log.Info("[VIP] Releasing the Virtual IP")
-					err = cluster.Network.DeleteIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-				close(cluster.completed)
-				return
-			}
-		}
-	}()
-	log.Infoln("Started Load Balancer and Virtual IP")
-	return nil
-}
-
-// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
-func (cluster *Cluster) StartLoadBalancerService(c *kubevip.Config, bgp *bgp.Server) error {
-	// Start a kube-vip loadbalancer service
-	log.Infof("Starting advertising address [%s] with kube-vip", c.VIP)
-
-	// use a Go context so we can tell the arp loop code when we
-	// want to step down
-	ctxArp, cancelArp := context.WithCancel(context.Background())
-	defer cancelArp()
-
-	cluster.stop = make(chan bool, 1)
-	cluster.completed = make(chan bool, 1)
-
-	err := cluster.Network.DeleteIP()
-	if err != nil {
-		log.Warnf("Attempted to clean existing VIP => %v", err)
-	}
-
-	err = cluster.Network.AddIP()
-	if err != nil {
-		log.Warnf("%v", err)
-	}
-
-	if c.EnableARP == true {
-		ctxArp, cancelArp = context.WithCancel(context.Background())
-
-		ipString := cluster.Network.IP()
-
-		var ndp *vip.NdpResponder
-		if vip.IsIPv6(ipString) {
-			ndp, err = vip.NewNDPResponder(c.Interface)
-			if err != nil {
-				log.Fatalf("failed to create new NDP Responder")
-			}
-		}
-		go func(ctx context.Context) {
-			if ndp != nil {
-				defer ndp.Close()
-			}
-
-			for {
-
-				select {
-				case <-ctx.Done(): // if cancel() execute
-					return
-				default:
-					// Ensure the address exists on the interface before attempting to ARP
-					set, err := cluster.Network.IsSet()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-					if !set {
-						log.Warnf("Re-applying the VIP configuration [%s] to the interface [%s]", ipString, c.Interface)
-						err = cluster.Network.AddIP()
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-					}
-
-					if vip.IsIPv4(ipString) {
-						// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
-						err := vip.ARPSendGratuitous(ipString, c.Interface)
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-					} else {
-						// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
-						err := ndp.SendGratuitous(ipString)
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-					}
-				}
-				time.Sleep(3 * time.Second)
-			}
-		}(ctxArp)
-	}
-
-	if c.EnableBGP {
-		// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
-		cidrVip := fmt.Sprintf("%s/%s", cluster.Network.IP(), c.VIPCIDR)
-		log.Debugf("Attempting to advertise the address [%s] over BGP", cidrVip)
-		err = bgp.AddHost(cidrVip)
-		if err != nil {
-			log.Error(err)
-		}
-	}
-
-	go func() {
-		for {
-			select {
-			case <-cluster.stop:
-				// Stop the Arp context if it is running
-				cancelArp()
-
-				log.Info("[LOADBALANCER] Stopping load balancers")
-				log.Infof("[VIP] Releasing the Virtual IP [%s]", c.VIP)
-				err = cluster.Network.DeleteIP()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				close(cluster.completed)
-				return
-			}
-		}
-	}()
-	log.Infoln("Started Load Balancer and Virtual IP")
-	return nil
-}

pkg/cluster/state.go

@@ -1,39 +0,0 @@
-package cluster
-
-import (
-	"io"
-
-	"github.com/hashicorp/raft"
-)
-
-// FSM - Finite State Machine for Raft
-type FSM struct {
-}
-
-// Apply - TODO
-func (fsm FSM) Apply(log *raft.Log) interface{} {
-	return nil
-}
-
-// Restore - TODO
-func (fsm FSM) Restore(snap io.ReadCloser) error {
-	return nil
-}
-
-// Snapshot - TODO, returns an empty snapshot
-func (fsm FSM) Snapshot() (raft.FSMSnapshot, error) {
-	return Snapshot{}, nil
-}
-
-// Snapshot -
-type Snapshot struct {
-}
-
-// Persist -
-func (snapshot Snapshot) Persist(sink raft.SnapshotSink) error {
-	return nil
-}
-
-// Release -
-func (snapshot Snapshot) Release() {
-}

pkg/debouncer/debouncer.go

@@ -0,0 +1,277 @@
+package debouncer
+
+import (
+	"context"
+	"fmt"
+	log "log/slog"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	"k8s.io/apimachinery/pkg/watch"
+)
+
+const (
+	DefaultTime = "0s"
+	minimalTime = time.Millisecond * 200
+)
+
+type debouncer struct {
+	input    <-chan watch.Event
+	output   chan watch.Event
+	stopChan chan any
+	stopOnce sync.Once
+	// events holds event per namespace
+	namespaces   sync.Map
+	debounceTime time.Duration
+}
+
+type ns struct {
+	sync.Map
+	cnt atomic.Int64
+}
+
+func (n *ns) get(name string) (*object, bool) {
+	value, exists := n.Load(name)
+	if !exists {
+		return nil, false
+	}
+	i, ok := value.(*object)
+	if !ok {
+		return nil, false
+	}
+	return i, true
+}
+
+func (n *ns) add(name string, output chan<- watch.Event) *object {
+	i := newObject(output)
+	n.Store(name, i)
+	n.cnt.Add(1)
+	return i
+}
+
+func (n *ns) del(name string, object *object) {
+	if n.CompareAndDelete(name, object) {
+		n.cnt.Add(-1)
+	}
+}
+
+func New(input <-chan watch.Event, debounceTime string) (*debouncer, error) {
+	dt, err := time.ParseDuration(debounceTime)
+	if err != nil {
+		// debouncer was configured with invalid unparsable value, return error
+		return nil, fmt.Errorf("failed to parse debounce time configuration: %w", err)
+	}
+	if dt < minimalTime {
+		if dt > 0 {
+			log.Warn("configured debounce time is less than the minimal threshold of 200ms, debouncer will remain disabled", "config value", dt.String())
+		}
+		return nil, nil
+	}
+	return &debouncer{
+		input:        input,
+		output:       make(chan watch.Event),
+		stopChan:     make(chan any),
+		debounceTime: dt,
+	}, nil
+}
+
+func (d *debouncer) Start(ctx context.Context) error {
+	wg := sync.WaitGroup{}
+	debouncerCtx, cancel := context.WithCancel(ctx)
+	defer func() {
+		cancel()
+		wg.Wait()
+		close(d.output)
+	}()
+
+	for {
+		select {
+		case <-debouncerCtx.Done():
+			// return if debouncer context was cancelled
+			return nil
+		case <-d.stopChan:
+			// return if Stop() was called
+			return nil
+		case tmp := <-d.input:
+			// event has no type, probably error
+			if tmp.Type == "" {
+				return fmt.Errorf("get undefined object (input channel probably closed)")
+			}
+
+			var namespace, name string
+
+			// type switch event object
+			switch v := tmp.Object.(type) {
+			case *discoveryv1.EndpointSlice:
+				namespace = v.Namespace
+				name = v.Name
+			case *v1.Endpoints: //nolint:staticcheck
+				namespace = v.Namespace
+				name = v.Name
+			case *v1.Service:
+				namespace = v.Namespace
+				name = v.Name
+			default:
+				return fmt.Errorf("objects of type %T are not supported", v)
+			}
+
+		processEvent:
+			for {
+				eventNs, exists := d.getNs(namespace)
+				if !exists {
+					// if not, create new map for the namespace
+					eventNs = d.addNs(namespace)
+				}
+
+				// check if the object was previously reconciled
+				eventObject, exists := eventNs.get(name)
+
+				// if not and the event is not of type 'Deleted', create new object
+				if !exists && tmp.Type != watch.Deleted {
+					eventObject = eventNs.add(name, d.output)
+
+					workerObject := eventObject
+					workerNs := eventNs
+					workerName := name
+					workerNamespace := namespace
+					workerObject.onStop = func() {
+						// Remove the object before its worker can become receiver-less.
+						workerNs.del(workerName, workerObject)
+					}
+
+					wg.Go(func() {
+						// start deboucing events for this object
+						workerObject.start(debouncerCtx, d.debounceTime)
+						// if debouncer for the object ended - e.g. object was deleted - clean the map of objects
+						workerNs.del(workerName, workerObject)
+						// if namespace is empty, delete the namespace map
+						if workerNs.cnt.Load() == 0 {
+							d.delNs(workerNamespace, workerNs)
+						}
+					})
+				}
+
+				if eventObject == nil {
+					break processEvent
+				}
+
+				// pass the watch event to the debouncer object
+				select {
+				case eventObject.input <- tmp:
+					break processEvent
+				case <-eventObject.stopChan:
+					// The object stopped after the map lookup. Retry the event
+					// against the newly-created object instead of dropping it.
+					continue processEvent
+				case <-debouncerCtx.Done():
+					return nil
+				}
+			}
+		}
+	}
+}
+
+func (d *debouncer) Stop() {
+	d.stopOnce.Do(func() {
+		close(d.stopChan)
+	})
+}
+
+func (d *debouncer) Output() chan watch.Event {
+	return d.output
+}
+
+func (d *debouncer) getNs(namespace string) (*ns, bool) {
+	value, exists := d.namespaces.Load(namespace)
+	if !exists {
+		return nil, false
+	}
+	n, ok := value.(*ns)
+	if !ok {
+		return nil, false
+	}
+	return n, true
+}
+
+func (d *debouncer) addNs(namespace string) *ns {
+	n := ns{}
+	d.namespaces.Store(namespace, &n)
+	return &n
+}
+
+func (d *debouncer) delNs(namespace string, ns *ns) {
+	d.namespaces.CompareAndDelete(namespace, ns)
+}
+
+type object struct {
+	input    chan watch.Event
+	output   chan<- watch.Event
+	stopChan chan any
+	stopOnce sync.Once
+	onStop   func()
+}
+
+func newObject(output chan<- watch.Event) *object {
+	return &object{
+		input:    make(chan watch.Event),
+		output:   output,
+		stopChan: make(chan any),
+	}
+}
+
+func (o *object) start(ctx context.Context, debounceTime time.Duration) {
+	t := time.NewTicker(debounceTime)
+
+	var last *watch.Event
+
+	defer func() {
+		if last != nil {
+			o.output <- *last
+			last = nil
+		}
+	}()
+
+	for {
+		select {
+		case <-ctx.Done():
+			// if context is done, return
+			return
+		case <-o.stopChan:
+			// return if Stop() was called
+			return
+		case tmp := <-o.input:
+			// if last event is known, but an event of another type arrived,
+			// send out the previous event
+			if last != nil && last.Type != tmp.Type {
+				o.output <- *last
+			}
+			// save current event as the last event
+			last = &tmp
+			// reset the ticker to wait for more events
+			t.Reset(debounceTime)
+		case <-t.C:
+			if last != nil {
+				// on tick, if we have an event, send it out
+				o.output <- *last
+				// if the event is of type 'Deleted', stop the debouncer for the object
+				if last.Type == watch.Deleted {
+					o.stop()
+				}
+				// reset last known event, so it won't be send out twice
+				last = nil
+			}
+		}
+	}
+}
+
+func (o *object) stop() {
+	o.stopOnce.Do(func() {
+		if o.onStop != nil {
+			o.onStop()
+		}
+		close(o.stopChan)
+	})
+}