Move this error to a warning

Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# Enable GitHub funding
+
+github: [kube-vip]
+

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: gomod
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: /
+    schedule:
+      interval: weekly

.github/workflows/anchore-syft.yml

@@ -0,0 +1,31 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# This workflow checks out code, builds an image, performs a container image
+# scan with Anchore's Syft tool, and uploads the results to the GitHub Dependency
+# submission API.
+
+# For more information on the Anchore sbom-action usage
+# and parameters, see https://github.com/anchore/sbom-action. For more
+# information about the Anchore SBOM tool, Syft, see
+# https://github.com/anchore/syft
+name: Anchore Syft SBOM scan
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  sbom:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.ref_name }}
+      - name: Anchore SBOM Action
+        uses: anchore/sbom-action@v0.20.10
+        with:
+          format: cyclonedx-json

.github/workflows/ci-pull-request.yaml

@@ -0,0 +1,71 @@
+name: For each PR
+on:
+  pull_request:
+jobs:
+  e2e-tests:
+    runs-on: ubuntu-latest
+    name: E2E tests
+    strategy:
+      matrix:
+        mode: ["arp", "rt", "bgp"]
+      fail-fast: true
+    steps:
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H-%M')"
+      - name: Ensure fs wont cause issues
+        run: sudo sysctl fs.inotify.max_user_instances=8192 &&  sudo sysctl fs.inotify.max_user_watches=524288
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Install Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+      - name: Build image locally
+        run: make dockerx86Local
+      - name: Run Manifest generation tests
+        run: make manifest-test
+      - name: Run ARP mode tests v1.29.0 onwards
+        run: E2E_KEEP_LOGS=true make e2e-tests129-arp
+        if: matrix.mode== 'arp'
+      - name: Run RT mode tests v1.29.0 onwards
+        run: E2E_KEEP_LOGS=true make e2e-tests129-rt
+        if: matrix.mode== 'rt'
+      - name: Get GoBGP binaries
+        run: make get-gobgp
+        if: matrix.mode== 'bgp'
+      - name: Run BGP mode tests v1.29.0 onwards
+        run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true make e2e-tests129-bgp
+        if: matrix.mode== 'bgp'
+      - name: Change log directory permissions
+        run: sudo chmod -R 755 /tmp/kube-vip-test-${{ matrix.mode }}*
+        if: matrix.mode== 'bgp' && always()
+      - name: Save logs
+        uses: actions/upload-artifact@v5
+        with:
+          name: e2e-test-logs-${{ matrix.mode }}-${{ steps.date.outputs.date }}
+          path: /tmp/kube-vip-test-${{ matrix.mode }}*
+        if: always()
+  service-e2e-tests:
+    runs-on: ubuntu-latest
+    name: E2E service tests
+    steps:
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H-%M')"
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Install Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+      - name: Build image with iptables
+        run: make dockerx86ActionIPTables
+      - name: Run tests
+        run: DOCKERTAG=action E2E_KEEP_LOGS=true make service-tests
+      - name: Save logs
+        uses: actions/upload-artifact@v5
+        with:
+          name: services-test-logs-${{ steps.date.outputs.date }}
+          path: /tmp/kube-vip-service-tests*
+        if: always()

.github/workflows/ci.yaml

@@ -1,25 +1,62 @@
-name: For each commit and PR
+name: For each commit
 on:
   push:
   pull_request:
-
 jobs:
   validation:
     runs-on: ubuntu-latest
-    env:
-      CGO_ENABLED: 0
+    name: Checks and linters
     steps:
-    - name: Init
-      run: sudo apt-get update && sudo apt-get install -y build-essential golint
-    - name: Checkout code
-      uses: actions/checkout@v2
-    - name: Install Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: '1.15'
-    - name: checks
-      run: make check
-    - name: test docker build
-      run: make dockerx86Action
-    - name: Manifest generate
-      run: ./testing/testing.sh
+      - name: Init
+        run: sudo apt-get update && sudo apt-get install -y build-essential &&  sudo sysctl fs.inotify.max_user_instances=8192 &&  sudo sysctl fs.inotify.max_user_watches=524288
+      - name: Install golangci-lint
+        run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.64.8
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Install Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+      - name: All checks
+        run: make check
+  unit-tests:
+    runs-on: ubuntu-latest
+    name: Unit tests
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Install Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+      - name: Run tests
+        run: make unit-tests
+  integration-tests:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Install Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+      - name: Run tests
+        run: make integration-tests
+  image-vul-check:
+    runs-on: ubuntu-latest
+    name: Image vulnerability scan
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Build image with iptables
+        run: make dockerx86ActionIPTables
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: "plndr/kube-vip:action"
+          format: "table"
+          exit-code: "1"
+          ignore-unfixed: true
+          vuln-type: "os,library"
+          severity: "CRITICAL,HIGH"

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,75 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '17 10 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v6
+      
+    - name: Install Go
+      uses: actions/setup-go@v6
+      with:
+        go-version-file: go.mod
+        
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v4
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v4
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v4

.github/workflows/main.yaml

@@ -0,0 +1,52 @@
+name: Build and publish main image regularly
+
+on:
+  schedule:
+    - cron: '25 0 * * *'
+  workflow_dispatch:
+
+
+jobs:
+  nightly_build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to Github Packages
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}   
+      - name: Build standard version
+        id: docker_build
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: >-
+            plndr/kube-vip:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip:${{ github.ref_name }}
+      - name: Build iptables version
+        id: docker_build_iptables
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile_iptables
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: >-
+            plndr/kube-vip-iptables:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip-iptables:${{ github.ref_name }}
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/release.yaml

@@ -0,0 +1,64 @@
+name: Publish Releases to Docker Hub and GitHub Container Registry
+
+on:
+  push:
+    tags:
+      - '*'
+  workflow_dispatch:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to Github Packages
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Generate Metadata
+        uses: docker/metadata-action@v5.9.0
+        id: metadata
+        with:
+          labels: |
+            org.opencontainers.image.documentation=https://kube-vip.io/docs/
+      - name: Build and push main branch
+        id: docker_build
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          labels: ${{ steps.metadata.outputs.labels }}
+          tags: >-
+            plndr/kube-vip:${{ github.ref_name }},
+            plndr/kube-vip:latest,
+            ghcr.io/kube-vip/kube-vip:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip:latest
+      - name: Build iptables version and push main branch
+        id: docker_build_iptables
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile_iptables
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          labels: ${{ steps.metadata.outputs.labels }}
+          tags: >-
+            plndr/kube-vip-iptables:${{ github.ref_name }},
+            plndr/kube-vip-iptables:latest,
+            ghcr.io/kube-vip/kube-vip-iptables:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip-iptables:latest
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

.gitignore

@@ -0,0 +1,7 @@
+.idea
+kube-vip
+.vscode
+bin
+testing/e2e/etcd/certs
+pkg/etcd/etcd.pid
+pkg/etcd/etcd-data

.golangci.yml

@@ -0,0 +1,13 @@
+run:
+  timeout: 10m
+  
+linters:
+  enable:
+  - bodyclose
+  - gofmt
+  - goimports
+  - revive
+  - gosec
+  - misspell
+  - unconvert
+  - unparam

CHANGELOG.md

@@ -0,0 +1,33 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- SIGUSR1 signal handler for runtime configuration dumps (#1301)
+  - Send SIGUSR1 to kube-vip process to dump current configuration to stdout
+  - Configuration dump includes:
+    - Basic configuration (VIP, interface, port, namespace settings)
+    - BGP configuration (enabled status, AS number, router ID, peers)
+    - ARP/NDP configuration (enabled status, broadcast rate)
+    - Services configuration (enabled status, load balancer settings)
+    - Network interfaces status
+    - Leader election configuration (type, lease details)
+    - Runtime statistics (load balancer, Prometheus, health check settings)
+  - Output format: Human-readable plaintext via fmt.Printf()
+  - Thread-safe implementation using mutex protection
+  - Non-disruptive: Process continues running after configuration dump
+  - Added comprehensive unit tests for all dump methods
+  - Added E2E tests for signal handling
+
+### Changed
+- Updated signal handlers in manager_arp.go, manager_bgp.go, manager_wireguard.go, and manager_table.go to use switch statement pattern for handling multiple signals (SIGUSR1, SIGINT, SIGTERM)
+
+## [v1.0.1] - Previous Release
+
+### Previous changes
+- See git history for changes prior to CHANGELOG.md introduction

CODEOWNERS

@@ -0,0 +1,10 @@
+
+# For more information on the syntax of the CODEOWNERS file, see:
+# https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners
+
+# The Kube-Vip maintainers team
+*   @thebsdbox @yastij
+
+# Emeritus Maintainers
+#
+# N/A

CODE_OF_CONDUCT.md

@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+conduct@kube-vip.io.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

CONTRIBUTING.md

@@ -0,0 +1,289 @@
+# Developer Guide
+
+Thank you for taking the time out to contribute to the **kube-vip** project!
+
+This guide will walk you through the process of making your first commit and how
+to effectively get it merged upstream.
+
+<!-- toc -->
+- [Getting Started](#getting-started)
+- [Contribute](#contribute)
+  - [GitHub Workflow](#github-workflow)
+  - [Getting reviewers](#getting-reviewers)
+  - [Inclusive Naming](#inclusive-naming)
+  - [Building and testing your change](#building-and-testing-your-change)
+  - [Reverting a commit](#reverting-a-commit)
+  - [Sign-off Your Work](#sign-off-your-work)
+- [Issue and PR Management](#issue-and-pr-management)
+  - [Filing An Issue](#filing-an-issue)
+  - [Issue Triage](#issue-triage)
+<!-- /toc -->
+
+## Getting Started
+
+To get started, let's ensure you have completed the following prerequisites for
+contributing to project **kube-vip**:
+
+1. Read and observe the [code of conduct](CODE_OF_CONDUCT.md).
+2. Check out the [Architecture documentation](https://kube-vip.io) for the **kube-vip**
+   architecture and design.
+3. Set up your [development environment](docs/contributors/manual-installation.md)
+
+Now that you're setup, skip ahead to learn how to [contribute](#contribute).
+
+**Also**, A GitHub account will be required in order to submit code changes and
+interact with the project. For committing any changes in  **Github** it is required for
+you to have a [github account](https://github.com/join).
+
+## Contribute
+
+There are multiple ways in which you can contribute, either by contributing
+code in the form of new features or bug-fixes or non-code contributions like
+helping with code reviews, triaging of bugs, documentation updates, filing
+[new issues](#filing-an-issue) or writing blogs/manuals etc.
+
+### GitHub Workflow
+
+Developers work in their own forked copy of the repository and when ready,
+submit pull requests to have their changes considered and merged into the
+project's repository.
+
+1. Fork your own copy of the repository to your GitHub account by clicking on
+   `Fork` button on [kube-vip's GitHub repository](https://github.com/kube-vip/kube-vip).
+2. Clone the forked repository on your local setup.
+
+    ```bash
+    git clone https://github.com/$user/kube-vip
+    ```
+
+    Add a remote upstream to track upstream kube-vip repository.
+
+    ```bash
+    git remote add upstream https://github.com/kube-vip/kube-vip
+    ```
+
+    Never push to upstream remote
+
+    ```bash
+    git remote set-url --push upstream no_push
+    ```
+
+3. Create a topic branch.
+
+    ```bash
+    git checkout -b branchName
+    ```
+
+4. Make changes and commit it locally. Make sure that your commit is
+   [signed](#sign-off-your-work).
+
+    ```bash
+    git add <modifiedFile>
+    git commit -s
+    ```
+
+5. Update the "Unreleased" section of the [CHANGELOG](CHANGELOG.md) for any
+   significant change that impacts users.
+6. Keeping branch in sync with upstream.
+
+    ```bash
+    git checkout branchName
+    git fetch upstream
+    git rebase upstream/main
+    ```
+
+7. Push local branch to your forked repository.
+
+    ```bash
+    git push -f $remoteBranchName branchName
+    ```
+
+8. Create a Pull request on GitHub.
+   Visit your fork at `https://github.com/kube-vip/kube-vip` and click
+   `Compare & Pull Request` button next to your `remoteBranchName` branch.
+
+### Getting reviewers
+
+Once you have opened a Pull Request (PR), reviewers will be assigned to your
+PR and they may provide review comments which you need to address.
+Commit changes made in response to review comments to the same branch on your
+fork. Once a PR is ready to merge, squash any *fix review feedback, typo*
+and *merged* sorts of commits.
+
+To make it easier for reviewers to review your PR, consider the following:
+
+1. Follow the golang [coding conventions](https://github.com/golang/go/wiki/CodeReviewComments).
+2. Format your code with `make golangci-fix`; if the [linters](ci/README.md) flag an issue that
+   cannot be fixed automatically, an error message will be displayed so you can address the issue.
+3. Follow [git commit](https://chris.beams.io/posts/git-commit/) guidelines.
+4. Follow [logging](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md) guidelines.
+
+If your PR fixes a bug or implements a new feature, add the appropriate test
+cases to our [automated test suite](ci/README.md) to guarantee enough
+coverage. A PR that makes significant code changes without contributing new test
+cases will be flagged by reviewers and will not be accepted.
+
+### Inclusive Naming
+
+For symbol names and documentation, do not introduce new usage of harmful
+language such as 'master / slave' (or 'slave' independent of 'master') and
+'blacklist / whitelist'. For more information about what constitutes harmful
+language and for a reference word replacement list, please refer to the
+[Inclusive Naming Initiative](https://inclusivenaming.org/).
+
+We are committed to removing all harmful language from the project. If you
+detect existing usage of harmful language in code or documentation, please
+report the issue to us or open a Pull Request to address it directly. Thanks!
+
+### Building and testing your change
+
+To build the **kube-vip** Docker image together with all **kube-vip** bits, you can simply
+do:
+
+1. Checkout your feature branch and `cd` into it.
+2. Run `make dockerx86`
+
+The second step will compile the **kube-vip** code in a `golang` container, and build
+a `Ubuntu 20.04` Docker image that includes all the generated binaries. [`Docker`](https://docs.docker.com/install)
+must be installed on your local machine in advance.
+
+Alternatively, you can build the **kube-vip** code in your local Go environment. The
+**kube-vip** project uses the [Go modules support](https://github.com/golang/go/wiki/Modules) which was introduced in Go 1.11. It
+facilitates dependency tracking and no longer requires projects to live inside
+the `$GOPATH`.
+
+To develop locally, you can follow these steps:
+
+ 1. [Install Go 1.19](https://golang.org/doc/install)
+ 2. Checkout your feature branch and `cd` into it.
+ 3. To build all Go files and install them under `bin`, run `make bin`
+ 4. To run all Go unit tests, run `make test-unit`
+ 5. To build the **kube-vip** Ubuntu Docker image separately with the binaries generated in step 2, run `make ubuntu`
+
+### CI testing
+
+For more information about the tests we run as part of CI, please refer to
+[ci/README.md](ci/README.md).
+
+### Reverting a commit
+
+1. Create a branch in your forked repo
+
+    ```bash
+    git checkout -b revertName
+    ```
+
+2. Sync the branch with upstream
+
+    ```bash
+    git fetch upstream
+    git rebase upstream/main
+    ```
+
+3. Create a revert based on the SHA of the commit. The commit needs to be
+   [signed](#sign-off-your-work).
+
+    ```bash
+    git revert -s SHA
+    ```
+
+4. Push this new commit.
+
+    ```bash
+    git push $remoteRevertName revertName
+    ```
+
+5. Create a Pull Request on GitHub.
+   Visit your fork at `https://github.com/kube-vip/kube-vip` and click
+   `Compare & Pull Request` button next to your `remoteRevertName` branch.
+
+### Sign-off Your Work
+
+It is recommended to sign your work when contributing to the **kube-vip**
+repository. 
+
+Git provides the `-s` command-line option to append the required line
+automatically to the commit message:
+
+```bash
+git commit -s -m 'This is my commit message'
+```
+
+For an existing commit, you can also use this option with `--amend`:
+
+```bash
+git commit -s --amend
+```
+
+If more than one person works on something it's possible for more than one
+person to sign-off on it. For example:
+
+```bash
+Signed-off-by: Some Developer somedev@example.com
+Signed-off-by: Another Developer anotherdev@example.com
+```
+
+We use the [DCO Github App](https://github.com/apps/dco) to enforce that all
+commits in a Pull Request include the required `Signed-off-by` line. If this is
+not the case, the app will report a failed status for the Pull Request and it
+will be blocked from being merged.
+
+Compared to our earlier CLA, DCO tends to make the experience simpler for new
+contributors. If you are contributing as an employee, there is no need for your
+employer to sign anything; the DCO assumes you are authorized to submit
+contributions (it's your responsibility to check with your employer).
+
+## Issue and PR Management
+
+We use labels and workflows (some manual, some automated with GitHub Actions) to
+help us manage triage, prioritize, and track issue progress. For a detailed
+discussion, see [docs/issue-management.md](docs/contributors/issue-management.md).
+
+### Filing An Issue
+
+Help is always appreciated. If you find something that needs fixing, please file
+an issue [here](https://github.com/kube-vip/kube-vip/issues). Please ensure
+that the issue is self explanatory and has enough information for an assignee to
+get started.
+
+Before picking up a task, go through the existing
+[issues](https://github.com/kube-vip/kube-vip/issues) and make sure that your
+change is not already being worked on. If it does not exist, please create a new
+issue and discuss it with other members.
+
+For simple contributions to **kube-vip**, please ensure that this minimum set of
+labels are included on your issue:
+
+* **kind** -- common ones are `kind/feature`, `kind/support`, `kind/bug`,
+  `kind/documentation`, or `kind/design`. For an overview of the different types
+  of issues that can be submitted, see [Issue and PR
+  Kinds](#issue-and-pr-kinds).
+  The kind of issue will determine the issue workflow.
+* **area** (optional) -- if you know the area the issue belongs in, you can assign it.
+  Otherwise, another community member will label the issue during triage. The
+  area label will identify the area of interest an issue or PR belongs in and
+  will ensure the appropriate reviewers shepherd the issue or PR through to its
+  closure. For an overview of areas, see the
+  [`docs/github-labels.md`](docs/contributors/github-labels.md).
+* **size** (optional) -- if you have an idea of the size (lines of code, complexity,
+  effort) of the issue, you can label it using a [size label](#size). The size
+  can be updated during backlog grooming by contributors. This estimate is used
+  to guide the number of features selected for a milestone.
+
+All other labels will be assigned during issue triage.
+
+### Issue Triage
+
+Once an issue has been submitted, the CI (GitHub actions) or a human will
+automatically review the submitted issue or PR to ensure that it has all relevant
+information. If information is lacking or there is another problem with the
+submitted issue, an appropriate `triage/<?>` label will be applied.
+
+After an issue has been triaged, the maintainers can prioritize the issue with
+an appropriate `priority/<?>` label.
+
+Once an issue has been submitted, categorized, triaged, and prioritized it
+is marked as `ready-to-work`. A ready-to-work issue should have labels
+indicating assigned areas, prioritization, and should not have any remaining
+triage labels.
+

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:experimental
 
-FROM golang:1.15-alpine as dev
+FROM golang:1.25.4-alpine3.22 as dev
 RUN apk add --no-cache git ca-certificates make
 RUN adduser -D appuser
 COPY . /src/
@@ -16,4 +16,4 @@ FROM scratch
 COPY --from=dev /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 # Add kube-vip binary
 COPY --from=dev /src/kube-vip /
-ENTRYPOINT ["/kube-vip"]
+ENTRYPOINT ["/kube-vip"]

Dockerfile_iptables

@@ -0,0 +1,21 @@
+# syntax=docker/dockerfile:experimental
+
+FROM golang:1.25.4-alpine3.22 as dev
+RUN apk add --no-cache git make
+RUN adduser -D appuser
+COPY . /src/
+WORKDIR /src
+
+ENV GO111MODULE=on
+RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
+    --mount=type=cache,sharing=locked,id=goroot,target=/root/.cache/go-build \
+    CGO_ENABLED=0 GOOS=linux make build
+
+FROM alpine:3.22.2
+# Update pkgs and add iptables
+RUN apk upgrade && \
+    apk add --no-cache iptables iptables-legacy
+
+# Add kube-vip binary
+COPY --from=dev /src/kube-vip /
+ENTRYPOINT ["/kube-vip"]

Makefile

@@ -2,10 +2,11 @@ SHELL := /bin/sh
 
 # The name of the executable (default is current directory name)
 TARGET := kube-vip
-.DEFAULT_GOAL: $(TARGET)
+.DEFAULT_GOAL := $(TARGET)
 
 # These will be provided to the target
-VERSION := 0.3.3
+VERSION := v1.0.2
+
 BUILD := `git rev-parse HEAD`
 
 # Operating System Default (LINUX)
@@ -14,9 +15,9 @@ TARGETOS=linux
 # Use linker flags to provide version/build settings to the target
 LDFLAGS=-ldflags "-s -w -X=main.Version=$(VERSION) -X=main.Build=$(BUILD) -extldflags -static"
 DOCKERTAG ?= $(VERSION)
-REPOSITORY = plndr
+REPOSITORY ?= docker.io/plndr
 
-.PHONY: all build clean install uninstall fmt simplify check run e2e-tests
+.PHONY: all build clean install uninstall simplify check run e2e-tests
 
 all: check install
 
@@ -36,17 +37,30 @@ install:
 uninstall: clean
 	@rm -f $$(which ${TARGET})
 
-fmt:
-	@gofmt -l -w ./...
-
 demo:
 	@cd demo
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 	@cd ..
 
 ## Remote (push of images)
 # This build a local docker image (x86 only) for quick testing
+
+dockerx86Dev:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 --push -t $(REPOSITORY)/$(TARGET):dev .
+	@echo New single x86 Architecture Docker image created
+
+dockerx86Iptables:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 -f ./Dockerfile_iptables --push -t $(REPOSITORY)/$(TARGET):dev .
+	@echo New single x86 Architecture Docker image created
+
+dockerx86IptablesLocal:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 -f ./Dockerfile_iptables -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@echo New single x86 Architecture Docker image created
+
 dockerx86:
 	@-rm ./kube-vip
 	@docker buildx build  --platform linux/amd64 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
@@ -54,7 +68,7 @@ dockerx86:
 
 docker:
 	@-rm ./kube-vip
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 ## Local (docker load of images)
@@ -69,33 +83,97 @@ dockerx86Action:
 	@docker buildx build  --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):action .
 	@echo New Multi Architecture Docker image created
 
+dockerx86ActionIPTables:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 -f ./Dockerfile_iptables --load -t $(REPOSITORY)/$(TARGET):action .
+	@echo New Multi Architecture Docker image created
+
 dockerLocal:
 	@-rm ./kube-vip
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7 --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 simplify:
-	@gofmt -s -l -w ./...
+	@gofmt -s -l -w *.go pkg cmd
 
 check:
 	go mod tidy
 	test -z "$(git status --porcelain)"
-	test -z $(shell gofmt -l main.go | tee /dev/stderr) || echo "[WARN] Fix formatting issues with 'make fmt'"
-	golint ./...
+	test -z $(shell gofmt -l *.go pkg cmd) || echo "[WARN] Fix formatting issues with 'make simplify'"
+	golangci-lint run
 	go vet ./...
-	
+
 run: install
 	@$(TARGET)
 
 manifests:
 	@make build
 	@mkdir -p ./docs/manifests/$(VERSION)/
-	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-arp.yaml
-	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --bgp --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-bgp.yaml
-	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
-	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
-	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster --provider-config /etc/cloud-sa/cloud-sa.json > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
+	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-arp.yaml
+	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-lb.yaml
+	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-bgp.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
 	@-rm ./kube-vip
 
+manifest-test: 
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --enableLoadBalancer
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --controlplane --services
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster
+
+unit-tests:
+	go test ./...
+
+integration-tests:
+	go test -tags=integration,e2e -v ./pkg/etcd
+
 e2e-tests:
-	E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/ginkgo -v -p testing/e2e
+	GOMAXPROCS=4 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd
+
+e2e-tests129-arp:
+	GOMAXPROCS=4 TEST_MODE=arp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
+
+e2e-tests129-rt:
+	GOMAXPROCS=4 TEST_MODE=rt V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
+
+e2e-tests129-bgp:
+	GOMAXPROCS=4 TEST_MODE=bgp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
+
+e2e-tests129: e2e-tests129-arp e2e-tests129-rt e2e-tests129-bgp
+
+service-tests:
+	E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/services -Services -simple -deployments -leaderActive -leaderFailover -localDeploy -egress -egressIPv6 -dualStack
+
+trivy: dockerx86ActionIPTables
+	docker run -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy:0.47.0 \
+		image  \
+		--format table \
+		--exit-code  1 \
+		--ignore-unfixed \
+		--vuln-type  'os,library' \
+		--severity  'CRITICAL,HIGH'  \
+		$(REPOSITORY)/$(TARGET):action
+
+kind-quick:
+	echo "Standing up your cluster"
+	kind create cluster --config ./testing/kind/kind.yaml --name kube-vip
+	kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
+	kubectl create configmap --namespace kube-system kubevip --from-literal range-global=172.18.100.10-172.18.100.30
+	kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
+	kind load docker-image --name kube-vip $(REPOSITORY)/$(TARGET):$(DOCKERTAG)
+	docker run --network host --rm $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --services --inCluster --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --servicesElection --interface  eth0 | kubectl apply -f -
+
+kind-reload:
+	kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name services
+	kubectl rollout restart -n kube-system daemonset/kube-vip-ds
+
+get-gobgp:
+	mkdir -p bin
+	wget -nc --directory-prefix=bin https://github.com/osrg/gobgp/releases/download/v3.37.0/gobgp_3.37.0_linux_amd64.tar.gz
+	tar -xvzf bin/gobgp_3.37.0_linux_amd64.tar.gz -C bin
+

README.md

@@ -1,8 +1,10 @@
 # kube-vip
 
-High Availability and Load-Balancing 
+High Availability and Load-Balancing
 
-![](https://kube-vip.io/kube-vip.png)
+![](https://github.com/kube-vip/kube-vip/raw/main/kube-vip.png)
+
+[![Build and publish main image regularly](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml/badge.svg)](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml) [![LFX Health Score](https://img.shields.io/static/v1?label=Health%20Score&message=Healthy&color=A7F3D0&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip) [![LFX Active Contributors](https://img.shields.io/static/v1?label=Active%20contributors%20(1Y)&message=212&color=0094FF&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip)
 
 ## Overview
 Kubernetes Virtual IP and Load-Balancer for both control plane and Kubernetes services
@@ -14,12 +16,13 @@ The idea behind `kube-vip` is a small self-contained Highly-Available option for
 - Virtualisation
 - Pretty much anywhere else :)
 
-**NOTE** All documentation of both usage and architecture are now available at [https://kube-vip.io](https://kube-vip.io)
+**NOTE:** All documentation of both usage and architecture are now available at [https://kube-vip.io](https://kube-vip.io).
 
 ## Features
 
 Kube-Vip was originally created to provide a HA solution for the Kubernetes control plane, over time it has evolved to incorporate that same functionality into Kubernetes service type [load-balancers](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer).
 
+- VIP addresses can be both IPv4 or IPv6
 - Control Plane with ARP (Layer 2) or BGP (Layer 3)
 - Control Plane using either [leader election](https://godoc.org/k8s.io/client-go/tools/leaderelection) or [raft](https://en.wikipedia.org/wiki/Raft_(computer_science))
 - Control Plane HA with kubeadm (static Pods)
@@ -29,7 +32,8 @@ Kube-Vip was originally created to provide a HA solution for the Kubernetes cont
 - Service LoadBalancer address pools per namespace or global
 - Service LoadBalancer address via (existing network DHCP)
 - Service LoadBalancer address exposure to gateway via UPNP
-- ... manifest generation, vendor API integrations and many nore... 
+- Egress! Kube-vip will utilise a service loadbalancer as both the ingress and **egress** for a pod. 
+- ... manifest generation, vendor API integrations and many more...
 
 ## Why?
 
@@ -41,7 +45,7 @@ The purpose of `kube-vip` is to simplify the building of HA Kubernetes clusters,
 
 **VIP**:
 - [Keepalived](https://www.keepalived.org/)
-- [Ucarp](https://ucarp.wordpress.com/)
+- [UCARP](https://ucarp.wordpress.com/)
 - Hardware Load-balancer (functionality differs per vendor)
 
 
@@ -54,4 +58,14 @@ All of these would require a separate level of configuration and in some infrast
 
 ## Troubleshooting and Feedback
 
-Please raise issues on the GitHub repository and as mentioned check the documentation at [https://kube-vip.io](https://kube-vip.io/)
+Please raise issues on the GitHub repository and as mentioned check the documentation at [https://kube-vip.io](https://kube-vip.io/).
+
+## Contributing
+
+Thanks for taking the time to join our community and start contributing! We welcome pull requests. Feel free to dig through the [issues](https://github.com/kube-vip/kube-vip/issues) and jump in.
+
+:warning: This project has issue compiling on MacOS, please compile it on linux distribution
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=kube-vip/kube-vip&type=Date)](https://star-history.com/#kube-vip/kube-vip&Date)

ROADMAP.md

@@ -0,0 +1,35 @@
+# Kube-Vip Roadmap
+
+This document outlines the roadmap for the **kube-vip** project and only covers the technologies within this particular project, other projects that augment or provide additional functionality (such as cloud-providers) may have their own roadmaps in future. The functionality for **kube-vip** has grown either been developed organically or through real-world needs, and this is the first attempt to put into words a plan for the future of **kube-vip** and will additional evolve over time. This means that items listed or detailed here are not necessarily set in stone and the roadmap can grow/shrink as the project matures. We definitely welcome suggestions and ideas from everyone about the roadmap and **kube-vip** features. Reach us through Issues, Slack or email <catch-all>@kube-vip.io.
+
+## Release methodology
+
+The **kube-vip** project attempts to follow a tick-tock release cycle, this typically means that one release will come **packed** with new features where the following release will come with fixes, code sanitation and performance enhancements.
+
+## Roadmap
+
+The **kube-vip** project offers two main areas of functionality:
+
+- HA Kubernetes clusters through a control-plane VIP
+- Kubernetes `service type:LoadBalancer`
+
+Whilst both of these functions share underlying technologies and code they will have slightly differing roadmaps.
+
+### HA Kubernetes Control Plane
+
+- **Re-implement LoadBalancing** - due to a previous request the HTTP loadbalancing was removed leaving just HA for the control plane. This functionality will be re-implemented either through the original round-robin HTTP requests or utilising IPVS.
+- **Utilise the Kubernetes API to determine additional Control Plane members** - Once a single node cluster is running **kube-vip** could use the API to determine the additional members, at this time a Cluster-API provider needs to drop a static manifest per CP node.
+- **Re-evaluate raft** - **kube-vip** is mainly designed to run within a Kubernetes cluster, however it's original design was a raft cluster external to Kubernetes. Unfortunately given some of the upgrade paths identified in things like CAPV moving to leaderElection within Kubernetes became a better idea.
+
+## Kubernetes `service type:LoadBalancer`
+
+- **`ARP` LeaderElection per loadBalancer** - Currently only one pod that is elected leader will field all traffic for a VIP.. extending this to generate a leaderElection token per service would allow services to proliferate across all pods across the cluster
+- **Aligning of `service` and `manager`** - The move to allow hybrid (be both HA control plane and offer load-balancer services at the same time) introduced a duplicate code path.. these need to converge as it's currently confusing for contributors.
+
+## Global **Kube-Vip** items
+
+- **Improved metrics** - At this time the scaffolding for monitoring exists, however this needs drastically extending to provide greater observability to what is happening within **kube-vip**
+- **Windows support** - The Go SDK didn't support the capability for low-levels sockets for ARP originally, this should be revisited.
+- **Additional BGP features** :
+  - Communities
+  - BFD

cmd/kube-vip-config.go

@@ -1,146 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-
-	"github.com/ghodss/yaml"
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	appv1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-// [sample configuration] - flags
-var cliConfig kubevip.Config
-var cliConfigLB kubevip.LoadBalancer
-var cliLocalPeer string
-var cliRemotePeers, cliBackends []string
-
-func init() {
-	kubeVipSampleConfig.Flags().StringVar(&cliConfig.Interface, "interface", "eth0", "Name of the interface to bind to")
-	kubeVipSampleConfig.Flags().StringVar(&cliConfig.VIP, "vip", "192.168.0.1", "The Virtual IP address")
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfig.SingleNode, "singleNode", false, "Start this instance as a single node")
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfig.StartAsLeader, "startAsLeader", false, "Start this instance as the cluster leader")
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfig.EnableARP, "arp", true, "Use ARP broadcasts to improve VIP re-allocations")
-	kubeVipSampleConfig.Flags().StringVar(&cliLocalPeer, "localPeer", "server1:192.168.0.1:10000", "Settings for this peer, format: id:address:port")
-	kubeVipSampleConfig.Flags().StringSliceVar(&cliRemotePeers, "remotePeers", []string{"server2:192.168.0.2:10000", "server3:192.168.0.3:10000"}, "Comma seperated remotePeers, format: id:address:port")
-	// Load Balancer flags
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfigLB.BindToVip, "lbBindToVip", false, "Bind example load balancer to VIP")
-	kubeVipSampleConfig.Flags().StringVar(&cliConfigLB.Type, "lbType", "tcp", "Type of load balancer instance (TCP/HTTP)")
-	kubeVipSampleConfig.Flags().StringVar(&cliConfigLB.Name, "lbName", "Example Load Balancer", "The name of a load balancer instance")
-	kubeVipSampleConfig.Flags().IntVar(&cliConfigLB.Port, "lbPort", 8080, "Port that load balancer will expose on")
-	kubeVipSampleConfig.Flags().StringSliceVar(&cliBackends, "lbBackends", []string{"192.168.0.1:8080", "192.168.0.2:8080"}, "Comma seperated backends, format: address:port")
-}
-
-var kubeVipSampleConfig = &cobra.Command{
-	Use:   "config",
-	Short: "Generate a Sample configuration",
-	Run: func(cmd *cobra.Command, args []string) {
-
-		// // Parse localPeer
-		// p, err := kubevip.ParsePeerConfig(cliLocalPeer)
-		// if err != nil {
-		// 	cmd.Help()
-		// 	log.Fatalln(err)
-		// }
-		// cliConfig.LocalPeer = *p
-
-		// // Parse remotePeers
-		// //Iterate backends
-		// for i := range cliRemotePeers {
-		// 	p, err := kubevip.ParsePeerConfig(cliRemotePeers[i])
-		// 	if err != nil {
-		// 		cmd.Help()
-		// 		log.Fatalln(err)
-		// 	}
-		// 	cliConfig.RemotePeers = append(cliConfig.RemotePeers, *p)
-		// }
-
-		// //Iterate backends
-		// for i := range cliBackends {
-		// 	b, err := kubevip.ParseBackendConfig(cliBackends[i])
-		// 	if err != nil {
-		// 		cmd.Help()
-		// 		log.Fatalln(err)
-		// 	}
-		// 	cliConfigLB.Backends = append(cliConfigLB.Backends, *b)
-		// }
-
-		// Add the basic Load-Balancer to the configuration
-		cliConfig.LoadBalancers = append(cliConfig.LoadBalancers, cliConfigLB)
-
-		err := cliConfig.ParseFlags(cliLocalPeer, cliRemotePeers, cliBackends)
-		if err != nil {
-			cmd.Help()
-			log.Fatalln(err)
-		}
-
-		err = kubevip.ParseEnvironment(&cliConfig)
-		if err != nil {
-			cmd.Help()
-			log.Fatalln(err)
-		}
-
-		cliConfig.PrintConfig()
-	},
-}
-
-var kubeVipSampleManifest = &cobra.Command{
-	Use:   "manifest",
-	Short: "Generate a Sample kubernetes manifest",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Generate the sample manifest specification
-		p := &appv1.Pod{
-			TypeMeta: metav1.TypeMeta{
-				Kind:       "Pod",
-				APIVersion: "v1",
-			},
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      "kube-vip",
-				Namespace: "kube-system",
-			},
-			Spec: appv1.PodSpec{
-				Containers: []appv1.Container{
-					{
-						Name:  "kube-vip",
-						Image: fmt.Sprintf("docker.io/plndr/kube-vip:%s", Release.Version),
-						SecurityContext: &appv1.SecurityContext{
-							Capabilities: &appv1.Capabilities{
-								Add: []appv1.Capability{
-									"NET_ADMIN",
-									"SYS_TIME",
-								},
-							},
-						},
-						Args: []string{
-							"start",
-							"-c",
-							"/etc/kube-vip/config.yaml",
-						},
-						VolumeMounts: []appv1.VolumeMount{
-							{
-								Name:      "config",
-								MountPath: "/etc/kube-vip/",
-							},
-						},
-					},
-				},
-				Volumes: []appv1.Volume{
-					{
-						Name: "config",
-						VolumeSource: appv1.VolumeSource{
-							HostPath: &appv1.HostPathVolumeSource{
-								Path: "/etc/kube-vip/",
-							},
-						},
-					},
-				},
-				HostNetwork: true,
-			},
-		}
-
-		b, _ := yaml.Marshal(p)
-		fmt.Printf(string(b))
-	},
-}

cmd/kube-vip-kubeadm.go

@@ -1,18 +1,13 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
-	"net"
 	"os"
 
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
 	"github.com/spf13/cobra"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
 )
 
 // kubeadm adds two subcommands for managing a vip during a kubeadm init/join
@@ -28,8 +23,8 @@ func init() {
 var kubeKubeadm = &cobra.Command{
 	Use:   "kubeadm",
 	Short: "Kubeadm functions",
-	Run: func(cmd *cobra.Command, args []string) {
-		cmd.Help()
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		_ = cmd.Help()
 		// TODO - A load of text detailing what's actually happening
 	},
 }
@@ -38,124 +33,84 @@ var kubeKubeadmInit = &cobra.Command{
 	Use:   "init",
 	Short: "kube-vip init",
 	Long:  "The \"init\" subcommand will generate the Kubernetes manifest that will be started by kubeadm through the kubeadm init process",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
 		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
+		err := kubevip.ParseEnvironment(&initConfig)
+		if err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+
 		// TODO - check for certain things VIP/interfaces
 		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+			_ = cmd.Help()
+			log.Error("No interface is specified for kube-vip to bind to")
+			return
 		}
 
 		if initConfig.VIP == "" && initConfig.Address == "" {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
-		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
 
-		fmt.Println(cfg)
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("generating VIPSubnet", "err", err)
+				return
+			}
+		}
+
+		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
+		fmt.Println(cfg) // output manifest to stdout
 	},
 }
 
 var kubeKubeadmJoin = &cobra.Command{
 	Use:   "join",
 	Short: "kube-vip join",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
 		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
+		err := kubevip.ParseEnvironment(&initConfig)
+		if err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+
 		// TODO - check for certain things VIP/interfaces
 		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+			_ = cmd.Help()
+			log.Error("No interface is specified for kube-vip to bind to")
+			return
 		}
 
 		if initConfig.VIP == "" && initConfig.Address == "" {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
 
 		if _, err := os.Stat(kubeConfigPath); os.IsNotExist(err) {
-			log.Fatalf("Unable to find file [%s]", kubeConfigPath)
+			log.Error("kubeConfig not found", "Path", kubeConfigPath)
+			return
 		}
 
-		// We will use kubeconfig in order to find all the master nodes
-		// use the current context in kubeconfig
-		config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
-		if err != nil {
-			log.Fatal(err.Error())
-		}
-
-		// create the clientset
-		clientset, err := kubernetes.NewForConfig(config)
-		if err != nil {
-			log.Fatal(err.Error())
-		}
-
-		opts := metav1.ListOptions{}
-		opts.LabelSelector = "node-role.kubernetes.io/master"
-		nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), opts)
-
-		// Iterate over all nodes that are masters and find the details to build a peer list
-		for x := range nodes.Items {
-			// Get hostname and address
-			var nodeAddress, nodeHostname string
-			for y := range nodes.Items[x].Status.Addresses {
-				switch nodes.Items[x].Status.Addresses[y].Type {
-				case corev1.NodeHostName:
-					nodeHostname = nodes.Items[x].Status.Addresses[y].Address
-				case corev1.NodeInternalIP:
-					nodeAddress = nodes.Items[x].Status.Addresses[y].Address
-				}
-			}
-
-			newPeer, err := kubevip.ParsePeerConfig(fmt.Sprintf("%s:%s:%d", nodeHostname, nodeAddress, 10000))
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
 			if err != nil {
-				panic(err.Error())
+				log.Error("generating VIPSubnet", "err", err)
+				return
 			}
-			initConfig.RemotePeers = append(initConfig.RemotePeers, *newPeer)
-
 		}
-		// Generate manifest and print
-		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
-		fmt.Println(cfg)
+
+		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
+		fmt.Println(cfg) // output manifest to stdout
 	},
 }
-
-func autoGenLocalPeer() (*kubevip.RaftPeer, error) {
-	// hostname // address // defaultport
-	h, err := os.Hostname()
-	if err != nil {
-		return nil, err
-	}
-
-	var a string
-	addrs, err := net.InterfaceAddrs()
-	if err != nil {
-		return nil, err
-	}
-	for _, address := range addrs {
-		// check the address type and if it is not a loopback the display it
-		if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
-			if ipnet.IP.To16() != nil {
-				a = ipnet.IP.String()
-				break
-			}
-		}
-	}
-	if a == "" {
-		return nil, fmt.Errorf("Unable to find local address")
-	}
-	return &kubevip.RaftPeer{
-		ID:      h,
-		Address: a,
-		Port:    10000,
-	}, nil
-
-}

cmd/kube-vip-manifests.go

@@ -3,8 +3,9 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
 	"github.com/spf13/cobra"
 )
 
@@ -12,23 +13,27 @@ import (
 // manifests will be used to generate:
 // - Pod spec manifest, mainly used for a static pod (kubeadm)
 // - Daemonset manifest, mainly used to run kube-vip as a deamonset within Kubernetes (k3s/rke)
+// - RBAC manifest, used to generate the RBAC permissions for kube-vip
 
-//var inCluster bool
-var taint bool
+var taint, role, rolebinding bool
 
 func init() {
 	kubeManifest.PersistentFlags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
+	kubeManifest.PersistentFlags().StringVar(&image, "image", "ghcr.io/kube-vip/kube-vip", "Define a hardcoded image with or without tag for the manifest")
 	kubeManifestDaemon.PersistentFlags().BoolVar(&taint, "taint", false, "Taint the manifest for only running on control planes")
+	kubeManifestRbac.PersistentFlags().BoolVar(&role, "role", false, "Generate only a Role inside the serviceNamespace access")
+	kubeManifestRbac.PersistentFlags().BoolVar(&rolebinding, "rolebinding", false, "Generate only a RoleBinding for namespaced access")
 
 	kubeManifest.AddCommand(kubeManifestPod)
 	kubeManifest.AddCommand(kubeManifestDaemon)
+	kubeManifest.AddCommand(kubeManifestRbac)
 }
 
 var kubeManifest = &cobra.Command{
 	Use:   "manifest",
 	Short: "Manifest functions",
-	Run: func(cmd *cobra.Command, args []string) {
-		cmd.Help()
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		_ = cmd.Help()
 		// TODO - A load of text detailing what's actually happening
 	},
 }
@@ -36,51 +41,111 @@ var kubeManifest = &cobra.Command{
 var kubeManifestPod = &cobra.Command{
 	Use:   "pod",
 	Short: "Generate a Pod Manifest",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		var err error
+
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
 		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
-		// TODO - check for certain things VIP/interfaces
-		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
+			log.Error("parsing environment", "err", err)
+			return
 		}
 
 		// The control plane has a requirement for a VIP being specified
-		if initConfig.EnableControlPane && (initConfig.VIP == "" && initConfig.Address == "") {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
-		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
 
-		fmt.Println(cfg)
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("config parse", "err", err)
+				return
+			}
+		}
+
+		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
+		fmt.Println(cfg) // output manifest to stdout
 	},
 }
 
 var kubeManifestDaemon = &cobra.Command{
 	Use:   "daemonset",
 	Short: "Generate a Daemonset Manifest",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		var err error
+
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
 		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
-		// TODO - check for certain things VIP/interfaces
-		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+		// The control plane has a requirement for a VIP being specified
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
+		}
+
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("config parse", "err", err)
+				return
+			}
+		}
+
+		cfg := kubevip.GenerateDaemonsetManifestFromConfig(&initConfig, image, Release.Version, inCluster, taint)
+		fmt.Println(cfg) // output manifest to stdout
+	},
+}
+
+var kubeManifestRbac = &cobra.Command{
+	Use:   "rbac",
+	Short: "Generate an RBAC Manifest",
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		var err error
+
+		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
+		// TODO - A load of text detailing what's actually happening
+		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
+			log.Error("parsing environment", "err", err)
+			return
 		}
 
 		// The control plane has a requirement for a VIP being specified
-		if initConfig.EnableControlPane && (initConfig.VIP == "" && initConfig.Address == "") {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
-		cfg := kubevip.GenerateDeamonsetManifestFromConfig(&initConfig, Release.Version, inCluster, taint)
 
-		fmt.Println(cfg)
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("generating VIPSubnet", "err", err)
+				return
+			}
+		}
+		saCfg := kubevip.GenerateSA(&initConfig)
+		roleCfg := kubevip.GenerateRole(&initConfig, role)
+		if role {
+			rolebinding = true
+		}
+		roleBindingCfg := kubevip.GenerateRoleBinding(rolebinding, saCfg, roleCfg)
+
+		// Output the YAML manifests to stdout
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(saCfg))
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(roleCfg))
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(roleBindingCfg))
 	},
 }

cmd/kube-vip-start.go

@@ -1,136 +0,0 @@
-package cmd
-
-import (
-	"os"
-	"os/signal"
-
-	"github.com/plunder-app/kube-vip/pkg/bgp"
-	"github.com/plunder-app/kube-vip/pkg/cluster"
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-)
-
-// Start as a single node (no cluster), start as a leader in the cluster
-var startConfig kubevip.Config
-var startConfigLB kubevip.LoadBalancer
-var startLocalPeer, startKubeConfigPath string
-var startRemotePeers, startBackends []string
-var inCluster bool
-
-func init() {
-	// Get the configuration file
-	kubeVipStart.Flags().StringVarP(&configPath, "config", "c", "", "Path to a kube-vip configuration")
-	kubeVipStart.Flags().BoolVarP(&disableVIP, "disableVIP", "d", false, "Disable the VIP functionality")
-
-	// Pointers so we can see if they're nil (and not called)
-	kubeVipStart.Flags().StringVar(&startConfig.Interface, "interface", "eth0", "Name of the interface to bind to")
-	kubeVipStart.Flags().StringVar(&startConfig.VIP, "vip", "192.168.0.1", "The Virtual IP address")
-	kubeVipStart.Flags().StringVar(&startConfig.Address, "address", "", "an address (IP or DNS name) to use as a VIP")
-	kubeVipStart.Flags().IntVar(&startConfig.Port, "port", 6443, "listen port for the VIP")
-	kubeVipStart.Flags().BoolVar(&startConfig.DDNS, "ddns", false, "use Dynamic DNS + DHCP to allocate VIP for address")
-	kubeVipStart.Flags().BoolVar(&startConfig.SingleNode, "singleNode", false, "Start this instance as a single node")
-	kubeVipStart.Flags().BoolVar(&startConfig.StartAsLeader, "startAsLeader", false, "Start this instance as the cluster leader")
-	kubeVipStart.Flags().BoolVar(&startConfig.EnableARP, "arp", false, "Use ARP broadcasts to improve VIP re-allocations")
-	kubeVipStart.Flags().StringVar(&startLocalPeer, "localPeer", "server1:192.168.0.1:10000", "Settings for this peer, format: id:address:port")
-	kubeVipStart.Flags().StringSliceVar(&startRemotePeers, "remotePeers", []string{"server2:192.168.0.2:10000", "server3:192.168.0.3:10000"}, "Comma seperated remotePeers, format: id:address:port")
-	// Load Balancer flags
-	kubeVipStart.Flags().BoolVar(&startConfigLB.BindToVip, "lbBindToVip", false, "Bind example load balancer to VIP")
-	kubeVipStart.Flags().StringVar(&startConfigLB.Type, "lbType", "tcp", "Type of load balancer instance (TCP/HTTP)")
-	kubeVipStart.Flags().StringVar(&startConfigLB.Name, "lbName", "Example Load Balancer", "The name of a load balancer instance")
-	kubeVipStart.Flags().IntVar(&startConfigLB.Port, "lbPort", 8080, "Port that load balancer will expose on")
-	kubeVipStart.Flags().IntVar(&startConfigLB.BackendPort, "lbBackEndPort", 6443, "A port that all backends may be using (optional)")
-	kubeVipStart.Flags().StringSliceVar(&startBackends, "lbBackends", []string{"192.168.0.1:8080", "192.168.0.2:8080"}, "Comma seperated backends, format: address:port")
-
-	// Cluster configuration
-	kubeVipStart.Flags().StringVar(&startKubeConfigPath, "kubeConfig", "/etc/kubernetes/admin.conf", "The path of a kubernetes configuration file")
-	kubeVipStart.Flags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
-	kubeVipStart.Flags().BoolVar(&startConfig.EnableLeaderElection, "leaderElection", false, "Use the Kubernetes leader election mechanism for clustering")
-
-	// This sets the namespace that the lock should exist in
-	kubeVipStart.Flags().StringVarP(&startConfig.Namespace, "namespace", "n", "kube-system", "The configuration map defined within the cluster")
-}
-
-var kubeVipStart = &cobra.Command{
-	Use:   "start",
-	Short: "Start the Virtual IP / Load balancer",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
-		var err error
-
-		// If a configuration file is loaded, then it will overwrite flags
-
-		if configPath != "" {
-			c, err := kubevip.OpenConfig(configPath)
-			if err != nil {
-				log.Fatalf("%v", err)
-			}
-			startConfig = *c
-		}
-
-		// parse environment variables, these will overwrite anything loaded or flags
-		err = kubevip.ParseEnvironment(&startConfig)
-		if err != nil {
-			log.Fatalln(err)
-		}
-
-		newCluster, err := cluster.InitCluster(&startConfig, disableVIP)
-		if err != nil {
-			log.Fatalf("%v", err)
-		}
-		var bgpServer *bgp.Server
-		if startConfig.SingleNode {
-			// If the Virtual IP isn't disabled then create the netlink configuration
-			// Start a single node cluster
-			newCluster.StartSingleNode(&startConfig, disableVIP)
-		} else {
-			if disableVIP {
-				log.Fatalln("Cluster mode requires the Virtual IP to be enabled, use single node with no VIP")
-			}
-
-			if startConfig.EnableLeaderElection {
-				cm, err := cluster.NewManager(startKubeConfigPath, inCluster, startConfig.Port)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-
-				if startConfig.EnableBGP {
-					log.Info("Starting the BGP server to adverise VIP routes to VGP peers")
-					bgpServer, err = bgp.NewBGPServer(&startConfig.BGPConfig)
-					if err != nil {
-						log.Fatalf("%v", err)
-					}
-
-					// Defer a function to check if the bgpServer has been created and if so attempt to close it
-					defer func() {
-						if bgpServer != nil {
-							bgpServer.Close()
-						}
-					}()
-				}
-
-				// Leader Cluster will block
-				err = newCluster.StartLeaderCluster(&startConfig, cm, bgpServer)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-			} else {
-
-				// // Start a multi-node (raft) cluster, this doesn't block so will wait on signal
-				err = newCluster.StartRaftCluster(&startConfig)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-				signalChan := make(chan os.Signal, 1)
-				signal.Notify(signalChan, os.Interrupt)
-
-				<-signalChan
-
-				newCluster.Stop()
-			}
-
-		}
-
-	},
-}

cmd/kube-vip.go

@@ -3,42 +3,38 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"net"
 	"net/http"
 	"os"
+	"slices"
+	"strings"
 	"time"
 
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	"github.com/plunder-app/kube-vip/pkg/manager"
-	"github.com/plunder-app/kube-vip/pkg/packet"
+	log "log/slog"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/manager"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
 )
 
-// Path to the configuration file
-var configPath string
+// Is an option to set the image
+var image string
 
-// Path to the configuration file
-var namespace string
-
-// Disable the Virtual IP (bind to the existing network stack)
-var disableVIP bool
-
-// Disable the Virtual IP (bind to the existing network stack)
-var controlPlane bool
-
-// Run as a load balancer service (within a pod / kubernetes)
-var serviceArp bool
+// Is kube-vip running within cluster
+var inCluster bool
 
 // ConfigMap name within a Kubernetes cluster
 var configMap string
 
-// Configure the level of loggin
-var logLevel uint32
-
-// Provider Config
-var providerConfig string
+// Points to a kubernetes configuration file
+var kubeConfigPath string
 
 // Release - this struct contains the release information populated when building kube-vip
 var Release struct {
@@ -47,11 +43,10 @@ var Release struct {
 }
 
 // Structs used via the various subcommands
-var initConfig kubevip.Config
-var initLoadBalancer kubevip.LoadBalancer
-
-// Points to a kubernetes configuration file
-var kubeConfigPath string
+var (
+	initConfig       kubevip.Config
+	initLoadBalancer kubevip.LoadBalancer
+)
 
 var kubeVipCmd = &cobra.Command{
 	Use:   "kube-vip",
@@ -59,83 +54,114 @@ var kubeVipCmd = &cobra.Command{
 }
 
 func init() {
-
-	localpeer, err := autoGenLocalPeer()
-	if err != nil {
-		log.Fatalln(err)
-	}
-	initConfig.LocalPeer = *localpeer
-	//initConfig.Peers = append(initConfig.Peers, *localpeer)
+	// Basic flags
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Interface, "interface", "", "Name of the interface to bind to")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesInterface, "serviceInterface", "", "Name of the interface to bind to (for services)")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIP, "vip", "", "The Virtual IP address")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPSubnet, "vipSubnet", "", "The Virtual IP address subnet e.g. /32 /24 /8 etc.. (Default to 32 for IPv4 and 128 for IPv6)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.NodeName, "nodeName", "", "Name to be used for lease holder. Must be unique for each node/instance")
+
+	// VIP flags
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Address, "address", "", "an address (IP or DNS name) to use as a VIP")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.Port, "port", 6443, "listen port for the VIP")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPCIDR, "cidr", "32", "The CIDR range for the virtual IP address")
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for Vip changes")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Annotations, "annotations", "", "Set Node annotations prefix for parsing")
+	kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.Port, "port", 6443, "Port for the VIP")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for VIP changes")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableWireguard, "wireguard", false, "Enable Wireguard for services VIPs")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableRoutingTable, "table", false, "Enable Routing Table for services VIPs")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.PreserveVIPOnLeadershipLoss, "preserveVipOnLeadershipLoss", false, "Preserve ARP VIP addresses on interface when leadership is lost (default: false for backward compatibility)")
+
+	// LoadBalancer flags
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "enableLoadBalancer", false, "enable loadbalancing on the VIP with IPVS")
+	kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.LoadBalancerPort, "lbPort", 6443, "loadbalancer port for the VIP")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerForwardingMethod, "lbForwardingMethod", "local", "loadbalancer forwarding method")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DDNS, "ddns", false, "use Dynamic DNS + DHCP to allocate VIP for address")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MirrorDestInterface, "mirrorDestInterface", "", "network interface where all traffic that traverses the service interface will be mirrored to. Source interface will use default interface is servicesInterface is not set.")
 
 	// Clustering type (leaderElection)
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLeaderElection, "leaderElection", false, "Use the Kubernetes leader election mechanism for clustering")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 5, "Length of time a Kubernetes leader lease can be held for")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time a Kubernetes leader can attempt to renew its lease")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Number of times the host will retry to hold a lease")
-
-	// Clustering type (raft)
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.StartAsLeader, "startAsLeader", false, "Start this instance as the cluster leader")
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.AddPeersAsBackends, "addPeersToLB", true, "Add raft peers to the load-balancer")
-
-	// Packet flags
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableMetal, "metal", false, "This will use the Equinix Metal API (requires the token ENV) to update the EIP <-> VIP")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalAPIKey, "metalKey", "", "The API token for authenticating with the Equinix Metal API")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalProject, "metalProject", "", "The name of project already created within Equinix Metal")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalProjectID, "metalrojectID", "", "The ID of project already created within Equinix Metal")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ProviderConfig, "provider-config", "", "The path to a provider configuration")
-
-	// Load Balancer flags
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "lbEnable", false, "Enable a load-balancer on the VIP")
-	kubeVipCmd.PersistentFlags().BoolVar(&initLoadBalancer.BindToVip, "lbBindToVip", true, "Bind example load balancer to VIP")
-	kubeVipCmd.PersistentFlags().StringVar(&initLoadBalancer.Type, "lbType", "tcp", "Type of load balancer instance (TCP/HTTP)")
-	kubeVipCmd.PersistentFlags().StringVar(&initLoadBalancer.Name, "lbName", "Kubeadm Load Balancer", "The name of a load balancer instance")
-	kubeVipCmd.PersistentFlags().IntVar(&initLoadBalancer.Port, "lbPort", 6443, "Port that load balancer will expose on")
-	kubeVipCmd.PersistentFlags().IntVar(&initLoadBalancer.BackendPort, "lbBackEndPort", 6444, "A port that all backends may be using (optional)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaderElectionType, "leaderElectionType", "kubernetes", "Defines the backend to run the leader election: kubernetes or etcd. Defaults to kubernetes.")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaseName, "leaseName", "plndr-cp-lock", "Name of the lease that is used for leader election")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 5, "Length of time (in seconds) a Kubernetes leader lease can be held for")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time (in seconds) a Kubernetes leader can attempt to renew its lease")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Length of time (in seconds) the LeaderElector clients should wait between tries of actions")
 
 	// BGP flags
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableBGP, "bgp", false, "This will enable BGP support within kube-vip")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.RouterID, "bgpRouterID", "", "The routerID for the bgp server")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.SourceIF, "sourceIF", "", "The source interface for bgp peering (not to be used with sourceIP)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.SourceIP, "sourceIP", "", "The source address for bgp peering (not to be used with sourceIF)")
 	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.AS, "localAS", 65000, "The local AS number for the bgp server")
+	kubeVipCmd.PersistentFlags().Uint64Var(&initConfig.BGPConfig.HoldTime, "bgpHoldTimer", 30, "The hold timer for all bgp peers (it defines the time a session is held)")
+	kubeVipCmd.PersistentFlags().Uint64Var(&initConfig.BGPConfig.KeepaliveInterval, "bgpKeepAliveInterval", 10, "The keepalive interval for all bgp peers (it defines the heartbeat of keepalive messages)")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPPeerConfig.Address, "peerAddress", "", "The address of a BGP peer")
 	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPPeerConfig.AS, "peerAS", 65000, "The AS number for a BGP peer")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPPeerConfig.Password, "peerPass", "", "The md5 password for a BGP peer")
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPPeerConfig.MultiHop, "multihop", false, "This will enable BGP multihop support")
-	kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.BGPPeers, "bgppeers", []string{}, "Comma seperated BGP Peer, format: address:as:password:multihop")
+	kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.BGPPeers, "bgppeers", []string{}, "Comma separated BGP Peer, format: address:as:password:multihop")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Annotations, "annotations", "", "Set Node annotations prefix for parsing")
 
-	// Control plane specific flags
-	kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The configuration map defined within the cluster")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPConfig.Zebra.Enabled, "zebra", false, "This will enable Zebra support within kube-vip")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.URL, "zebraUrl", "unix:/var/run/frr/zserv.api", "Path to the unix domain socket for connecting to Zebra daemon")
+	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.Zebra.Version, "zebraVersion", 6, "Zebra API Version")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.SoftwareName, "zebraSoftwareName", "frr8.3", "Software Name for Zebra")
+
+	// Namespace for kube-vip
+	kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The namespace for the configmap defined within the cluster")
 
 	// Manage logging
-	kubeVipCmd.PersistentFlags().Uint32Var(&logLevel, "log", 4, "Set the level of logging")
+	kubeVipCmd.PersistentFlags().Int32Var(&initConfig.Logging, "log", 0, "Set the level of logging")
 
 	// Service flags
 	kubeVipService.Flags().StringVarP(&configMap, "configMap", "c", "plndr", "The configuration map defined within the cluster")
 
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableControlPane, "controlplane", false, "Enable HA for control plane, hybrid mode")
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServices, "services", false, "Enable Kubernetes services, hybrid mode")
+	// Routing Table flags
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableID, "tableID", 198, "The routing table used for all table entries")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableType, "tableType", unix.RTN_UNICAST, "The type of route that will be added to the routing table")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingProtocol, "routingProtocol", 248, "The routing protocol value used to create routes")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.CleanRoutingTable, "cleanRoutingTable", false, "Clean routing table of redundant routes on start")
+
+	// Behaviour flags
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableControlPlane, "controlplane", false, "Enable HA for control plane")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DetectControlPlane, "autodetectcp", false, "Determine working address for control plane (from loopback)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServices, "services", false, "Enable Kubernetes services")
+
+	// Extended behaviour flags
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServicesElection, "servicesElection", false, "Enable leader election per kubernetes service")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassOnly, "lbClassOnly", false, fmt.Sprintf("Enable load balancing only for services with LoadBalancerClass %q", kubevip.LBClassName))
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerClassName, "lbClassName", kubevip.LBClassName, fmt.Sprintf("Name of load balancer class for kube-VIP, defaults to %q", kubevip.LBClassName))
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassLegacyHandling, "lbClassNameLegacyHandling", true, "Use legacy LoadBalancer class name handling (e.g. accepting services both with empty and non-empty class)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServiceSecurity, "onlyAllowTrafficServicePorts", false, "Only allow traffic to service ports, others will be dropped, defaults to false")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableNodeLabeling, "enableNodeLabeling", false, fmt.Sprintf("Enable leader node labeling with %q, defaults to false", kubevip.HasIP))
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesLeaseName, "servicesLeaseName", "plndr-svcs-lock", "Name of the lease that is used for leader election for services (in arp mode)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.DNSMode, "dnsMode", "first", "Name of the mode that DNS lookup will be performed (first, ipv4, ipv6, dual)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DisableServiceUpdates, "disableServiceUpdates", false, "If true, kube-vip will process services as usual, but will not update service's Status.LoadBalancer.Ingress slice")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpoints, "enableEndpoints", false, "If enabled, kube-vip will only advertise services, but will use the (deprecated since v1.33) endpoints for IP addresses")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoInterfaceGlobalScope, "loInterfaceGlobalScope", false, "If true, kube-vip will set global scope when using the lo interface, otherwise a host scope will be used by default")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.HealthCheckPort, "healthCheckPort", 0, "If set to non-zero (> 1024), then this is the port that the healthcheck will listen on")
 
 	// Prometheus HTTP Server
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.PrometheusHTTPServer, "promethuesHTTPServer", ":2112", "Host and port used to expose Promethues metrics via an HTTP server")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.PrometheusHTTPServer, "prometheusHTTPServer", ":2112", "Host and port used to expose Prometheus metrics via an HTTP server")
+
+	// Etcd
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.CAFile, "etcdCACert", "", "Verify certificates of TLS-enabled secure servers using this CA bundle file")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.ClientCertFile, "etcdCert", "", "Identify secure client using this TLS certificate file")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.ClientKeyFile, "etcdKey", "", "Identify secure client using this TLS key file")
+	kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.Etcd.Endpoints, "etcdEndpoints", nil, "Etcd member endpoints")
+
+	// Kubernetes client specific flags
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.K8sConfigFile, "k8sConfigPath", "/etc/kubernetes/admin.conf", "Path to the configuration file used with the Kubernetes client")
+
+	// Configuration file flag
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ConfigFile, "config-file", "", "Path to a JSON/YAML configuration file to load settings from")
 
 	kubeVipCmd.AddCommand(kubeKubeadm)
 	kubeVipCmd.AddCommand(kubeManifest)
 	kubeVipCmd.AddCommand(kubeVipManager)
 	kubeVipCmd.AddCommand(kubeVipSample)
 	kubeVipCmd.AddCommand(kubeVipService)
-	kubeVipCmd.AddCommand(kubeVipStart)
 	kubeVipCmd.AddCommand(kubeVipVersion)
 
-	// Sample commands
-	kubeVipSample.AddCommand(kubeVipSampleConfig)
-	kubeVipSample.AddCommand(kubeVipSampleManifest)
-
+	// Set the logging level for all subsequent functions
+	log.SetLogLoggerLevel(log.Level(initConfig.Logging))
 }
 
 // Execute - starts the command parsing process
@@ -149,7 +175,7 @@ func Execute() {
 var kubeVipVersion = &cobra.Command{
 	Use:   "version",
 	Short: "Version and Release information about the Kubernetes Virtual IP Server",
-	Run: func(cmd *cobra.Command, args []string) {
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 		fmt.Printf("Kube-VIP Release Information\n")
 		fmt.Printf("Version:  %s\n", Release.Version)
 		fmt.Printf("Build:    %s\n", Release.Build)
@@ -159,22 +185,43 @@ var kubeVipVersion = &cobra.Command{
 var kubeVipSample = &cobra.Command{
 	Use:   "sample",
 	Short: "Generate a Sample configuration",
-	Run: func(cmd *cobra.Command, args []string) {
-		cmd.Help()
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		_ = cmd.Help()
 	},
 }
 
 var kubeVipService = &cobra.Command{
 	Use:   "service",
 	Short: "Start the Virtual IP / Load balancer as a service within a Kubernetes cluster",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 
-		// parse environment variables, these will overwrite anything loaded or flags
+		// Load configuration from file if specified (lowest priority)
+		if initConfig.ConfigFile != "" {
+			err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
+			if err != nil {
+				log.Error("loading config file", "err", err)
+				return
+			}
+		}
+
+		// parse environment variables, these will overwrite anything loaded from config file
 		err := kubevip.ParseEnvironment(&initConfig)
 		if err != nil {
-			log.Fatalln(err)
+			log.Error("parsing env", "err", err)
+			return
+		}
+
+		// Change RTN_UNSPEC to default type
+		if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
+			initConfig.RoutingProtocol = unix.RTN_UNICAST
+		}
+
+		// Set the logging level for all subsequent functions
+		log.SetLogLoggerLevel(log.Level(initConfig.Logging))
+
+		if err := initConfig.CheckInterface(); err != nil {
+			log.Error("checking interface", "err", err)
+			return
 		}
 
 		// User Environment variables as an option to make manifest clearer
@@ -183,16 +230,27 @@ var kubeVipService = &cobra.Command{
 			configMap = envConfigMap
 		}
 
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("generating CIDR", "err", err)
+				return
+			}
+		}
+
 		// Define the new service manager
 		mgr, err := manager.New(configMap, &initConfig)
 		if err != nil {
-			log.Fatalf("%v", err)
+			log.Error("new manager", "err", err)
+			return
 		}
 
 		// Start the service manager, this will watch the config Map and construct kube-vip services for it
 		err = mgr.Start()
 		if err != nil {
-			log.Fatalf("%v", err)
+			log.Error("manager start", "err", err)
+			return
 		}
 	},
 }
@@ -200,18 +258,143 @@ var kubeVipService = &cobra.Command{
 var kubeVipManager = &cobra.Command{
 	Use:   "manager",
 	Short: "Start the kube-vip manager",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		// Load configuration from file if specified (lowest priority)
+		if initConfig.ConfigFile != "" {
+			err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
+			if err != nil {
+				log.Error("loading config file", "err", err)
+				return
+			}
+		}
 
-		go servePrometheusHTTPServer(cmd.Context(), PrometheusHTTPServerConfig{
-			Addr: initConfig.PrometheusHTTPServer,
-		})
-
-		// parse environment variables, these will overwrite anything loaded or flags
+		// parse environment variables, these will overwrite anything loaded from config file
 		err := kubevip.ParseEnvironment(&initConfig)
 		if err != nil {
-			log.Fatalln(err)
+			log.Error("parsing environment", "err", err)
+			return
+		}
+
+		// Change RTN_UNSPEC to default type
+		if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
+			initConfig.RoutingProtocol = unix.RTN_UNICAST
+		}
+
+		// Set the logging level for all subsequent functions
+		log.SetLogLoggerLevel(log.Level(initConfig.Logging))
+
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("No interface is specified for kube-vip to bind to")
+				return
+			}
+		}
+
+		// Welome messages
+		log.Info("kube-vip.io", "version", Release.Version, "build", Release.Build)
+
+		// start prometheus server
+		if initConfig.PrometheusHTTPServer != "" {
+			go servePrometheusHTTPServer(cmd.Context(), PrometheusHTTPServerConfig{
+				Addr: initConfig.PrometheusHTTPServer,
+			})
+		}
+
+		// Determine the kube-vip mode
+		var mode string
+		if initConfig.EnableARP {
+			mode = "ARP"
+		}
+
+		if initConfig.EnableBGP {
+			mode = "BGP"
+		}
+
+		if initConfig.EnableWireguard {
+			mode = "Wireguard"
+		}
+
+		if initConfig.EnableRoutingTable {
+			mode = "Routing Table"
+		}
+
+		// Provide configuration to output/logging
+		log.Info("starting", "namespace", initConfig.Namespace, "Mode", mode, "Control Plane", initConfig.EnableControlPlane, "Services", initConfig.EnableServices)
+
+		// End if nothing is enabled
+		if !initConfig.EnableServices && !initConfig.EnableControlPlane {
+			log.Error("no features are enabled")
+			return
+		}
+
+		if !initConfig.EnableARP && strings.Contains(initConfig.VIPSubnet, kubevip.Auto) {
+			log.Error("auto subnet discovery cannot be used outside ARP mode")
+			return
+		}
+
+		if strings.Contains(initConfig.VIPSubnet, kubevip.Auto) && initConfig.Address != "" {
+			log.Error("auto subnet discovery cannot be used if VIP address was provided")
+			return
+		}
+
+		// If we're using wireguard then all traffic goes through the wg0 interface
+		if initConfig.EnableWireguard {
+			if initConfig.Interface == "" {
+				// Set the vip interface to the wireguard interface
+				initConfig.Interface = "wg0"
+			}
+
+			log.Info("configuring Wireguard networking")
+			l, err := netlink.LinkByName(initConfig.Interface)
+			if err != nil {
+				if strings.Contains(err.Error(), "Link not found") {
+					log.Warn("attempting to create wireguard interface", "interface not found", initConfig.Interface)
+					err = netlink.LinkAdd(&netlink.Wireguard{LinkAttrs: netlink.LinkAttrs{Name: initConfig.Interface}})
+					if err != nil {
+						log.Error("adding link", "err", err)
+						return
+					}
+					l, err = netlink.LinkByName(initConfig.Interface)
+					if err != nil {
+						log.Error("finding link", "err", err)
+						return
+					}
+				}
+			}
+			err = netlink.LinkSetUp(l)
+			if err != nil {
+				log.Error("setting link UP", "err", err)
+				return
+			}
+
+		} else { // if we're not using Wireguard then we'll need to use an actual interface
+			// Check if the interface needs auto-detecting
+			if initConfig.Interface == "" {
+				log.Info("No interface is specified for VIP in config, auto-detecting default Interface")
+				defaultIF, err := vip.GetDefaultGatewayInterface()
+				if err != nil {
+					_ = cmd.Help()
+					log.Error("detecting interface", "err", err)
+					return
+				}
+				initConfig.Interface = defaultIF.Name
+				log.Info("kube-vip bind", "interface", initConfig.Interface)
+
+				go func() {
+					if err := vip.MonitorDefaultInterface(context.TODO(), defaultIF); err != nil {
+
+						log.Error("interface monitor", "err", err)
+						return
+					}
+				}()
+			}
+		}
+		// Perform a check on the state of the interface
+		if err := initConfig.CheckInterface(); err != nil {
+			log.Error("checking interface", "err", err)
+			return
 		}
 
 		// User Environment variables as an option to make manifest clearer
@@ -220,34 +403,25 @@ var kubeVipManager = &cobra.Command{
 			configMap = envConfigMap
 		}
 
-		// If Packet is enabled and there is a provider configuration passed
-		if initConfig.EnableMetal {
-			if providerConfig != "" {
-				providerAPI, providerProject, err := packet.GetPacketConfig(providerConfig)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-				initConfig.MetalAPIKey = providerAPI
-				initConfig.MetalProject = providerProject
-			}
-		}
-
 		// Define the new service manager
 		mgr, err := manager.New(configMap, &initConfig)
 		if err != nil {
-			log.Fatalf("%v", err)
+			log.Error("new manager", "err", err)
+			return
 		}
 
-		prometheus.MustRegister(mgr.PromethesCollector()...)
+		prometheus.MustRegister(mgr.PrometheusCollector()...)
 
 		// Start the service manager, this will watch the config Map and construct kube-vip services for it
 		err = mgr.Start()
 		if err != nil {
-			log.Fatalf("%v", err)
+			log.Error("start manager", "err", err)
+			return
 		}
 	},
 }
 
+// PrometheusHTTPServerConfig defines the Prometheus server configuration.
 type PrometheusHTTPServerConfig struct {
 	// Addr sets the http server address used to expose the metric endpoint
 	Addr string
@@ -257,23 +431,34 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
 	var err error
 	mux := http.NewServeMux()
 	mux.Handle("/metrics", promhttp.Handler())
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { //nolint TODO
+		_, _ = w.Write([]byte(`<html>
+			<head><title>kube-vip</title></head>
+			<body>
+			<h1>kube-vip Metrics</h1>
+			<p><a href="` + "/metrics" + `">Metrics</a></p>
+			</body>
+			</html>`))
+	})
 
 	srv := &http.Server{
-		Addr:    config.Addr,
-		Handler: mux,
+		Addr:              config.Addr,
+		Handler:           mux,
+		ReadHeaderTimeout: 2 * time.Second,
 	}
 
 	go func() {
 		if err = srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
-			log.Fatalf("listen:%+s\n", err)
+			log.Error("prometheus HTTP server", "err", err)
+			return
 		}
 	}()
 
-	log.Printf("server started")
+	log.Info("prometheus HTTP server started")
 
 	<-ctx.Done()
 
-	log.Printf("server stopped")
+	log.Info("prometheus HTTP server stopped")
 
 	ctxShutDown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer func() {
@@ -281,11 +466,46 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
 	}()
 
 	if err = srv.Shutdown(ctxShutDown); err != nil {
-		log.Fatalf("server Shutdown Failed:%+s", err)
+		log.Error("shutting down prometheus HTTP server", "err", err)
+		return
 	}
 
 	if err == http.ErrServerClosed {
 		err = nil
 	}
-
+}
+
+func GenerateCidrRange(address string, dnsMode string) (string, error) {
+	var cidrs []string
+
+	addresses := strings.Split(address, ",")
+	for _, a := range addresses {
+		ip := net.ParseIP(a)
+		if ip == nil {
+			// we probably are a DNS name
+			ips, err := utils.LookupHost(a, dnsMode)
+			if len(ips) == 0 || err != nil {
+				return "", fmt.Errorf("invalid IP address: %s from [%s], %v", a, address, err)
+			}
+			for _, addr := range ips {
+				ip = net.ParseIP(addr)
+				if ip.To4() != nil {
+					cidrs = append(cidrs, "32")
+				} else {
+					cidrs = append(cidrs, "128")
+				}
+			}
+		} else {
+			if ip.To4() != nil {
+				cidrs = append(cidrs, "32")
+			} else {
+				cidrs = append(cidrs, "128")
+			}
+		}
+	}
+	// compact as DNS could have a lot of addresses
+	slices.Sort(cidrs)
+	cidrs = slices.Compact(cidrs)
+	slices.Reverse(cidrs)
+	return strings.Join(cidrs, ","), nil
 }

demo/README.md

@@ -0,0 +1,46 @@
+# Demo client-server
+
+This contains some example code to determine how long "failovers" are taking within kube-vip, the server component should live within the cluster and the client should be externally.
+
+## Deploy the server
+
+Simply apply the manifest to a working cluster that has kube-vip deployed:
+
+```
+kubectl apply -f ./demo/server/deploy.yaml
+```
+
+Retrieve the loadBalancer IP that is fronting the service:
+
+```
+kubectl get svc demo-service
+NAME           TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)           AGE
+demo-service   LoadBalancer   10.104.18.147   192.168.0.217   10002:32529/UDP   117m
+```
+
+## Connect the client
+
+From elsewhere, clone the kube-vip repository and connect the client to the server endpoint (loadBalancer IP) with the following command:
+
+```
+go run ./demo/client/main.go -address=<vip>
+```
+
+You will only see output when the client has reconcilled the connection to a pod beneath the service, where it will print the timestamp to reconnection along with the time in milliseconds it took:
+
+```
+15:58:35.916952 3008
+15:58:45.947506 2005
+15:58:57.983151 3007
+15:59:08.013450 2005
+15:59:20.046491 3008
+15:59:30.076341 2507
+15:59:42.110747 3008
+```
+
+## Kill some pods to test
+
+On a machine or control plane that has `kubectl` and has the credentials to speak to the cluster we will run a command to find the demo pod and kill it every 10 seconds:
+
+`while true ; do kubectl delete pod $(kubectl get pods | grep -v NAME | grep vip| awk '{ print $1 }'); sleep 10; done`
+

demo/client/main.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"net"
+	"time"
+)
+
+const udpdata = "a3ViZS12aXAK=kube-vip"
+
+func main() {
+	address := flag.String("address", "127.0.0.1", "The address of the server")
+	port := flag.Int("port", 10002, "the port of the server")
+	interval := flag.Float64("interval", 1000, "Interval in milliseconds")
+	flag.Parse()
+	var errorTime time.Time
+	var errorOccurred bool
+	for {
+		p := make([]byte, 2048)
+		conn, err := net.Dial("udp", fmt.Sprintf("%s:%d", *address, *port))
+		if err != nil {
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			continue
+		}
+
+		err = conn.SetDeadline(time.Now().Add(time.Duration(*interval) * time.Millisecond))
+		if err != nil {
+			//fmt.Printf("Connectivity error [%v]", err)
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			if err = conn.Close(); err != nil {
+				fmt.Printf("Error closing connection [%v]", err)
+			}
+			continue
+		}
+
+		_, err = fmt.Fprint(conn, udpdata)
+		if err != nil {
+			fmt.Printf("Error writing data [%v]", err)
+		}
+
+		_, err = bufio.NewReader(conn).Read(p)
+		if err != nil {
+			//fmt.Printf("read error %v\n", err)
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			if err = conn.Close(); err != nil {
+				fmt.Printf("Error closing connection [%v]", err)
+			}
+			continue
+		}
+		time.Sleep(time.Duration(*interval) * time.Millisecond)
+		if errorOccurred {
+			finishTime := time.Since(errorTime)
+			//fmt.Printf("connectivity reconciled in %dms\n", finishTime.Milliseconds())
+			//t :=time.Now().Format("15:04:05.000000")
+			fmt.Printf("%s %d\n", time.Now().Format("15:04:05.000000"), finishTime.Milliseconds())
+
+			errorOccurred = false
+		}
+	}
+}

demo/go.mod

@@ -1,3 +0,0 @@
-module github.com/plunder-app/kube-vip/demo
-
-go 1.13

demo/server/Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:experimental
 
-FROM golang:1.13-alpine as dev
+FROM golang:1.19-alpine as dev
 RUN apk add --no-cache git ca-certificates
 RUN adduser -D appuser
 COPY main.go /src/
@@ -13,4 +13,4 @@ RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
 
 FROM scratch
 COPY --from=dev /src/demo /
-CMD ["/demo"]
+CMD ["/demo"]

demo/server/Makefile

@@ -45,7 +45,7 @@ fmt:
 	@gofmt -l -w $(SRC)
 
 docker:
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 simplify:

demo/server/deploy.yaml

@@ -6,7 +6,7 @@ metadata:
     app: kube-vip-demo
   name: kube-vip-demo
 spec:
-  replicas: 3
+  replicas: 1
   selector:
     matchLabels:
       app: kube-vip-demo
@@ -30,4 +30,25 @@ spec:
         ports:
             - containerPort: 10001
             - containerPort: 10002
-status: {}
+status: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: demo-service
+  namespace: default
+  labels:
+    app: demo-service
+  annotations:
+    kube-vip.io/egress: "true"
+spec:
+  type: LoadBalancer
+  # "Local" preserves the client source IP and avoids a second hop for
+  # LoadBalancer and NodePort
+  externalTrafficPolicy: Local
+  ports:
+  - name: demo-udp
+    port: 10002
+    protocol: UDP
+  selector:
+    app: kube-vip-demo

demo/server/go.mod

@@ -0,0 +1,3 @@
+module github.com/kube-vip/kube-vip/demo
+
+go 1.13

demo/server/main.go

@@ -32,7 +32,7 @@ func main() {
 		}
 	}
 
-	if serverType == strings.ToLower("udp") {
+	if serverType == "udp" {
 		// Start the UDP echo server
 
 		ServerAddr, err := net.ResolveUDPAddr("udp", ":10002")

docs/architecture/index.md

@@ -1,135 +0,0 @@
-# **kube-vip** architecture
-
-This section covers two parts of the architecture: 
-
-1. The technical capabilities of `kube-vip`
-2. The components to build a load-balancing service within [Kubernetes](https://kubernetes.io)
-
-The `kube-vip` project is designed to provide both a highly available networking endpoint and load-balancing functionality for underlying networking services. The project was originally designed for the purpose of providing a resilient control-plane for Kubernetes, it has since expanded to provide the same functionality for applications within a Kubernetes cluster.
-
-Additionally `kube-vip` is designed to be lightweight and **multi-architecture**, all of the components are built for Linux but are also built for both `x86` and `armv7`,`armhvf`. This means that `kube-vip` will run fine in **bare-metal**, **virtual** and **edge** (raspberry pi or small arm SoC devices). 
-
-## Technologies
-
-There are a number of technologies or functional design choices that provide high-availability or networking functions as part of a VIP/Load-balancing solution.
-
-### Cluster
-
-The `kube-vip` service builds a multi-node or multi-pod cluster to provide High-Availability. In ARP mode a leader is elected, this node will inherit the Virtual IP and become the leader of the load-balancing within the cluster, whereas with BGP all nodes will advertise the VIP address.
-
-When using ARP or layer2 it will use [leader election](https://godoc.org/k8s.io/client-go/tools/leaderelection)
-
-It is also possible to use [raft](https://en.wikipedia.org/wiki/Raft_(computer_science) clustering technology, but this approach has largely been superseded by leader election especially when running in cluster.
-
-### Virtual IP
-
-The leader within the cluster will assume the **vip** and will have it bound to the selected interface that is declared within the configuration. When the leader changes it will evacuate the **vip** first or in failure scenarios the **vip** will be directly assumed by the next elected leader.
-
-When the **vip** moves from one host to another any host that has been using the **vip** will retain the previous `vip <-> MAC address` mapping until the ARP (Address resolution protocol) expires the old entry (typically 30 seconds) and retrieves a new `vip <-> MAC` mapping. This can be improved using Gratuitous ARP broadcasts (when enabled), this is detailed below.
-
-### ARP
-
-(Optional) The `kube-vip` can be configured to broadcast a [gratuitous arp](https://wiki.wireshark.org/Gratuitous_ARP) that will typically immediately notify all local hosts that the `vip <-> MAC` has changed.
-
-**Below** we can see that the failover is typically done within a few seconds as the ARP broadcast is recieved.
-
-```
-64 bytes from 192.168.0.75: icmp_seq=146 ttl=64 time=0.258 ms
-64 bytes from 192.168.0.75: icmp_seq=147 ttl=64 time=0.240 ms
-92 bytes from 192.168.0.70: Redirect Host(New addr: 192.168.0.75)
-Vr HL TOS  Len   ID Flg  off TTL Pro  cks      Src      Dst
- 4  5  00 0054 bc98   0 0000  3f  01 3d16 192.168.0.95  192.168.0.75 
-
-Request timeout for icmp_seq 148
-92 bytes from 192.168.0.70: Redirect Host(New addr: 192.168.0.75)
-Vr HL TOS  Len   ID Flg  off TTL Pro  cks      Src      Dst
- 4  5  00 0054 75ff   0 0000  3f  01 83af 192.168.0.95  192.168.0.75 
-
-Request timeout for icmp_seq 149
-92 bytes from 192.168.0.70: Redirect Host(New addr: 192.168.0.75)
-Vr HL TOS  Len   ID Flg  off TTL Pro  cks      Src      Dst
- 4  5  00 0054 2890   0 0000  3f  01 d11e 192.168.0.95  192.168.0.75 
-
-Request timeout for icmp_seq 150
-64 bytes from 192.168.0.75: icmp_seq=151 ttl=64 time=0.245 ms
-```
-
-### Load Balancing
-
-Within a Kubernetes cluster, the load-balancing is managed by the `plndr-cloud-provider` which watches all service that are created, and for those of `type=LoadBalancer` will create the configuration for `kube-vip` to consume.
-
-#### Load Balancing (Inside a cluster)
-
-When using `type=LoadBalancer` within a Kubernetes cluster `kube-vip` will assign the VIP to the leader (when using ARP) or to all running Pods (when using BGP). When traffic is directed to a node with the VIP then the rules configured by `kube-proxy` will redirect the traffic to one of the pods running in the service.
-
-#### Load Balancing (Outside a cluster)
-
-Within the configuration of `kube-vip` multiple load-balancers can be created, below is the example load-balancer for a Kubernetes Control-plane:
-
-```
-loadBalancers:
-- name: Kubernetes Control Plane
-  type: tcp
-  port: 6443
-  bindToVip: true
-  backends:
-  - port: 6444
-    address: 192.168.0.70
-  - port: 6444
-    address: 192.168.0.71
-  - port: 6444
-    address: 192.168.0.72
-```
-
-The above load balancer will create an instance that listens on port `6443` and will forward traffic to the array of backend addresses. If the load-balancer type is `tcp` then the backends will be IP addresses, however if the backend is set to `http` then the backends should be URLs:
-
-```
-  type: http
-  port: 6443
-  bindToVip: true
-  backends:
-  - port: 6444
-    address: https://192.168.0.70
-```
-
-Additionally the load-balancing within `kibe-vip` has two modes of operation:
-
-`bindToVip: false` - will result in every node in the cluster binding all load-balancer port(s) to all interfaces on the host itself
-
-`bindToVip: true` - The load-balancer will only **bind** to the VIP address.
-
-
-## Components within a Kubernetes Cluster
-
-The `kube-vip` kubernetes load-balancer requires a number of components in order to function:
-
-- The Plunder Cloud Provider -> [https://github.com/plunder-app/plndr-cloud-provider](https://github.com/plunder-app/plndr-cloud-provider)
-- The Kube-Vip Deployment -> [https://github.com/plunder-app/kube-vip](https://github.com/plunder-app/kube-vip)
-
-### Architecture overview
-
-![kubernetes-vip-architecture.png](kubernetes-vip-architecture.png)
-
-### Plunder Cloud Provider
-
-The cloud provider works like all Kubernetes cloud providers and is built using the Kubernetes cloud-provider SDK. It's role is to provide the same cloud "like" services one would expect from services such as AWS / Azure / GCP etc.. in that when a user requests functionality then the cloud provider can speak natively to the underlying vendor and provision the required service 
-
-e.g. _In AWS when requesting a Kubernetes LoadBalancer, the cloud provider will provision an **ELB**_ 
-
-The `Plunder cloud Provider` is *currently* only designed to intercept the creation of LoadBalancers and translate that into a `kube-vip` load balancer. 
-
-It is configured by a `configMap` within the `kube-system` namespace that contains the ranges of addresses that the other `kube-vip` load-balancers can use, it will also manage the allocation of addresses and then build the configMap configurations in these namespaces for consumption by `kube-vip`. The IP addresses for each namespace should be in the structure `cidr-<namespace>` followed by the cidr range for the address pool.
-
-**Example `ConfigMap`**
-
-```
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: plndr
-  namespace: kube-system
-data:
-  cidr-default: 192.168.0.200/29
-  cidr-plunder: 192.168.0.210/29
-  cidr-testing: 192.168.0.220/29
-``` 

docs/control-plane/0.1.4/index.md

@@ -1,147 +0,0 @@
-# Load Balancing a Kubernetes Cluster (Control-Plane) (pre 0.1.5)
-
-This document covers all of the details for using `kube-vip` to build a HA Kubernetes cluster
-
-`tl;dr version`
-- Generate/modify first node `kube-vip` config/manifest
-- `init` first node
-- `join` remaining nodes
-- Add remaining config/manifests
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.75 |
-| controlPlane01 | 10.0.0.70 |
-| controlPlane02 | 10.0.0.71 |
-| controlPlane03 | 10.0.0.72 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.17.0.
-
-### Generate the `kube-vip` configuration
-
-Make sure that the config directory exists: `sudo mkdir -p /etc/kube-vip/`, this directory can be any directory however the `hostPath` in the manifest will need modifying to point to the correct path.
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1 /kube-vip sample config | sudo tee /etc/kube-vip/config.yaml
-```
-
-### Modify the configuration
-
-**Cluster Configuration**
-Modify the `remotePeers` to point to the correct addresses of the other two nodes, ensure that their `id` is unique otherwise this will confuse the raft algorithm. The `localPeer` should be the configuration of the current node (`controlPlane01`), which is where this instance of the cluster will run. 
-
-As this node will be the first node, it will need to elect itself leader as until this occurs the VIP won’t be activated!
-
-`startAsLeader: true`
-
-**VIP Config**
-We will need to set our VIP address to `192.168.0.75` and to ensure all hosts are updated when the VIP moves we will enable ARP broadcasts `gratuitousARP: true`
-
-**Load Balancer**
-We will configure the load balancer to sit on the standard API-Server port `6443` and we will configure the backends to point to the API-servers that will be configured to run on port `6444`. Also for the Kubernetes Control Plane we will configure the load balancer to be of `type: tcp`.
-
-We can also use `6443` for both the VIP and the API-Servers, in order to do this we need to specify that the api-server is bound to it's local IP. To do this we use the `--apiserver-advertise-address` flag as part of the `init`, this means that we can then bind the same port to the VIP and we wont have a port conflict.
-
-**config.yaml**
-
-`user@controlPlane01:/etc/kube-vip$ cat config.yaml`
-
-...
-
-``` 
-remotePeers:
-- id: server2
-  address: 192.168.0.71
-  port: 10000
-- id: server3
-  address: 192.168.0.72
-  port: 10000
-localPeer:
-  id: server1
-  address: 192.168.0.70
-  port: 10000
-vip: 192.168.0.75
-gratuitousARP: true
-singleNode: false
-startAsLeader: true
-interface: ens192
-loadBalancers:
-- name: Kubernetes Control Plane
-  type: tcp
-  port: 6443
-  bindToVip: true
-  backends:
-  - port: 6444
-    address: 192.168.0.70
-  - port: 6444
-    address: 192.168.0.71
-  - port: 6444
-    address: 192.168.0.72
-```
-
-### First Node
-
-To generate the basic Kubernetes static pod `yaml` configuration:
-
-Make sure that the manifest directory exists: `sudo mkdir -p /etc/kubernetes/manifests/`
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1 /kube-vip sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Ensure that `image: plndr/kube-vip:<x>` is modified to point to a specific version (`0.1` at the time of writing), refer to [docker hub](https://hub.docker.com/r/plndr/kube-vip/tags) for details. Also ensure that the `hostPath` points to the correct `kube-vip` configuration, if it isn’t the above path. 
-
-The **vip** is set to `192.168.0.75` and this first node will elect itself as leader, and as part of the `kubeadm init` it will use the VIP in order to speak back to the initialising api-server.
-
-`sudo kubeadm init --control-plane-endpoint “192.168.0.75:6443” --apiserver-bind-port 6444 --upload-certs --kubernetes-version “v1.17.0”`
-
-Once this node is up and running we will be able to see the control-plane pods, including the `kube-vip` pod:
-
-```
-$ kubectl get pods -A
-NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
-<...>
-kube-system   kube-vip-controlplane01                  1/1     Running   0          10m
-```
-
-### Remaining Nodes
-
-We first will need to create the `kube-vip` configuration that resides in `/etc/kube-vip/config.yaml` or we can regenerate it from scratch using the above example. Ensure that the configuration is almost identical with the `localPeer` and `remotePeers` sections are updated for each node. Finally, ensure that the remaining nodes will behave as standard cluster nodes by setting `startAsLeader: false`.
-
-At this point **DON’T** generate the manifests, this is due to some bizarre `kubeadm/kubelet` behaviour.
-
-```
-  kubeadm join 192.168.0.75:6443 --token <tkn> \
-    --discovery-token-ca-cert-hash sha256:<hash> \
-    --control-plane --certificate-key <key> 
-
-```
-
-**After** this node has been added to the cluster, we can add the manifest to also add this node as a `kube-vip` member. (Adding the manifest afterwards doesn’t interfere with `kubeadm`). 
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1 /kube-vip sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Once this node is added we will be able to see that the `kube-vip` pod is up and running as expected:
-
-```
-user@controlPlane01:~$ kubectl get pods -A | grep vip
-kube-system   kube-vip-controlplane01                  1/1     Running             1          16m
-kube-system   kube-vip-controlplane02                  1/1     Running             0          18m
-kube-system   kube-vip-controlplane03                  1/1     Running             0          20m
-
-```
-
-If we look at the logs, we can see that the VIP is running on the second node and we’re waiting for our third node to join the cluster:
-
-```
-$ kubectl logs kube-vip-controlplane02  -n kube-system
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-
-```

docs/control-plane/0.1.5/index.md

@@ -1,195 +0,0 @@
-# Load Balancing a Kubernetes Cluster (Control-Plane)
-
-This document covers the newer (post `0.1.5`) method for using `kube-vip` to provide HA for a Kubernetes Cluster. The documentation for older releases can be found [here](./0.1.4/)
-
-This document covers all of the details for using `kube-vip` to build a HA Kubernetes cluster
-
-`tl;dr version`
-- Generate/modify first node `kube-vip` config/manifest
-- `init` first node
-- `join` remaining nodes
-- Add remaining config/manifests
-
-Below are examples of the steps required:
-
-```
-# First Node
-sudo docker run --network host --rm plndr/kube-vip:0.1.5 kubeadm init --interface ens192 --vip 192.168.0.81 --startAsLeader=true | sudo tee /etc/kubernetes/manifests/vip.yaml
-
-sudo kubeadm init --kubernetes-version 1.17.0 --control-plane-endpoint 192.168.0.81 --upload-certs
-
-# Additional Node(s)
-
-sudo kubeadm join 192.168.0.81:6443 --token w5atsr.blahblahblah --control-plane --certificate-key abc123
-
-sudo docker run -v /etc/kubernetes/admin.conf:/etc/kubernetes/admin.conf --network host --rm plndr/kube-vip:0.1.5 kubeadm join --interface ens192 --vip 192.168.0.81 --startAsLeader=false | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.75 |
-| controlPlane01 | 10.0.0.70 |
-| controlPlane02 | 10.0.0.71 |
-| controlPlane03 | 10.0.0.72 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.17.0.
-
-### Generate the `kube-vip` configuration
-
-Kube-Vip no longer requires storing it's configuration in a seperate directory and will now store its configuration in the actual manifest that defines the static pods. 
-
-```
-sudo docker run --network host \
-	--rm plndr/kube-vip:0.1.5 \
-	kubeadm init \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--startAsLeader=true | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-The above command will "initialise" the manifest within the `/etc/kubernetes/manifests` directory, that will be started when we actually initialise our Kubernetes cluster with `kubeadm init`
-
-### Modify the configuration
-
-**Cluster Configuration**
-As this node will be the first node, it will need to elect itself leader as until this occurs the VIP won’t be activated!
-
-`--startAsLeader=true`
-
-**VIP Config**
-We will need to set our VIP address to `192.168.0.75` with `--vip 192.168.0.75` and to ensure all hosts are updated when the VIP moves we will enable ARP broadcasts `--arp` (defaults to `true`)
-
-**Load Balancer**
-We will configure the load balancer to sit on the standard API-Server port `6443` and we will configure the backends to point to the API-servers that will be configured to run on port `6444`. Also for the Kubernetes Control Plane we will configure the load balancer to be of `type: tcp`.
-
-We can also use `6443` for both the VIP and the API-Servers, in order to do this we need to specify that the api-server is bound to it's local IP. To do this we use the `--apiserver-advertise-address` flag as part of the `init`, this means that we can then bind the same port to the VIP and we wont have a port conflict.
-
-**vip.yaml** Static-pod Manifest
-
-`$ sudo cat /etc/kubernetes/manifests/vip.yaml`
-
-
-``` 
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - start
-    env:
-    - name: vip_arp
-      value: "true"
-    - name: vip_interface
-      value: ens192
-    - name: vip_address
-      value: 192.168.0.81
-    - name: vip_startleader
-      value: "true"
-    - name: vip_addpeerstolb
-      value: "true"
-    - name: vip_localpeer
-      value: controlPlane01:192.168.0.70:10000
-    - name: lb_backendport
-      value: "6443"
-    - name: lb_name
-      value: Kubeadm Load Balancer
-    - name: lb_type
-      value: tcp
-    - name: lb_bindtovip
-      value: "true"
-    image: plndr/kube-vip:0.1.5
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - SYS_TIME
-  hostNetwork: true
-status: {}
-```
-
-### First Node
-
-To generate the basic Kubernetes static pod `yaml` configuration:
-
-Make sure that the manifest directory exists: `sudo mkdir -p /etc/kubernetes/manifests/`
-
-```
-sudo docker run --network host \
-	--rm plndr/kube-vip:0.1.5 \
-	kubeadm init \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--startAsLeader=true | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-Ensure that `image: plndr/kube-vip:<x>` is modified to point to a specific version (`0.1.5` at the time of writing), refer to [docker hub](https://hub.docker.com/r/plndr/kube-vip/tags) for details. 
-
-The **vip** is set to `192.168.0.75` and this first node will elect itself as leader, and as part of the `kubeadm init` it will use the VIP in order to speak back to the initialising api-server.
-
-`sudo kubeadm init --control-plane-endpoint “192.168.0.75:6443” --apiserver-bind-port 6444 --upload-certs --kubernetes-version “v1.17.0”`
-
-Once this node is up and running we will be able to see the control-plane pods, including the `kube-vip` pod:
-
-```
-$ kubectl get pods -A
-NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
-<...>
-kube-system   kube-vip-controlplane01                  1/1     Running   0          10m
-```
-
-### Remaining Nodes
-
-
-At this point **DON’T** generate the manifests, this is due to some bizarre `kubeadm/kubelet` behaviour.
-
-```
-  kubeadm join 192.168.0.75:6443 --token <tkn> \
-    --discovery-token-ca-cert-hash sha256:<hash> \
-    --control-plane --certificate-key <key> 
-
-```
-
-**After** this node has been added to the cluster, we can add the manifest to also add this node as a `kube-vip` member. (Adding the manifest afterwards doesn’t interfere with `kubeadm`). 
-
-```
-sudo docker run \
-	-v /etc/kubernetes/admin.conf:/etc/kubernetes/admin.conf \
-	--network host \
-	--rm plndr/kube-vip:0.1.5 \
-	kubeadm join \
-	--interface ens192 \
-	--vip 192.168.0.81 \
-	--startAsLeader=false | sudo tee /etc/kubernetes/manifests/vip.yaml
-
-```
-
-Once this node is added we will be able to see that the `kube-vip` pod is up and running as expected:
-
-```
-user@controlPlane01:~$ kubectl get pods -A | grep vip
-kube-system   kube-vip-controlplane01                  1/1     Running             1          16m
-kube-system   kube-vip-controlplane02                  1/1     Running             0          18m
-kube-system   kube-vip-controlplane03                  1/1     Running             0          20m
-
-```
-
-If we look at the logs, we can see that the VIP is running on the second node and we’re waiting for our third node to join the cluster:
-
-```
-$ kubectl logs kube-vip-controlplane02  -n kube-system
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-
-```

docs/control-plane/index.md

@@ -1,421 +0,0 @@
-# Load Balancing a Kubernetes Cluster (Control-Plane)
-
-**Note**: The most common deployment currently for HA Kubernetes clusters w/`kube-vip` involved `kubeadm`, however recently we've worked to bring a method of bringing `kube-vip` to other types of Kubernetes cluster. Typically this deployment method makes use of a daemonset that is usually brought up during the cluster instantiation.. So for those wanting to deploy [k3s](https://k3s.io), we now have installation steps available [here](https://kube-vip.io/control-plane/#k3s),
-
-This document covers the newer (post `0.1.6`) method for using `kube-vip` to provide HA for a Kubernetes Cluster. The documentation for older releases can be found [here](./0.1.5/)
-
-From version `0.1.6` we've moved `kube-vip` from raft to leaderElection within the Kubernetes cluster. After a lot of testing it became clear that the leaderElection gave quicker reconciliation when removing nodes etc.. during upgrades and failures.
-
-For **more** configuration around LeaderElection click [here](https://kube-vip.io/control-plane/#leaderelection-configuration).
-
-This document covers all of the details for using `kube-vip` to build a HA Kubernetes cluster
-
-`tl;dr version`
-- Generate/modify first node `kube-vip` config/manifest
-- `init` first node
-- `join` remaining nodes
-- Add remaining config/manifests
-
-Below are examples of the steps required:
-
-```
-# First Node
-sudo docker run --network host --rm plndr/kube-vip:0.2.1 manifest pod \
---interface ens192 \
---vip 192.168.0.75 \
---arp \
---leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-
-sudo kubeadm init --kubernetes-version 1.17.0 --control-plane-endpoint 192.168.0.75 --upload-certs
-
-# Additional Node(s)
-
-sudo kubeadm join 192.168.0.75:6443 --token w5atsr.blahblahblah --control-plane --certificate-key abc123
-
-sudo docker run --network host --rm plndr/kube-vip:0.2.1 manifest pod \
---interface ens192 \
---vip 192.168.0.75 \
---arp \
---leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.75 |
-| controlPlane01 | 10.0.0.70 |
-| controlPlane02 | 10.0.0.71 |
-| controlPlane03 | 10.0.0.72 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.17.0.
-
-### Generate the `kube-vip` configuration
-
-`kube-vip` no longer requires storing its configuration in a separate directory and will now store its configuration in the actual manifest that defines the static pods. 
-
-```
-sudo docker run --network host \
-	--rm plndr/kube-vip:0.2.1 \
-	manifest pod \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--arp \
-	--leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-The above command will "initialise" the manifest within the `/etc/kubernetes/manifests` directory, that will be started when we actually initialise our Kubernetes cluster with `kubeadm init`
-
-### Modify the configuration
-
-**Cluster Configuration**
-To enable Kubernetes leader Election passing the `--leaderElection` flag will enable `kube-vip` to use the Kubernetes leaderElection functionality to work out which member is the leader.
-
-**VIP Config**
-We will need to set our VIP address to `192.168.0.75` with `--vip 192.168.0.75` and to ensure all hosts are updated when the VIP moves we will enable ARP broadcasts `--arp` (defaults to `true`)
-
-**vip.yaml** Static-pod Manifest
-
-`$ sudo cat /etc/kubernetes/manifests/vip.yaml`
-
-
-``` 
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - start
-    env:
-    - name: vip_arp
-      value: "true"
-    - name: vip_interface
-      value: ens160
-    - name: vip_leaderelection
-      value: "true"
-    - name: vip_leaseduration
-      value: "5"
-    - name: vip_renewdeadline
-      value: "3"
-    - name: vip_retryperiod
-      value: "1"
-    - name: vip_address
-      value: 192.168.0.75
-    image: plndr/kube-vip:0.2.1
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - SYS_TIME
-  hostNetwork: true
-status: {}
-```
-
-### First Node
-
-To generate the basic Kubernetes static pod `yaml` configuration:
-
-Make sure that the manifest directory exists: `sudo mkdir -p /etc/kubernetes/manifests/`
-
-```
-sudo docker run --network host \
-	--rm plndr/kube-vip:0.2.1 \
-	manifest pod \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--arp \
-	--leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-Ensure that `image: plndr/kube-vip:<x>` is modified to point to a specific version (`0.1.8` at the time of writing), refer to [docker hub](https://hub.docker.com/r/plndr/kube-vip/tags) for details. 
-
-The **vip** is set to `192.168.0.75` and this first node will elect itself as leader, and as part of the `kubeadm init` it will use the VIP in order to speak back to the initialising api-server.
-
-`sudo kubeadm init --control-plane-endpoint “192.168.0.75:6443” --upload-certs --kubernetes-version “v1.17.0”`
-
-Once this node is up and running we will be able to see the control-plane pods, including the `kube-vip` pod:
-
-```
-$ kubectl get pods -A
-NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
-<...>
-kube-system   kube-vip-controlplane01                  1/1     Running   0          10m
-```
-
-### Remaining Nodes
-
-
-At this point **DON’T** generate the manifests, this is due to some bizarre `kubeadm/kubelet` behaviour.
-
-```
-  kubeadm join 192.168.0.75:6443 --token <tkn> \
-    --discovery-token-ca-cert-hash sha256:<hash> \
-    --control-plane --certificate-key <key> 
-
-```
-
-**After** this node has been added to the cluster, we can add the manifest to also add this node as a `kube-vip` member. (Adding the manifest afterwards doesn’t interfere with `kubeadm`). 
-
-```
-sudo docker run --network host \
-	--rm plndr/kube-vip:0.2.1 \
-	manifest pod \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--arp \
-	--leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-
-```
-
-Once this node is added we will be able to see that the `kube-vip` pod is up and running as expected:
-
-```
-user@controlPlane01:~$ kubectl get pods -A | grep vip
-kube-system   kube-vip-controlplane01                  1/1     Running             1          16m
-kube-system   kube-vip-controlplane02                  1/1     Running             0          18m
-kube-system   kube-vip-controlplane03                  1/1     Running             0          20m
-
-```
-
-## DNS Support
-
-### Static DNS Support (added in 0.2.0)
-
-A new flag `--address` is introduced to support using a DNS record as the control plane endpoint. `kube-vip` will do a dns lookup to retrieve the IP for the DNS record, and use that IP as the VIP. An `dnsUpdater` periodically checks and updates the system if IP changes for the DNS record.
-
-### Dynamic DNS Support (added in 0.2.1)
-
-`kube-vip` was also updated to support DHCP + [Dynamic DNS](https://en.wikipedia.org/wiki/Dynamic_DNS), for the use case where it's not able to reserve a static IP for the control plane endpoint.
-
-A new flag `--ddns` is introduced. Once enabled, `kube-vip` expects the input `--address` will be a FQDN without binding to an IP. Then `kube-vip` will start a dhcp client to allocate an IP for the hostname of FQDN, and maintain the lease for it.
-
-Once DHCP returns an IP for the FQDN, the same `dnsUpdater` runs to periodically checks and updates if IP got changed.
-
-## BGP Support (added in 0.1.8)
-
-In version `0.1.8`+ `kube-vip` was updated to support [BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol) as a VIP failover mechanism. When a node is elected as a leader then it will update it's peers so that they are aware to route traffic to that node in order to access the VIP. 
-
-The following new flags are used:
-
-- `--bgp` This will enable BGP support within kube-vip
-- `--localAS` The local AS number
-- `--bgpRouterID` The local router address
-- `--peerAS` The AS number for a BGP peer
-- `--peerAddress` The address of a BGP peer
-
-### BGP Packet support
-
-If the `--bgp` flag is passed alone with the Packet flags `packet, packetKey and packetProject`, then the Packet API will be used in order to determine the BGP configuration for the nodes being used in the cluster. This automates a lot of the process and makes using BGP within Packet much simpler.
-
-## Packet Support (added in 0.1.7)
-
-Recently in version `0.1.7` of `kube-vip` we added the functionality to use a Packet Elastic IP as the virtual IP fronting the Kubernetes Control plane cluster. In order to first get out virtual IP we will need to use our Packet account and create a EIP (either public (eek) or private). We will only need a single address so a `/32` will suffice, once this is created as part of a Packet project we can now apply this address to the servers that live in the same project. 
-
-In this example we've logged into the UI can created a new EIP of `147.75.1.2`, and we've deployed three small server instances with Ubuntu.
-
-The following new flags are used:
-
-- `--packet` which enables the use of the Packet API
-- `--packetKey` which is our API key
-- `--packetProject`which is the name of our Packet project where our servers and EIP are located.
-
-*Also* the `--arp` flag should NOT be used as it wont work within the Packet network.
-
-### Variables
-
-```
-export EIP=1.1.1.1
-export PACKET_AUTH_TOKEN=XYZ
-```
-
-### First node
-
-```
-# Generate the manifest
-
-sudo docker run --network host --rm plndr/kube-vip:0.2.1 manifest pod \
---arp=false \
---interface lo \
---vip $EIP \
---leaderElection \
---packet \
---packetKey $PACKET_AUTH_TOKEN \
---packetProject vipTest | sudo tee /etc/kubernetes/manifests/vip.yaml\
-
-# Init Kubernetes
-
-sudo kubeadm init --kubernetes-version 1.18.5 --control-plane-endpoint $EIP --upload-certs
-```
-
-### Other nodes
-
-```
-# Join
-kubeadm join $EIP:6443 --token BLAH --control-plane --certificate-key BLAH --discovery-token-ca-cert-hash sha:blah
-
-# Generate Manifest
-
-sudo docker run --network host --rm plndr/kube-vip:0.2.1 manifest pod \
---arp=false \
---interface lo \
---vip $EIP \
---leaderElection \
---packet \
---packetKey $PACKET_AUTH_TOKEN \
---packetProject vipTest | sudo tee /etc/kubernetes/manifests/vip.yaml\
-```
-
-The Elastic IP failover takes some time (30+ seconds) to move from a failed host to a new leader, so in this release it is mainly for testing. 
-
-
-## Upgrades
-
-From above we have a 3 node cluster and the controlPlane01 is leader:
-
-```
-$ kubectl logs -n kube-system kube-vip-controlplane01 -f
-time="2020-07-04T15:12:52Z" level=info msg="Beginning cluster membership, namespace [kube-system], lock name [plunder-lock], id [controlPlane01]"
-I0704 15:12:52.290420       1 leaderelection.go:242] attempting to acquire leader lease  kube-system/plunder-lock...
-I0704 15:12:56.373113       1 leaderelection.go:252] successfully acquired lease kube-system/plunder-lock
-time="2020-07-04T15:12:56Z" level=info msg="This node is assuming leadership of the cluster"
-time="2020-07-04T15:12:56Z" level=error msg="This node is leader and is adopting the virtual IP"
-time="2020-07-04T15:12:56Z" level=info msg="Starting TCP Load Balancer for service [192.168.0.81:0]"
-time="2020-07-04T15:12:56Z" level=info msg="Load Balancer [Kubeadm Load Balancer] started"
-time="2020-07-04T15:12:56Z" level=info msg="Broadcasting ARP update for 192.168.0.81 (00:50:56:a5:69:a1) via ens192"
-time="2020-07-04T15:12:56Z" level=info msg="Starting TCP Load Balancer for service [192.168.0.81:0]"
-time="2020-07-04T15:12:56Z" level=info msg="Load Balancer [Kubeadm Load Balancer] started"
-time="2020-07-04T15:12:56Z" level=info msg="Broadcasting ARP update for 192.168.0.81 (00:50:56:a5:69:a1) via ens192"
-time="2020-07-04T15:12:56Z" level=info msg="new leader elected: controlPlane01"
-```
-
-We will kill this node and watch `kube-vip` logs from another node:
-
-#### Pinging VIP
-
-```
-64 bytes from 192.168.0.81: icmp_seq=667 ttl=64 time=0.387 ms
-Request timeout for icmp_seq 668
-Request timeout for icmp_seq 669
-Request timeout for icmp_seq 670
-Request timeout for icmp_seq 671
-Request timeout for icmp_seq 672
-64 bytes from 192.168.0.81: icmp_seq=673 ttl=64 time=0.453 ms
-```
-
-#### Logs 
-```
-$ kubectl logs -n kube-system kube-vip-controlplane03 -f
-time="2020-07-04T15:17:53Z" level=info msg="Beginning cluster membership, namespace [kube-system], lock name [plunder-lock], id [controlPlane03]"
-I0704 15:17:53.484698       1 leaderelection.go:242] attempting to acquire leader lease  kube-system/plunder-lock...
-time="2020-07-04T15:17:53Z" level=info msg="new leader elected: controlPlane01"
-E0704 15:20:18.864141       1 leaderelection.go:331] error retrieving resource lock kube-system/plunder-lock: etcdserver: request timed out
-time="2020-07-04T15:20:20Z" level=info msg="new leader elected: controlPlane02"
-```
-
-#### Adding `controlPlane04`
-
-A kubeadm join will fail as the `controlPlane01` still exists as an endpoint, so we have two options (manual steps and configmap edit to remove all mention of this node, or we can bring this node up and `kubeadm reset` the node (which we will do)). 
-
-```
-$ kubectl get nodes
-NAME             STATUS     ROLES    AGE   VERSION
-controlplane01   NotReady   master   14m   v1.17.0
-controlplane02   Ready      master   13m   v1.17.2
-controlplane03   Ready      master   13m   v1.17.0
-controlplane04   NotReady   master   9s    v1.17.0
-```
- After this we can add this node into `kube-vip` with the same manifest created by `docker run`.
-
-## LeaderElection configuration
-
-The Kubernetes LeaderElection that is used to manage the election of a new leader now supports having it's settings managed through flags. 
-
- - `--leaseDuration` Length of time a Kubernetes leader lease can be held for
- - `--leaseRenewDuration` Length of time a Kubernetes leader can attempt to renew its lease
- - `--leaseRetry` Number of times the host will retry to hold a lease
-
-For larger clusters the `--leaseDuration` and `--leaseRenewDuration` may need extending due to slower `etcd` performance. (Tested with 2000 nodes)
-
-## k3s
-
-This section details the steps required to deploye `k3s` in a Highly available manner, using kube-vip deployed within k3s as a daemonset on the control plane nodes. As of `k3s` v1 the persistent datastore is back to etcd, however this guide will also include the steps for using `mysql`.
-
-### Example MySQL deployment (optional)
-
-To quickly validate this we can use docker on a host to quickly spin up a mysql database to store the persistent Kubernetes data.
-
-#### Create local directory for BD storage
-`mkdir mysql`
-
-#### Start Docker MySQL container
-`sudo docker run --cap-add SYS_NICE -p 3306:3306 --name k3s-mysql -v /home/dan/mysql:/var/lib/mysql -e MYSQL_ROOT_PASSWORD=k3s-password -d mysql:8`
-
-### Create `kube-vip` manifest
-
-The `kube-vip` manifest contains all the configuration for starting up `kube-vip` within the `k3s` cluster, it runs as a daemonset with affinity/taints for the control-plane nodes. As `k3s` starts it will parse all manifests in the manifests folder and start the highly available VIP across all control plane nodes in the cluster.
-
-#### Create the `k3` manifests directory
-
-Create the manifests directory, this directory is used by `k3s` for all of it's other deployments once it's up and running.
-`sudo mkdir -p /var/lib/rancher/k3s/server/manifests/`
-
-#### Generate the manifest
-
-Modify the `vipAddress` and `vipInterface` to match the floating IP address you'd like to use and the interface it should bind to.
-
-`curl -sL kube-vip.io/k3s | vipAddress=192.168.0.10 vipInterface=ens192 sh | sudo tee /var/lib/rancher/k3s/server/manifests/vip.yaml`
-
-### Start `k3s`
-
-Set the VIP **first**
-
-`export VIP=192.168.0.10`
-
-
-From online `-->`
-
-```
-curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--write-kubeconfig-mode 644 \
--t agent-secret --tls-san $VIP" sh -
-```
-
-From local `-->`
-
-```
-sudo ./k3s server --tls-san $VIP
-```
-
-#### With MySQL
-
-From online `-->`
-
-```
-curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--write-kubeconfig-mode 644 \
---datastore-endpoint mysql://root:k3s-password@tcp(192.168.0.43:3306)/kubernetes \
--t agent-secret --tls-san $VIP" sh -
-```
-
-From local `-->`
-
-```
-sudo ./k3s server --tls-san $VIP \
---datastore-endpoint="mysql://root:k3s-password@tcp(192.168.0.43:3306)/kubernetes"
-```
-
-### Get a `kubeconfig` that uses the vip
-
-````
-mkdir -p $HOME/.kube
-sudo cat /etc/rancher/k3s/k3s.yaml | sed 's/127.0.0.1/'$VIP'/g' > $HOME/.kube/config
-sudo chown $(id -u):$(id -g) $HOME/.kube/config
-```

docs/hybrid/daemonset/index.md

@@ -1,226 +0,0 @@
-# Kube-Vip as a daemonset
-
-In Hybrid mode `kube-vip` will manage a virtual IP address that is passed through it's configuration for a Highly Available Kubernetes cluster, it will also "watch" services of `type:LoadBalancer` and once their `spec.LoadBalancerIP` is updated (typically by a cloud controller) it will advertise this address using BGP/ARP.
-
-
-**Note about Daemonsets**
-
-The "hybrid" mode is now the default mode in `kube-vip` from `0.2.3` onwards, and allows both modes to be enabled at the same time. 
-
-If the Kubernetes installer allows for adding a Virtual IP as an additional [SAN](https://en.wikipedia.org/wiki/Subject_Alternative_Name) to the API server certificate then we can apply `kube-vip` to the cluster once the first node has been brought up. 
-
-Unlike generating the static manifest there are a few more things that may need configuring, this page will cover most scenarios.
-
-## Create the RBAC settings
-
-As a daemonSet runs within the Kubernetes cluster it needs the correct access to be able to watch Kubernetes services and other objects. In order to do this we create a User, Role, and a binding.. we can apply this with the command:
-
-```
-kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
-```
-
-## Generating a Manifest
-
-This section only covers generating a simple *BGP* configuration, as the main focus is will be on additional changes to the manifest. For more examples we can look at [here](/hybrid/static/).
-
-**Note:** Pay attention if using the "static" examples, as the `manifest` subcommand should use `daemonset` and NOT `pod`.
-
-### Set configuration details
-
-`export VIP=192.168.0.40`
-
-`export INTERFACE=<interface>`
-
-### Configure to use a container runtime
-
-The easiest method to generate a manifest is using the container itself, below will create an alias for different container runtimes.
-
-#### containerd
-`alias kube-vip="ctr run --rm --net-host docker.io/plndr/kube-vip:0.3.1 vip"`
-
-#### Docker
-`alias kube-vip="docker run --network host --rm plndr/kube-vip:0.3.1"`
-
-### BGP Example
-
-This configuration will create a manifest that will start `kube-vip` providing **controlplane** and **services** management. **Unlike** ARP, all nodes in the BGP configuration will advertise virtual IP addresses. 
-
-**Note** we bind the address to `lo` as we don't want multiple devices that have the same address on public interfaces. We can specify all the peers in a comma seperate list in the format of `address:AS:password:multihop`.
-
-**Note 2** we pass the `--inCluster` flag as this is running as a daemonSet within the Kubernetes cluster and therefore will have access to the token inside the running pod.
-
-**Note 2** we pass the `--taint` flag as we're deploying `kube-vip` as both a daemonset and as advertising controlplane, we want to taint this daemonset to only run on the worker nodes.
-
-`export INTERFACE=lo`
-
-```
-kube-vip manifest daemonset \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --inCluster \
-    --taint \
-    --bgp \
-    --bgppeers 192.168.0.10:65000::false,192.168.0.11:65000::false
-```
-
-### Generated Manifest
-
-```
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: lo
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: "true"
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_peers
-          value: "192.168.0.10:65000::false,192.168.0.11:65000::false"
-        - name: vip_address
-          value: 192.168.0.40
-        image: plndr/kube-vip:0.2.3
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      nodeSelector:
-        node-role.kubernetes.io/master: "true"
-      tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
-  updateStrategy: {}
-```
-
-### Manifest Overview
-
-- `nodeSelector` - Ensures that this particular daemonset only runs on control plane nodes
-- `serviceAccountName: kube-vip` - this specifies the user in the `rbac` that will give us the permissions to get/update services.
-- `hostNetwork: true` - This pod will need to modify interfaces (for VIPs)
-- `env {...}` - We pass the configuration into the kube-vip pod through environment variables.
-
-## Equinix Metal Overview (using the [Equinix Metal CCM](https://github.com/packethost/packet-ccm))
-
-The below example is for running `type:LoadBalancer` services on worker nodes only and will create a daemonset that will run `kube-vip`. 
-
-**NOTE** This use-case requires the [Equinix Metal CCM](https://github.com/packethost/packet-ccm) to be installed and that the cluster/kubelet is configured to use an "external" cloud provider.
-
-This is important as the CCM will apply the BGP configuration to the [node annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) making it easy for `kube-vip` to find the networking configuration it needs to expose load balancer addresses. The `--annotations metal.equinix.com` will cause kube-vip to "watch" the annotations of the worker node that it is running on, once all of the configuarion has been applied by the CCM then the `kube-vip` pod is ready to advertise BGP addresses for the service.
-
-```
-kube-vip manifest daemonset \
-  --interface $INTERFACE \
-  --services \
-  --bgp \
-  --annotations metal.equinix.com \
-  --inCluster | k apply -f -
-```
-
-### Troubleshooting
-
-If `kube-vip` has been sat waiting for a long time then you may need to investigate that the annotations have been applied correctly by doing running the `describe` on the node:
-
-```
-kubectl describe node k8s.bgp02
-...
-Annotations:        kubeadm.alpha.kubernetes.io/cri-socket: /var/run/dockershim.sock
-                    node.alpha.kubernetes.io/ttl: 0
-                    metal.equinix.com/node-asn: 65000
-                    metal.equinix.com/peer-asn: 65530
-                    metal.equinix.com/peer-ip: x.x.x.x
-                    metal.equinix.com/src-ip: x.x.x.x
-```
-
-If there are errors regarding `169.254.255.1` or `169.254.255.2` in the `kube-vip` logs then the routes to the ToR switches that provide BGP peering may by missing from the nodes. They can be replaced with the below command:
-
-```
-GATEWAY_IP=$(curl https://metadata.platformequinix.com/metadata | jq -r ".network.addresses[] | select(.public == false) | .gateway")
-ip route add 169.254.255.1 via $GATEWAY_IP
-ip route add 169.254.255.2 via $GATEWAY_IP
-```
-
-Additionally examining the logs of the Packet CCM may reveal why the node is not yet ready.
-
-## K3s overview (on Equinix Metal)
-
-### Step 1: TIDY (best if something was running before)
-`rm -rf /var/lib/rancher /etc/rancher ~/.kube/*; ip addr flush dev lo; ip addr add 127.0.0.1/8 dev lo; mkdir -p /var/lib/rancher/k3s/server/manifests/`
-
-### Step 2: Get rbac
-`curl https://kube-vip.io/manifests/rbac.yaml > /var/lib/rancher/k3s/server/manifests/rbac.yaml`
-
-### Step 3: Generate kube-vip (get EIP from CLI or UI)
-
-```
-export EIP=x.x.x.x
-export INTERFACE=lo
-```
-
-```
-kube-vip manifest daemonset \
-  --interface $INTERFACE \
-  --vip $EIP \
-  --controlplane \
-  --services \
-  --inCluster \
-  --taint \
-  --bgp \
-  --metal \
-  --provider-config /etc/cloud-sa/cloud-sa.json | tee /var/lib/rancher/k3s/server/manifests/vip.yaml
-```
-
-NOTE: the `—provider-config` actually comes from the secret we apply in step 5 (this will leave kube-vip waiting to start)
-
-### Step 4: Up Cluster
-`K3S_TOKEN=SECRET k3s server --cluster-init --tls-san $EIP --no-deploy servicelb --disable-cloud-controller`
-
-### Step 5: Add CCM
-
-`alias k="k3s kubectl"`
-`k apply -f ./secret.yaml`
-
-(^ https://github.com/packethost/packet-ccm/blob/master/deploy/template/secret.yaml)
-
-`k apply -f https://gist.githubusercontent.com/thebsdbox/c86dd970549638105af8d96439175a59/raw/4abf90fb7929ded3f7a201818efbb6164b7081f0/ccm.yaml`
-
-### Step 6: Demo !
-`k apply -f https://k8s.io/examples/application/deployment.yaml`
-`k expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx`
-
-### Step 7 watch and test:
-`k get svc --watch`

docs/hybrid/index.md

@@ -1,141 +0,0 @@
-# Using `kube-vip` in Hybrid Mode
-
-We can deploy kube-vip in two different methods, which completely depends on your use-case and method for installing Kubernetes:
-
-- Static Pods (hybrid)
-- Daemonset (hybrid, requires taint)
-
-## **Prerequisites**
-
-In order for `kube-vip` to be able to speak with the Kubernetes API server, we need to be able to resolve the hostname within the pod. In order to ensure this will work as expected the `/etc/hosts` file should have the `hostname` of the server within it. The `/etc/hosts` file is passed into the running container and will ensure that the pod isn't "confused" by any Kubernetes networking.
-
-## Kubernetes Services (`type:LoadBalancer`)
-
-To learn more about how `kube-vip` in hybrid works with the LoadBalancer services within a kubernetes cluster the documentation is [here](./services/). To get `kube-vip` deployed read on !
-
-## Static Pods
-
-Static pods are a Kubernetes pod that is ran by the `kubelet` on a single node, and is **not** managed by the Kubernetes cluster itself. This means that whilst the pod can appear within Kubernetes it can't make use of a variety of kubernetes functionality (such as the kubernetes token or `configMaps`). The static pod approach is primarily required for [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/), this is due to the sequence of actions performed by `kubeadm`. Ideally we want `kube-vip` to be part of the kubernetes cluster, for various bits of functionality we also need `kube-vip` to provide a HA virtual IP as part of the installation. 
-
-The sequence of events for this to work follows:
-1. Generate a `kube-vip` manifest in the static pods manifest folder
-2. Run `kubeadm init`, this generates the manifests for the control plane and wait to connect to the VIP
-3. The `kubelet` will parse and execute all manifest, including the `kube-vip` manifest
-4. `kube-vip` starts and advertises our VIP
-5. The `kubeadm init` finishes succesfully.
-
-## Daemonset
-
-Other Kubernetes distributions can bring up a Kubernetes cluster, without depending on a VIP (BUT they are configured to support one). A prime example of this would be k3s, that can be configured to start and also sign the certificates to allow incoming traffic to a virtual IP. Given we don't need the VIP to exist **before** the cluster, we can bring up the k3s node(s) and then add `kube-vip` as a daemonset for all control plane nodes.
-
-# Deploying `kube-vip`
-
-The simplest method for generating the Kubernetes manifests is with `kube-vip` itself.. The subcommand `manifest pod|daemonset` can be used to generate specific types of Kubernetes manifests for use in a cluster. These subcommands can be configured with additional flags to enable/disable BGP/ARP/LeaderElection and a host of other options.
-
-Both Examples will use the same Architecture:
-
-#### Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.40 |
-| controlPlane01 | 10.0.0.41 |
-| controlPlane02 | 10.0.0.42 |
-| controlPlane03 | 10.0.0.43 |
-| worker01       | 10.0.0.44 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.19.0, we only have one worker as we're going to use our controlPlanes in "hybrid" mode.
-
-## As a static Pod (for kubeadm)
-
-The details for creating a static pod are available [here](./static/)
-
-## As a daemonset
-
-When using `kube-vip` as a daemonset the details are available [here](./daemonset/)
-
-# Kube-Vip flag reference
-
-| Category     | Flag | Usage | Notes |
-|--------------|------|-------|-------|
-|**Mode**          ||||
-|              |`--controlPlane`|Enables `kube-vip` control-plane functionality||
-|              |`--services`|Enables `kube-vip` to watch services of type:LoadBalancer||
-|**Vip Config**   ||||
-|              |`--arp`|Enables ARP brodcasts from Leader||
-|              |`--bgp`|Enables BGP peering from `kube-vip`||
-|              |`--vip`|`<IP Address>`|(deprecated)|
-|              |`--address`|`<IP Address>` or `<DNS name>`||
-|              |`--interface`|`<linux interface>`||
-|              |`--leaderElection`|Enables Kubernetes LeaderElection|Used by ARP, as only the leader can broadcast|
-|**Services**||||
-|              |`--cidr`|Defaults "32"|Used when advertising BGP addresses (typically as `x.x.x.x/32`)|
-|**Kubernetes**||||
-|              |`--inCluster`|Defaults to looking inside the Pod for the token||
-|              |`--taint`|Enables a taint, stopping control plane daemonset being on workers||
-|**LeaderElection**||||
-|              |`--leaseDuration`|default 5|Seconds a lease is held for|
-|              |`--leaseRenewDuration`|default 3|Seconds a leader can attempt to renew the lease|
-|              |`--leaseRetry`|default 1|Number of times the leader will hold the lease for|
-|              |`--namespace`|"kube-vip"|The namespace where the lease will reside|
-|**BGP**||||
-|              |`--bgpRouterID`|`<IP Address>`|Typically the address of the local node|
-|              |`--localAS`|default 65000|The AS we peer from|
-|              |`--bgppeers`|`<address:AS:password:mutlihop>`|Comma seperate list of BGP peers|
-|              |`--peerAddress`|`<IP Address>`|Address of a single BGP Peer|
-|              |`--peerAS`|default 65000|AS of a single BGP Peer|
-|              |`--peerPass`|""| Password to work with a single BGP Peer|
-|              |`--multiHop`|Enables eBGP MultiHop| Enable multiHop with a single BGP Peer|
-|              |`--annotaions`|`<provider string>`|Startup will be paused until the node annotaions contain the BGP configuration|
-|**Equinix Metal**|||(May be deprecated)|
-|              |`--metal`|Enables Equinix Metal API calls||
-|              |`--metalKey`|Equinix Metal API token||
-|              |`--metalProject`|Equinix Metal Project (Name)||
-|              |`--metalProjectID`|Equinix Metal Project (UUID)||
-|              |`--provider-config`|Path to the Equinix Metal provider configuration|Requires the Equinix Metal CCM|
-
-## Changelog
-
-### Static DNS Support (added in 0.2.0)
-
-A new flag `--address` is introduced to support using a DNS record as the control plane endpoint. `kube-vip` will do a dns lookup to retrieve the IP for the DNS record, and use that IP as the VIP. An `dnsUpdater` periodically checks and updates the system if IP changes for the DNS record.
-
-### Dynamic DNS Support (added in 0.2.1)
-
-`kube-vip` was also updated to support DHCP + [Dynamic DNS](https://en.wikipedia.org/wiki/Dynamic_DNS), for the use case where it's not able to reserve a static IP for the control plane endpoint.
-
-A new flag `--ddns` is introduced. Once enabled, `kube-vip` expects the input `--address` will be a FQDN without binding to an IP. Then `kube-vip` will start a dhcp client to allocate an IP for the hostname of FQDN, and maintain the lease for it.
-
-Once DHCP returns an IP for the FQDN, the same `dnsUpdater` runs to periodically checks and updates if IP got changed.
-
-## BGP Support (added in 0.1.8)
-
-In version `0.1.8` `kube-vip` was updated to support [BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol) as a VIP failover mechanism. When a node is elected as a leader then it will update it's peers so that they are aware to route traffic to that node in order to access the VIP. 
-
-The following new flags are used:
-
-- `--bgp` This will enable BGP support within kube-vip
-- `--localAS` The local AS number
-- `--bgpRouterID` The local router address
-- `--peerAS` The AS number for a BGP peer
-- `--peerAddress` The address of a BGP peer
-
-### Equinix Metal BGP support
-
-If the `--bgp` flag is passed along with the Equinix Metal flags `metal, metalKey and metalProject`, then Equinix Metal  API will be used in order to determine the BGP configuration for the nodes being used in the cluster. This automates a lot of the process and makes using BGP within Equinix Metal much simpler.
-
-## Equinix Metal Control Plane Support (added in 0.1.8)
-
-Recently in version `0.1.7` of `kube-vip` we added the functionality to use a Equinix Metal Elastic IP as the virtual IP fronting the Kubernetes Control plane cluster. In order to first get out virtual IP we will need to use our Equinix Metal account and create a EIP (either public or private). We will only need a single address so a `/32` will suffice, once this is created as part of a Equinix Metal project we can now apply this address to the servers that live in the same project. 
-
-In this example we've logged into the UI can created a new EIP of `147.75.1.2`, and we've deployed three small server instances with Ubuntu.
-
-The following new flags are used:
-
-- `--metal` which enables the use of the Equinix Metal API
-- `--metalKey` which is our API key
-- `--metalProject`which is the name of our Equinix Metal project where our servers and EIP are located.
-
-*Also* the `--arp` flag should NOT be used as it wont work within the Equinix Metal network.

docs/hybrid/services/index.md

@@ -1,214 +0,0 @@
-# Kube-vip services 
-
-We've designed `kube-vip` to be as de-coupled or agnostic from other components that may exist within a Kubernetes cluster as possible. This has lead to `kube-vip` having a very simplistic but robust approach to advertising Kubernetes services to the outside world and marking these services as ready to use.
-
-## Flow
-
-This section details the flow of events in order for `kube-vip` to advertise a Kubernetes service:
-
-1. An end user exposes a application through Kubernetes as a LoadBalancer => `kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx`
-2. Within the Kubernetes cluster a service object is created with the `svc.Spec.Type = ServiceTypeLoadBalancer`
-3. A controller (typically a Cloud Controller) has a loop that "watches" for services of the type `LoadBalancer`.
-4. The controller now has the responsibility of providing an IP address for this service along with doing anything that is network specific for the environment where the cluster is running.
-5. Once the controller has an IP address it will update the service `svc.Spec.LoadBalancerIP` with it's new IP address.
-6. The `kube-vip` pods also implement a "watcher" for services that have a `svc.Spec.LoadBalancerIP` address attached.
-7. When a new service appears `kube-vip` will start advertising this address to the wider network (through BGP/ARP) which will allow traffic to come into the cluster and hit the service network.
-8. Finally `kube-vip` will update the service status so that the API reflects that this LoadBalancer is ready. This is done by updating the `svc.Status.LoadBalancer.Ingress` with the VIP address.
-
-## CCM
-
-We can see from the [flow](#Flow) above that `kube-vip` isn't coupled to anything other than the Kubernetes API, and will only act upon an existing Kubernetes primative (in this case the object of type `Service`). This makes it easy for exist CCMs to simply apply their logic to services of type LoadBalancer and leave `kube-vip` to take the next steps to advertise these load-balancers to the outside world. 
-
-The below instructions *should just work* on Kubernetes regardless of architecture, Linux as the Operating System is the only requirement.
-
-## Using the Plunder Cloud Provider (CCM)
-
-If you just want things to "work", then you can quickly install the "latest" components:
-
-**Install the `plndr-cloud-provider`
-
-```
-kubectl apply -f https://kube-vip.io/manifests/controller.yaml
-```
-
-**Create the `cidr` for the `global` namespace**
-
-```
-kubectl create configmap --namespace kube-system plndr --from-literal cidr-global=192.168.0.200/29
-```
-
-Creating services of `type: LoadBalancer` in the default namespace will now take addresses from the **global** cidr defined in the `configmap`.
-
-**Additional namespaces**
-
-Edit the `configmap` and add in the cidr ranges for those namespaces, the key in the cidr should be `cidr-<namespace>`, then ensure that `kube-vip` is deployed into that namespac
-e with the above `apply` command with the `-n namespace` flag.
-
-## The Detailed guide
-
-### Deploy the `plndr-cloud-provider`
-
-To deploy the [latest] then `kubectl apply -f https://kube-vip.io/manifests/controller.yaml`, specific versions should be found in the repository as detailed below:
-
-From the GitHub repository [https://github.com/plunder-app/plndr-cloud-provider/tree/master/example/pod](https://github.com/plunder-app/plndr-cloud-provider/tree/master/example/p
-od), find the version of the plunder cloud provider manifest (although typically the highest version number will provider more functionality/stability). The [raw] option in Githu
-b will provide the url that can be applied directly with a `kubectl apply -f <url>`.
-
-The following output should appear when the manifest is applied: 
-
-```
-serviceaccount/plunder-cloud-controller created
-clusterrole.rbac.authorization.k8s.io/system:plunder-cloud-controller-role created
-clusterrolebinding.rbac.authorization.k8s.io/system:plunder-cloud-controller-binding created
-pod/plndr-cloud-provider created
-```
-
-We can validate the cloud-provider by examining the pods:
-
-`kubectl logs -n kube-system plndr-cloud-provider-0 -f`
-
-#### The `plndr-cloud-provider` `configmap`
-
-The `configmap` details a CIDR range *per* namespace, however as of (`kube-vip 0.2.1` and `plnder-cloud-provider 0.1.4`), there is now the option of having a **global** CIDR rang
-e (`cidr-global)`.  
-
-To manage the ranges for the load-balancer instances, the `plndr-cloud-provider` has a `configmap` held in the `kube-system` namespace. The structure for the key/values within th
-e `configmap` should be that the key is in the format `cidr-<namespace>` and the value should be the cidr range.
-
-Example Configmap:
-
-```
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: plndr
-  namespace: kube-system
-data:
-  cidr-default: 192.168.0.200/29
-  cidr-global: 192.168.0.210/29
-```
-
-### Expose a service
-
-We can now expose a service and once the cloud provider has provided an address `kube-vip` will start to advertise that address to the outside world as shown below!
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx
-```
-
-We can also expose a specific address by specifying it on the command line:
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```
-
-### Using DHCP for Load Balancers (experimental)
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to use the local network DHCP server to provide `kube-vip` with a load-balancer address that can be used to access a
- Kubernetes service on the network. 
-
-In order to do this we need to signify to `kube-vip` and the cloud-provider that we don't need one of their managed addresses. We do this by explicitly exposing a service on the 
-address `0.0.0.0`. When `kube-vip` sees a service on this address it will create a `macvlan` interface on the host and request a DHCP address, once this address is provided it wi
-ll assign it as the VIP and update the Kubernetes service!
-
-```
-$ k expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx-dhcp --load-balancer-ip=0.0.0.0; k get svc
-service/nginx-dhcp exposed
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   0.0.0.0         80:31184/TCP   0s
-
-{ ... a second or so later ... }
-
-$ k get svc
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   192.168.0.155   80:31184/TCP   3s
-```
-
-### Using UPNP to expose a service to the outside world
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to expose a load-balancer on a specific port and using UPNP (on a supported gateway) expose this service to the inte
-rnet.
-
-Most simple networks look something like the following:
-
-`<----- <internal network 192.168.0.0/24> <Gateway / router> <external network address> ----> Internet`
-
-Using UPNP we can create a matching port on the `<external network address>` allowing your service to be exposed to the internet.
-
-#### Enable UPNP
-
-Add the following to the `kube-vip` `env:` section, and the rest should be completely automated. 
-
-**Note** some environments may require (Unifi) will require `Secure mode` being `disabled` (this allows a host with a different address to register a port)
-
-```
-- name: enableUPNP
-  value: "true"
-```
-
-#### Exposing a service
-
-To expose a port successfully we'll need to change the command slightly:
-
-`--target-port=80` the port of the application in the pods (HTT/NGINX)
-`--port=32380` the port the service will be exposed on (and what you should connect to in order to receive traffic from the service)
-
-`kubectl expose deployment plunder-nginx --port=32380 --target-port=80 --type=LoadBalancer --namespace plunder`
-
-The above example should expose a port on your external (internet facing address), that can be tested externally with:
-
-```
-$ curl externalIP:32380
-<!DOCTYPE html>
-<html>
-...
-```
-
-### Expose with Equinix Metal (using the `plndr-cloud-provider`)
-
-Either through the CLI or through the UI, create a public IPv4 EIP address.. and this is the address you can expose through BGP!
-
-```
-# packet ip request -p xxx-bbb-ccc -f ams1 -q 1 -t public_ipv4                                                                   
-+-------+---------------+--------+----------------------+
-|   ID  |    ADDRESS    | PUBLIC |       CREATED        |
-+-------+---------------+--------+----------------------+
-| xxxxx |     1.1.1.1   | true   | 2020-11-10T15:57:39Z |
-+-------+---------------+--------+----------------------+
-
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```
-
-## Equinix Metal Overview (using the [Equinix Metal CCM](https://github.com/packethost/packet-ccm))
-
-Below are two examples for running `type:LoadBalancer` services on worker nodes only and will create a daemonset that will run `kube-vip`. 
-
-**NOTE** This use-case requires the [Equinix Metal CCM](https://github.com/packethost/packet-ccm) to be installed and that the cluster/kubelet is configured to use an "external" cloud provider.
-
-### Using Annotations
-
-This is important as the CCM will apply the BGP configuration to the [node annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) making it easy for `kube-vip` to find the networking configuration it needs to expose load balancer addresses. The `--annotations metal.equinix.com` will cause kube-vip to "watch" the annotations of the worker node that it is running on, once all of the configuarion has been applied by the CCM then the `kube-vip` pod is ready to advertise BGP addresses for the service.
-
-```
-kube-vip manifest daemonset \
-  --interface $INTERFACE \
-  --services \
-  --bgp \
-  --annotations metal.equinix.com \
-  --inCluster | k apply -f -
-```
-
-### Using the existing CCM secret 
-
-Alternatively it is possible to create a daemonset that will use the existing CCM secret to do an API lookup, this will allow for discovering the networking configuration needed to advertise loadbalancer addresses through BGP.
-
-```
-kube-vip manifest daemonset --interface $INTERFACE \
---services \
---inCluster \
---bgp \
---metal \
---provider-config /etc/cloud-sa/cloud-sa.json | kubectl apply -f -
-```

docs/hybrid/static/index.md

@@ -1,129 +0,0 @@
-# Kube-vip as a Static Pod
-
-In Hybrid mode `kube-vip` will manage a virtual IP address that is passed through it's configuration for a Highly Available Kubernetes cluster, it will also "watch" services of `type:LoadBalancer` and once their `spec.LoadBalancerIP` is updated (typically by a cloud controller) it will advertise this address using BGP/ARP.
-
-The "hybrid" mode is now the default mode in `kube-vip` from `0.2.3` onwards, and allows both modes to be enabled at the same time. 
-
-## Generating a Manifest
-
-This section details creating a number of manifests for various use cases
-
-### Set configuration details
-
-`export VIP=192.168.0.40`
-
-`export INTERFACE=<interface>`
-
-### Configure to use a container runtime
-
-The easiest method to generate a manifest is using the container itself, below will create an alias for different container runtimes.
-
-#### containerd
-`alias kube-vip="ctr run --rm --net-host docker.io/plndr/kube-vip:0.3.1 vip /kube-vip"`
-
-#### Docker
-`alias kube-vip="docker run --network host --rm plndr/kube-vip:0.3.1"`
-
-
-### ARP
-
-This configuration will create a manifest that starts `kube-vip` providing **controlplane** and **services** management, using **leaderElection**. When this instance is elected as the leader it will bind the `vip` to the specified `interface`, this is also the same for services of `type:LoadBalancer`.
-
-`export INTERFACE=eth0`
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --arp \
-    --leaderElection | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-### BGP
-
-This configuration will create a manifest that will start `kube-vip` providing **controlplane** and **services** management. **Unlike** ARP, all nodes in the BGP configuration will advertise virtual IP addresses. 
-
-**Note** we bind the address to `lo` as we don't want multiple devices that have the same address on public interfaces. We can specify all the peers in a comma seperate list in the format of `address:AS:password:multihop`.
-
-`export INTERFACE=lo`
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --localAS 65000 \
-    --bgpRouterID 192.168.0.2 \
-    --bgppeers 192.168.0.10:65000::false,192.168.0.11:65000::false | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-### BGP with Equinix Metal
-
-When deploying Kubernetes with Equinix Metal with the `--controlplane` functionality we need to pre-populate the BGP configuration in order for the control plane to be advertised and work in a HA scenario. Luckily Equinix Metal provides the capability to "look up" the configuration details (for BGP) that we need in order to advertise our virtual IP for HA functionality. We can either make use of the [Equinix Metal API](https://metal.equinix.com/developers/api/) or we can parse the [Equinix Metal Metadata service](https://metal.equinix.com/developers/docs/servers/metadata/).
-
-**Note** If this cluster will be making use of Equinix Metal for `type:LoadBalancer` (by using the [Equinix Metal CCM](https://github.com/packethost/packet-ccm)) then we will need to ensure that nodes are set to use an external cloud-provider. Before doing a `kubeadm init|join` ensure the kubelet has the correct flags by using the following command `echo KUBELET_EXTRA_ARGS=\"--cloud-provider=external\" > /etc/default/kubelet`.
-
-#### Creating a manifest using the API
-
-We can enable `kube-vip` with the capability to discover the required configuration for BGP by passing the `--metal` flag and the API Key and our project ID.
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE\
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --metal \
-    --metalKey xxxxxxx \
-    --metalProjectID xxxxx | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-#### Creating a manifest using the metadata
-
-We can parse the metadata, *however* it requires that the tools `curl` and `jq` are installed. 
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE\
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --peerAS $(curl https://metadata.platformequinix.com/metadata | jq '.bgp_neighbors[0].peer_as') \
-    --peerAddress $(curl https://metadata.platformequinix.com/metadata | jq -r '.bgp_neighbors[0].peer_ips[0]') \
-    --localAS $(curl https://metadata.platformequinix.com/metadata | jq '.bgp_neighbors[0].customer_as') \
-    --bgpRouterID $(curl https://metadata.platformequinix.com/metadata | jq -r '.bgp_neighbors[0].customer_ip') | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-## Deploy your Kubernetes Cluster
-
-### First node
-
-```
-sudo kubeadm init \
-    --kubernetes-version 1.19.0 \
-    --control-plane-endpoint $VIP \
-    --upload-certs
-```
-
-### Additional Node(s)
-
-Due to an oddity with `kubeadm` we can't have our `kube-vip` manifest present **before** joining our additional nodes. So on these control plane nodes we will add them first to the cluster.
-
-```
-sudo kubeadm join $VIP:6443 \
-    --token w5atsr.blahblahblah 
-    --control-plane \
-    --certificate-key abc123
-```
-
-**Once**, joined these nodes can have the same command that we ran on the first node to populate the `/etc/kubernetes/manifests/` folder with the `kube-vip` manifest.
-
-## Services
-
-At this point your `kube-vip` static pods will be up and running and where used with the `--services` flag will also be watching for Kubernetes services that they can advertise. In order for `kube-vip` to advertise a service it needs a CCM or other controller to apply an IP address to the `spec.LoadBalancerIP`, which marks the loadbalancer as defined. 

docs/index/index.md

@@ -1,27 +0,0 @@
-# Kube-vip
-
-The **kube-vip** project provides High-Availability and load-balancing for both **inside** and **outside** a Kubernetes cluster
-
-![kube-vip.png](kube-vip.png)
-
-## Architecture
-
-The architecture for `kube-vip` (and associated kubernetes components) is covered in detail [here](/architecture/)
-
-
-## (New) Hybrid Control-plane HA and Kubernetes service `type=LoadBalancer`
-
-With the newest release of `kube-vip` the internal "manager" can handle the lifecycle of VIPs for both HA and for Kubernetes Load-Balancing. The main driver for this is being most effective for large nodes that can run control-plane components and run applications. The details for hybrid mode are [here](/hybrid/)
-
-## (Legacy) Control-plane load balancer
-
-The details are [here](/control-plane/)
-
-## (Legacy) Kubernetes service `"type: LoadBalancer"`
-
-The details are [here](/kubernetes/)
-
-## GitHub Repositories
-
-- The Plunder Cloud Provider -> [https://github.com/plunder-app/plndr-cloud-provider](https://github.com/plunder-app/plndr-cloud-provider)
-- The Kube-Vip Deployment -> [https://github.com/plunder-app/kube-vip](https://github.com/plunder-app/kube-vip)

docs/kubernetes/arp/index.md

@@ -1,181 +0,0 @@
-# Kube-vip (Layer 2 / ARP)
-
-**BEFORE** we begin we should ensure that ipvs has `strict` ARP enabled:
-
-```
-$ kubectl describe configmap -n kube-system kube-proxy | grep ARP
-  strictARP: false
-```
-
-If this is false we can enable it with the command:
-
-```
-$ kubectl get configmap kube-proxy -n kube-system -o yaml | \
- sed -e "s/strictARP: false/strictARP: true/" | \
- kubectl apply -f - -n kube-system
-```
-
-and confirm with:
-
-```
-$ kubectl describe configmap -n kube-system kube-proxy | grep ARP
-  strictARP: true
-```
-
-## Deploy `kube-vip`
-
-To deploy the [latest] then `kubectl apply -f https://kube-vip.io/manifests/kube-vip.yaml`, specific versions should be found in the repository as detailed below:
-
-From the GitHub repository [https://github.com/plunder-app/kube-vip/tree/master/example/deploy](https://github.com/plunder-app/kube-vip/tree/master/example/deploy) find the version of the `kube-vip` to deploy (although typically the highest version number will provider more functionality/stability). The [raw] option in Github will provide the url that can be applied directly with a `kubectl apply -f <url>`.
-
-The following output should appear when the manifest is applied: 
-```
-serviceaccount/vip created
-role.rbac.authorization.k8s.io/vip-role created
-rolebinding.rbac.authorization.k8s.io/vip-role-bind created
-deployment.apps/kube-vip-cluster created
-```
-
-*NOTE* The manifest for the `kube-vip` deployment has rules to ensure affinity (pods are always distributed to different nodes for HA). By default the replicas are set to `3` in the event you have less than `3` worker nodes then those replicas will sit as `pending`. This in itself isn't an issue, it means when new workers are added then they will be scheduled. *However*, tooling such as `kapps` will inspect the manifest before it's applied an error because of issues such as this.
-
-### Editing `kube-vip` configuration
-
-Either download and edit the manifest locally or apply as above and edit the deployment with `kubectl edit deploy/kube-vip-cluster` (change namespace where appropriate `-n`)
-
-```
-		- name: vip_interface
-		  value: ens192
-		- name: vip_configmap
-		  value: plndr
-		- name: vip_arp
-		  value: "true"
-		- name: vip_loglevel
-		  value: "5"
-```
-
-- `vip_interface` - defines the interface that the VIP will bind to
-- `vip_configmap` - defines the `configmap` that `kube-vip` will watch for service configuration
-- `vip_arp` - determines if ARP broadcasts are enabled
-- `vip_loglevel` - determines the verbosity of logging
-
-## Using other namespaces
-
-In this example we'll deploy and load-balance within the namespace `plunder`
-
-### Create the namespace
-
-`kubectl create namespace plunder`
-
-### Add a network range/cidr for this namespace
-
-`kubectl edit -n kube-system configmap/plndr`
-
-We will add the range 192.168.0.210/29 for the namespace plunder underneath the existing range for the namespace default:
-
-```
-apiVersion: v1
-data:
-  cidr-default: 192.168.0.200/29
-  cidr-global: 192.168.0.210/29
-  cidr-plunder: 192.168.0.220/29
-<...>
-```
-
-### Deploy `kube-vip` in the namespace **plunder**
-
-In the same way we deployed `kube-vip` into the default namespace we can deploy the same manifest into a different namespace using `-n namespace` e.g.
-
-**Note** change the version of manifest when actually deploying!
-
-```
-kubectl apply -f https://kube-vip.io/manifests/kube-vip.yaml -n plunder
-```
-
-## Usage
-
-This example will deploy into the namespace `plunder` as mention in the [Using other namespaces](Using other namespaces) example. Remove the `-n plunder` to deploy within the `default` namespace.
-
-### Deploy nginx
-
-```
-kubectl create deployment --image nginx plunder-nginx --namespace plunder
-```
-
-### Create a load balancer
-
-```
-kubectl expose deployment plunder-nginx --port=80 --type=LoadBalancer --namespace plunder
-```
-
-## Using DHCP for Load Balancers (experimental)
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to use the local network DHCP server to provide `kube-vip` with a load-balancer address that can be used to access a Kubernetes service on the network. 
-
-In order to do this we need to signify to `kube-vip` and the cloud-provider that we don't need one of their managed addresses. We do this by explicitly exposing a service on the address `0.0.0.0`. When `kube-vip` sees a service on this address it will create a `macvlan` interface on the host and request a DHCP address, once this address is provided it will assign it as the VIP and update the Kubernetes service!
-
-```
-$ k expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx-dhcp --load-balancer-ip=0.0.0.0; k get svc
-service/nginx-dhcp exposed
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   0.0.0.0         80:31184/TCP   0s
-
-{ ... a second or so later ... }
-
-$ k get svc
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   192.168.0.155   80:31184/TCP   3s
-```
-
-## Using UPNP to expose a service to the outside world
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to expose a load-balancer on a specific port and using UPNP (on a supported gateway) expose this service to the internet.
-
-Most simple networks look something like the following:
-
-`<----- <internal network 192.168.0.0/24> <Gateway / router> <external network address> ----> Internet`
-
-Using UPNP we can create a matching port on the `<external network address>` allowing your service to be exposed to the internet.
-
-### Enable UPNP
-
-Add the following to the `kube-vip` `env:` section, and the rest should be completely automated. 
-
-**Note** some environments may require (Unifi) will require `Secure mode` being `disabled` (this allows a host with a different address to register a port)
-
-```
-- name: enableUPNP
-  value: "true"
-```
-
-### Exposing a service
-
-To expose a port successfully we'll need to change the command slightly:
-
-`--target-port=80` the port of the application in the pods (HTT/NGINX)
-`--port=32380` the port the service will be exposed on (and what you should connect to in order to receive traffic from the service)
-
-`kubectl expose deployment plunder-nginx --port=32380 --target-port=80 --type=LoadBalancer --namespace plunder`
-
-The above example should expose a port on your external (internet facing address), that can be tested externally with:
-
-```
-$ curl externalIP:32380
-<!DOCTYPE html>
-<html>
-...
-```
-
-## Troubleshooting
-
-Typically the logs from the `kube-vip` controller will reveal the most clues as to where a problem may lie.
-
-The `ClusterRoleBinding` is missing will result in the following:
-
-```
-E0229 17:36:38.014351       1 retrywatcher.go:129] Watch failed: unknown (get endpoints)
-E0229 17:36:38.014352       1 retrywatcher.go:129] Watch failed: unknown (get endpoints)
-```
-
-Additionally ensure that the vip_interface matches the correct interface from `ip addr`

docs/kubernetes/bgp/index.md

@@ -1,87 +0,0 @@
-# Kube-vip (Layer 3 / BGP)
-
-## Deploy `kube-vip`
-
-To deploy the [latest] then `kubectl apply -f https://kube-vip.io/manifests/kube-vip.yaml`, specific versions should be found in the repository as detailed below:
-
-From the GitHub repository [https://github.com/plunder-app/kube-vip/tree/master/example/deploy](https://github.com/plunder-app/kube-vip/tree/master/example/deploy) find the version of the `kube-vip` to deploy (although typically the highest version number will provider more functionality/stability). The [raw] option in Github will provide the url that can be applied directly with a `kubectl apply -f <url>`.
-
-The following output should appear when the manifest is applied: 
-```
-serviceaccount/vip created
-role.rbac.authorization.k8s.io/vip-role created
-rolebinding.rbac.authorization.k8s.io/vip-role-bind created
-deployment.apps/kube-vip-cluster created
-```
-
-*NOTE* The manifest for the `kube-vip` deployment has rules to ensure affinity (pods are always distributed to different nodes for HA). By default the replicas are set to `3` in the event you have less than `3` worker nodes then those replicas will sit as `pending`. This in itself isn't an issue, it means when new workers are added then they will be scheduled. *However*, tooling such as `kapps` will inspect the manifest before it's applied an error because of issues such as this.
-
-### Editing `kube-vip` configuration
-
-Either download and edit the manifest locally or apply as above and edit the deployment with `kubectl edit deploy/kube-vip-cluster` (change namespace where appropriate `-n`)
-
-Ensure the `vip_arp` isn't enabled as ARP and BGP can't be used at the same time (today), also that the `vip_interface` is set to localhost (`lo`).
-
-```
-        - name: vip_interface
-		  value: "lo"
-		- name: vip_configmap
-		  value: "plndr" 
-		- name: bgp_enable
-		  value: "true"
-		- name: vip_loglevel
-		  value: "5"
-```
-
-### BGP Specific configuration
-
-Additionally for BGP we'll need some configuration details, your local friendly network admin should be able to help here:
-
-```
-		- name: bgp_routerid
-		  value: "192.168.0.45"
-		- name: bgp_as
-          value: "65000" 
-		- name: bgp_peeraddress
-    	  value: "10.0.0.1"
-		- name: bgp_peeras
-    	  value: "65522"
-```
-
-### BGP on Packet
-
-If you're lucky enough to be running services on Packet then The above BGP information can be found from the API, instead of specifying the above we need to use the following:
-
-```
-		- name: vip_packet
-		  value: "true"
-		- name: vip_packetproject
-		  value: "My Project" 
-		- name: PACKET_AUTH_TOKEN
-		  value: "XXYZZYVVY"
-```
-
-With the above configuration in place, all `kube-vip` pods will start in active mode and when a service is exposed then all nodes will advertise the VIP to the routers.
-
-## Expose a service
-
-Given that `kube-vip` doesn't know your network (at this point) ask your local friendly network OPs for an address you can advertise. That is the address you can expose to the outside world as shown below!
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```
-
-## Expose with packet 
-
-Either through the CLI or through the UI, create a public IPv4 EIP address.. and this is the address you can expose through BGP!
-
-```
-# packet ip request -p xxx-bbb-ccc -f ams1 -q 1 -t public_ipv4                                                                   
-+-------+---------------+--------+----------------------+
-|   ID  |    ADDRESS    | PUBLIC |       CREATED        |
-+-------+---------------+--------+----------------------+
-| xxxxx |     1.1.1.1   | true   | 2020-11-10T15:57:39Z |
-+-------+---------------+--------+----------------------+
-
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```

docs/kubernetes/index.md

@@ -1,75 +0,0 @@
-# Usage
-
-The below instructions *should just work* on Kubernetes regardless of architecture, Linux as the Operating System is the only requirement.
-
-## The `tl;dr` guide
-
-If you just want things to "work", then you can quickly install the "latest" components:
-
-**NOTE** the `kube-vip.yaml` may need customising to set ARP/BGP OR to configure which interface to bind VIPs too.
-
-**Install the `plndr-cloud-provider`, and `kube-vip`**
-
-```
-kubectl apply -f https://kube-vip.io/manifests/controller.yaml
-kubectl apply -f https://kube-vip.io/manifests/kube-vip.yaml
-```
-
-**Create the `cidr` for the `global` namespace**
-
-```
-kubectl create configmap --namespace kube-system plndr --from-literal cidr-global=192.168.0.200/29
-```
-
-Creating services of `type: LoadBalancer` in the default namespace will now take addresses from the **global** cidr defined in the `configmap`.
-
-**Additional namespaces**
-
-Edit the `configmap` and add in the cidr ranges for those namespaces, the key in the cidr should be `cidr-<namespace>`, then ensure that `kube-vip` is deployed into that namespace with the above `apply` command with the `-n namespace` flag.
-
-## The Detailed guide
-
-### Deploy the `plndr-cloud-provider`
-
-To deploy the [latest] then `kubectl apply -f https://kube-vip.io/manifests/controller.yaml`, specific versions should be found in the repository as detailed below:
-
-From the GitHub repository [https://github.com/plunder-app/plndr-cloud-provider/tree/master/example/pod](https://github.com/plunder-app/plndr-cloud-provider/tree/master/example/pod), find the version of the plunder cloud provider manifest (although typically the highest version number will provider more functionality/stability). The [raw] option in Github will provide the url that can be applied directly with a `kubectl apply -f <url>`.
-
-The following output should appear when the manifest is applied: 
-
-```
-serviceaccount/plunder-cloud-controller created
-clusterrole.rbac.authorization.k8s.io/system:plunder-cloud-controller-role created
-clusterrolebinding.rbac.authorization.k8s.io/system:plunder-cloud-controller-binding created
-pod/plndr-cloud-provider created
-```
-
-We can validate the cloud-provider by examining the pods:
-
-`kubectl logs -n kube-system plndr-cloud-provider-0 -f`
-
-#### The `plndr-cloud-provider` `configmap`
-
-The `configmap` details a CIDR range *per* namespace, however as of (`kube-vip 0.2.1` and `plnder-cloud-provider 0.1.4`), there is now the option of having a **global** CIDR range (`cidr-global)`.  
-
-To manage the ranges for the load-balancer instances, the `plndr-cloud-provider` has a `configmap` held in the `kube-system` namespace. The structure for the key/values within the `configmap` should be that the key is in the format `cidr-<namespace>` and the value should be the cidr range.
-
-Example Configmap:
-
-```
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: plndr
-  namespace: kube-system
-data:
-  cidr-default: 192.168.0.200/29
-  cidr-global: 192.168.0.210/29
-```
-
-### Deploying `kube-vip`
-
-To use `kube-vip` in Layer2/ARP the follow this [guide](/kubernetes/arp/)
-
-To use `kube-vip` in Layer3/BGP the follow this [guide](/kubernetes/bgp/)
-

docs/manifests/0.3.1/kube-vip-bgp-ds.yaml

@@ -1,72 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: eth0
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: vip_address
-          value: 192.168.0.1
-        image: plndr/kube-vip:0.3.1
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/0.3.1/kube-vip-bgp-em-ds.yaml

@@ -1,82 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: lo
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "false"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: provider_config
-          value: /etc/cloud-sa/cloud-sa.json
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: vip_address
-          value: 192.168.0.1
-        image: plndr/kube-vip:0.3.1
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-        volumeMounts:
-        - mountPath: /etc/cloud-sa
-          name: cloud-sa-volume
-          readOnly: true
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      volumes:
-      - name: cloud-sa-volume
-        secret:
-          secretName: packet-cloud-config
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/0.3.1/kube-vip-bgp.yaml

@@ -1,62 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - manager
-    env:
-    - name: vip_arp
-      value: "false"
-    - name: vip_interface
-      value: eth0
-    - name: port
-      value: "6443"
-    - name: vip_cidr
-      value: "32"
-    - name: cp_enable
-      value: "true"
-    - name: cp_namespace
-      value: kube-system
-    - name: svc_enable
-      value: "true"
-    - name: vip_startleader
-      value: "false"
-    - name: vip_addpeerstolb
-      value: "true"
-    - name: vip_localpeer
-      value: code:192.168.0.22:10000
-    - name: bgp_enable
-      value: "true"
-    - name: bgp_routerid
-    - name: bgp_as
-      value: "65000"
-    - name: bgp_peeraddress
-    - name: bgp_peerpass
-    - name: bgp_peeras
-      value: "65000"
-    - name: vip_address
-      value: 192.168.0.1
-    image: plndr/kube-vip:0.3.1
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_RAW
-        - SYS_TIME
-    volumeMounts:
-    - mountPath: /etc/kubernetes/admin.conf
-      name: kubeconfig
-  hostNetwork: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/admin.conf
-    name: kubeconfig
-status: {}
-

docs/manifests/controller.yaml

@@ -1,67 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: plunder-cloud-controller
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:plunder-cloud-controller-role
-rules:
-  - apiGroups: [""]
-    resources: ["configmaps", "endpoints","events","services/status"]
-    verbs: ["*"]
-  - apiGroups: [""]
-    resources: ["nodes", "services"]
-    verbs: ["list","get","watch","update"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:plunder-cloud-controller-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:plunder-cloud-controller-role
-subjects:
-- kind: ServiceAccount
-  name: plunder-cloud-controller
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: plndr-cloud-provider
-  namespace: kube-system
-spec:
-  serviceName: plndr-cloud-provider
-  podManagementPolicy: OrderedReady
-  replicas: 1
-  revisionHistoryLimit: 10
-  selector:
-    matchLabels:
-      app: kube-vip
-      component: plndr-cloud-provider
-  template:
-    metadata:
-      labels:
-        app: kube-vip
-        component: plndr-cloud-provider
-    spec:
-      containers:
-      - command:
-        - /plndr-cloud-provider
-        - --leader-elect-resource-name=plndr-cloud-controller
-        image: plndr/plndr-cloud-provider:0.1.5
-        name: plndr-cloud-provider
-        imagePullPolicy: Always
-        resources: {}
-      dnsPolicy: ClusterFirst
-      restartPolicy: Always
-      schedulerName: default-scheduler
-      securityContext: {}
-      terminationGracePeriodSeconds: 30
-      serviceAccountName: plunder-cloud-controller

docs/manifests/kube-vip-arp.yaml

@@ -1,85 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "create", "update", "list", "put"]
-  - apiGroups: [""]
-    resources: ["services"]
-    verbs: ["list","get","watch", "update"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-workers
-  name: kube-vip-workers
-  namespace: kube-system
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-workers
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-workers
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-workers
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: plndr/kube-vip:0.2.2
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        env:
-          - name: vip_interface
-            value: "ens160"
-          - name: vip_arp
-            value: "true"
-          - name: vip_loglevel
-            value: "5"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-      serviceAccountName: kube-vip
-status: {}

docs/manifests/kube-vip-bgp.yaml

@@ -1,90 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: [""]
-    resources: ["services"]
-    verbs: ["list","get","watch", "update"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-workers
-  name: kube-vip-workers
-  namespace: kube-system
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-workers
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-workers
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-workers
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: plndr/kube-vip:0.2.2
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        env:
-          - name: vip_interface
-            value: "lo"
-          - name: bgp_enable
-            value: "true"
-          - name: vip_loglevel
-            value: "5"
-          - name: bgp_routerinterface
-            value: "ens160"
-          - name: bgp_as
-            value: "64512"
-          - name: bgp_peeraddress
-            value: "192.168.0.1"
-          - name: bgp_peeras
-            value: "64512"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-      serviceAccountName: kube-vip
-status: {}

docs/manifests/kube-vip-em.yaml

@@ -1,91 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: [""]
-    resources: ["services", "services/status"]
-    verbs: ["list","get","watch", "update"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: lo
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: svc_enable
-          value: "true"
-        - name: provider_config
-          value: /etc/cloud-sa/cloud-sa.json
-        - name: vip_packet
-          value: "true"
-        - name: bgp_enable
-          value: "true"
-        image: plndr/kube-vip:0.2.3
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - SYS_TIME
-        volumeMounts:
-        - mountPath: /etc/cloud-sa
-          name: cloud-sa-volume
-          readOnly: true
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      volumes:
-      - name: cloud-sa-volume
-        secret:
-          secretName: packet-cloud-config
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0

docs/manifests/kube-vip.yaml

@@ -1,83 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: vip
----
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: vip-role
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "create", "update", "list", "put"]
-  - apiGroups: [""]
-    resources: ["configmaps", "endpoints"]
-    verbs: ["watch", "get"]
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: vip-role-bind
-subjects:
-  - kind: ServiceAccount
-    name: vip 
-    apiGroup: ""
-roleRef:
-  kind: Role
-  name: vip-role
-  apiGroup: "" 
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-cluster
-  name: kube-vip-cluster
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-cluster
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-cluster
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-cluster
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: plndr/kube-vip:0.1.3
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        env:
-          - name: vip_interface
-            value: "ens192"
-          - name: vip_configmap
-            value: "plndr" 
-          - name: vip_arp
-            value: "true"
-          - name: vip_loglevel
-            value: "5"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-      serviceAccountName: vip
-status: {}

docs/manifests/rbac.yaml

@@ -1,32 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: [""]
-    resources: ["services", "services/status", "nodes"]
-    verbs: ["list","get","watch", "update"]
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["list", "get", "watch", "update", "create"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system

docs/publish.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-echo Deploying updated documentation
-
-generate-md --layout github --input ./index/ --output /var/www/kube-vip/
-generate-md --layout github --input ./architecture/ --output /var/www/kube-vip/architecture/
-generate-md --layout github --input ./control-plane/ --output /var/www/kube-vip/control-plane/
-generate-md --layout github --input ./hybrid/ --output /var/www/kube-vip/hybrid/
-generate-md --layout github --input ./kubernetes/ --output /var/www/kube-vip/kubernetes/
-generate-md --layout github --input ./manifests/ --output /var/www/kube-vip/manifests/

example/deploy/0.1.2.yaml

@@ -28,7 +28,7 @@ spec:
                   - kube-vip-cluster
             topologyKey: "kubernetes.io/hostname"
       containers:
-      - image: plndr/kube-vip:0.1.2
+      - image: ghcr.io/kube-vip/kube-vip:0.3.7
         imagePullPolicy: Always
         name: kube-vip
         command:

example/deploy/0.1.3.yaml

@@ -58,7 +58,7 @@ spec:
                   - kube-vip-cluster
             topologyKey: "kubernetes.io/hostname"
       containers:
-      - image: plndr/kube-vip:0.1.3
+      - image: ghcr.io/kube-vip/kube-vip:0.3.7
         imagePullPolicy: Always
         name: kube-vip
         command:

example/deploy/0.3.5.yaml

@@ -26,23 +26,16 @@ spec:
           value: "6443"
         - name: vip_cidr
           value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
         - name: svc_enable
           value: "true"
-        - name: vip_leaderelection
+        - name: vip_startleader
+          value: "false"
+        - name: vip_addpeerstolb
           value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
+        - name: vip_localpeer
+          value: ip-172-20-40-207:172.20.40.207:10000
         - name: vip_address
-          value: 192.168.0.1
-        image: plndr/kube-vip:0.3.1
+        image: plndr/kube-vip:v0.3.5
         imagePullPolicy: Always
         name: kube-vip
         resources: {}
@@ -60,4 +53,3 @@ status:
   desiredNumberScheduled: 0
   numberMisscheduled: 0
   numberReady: 0
-

example/deployment.yaml

@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:latest
+        ports:
+        - containerPort: 80

example/service-interface-annotation.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-interface-ens192-service
+  annotations:
+    kube-vip.io/serviceInterface: ens192
+spec:
+  selector:
+    app: nginx
+  type: LoadBalancer
+  ports:
+  - name: http
+    port: 80
+    targetPort: 80
+    protocol: TCP

example/service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-service
+spec:
+  selector:
+    app: nginx
+  type: LoadBalancer
+  ports:
+  - name: http
+    port: 80
+    targetPort: 80
+    protocol: TCP

go.mod

@@ -1,65 +1,165 @@
-module github.com/plunder-app/kube-vip
+module github.com/kube-vip/kube-vip
 
-go 1.14
+go 1.24.4
 
 require (
-	github.com/armon/go-metrics v0.3.4 // indirect
-	github.com/davecgh/go-spew v1.1.1
-	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
-	github.com/digineo/go-dhclient v1.0.2
-	github.com/eapache/queue v1.1.0 // indirect
-	github.com/fatih/color v1.9.0 // indirect
-	github.com/ghodss/yaml v1.0.0
-	github.com/go-logr/logr v0.2.1 // indirect
-	github.com/golang/protobuf v1.4.2
-	github.com/google/go-cmp v0.5.2 // indirect
-	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/gopacket v1.1.18
-	github.com/google/uuid v1.1.2 // indirect
-	github.com/googleapis/gnostic v0.5.1 // indirect
-	github.com/hashicorp/go-hclog v0.14.1 // indirect
-	github.com/hashicorp/go-immutable-radix v1.2.0 // indirect
-	github.com/hashicorp/go-msgpack v1.1.5 // indirect
-	github.com/hashicorp/go-retryablehttp v0.6.7 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
-	github.com/hashicorp/raft v1.1.2
-	github.com/imdario/mergo v0.3.11 // indirect
-	github.com/k-sone/critbitgo v1.4.0 // indirect
-	github.com/kamhlos/upnp v0.0.0-20171112074648-2713e75d9aef
-	github.com/magiconair/properties v1.8.3 // indirect
-	github.com/mattn/go-colorable v0.1.7 // indirect
-	github.com/mdlayher/ndp v0.0.0-20200602162440-17ab9e3e5567
-	github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065 // indirect
-	github.com/mitchellh/mapstructure v1.3.3 // indirect
-	github.com/onsi/ginkgo v1.11.0
-	github.com/onsi/gomega v1.7.0
-	github.com/osrg/gobgp v2.0.0+incompatible
-	github.com/packethost/packngo v0.5.1
+	github.com/cloudflare/ipvs v0.11.0
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
+	github.com/docker/docker v28.5.2+incompatible
+	github.com/florianl/go-conntrack v0.4.0
+	github.com/google/go-cmp v0.7.0
+	github.com/google/go-containerregistry v0.20.6
+	github.com/google/nftables v0.3.0
+	github.com/gookit/slog v0.6.0
+	github.com/huin/goupnp v1.3.0
+	github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d
+	github.com/jpillora/backoff v1.0.0
+	github.com/mdlayher/ndp v1.1.0
+	github.com/onsi/ginkgo/v2 v2.27.2
+	github.com/onsi/gomega v1.38.2
+	github.com/osrg/gobgp/v3 v3.37.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.4.0
-	github.com/sirupsen/logrus v1.6.0
-	github.com/spf13/afero v1.4.0 // indirect
-	github.com/spf13/cast v1.3.1 // indirect
-	github.com/spf13/cobra v1.0.0
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/viper v1.7.1 // indirect
-	github.com/vishvananda/netlink v1.1.0
-	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect
-	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
-	golang.org/x/net v0.0.0-20201021035429-f5854403a974
-	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
-	golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4
-	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
-	google.golang.org/genproto v0.0.0-20200917134801-bb4cff56e0d0 // indirect
-	google.golang.org/grpc v1.32.0 // indirect
-	gopkg.in/ini.v1 v1.61.0 // indirect
-	k8s.io/api v0.19.2
-	k8s.io/apimachinery v0.19.2
-	k8s.io/client-go v0.19.0
-	k8s.io/klog v1.0.0
-	k8s.io/klog/v2 v2.3.0
-	k8s.io/utils v0.0.0-20200912215256-4140de9c8800 // indirect
-	sigs.k8s.io/kind v0.10.0
+	github.com/prometheus/client_golang v1.23.2
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cobra v1.10.1
+	github.com/stretchr/testify v1.11.1
+	github.com/vishvananda/netlink v1.3.1
+	go.etcd.io/etcd/api/v3 v3.6.6
+	go.etcd.io/etcd/client/pkg/v3 v3.6.6
+	go.etcd.io/etcd/client/v3 v3.6.6
+	go.uber.org/zap v1.27.1
+	golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
+	golang.org/x/sync v0.18.0
+	golang.org/x/sys v0.38.0
+	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
+	google.golang.org/grpc v1.77.0
+	google.golang.org/protobuf v1.36.10
+	k8s.io/api v0.34.2
+	k8s.io/apimachinery v0.34.2
+	k8s.io/client-go v0.34.1
+	k8s.io/klog/v2 v2.130.1
+	sigs.k8s.io/kind v0.30.0
+	sigs.k8s.io/yaml v1.6.0
 )
 
-replace github.com/osrg/gobgp v2.0.0+incompatible => github.com/osrg/gobgp v0.0.0-20191101114856-a42a1a5f6bf0
+require (
+	al.essio.dev/pkg/shellescape v1.5.1 // indirect
+	github.com/BurntSushi/toml v1.4.0 // indirect
+	github.com/Masterminds/semver/v3 v3.4.0 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/coreos/go-semver v0.3.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/dlclark/regexp2 v1.11.4 // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/eapache/channels v1.1.0 // indirect
+	github.com/eapache/queue v1.1.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.12.2 // indirect
+	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/gnostic-models v0.7.0 // indirect
+	github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gookit/color v1.6.0 // indirect
+	github.com/gookit/goutil v0.7.1 // indirect
+	github.com/gookit/gsr v0.1.1 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/josharian/native v1.1.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/k-sone/critbitgo v1.4.0 // indirect
+	github.com/magiconair/properties v1.8.9 // indirect
+	github.com/mailru/easyjson v0.9.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mdlayher/genetlink v1.3.2 // indirect
+	github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect
+	github.com/mdlayher/packet v1.1.2 // indirect
+	github.com/mdlayher/socket v0.5.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
+	github.com/moby/term v0.5.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/pelletier/go-toml v1.9.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.66.1 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/sagikazarmark/locafero v0.6.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/spf13/pflag v1.0.9 // indirect
+	github.com/spf13/viper v1.19.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tj/go-spin v1.1.0 // indirect
+	github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/vishvananda/netns v0.0.5 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xlab/c-for-go v1.3.0 // indirect
+	github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
+	go.opentelemetry.io/otel v1.38.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 // indirect
+	go.opentelemetry.io/otel/metric v1.38.0 // indirect
+	go.opentelemetry.io/otel/trace v1.38.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/crypto v0.45.0 // indirect
+	golang.org/x/mod v0.29.0 // indirect
+	golang.org/x/net v0.47.0 // indirect
+	golang.org/x/oauth2 v0.32.0 // indirect
+	golang.org/x/term v0.37.0 // indirect
+	golang.org/x/text v0.31.0 // indirect
+	golang.org/x/time v0.9.0 // indirect
+	golang.org/x/tools v0.38.0 // indirect
+	golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
+	k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect
+	modernc.org/cc/v4 v4.24.4 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/opt v0.1.4 // indirect
+	modernc.org/sortutil v1.2.1 // indirect
+	modernc.org/strutil v1.2.1 // indirect
+	modernc.org/token v1.1.0 // indirect
+	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
+)

kubernetes-control-plane.md

@@ -1,147 +0,0 @@
-# Load Balancing a Kubernetes Cluster (Control-Plane)
-
-This document covers all of the details for using `kube-vip` to build a HA Kubernetes cluster
-
-`tl;dr version`
-- Generate/modify first node `kube-vip` config/manifest
-- `init` first node
-- `join` remaining nodes
-- Add remaining config/manifests
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.75 |
-| controlPlane01 | 10.0.0.70 |
-| controlPlane02 | 10.0.0.71 |
-| controlPlane03 | 10.0.0.72 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.17.0.
-
-### Generate the `kube-vip` configuration
-
-Make sure that the config directory exists: `sudo mkdir -p /etc/kube-vip/`, this directory can be any directory however the `hostPath` in the manifest will need modifying to point to the correct path.
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1.5 sample config | sudo tee /etc/kube-vip/config.yaml
-```
-
-### Modify the configuration
-
-**Cluster Configuration**
-Modify the `remotePeers` to point to the correct addresses of the other two nodes, ensure that their `id` is unique otherwise this will confuse the raft algorithm. The `localPeer` should be the configuration of the current node (`controlPlane01`), which is where this instance of the cluster will run. 
-
-As this node will be the first node, it will need to elect itself leader as until this occurs the VIP won’t be activated!
-
-`startAsLeader: true`
-
-**VIP Config**
-We will need to set our VIP address to `192.168.0.75` and to ensure all hosts are updated when the VIP moves we will enable ARP broadcasts `gratuitousARP: true`
-
-**Load Balancer**
-We will configure the load balancer to sit on the standard API-Server port `6443` and we will configure the backends to point to the API-servers that will be configured to run on port `6444`. Also for the Kubernetes Control Plane we will configure the load balancer to be of `type: tcp`.
-
-We can also use `6443` for both the VIP and the API-Servers, in order to do this we need to specify that the api-server is bound to it's local IP. To do this we use the `--apiserver-advertise-address` flag as part of the `init`, this means that we can then bind the same port to the VIP and we wont have a port conflict.
-
-**config.yaml**
-
-`user@controlPlane01:/etc/kube-vip$ cat config.yaml`
-
-...
-
-``` 
-remotePeers:
-- id: server2
-  address: 192.168.0.71
-  port: 10000
-- id: server3
-  address: 192.168.0.72
-  port: 10000
-localPeer:
-  id: server1
-  address: 192.168.0.70
-  port: 10000
-vip: 192.168.0.75
-gratuitousARP: true
-singleNode: false
-startAsLeader: true
-interface: ens192
-loadBalancers:
-- name: Kubernetes Control Plane
-  type: tcp
-  port: 6443
-  bindToVip: true
-  backends:
-  - port: 6444
-    address: 192.168.0.70
-  - port: 6444
-    address: 192.168.0.71
-  - port: 6444
-    address: 192.168.0.72
-```
-
-### First Node
-
-To generate the basic Kubernetes static pod `yaml` configuration:
-
-Make sure that the manifest directory exists: `sudo mkdir -p /etc/kubernetes/manifests/`
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1.5 sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Ensure that `image: plndr/kube-vip:<x>` is modified to point to a specific version (`0.1.5` at the time of writing), refer to [docker hub](https://hub.docker.com/r/plndr/kube-vip/tags) for details. Also ensure that the `hostPath` points to the correct `kube-vip` configuration, if it isn’t the above path.
-
-The **vip** is set to `192.168.0.75` and this first node will elect itself as leader, and as part of the `kubeadm init` it will use the VIP in order to speak back to the initialising api-server.
-
-`sudo kubeadm init --control-plane-endpoint “192.168.0.75:6443” --apiserver-bind-port 6444 --upload-certs --kubernetes-version “v1.17.0”`
-
-Once this node is up and running we will be able to see the control-plane pods, including the `kube-vip` pod:
-
-```
-$ kubectl get pods -A
-NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
-<...>
-kube-system   kube-vip-controlplane01                  1/1     Running   0          10m
-```
-
-### Remaining Nodes
-
-We first will need to create the `kube-vip` configuration that resides in `/etc/kube-vip/config.yaml` or we can regenerate it from scratch using the above example. Ensure that the configuration is almost identical with the `localPeer` and `remotePeers` sections are updated for each node. Finally, ensure that the remaining nodes will behave as standard cluster nodes by setting `startAsLeader: false`.
-
-At this point **DON’T** generate the manifests, this is due to some bizarre `kubeadm/kubelet` behaviour.
-
-```
-  kubeadm join 192.168.0.75:6443 --token <tkn> \
-    --discovery-token-ca-cert-hash sha256:<hash> \
-    --control-plane --certificate-key <key> 
-
-```
-
-**After** this node has been added to the cluster, we can add the manifest to also add this node as a `kube-vip` member. (Adding the manifest afterwards doesn’t interfere with `kubeadm`). 
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1.5 sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Once this node is added we will be able to see that the `kube-vip` pod is up and running as expected:
-
-```
-user@controlPlane01:~$ kubectl get pods -A | grep vip
-kube-system   kube-vip-controlplane01                  1/1     Running             1          16m
-kube-system   kube-vip-controlplane02                  1/1     Running             0          18m
-kube-system   kube-vip-controlplane03                  1/1     Running             0          20m
-
-```
-
-If we look at the logs, we can see that the VIP is running on the second node and we’re waiting for our third node to join the cluster:
-
-```
-$ kubectl logs kube-vip-controlplane02  -n kube-system
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-
-```

main.go

@@ -1,6 +1,8 @@
 package main
 
-import "github.com/plunder-app/kube-vip/cmd"
+import (
+	"github.com/kube-vip/kube-vip/cmd"
+)
 
 // Version is populated from the Makefile and is tied to the release TAG
 var Version string
@@ -9,6 +11,7 @@ var Version string
 var Build string
 
 func main() {
+
 	cmd.Release.Version = Version
 	cmd.Release.Build = Build
 	cmd.Execute()

pkg/arp/arp.go

@@ -0,0 +1,207 @@
+package arp
+
+import (
+	"context"
+	"fmt"
+	log "log/slog"
+	"sync"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
+
+type Manager struct {
+	instances sync.Map
+	config    *kubevip.Config
+}
+
+type Instance struct {
+	network vip.Network
+	ndp     *vip.NdpResponder
+	mu      sync.Mutex
+	counter int
+}
+
+func NewManager(config *kubevip.Config) *Manager {
+	return &Manager{
+		config: config,
+	}
+}
+
+func NewInstance(network vip.Network, ndp *vip.NdpResponder) *Instance {
+	return &Instance{
+		ndp:     ndp,
+		network: network,
+		counter: 1,
+	}
+}
+
+func (i *Instance) Name() string {
+	return i.network.ARPName()
+}
+
+func (m *Manager) Insert(instance *Instance) {
+	i, err := m.get(instance.Name())
+	if err != nil {
+		log.Error("[ARP manager] unable to insert instance", "err", err)
+		return
+	}
+	if i == nil {
+		log.Info("[ARP manager] inserting ARP/NDP instance", "name", instance.Name())
+		m.instances.Store(instance.Name(), instance)
+	} else {
+		i.mu.Lock()
+		defer i.mu.Unlock()
+		i.counter++
+	}
+}
+
+func (m *Manager) Remove(instance *Instance) {
+	m.RemoveWithIPDelete(instance, true)
+}
+
+// RemoveOnLeadershipLoss removes an ARP instance when leadership is lost
+func (m *Manager) RemoveOnLeadershipLoss(instance *Instance) {
+	// Use the inverse of PreserveVIPOnLeadershipLoss to decide whether to delete the IP
+	// If preserve is true, don't delete IP (deleteIP = false)
+	// If preserve is false, delete IP (deleteIP = true), This is the legacy behavior
+	deleteIP := !m.config.PreserveVIPOnLeadershipLoss
+	m.RemoveWithIPDelete(instance, deleteIP)
+}
+
+func (m *Manager) RemoveWithIPDelete(instance *Instance, deleteIP bool) {
+	i, err := m.get(instance.Name())
+	if err != nil {
+		log.Error("[ARP manager] unable to remove the instance", "err", err)
+		return
+	}
+	if i != nil {
+		i.mu.Lock()
+		defer i.mu.Unlock()
+		if i.counter > 1 {
+			i.counter--
+		} else {
+			log.Info("[ARP manager] removing ARP/NDP instance", "name", instance.Name())
+			if deleteIP {
+				if _, err := instance.network.DeleteIP(); err != nil {
+					log.Error("failed to delete IP", "address", instance.network.IP(), "err", err)
+				}
+			}
+			m.instances.Delete(instance.Name())
+		}
+	} else {
+		log.Warn("[ARP manager] unable to remove the instance - instance not found", "name", instance.Name())
+	}
+}
+
+func (m *Manager) Count(name string) int {
+	i, err := m.get(name)
+	if err != nil {
+		log.Error("[ARP manager] unable to count instance", "err", err)
+		return -1
+	}
+	if i != nil {
+		i.mu.Lock()
+		defer i.mu.Unlock()
+		return i.counter
+	}
+	return 0
+}
+
+func (m *Manager) StartAdvertisement(ctx context.Context) {
+	log.Info("[ARP manager] starting ARP/NDP advertisement")
+	for {
+		select {
+		case <-ctx.Done(): // if cancel() execute
+			return
+		default:
+			m.instances.Range(func(_ any, instance any) bool {
+				if i, ok := instance.(*Instance); ok {
+					if i.counter > 0 {
+						ensureIPAndSendGratuitous(i)
+					} else {
+						// this instance should not be advertised - delete the IP just in case...
+						if _, err := i.network.DeleteIP(); err != nil {
+							log.Error("[ARP manager] failed to delete IP", "address", i.network.IP(), "err", err)
+						}
+					}
+				}
+				return true
+			})
+		}
+		if m.config.ArpBroadcastRate < 500 {
+			log.Warn("[ARP manager] arp broadcast rate is too low", "rate (ms)", m.config.ArpBroadcastRate, "setting to (ms)", "3000")
+			m.config.ArpBroadcastRate = 3000
+		}
+		time.Sleep(time.Duration(m.config.ArpBroadcastRate) * time.Millisecond)
+	}
+}
+
+func (m *Manager) get(name string) (*Instance, error) {
+	i, exists := m.instances.Load(name)
+	if !exists {
+		return nil, nil
+	}
+	inst, ok := i.(*Instance)
+	if !ok {
+		return nil, fmt.Errorf("value for name %q is not of Instance pointer type", name)
+	}
+	return inst, nil
+}
+
+// ensureIPAndSendGratuitous - adds IP to the interface if missing, and send
+// either a gratuitous ARP or gratuitous NDP. Re-adds the interface if it is IPv6
+// and in a dadfailed state.
+func ensureIPAndSendGratuitous(instance *Instance) {
+	iface := instance.network.Interface()
+	ipString := instance.network.IP()
+
+	// Check if IP is dadfailed
+	if instance.network.IsDADFAIL() {
+		log.Warn("IP address is in dadfailed state, removing config", "ip", ipString, "interface", iface)
+		deleted, err := instance.network.DeleteIP()
+		if err != nil {
+			log.Warn(err.Error())
+		}
+		if deleted {
+			log.Info("deleted and recreating address with NODAD flag to skip DAD", "IP", ipString, "interface", iface)
+			// Re-add immediately without DAD check since we're recovering from DADFAILED
+			// The AddIP function will set IFA_F_NODAD flag for IPv6 addresses when skipDAD=true
+			if _, err := instance.network.AddIP(false, true); err != nil {
+				log.Error("failed to recreate address after DADFAILED", "IP", ipString, "interface", iface, "err", err)
+			} else {
+				log.Info("successfully recreated address after DADFAILED recovery", "IP", ipString, "interface", iface)
+			}
+		}
+		// Return early after DADFAILED recovery to avoid double IP addition
+		return
+	}
+
+	// Normal case: add IP with precheck and normal DAD process
+	if added, err := instance.network.AddIP(true, false); err != nil {
+		log.Warn(err.Error())
+	} else if added {
+		log.Warn("Re-applied the VIP configuration", "ip", ipString, "interface", iface)
+	}
+
+	if utils.IsIPv6(ipString) {
+		// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
+		if instance.ndp == nil {
+			log.Error("NDP responder was not created")
+		} else {
+			err := instance.ndp.SendGratuitous(ipString)
+			if err != nil {
+				log.Warn(err.Error())
+			}
+		}
+
+	} else {
+		// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
+		err := vip.ARPSendGratuitous(ipString, iface)
+		if err != nil {
+			log.Warn(err.Error())
+		}
+	}
+}

pkg/backend/backend.go

@@ -0,0 +1,87 @@
+package backend
+
+import (
+	"fmt"
+	"time"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/k8s"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+type Entry struct {
+	Addr    string
+	Port    uint16
+	IsLocal bool
+}
+
+type Map map[Entry]bool
+
+func (e *Entry) Check() bool {
+	var client *kubernetes.Clientset
+	var err error
+	var config *rest.Config
+
+	adminConfigPath := "/etc/kubernetes/admin.conf"
+	// TODO: add one more switch case of homeConfigPath if there is such scenario in future
+	// homeConfigPath := filepath.Join(os.Getenv("HOME"), ".kube", "config")
+
+	var k8sAddr string
+	if utils.IsIPv4(e.Addr) {
+		k8sAddr = fmt.Sprintf("%s:%v", e.Addr, e.Port)
+	} else {
+		k8sAddr = fmt.Sprintf("[%s]:%v", e.Addr, e.Port)
+	}
+
+	switch {
+	case utils.FileExists(adminConfigPath):
+		config, err = k8s.NewRestConfig(adminConfigPath, false, k8sAddr)
+		if err != nil {
+			log.Error("create k8s REST config", "path", adminConfigPath, "err", err)
+			return false
+		}
+	default:
+		config, err = k8s.NewRestConfig("", true, k8sAddr)
+		if err != nil {
+			log.Error("create k8s REST config", "err", err)
+			return false
+		}
+	}
+
+	client, err = k8s.NewClientset(config)
+	if err != nil {
+		log.Error("create k8s client", "err", err)
+		return false
+	}
+
+	_, err = client.DiscoveryClient.ServerVersion()
+	if err != nil {
+		log.Error("discover k8s version", "err", err)
+		return false
+	}
+	return true
+}
+
+func Watch(tickAction func(), interval int, stop chan struct{}) {
+	if interval <= 0 {
+		interval = 5
+	}
+
+	ticker := time.NewTicker(time.Second * time.Duration(interval))
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-stop:
+			ticker.Stop()
+			return
+		case <-ticker.C:
+			ticker.Stop()
+			tickAction()
+			ticker.Reset(time.Second * time.Duration(interval))
+		}
+	}
+}

pkg/bgp/hosts.go

@@ -2,9 +2,10 @@ package bgp
 
 import (
 	"context"
+	"fmt"
 	"net"
 
-	api "github.com/osrg/gobgp/api"
+	api "github.com/osrg/gobgp/v3/api"
 )
 
 // AddHost will update peers of a host
@@ -13,15 +14,20 @@ func (b *Server) AddHost(addr string) (err error) {
 	if err != nil {
 		return err
 	}
+
 	p := b.getPath(ip)
 	if p == nil {
-		return err
+		return fmt.Errorf("failed to get path for %v", ip)
 	}
 
 	_, err = b.s.AddPath(context.Background(), &api.AddPathRequest{
 		Path: p,
 	})
 
+	if err != nil {
+		return err
+	}
+
 	return
 }
 

pkg/bgp/peers.go

@@ -4,36 +4,31 @@ import (
 	"context"
 	"fmt"
 	"net"
-	"strconv"
-	"strings"
 
-	"github.com/golang/protobuf/ptypes"
-	"github.com/golang/protobuf/ptypes/any"
-	api "github.com/osrg/gobgp/api"
+	//nolint
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	api "github.com/osrg/gobgp/v3/api"
+
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/osrg/gobgp/v3/pkg/server"
+	"google.golang.org/protobuf/types/known/anypb"
 )
 
-//AddPeer will add peers to the BGP configuration
-func (b *Server) AddPeer(peer Peer) (err error) {
-	port := 179
-
-	if t := strings.SplitN(peer.Address, ":", 2); len(t) == 2 {
-		peer.Address = t[0]
-
-		if port, err = strconv.Atoi(t[1]); err != nil {
-			return fmt.Errorf("Unable to parse port '%s' as int: %s", t[1], err)
-		}
-	}
-
+// AddPeer will add peers to the BGP configuration
+func (b *Server) AddPeer(peer kubevip.BGPPeer) (err error) {
 	p := &api.Peer{
 		Conf: &api.PeerConf{
 			NeighborAddress: peer.Address,
-			PeerAs:          peer.AS,
+			PeerAsn:         peer.AS,
 			AuthPassword:    peer.Password,
 		},
 
 		Timers: &api.Timers{
 			Config: &api.TimersConfig{
-				ConnectRetry: 10,
+				ConnectRetry:      10,
+				HoldTime:          b.c.HoldTime,
+				KeepaliveInterval: b.c.KeepaliveInterval,
 			},
 		},
 
@@ -46,95 +41,204 @@ func (b *Server) AddPeer(peer Peer) (err error) {
 		Transport: &api.Transport{
 			MtuDiscovery:  true,
 			RemoteAddress: peer.Address,
-			RemotePort:    uint32(port),
+			RemotePort:    uint32(179),
 		},
 	}
 
-	// if b.c.SourceIP != "" {
-	// 	p.Transport.LocalAddress = b.c.SourceIP
-	// }
-
-	// if b.c.SourceIF != "" {
-	// 	p.Transport.BindInterface = b.c.SourceIF
-	// }
-
-	return b.s.AddPeer(context.Background(), &api.AddPeerRequest{
-		Peer: p,
-	})
-}
-
-func (b *Server) getPath(ip net.IP) *api.Path {
-	var pfxLen uint32 = 32
-	if ip.To4() == nil {
-		if !b.c.IPv6 {
-			return nil
+	if b.c.MpbgpNexthop != "" {
+		p.AfiSafis = []*api.AfiSafi{
+			{
+				Config: &api.AfiSafiConfig{
+					Family: &api.Family{
+						Afi:  api.Family_AFI_IP,
+						Safi: api.Family_SAFI_UNICAST,
+					},
+					Enabled: true,
+				},
+			},
+			{
+				Config: &api.AfiSafiConfig{
+					Family: &api.Family{
+						Afi:  api.Family_AFI_IP6,
+						Safi: api.Family_SAFI_UNICAST,
+					},
+					Enabled: true,
+				},
+			},
 		}
 
-		pfxLen = 128
+		peer.SetMpbgpOptions(b.c)
+
+		ipv4Address, ipv6Address, err := peer.FindMpbgpAddresses(p, b.c)
+		if err != nil {
+			return fmt.Errorf("failed to get MP-BGP addresses: %w", err)
+		}
+
+		mask := "128"
+		address := ipv4Address
+		family := api.Family_AFI_IP
+		if utils.IsIPv4(p.Conf.NeighborAddress) {
+			mask = "32"
+			address = ipv6Address
+			family = api.Family_AFI_IP6
+		}
+
+		err = b.s.AddDefinedSet(context.Background(), &api.AddDefinedSetRequest{
+			DefinedSet: &api.DefinedSet{
+				DefinedType: api.DefinedType_NEIGHBOR,
+				Name:        fmt.Sprintf("peer-%s", p.Conf.NeighborAddress),
+				List:        []string{fmt.Sprintf("%s/%s", p.Conf.NeighborAddress, mask)},
+			},
+		})
+		if err != nil {
+			return fmt.Errorf("failed to add defined set: %v", err)
+		}
+
+		if address != "" {
+			if err := insertPolicy(b.s, address, p, family); err != nil {
+				return fmt.Errorf("failed to add policy: %w", err)
+			}
+		}
+	} else {
+		if b.c.SourceIP != "" {
+			p.Transport.LocalAddress = b.c.SourceIP
+		}
+
+		if b.c.SourceIF != "" {
+			p.Transport.BindInterface = b.c.SourceIF
+		}
 	}
 
-	nlri, _ := ptypes.MarshalAny(&api.IPAddressPrefix{
-		Prefix:    ip.String(),
-		PrefixLen: pfxLen,
-	})
+	if err := b.s.AddPeer(context.Background(), &api.AddPeerRequest{Peer: p}); err != nil {
+		return fmt.Errorf("failed to add peer: %v", err)
+	}
 
-	a1, _ := ptypes.MarshalAny(&api.OriginAttribute{
+	return nil
+}
+
+func (b *Server) getPath(ip net.IP) (path *api.Path) {
+	isV6 := ip.To4() == nil
+
+	//nolint
+	originAttr, _ := anypb.New(&api.OriginAttribute{
 		Origin: 0,
 	})
 
-	var nh string
-	if b.c.NextHop != "" {
-		nh = b.c.NextHop
-	} else if b.c.SourceIP != "" {
-		nh = b.c.SourceIP
+	if !isV6 {
+		//nolint
+		nlri, _ := anypb.New(&api.IPAddressPrefix{
+			Prefix:    ip.String(),
+			PrefixLen: 32,
+		})
+
+		//nolint
+		nhAttr, _ := anypb.New(&api.NextHopAttribute{
+			NextHop: "0.0.0.0", // gobgp will fill this
+		})
+
+		path = &api.Path{
+			Family: &api.Family{
+				Afi:  api.Family_AFI_IP,
+				Safi: api.Family_SAFI_UNICAST,
+			},
+			Nlri:   nlri,
+			Pattrs: []*anypb.Any{originAttr, nhAttr},
+		}
 	} else {
-		nh = b.c.RouterID
-	}
+		//nolint
+		nlri, _ := anypb.New(&api.IPAddressPrefix{
+			Prefix:    ip.String(),
+			PrefixLen: 128,
+		})
 
-	a2, _ := ptypes.MarshalAny(&api.NextHopAttribute{
-		NextHop: nh,
-	})
-
-	return &api.Path{
-		Family: &api.Family{
-			Afi:  api.Family_AFI_IP,
+		v6Family := &api.Family{
+			Afi:  api.Family_AFI_IP6,
 			Safi: api.Family_SAFI_UNICAST,
-		},
-		Nlri:   nlri,
-		Pattrs: []*any.Any{a1, a2},
-	}
-}
-
-// ParseBGPPeerConfig - take a string and parses it into an array of peers
-func ParseBGPPeerConfig(config string) (bgpPeers []Peer, err error) {
-	peers := strings.Split(config, ",")
-	if len(peers) == 0 {
-		return nil, fmt.Errorf("No BGP Peer configurations found")
-	}
-
-	for x := range peers {
-		peer := strings.Split(peers[x], ":")
-		if len(peer) != 4 {
-			return nil, fmt.Errorf("BGP Peer configuration format error <host>:<AS>:<password>:<multihop>")
-		}
-		ASNumber, err := strconv.Atoi(peer[1])
-		if err != nil {
-			return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
-
-		}
-		multiHop, err := strconv.ParseBool(peer[3])
-		if err != nil {
-			return nil, fmt.Errorf("BGP MultiHop format error (true/false) [%s]", peer[1])
 		}
 
-		peerConfig := Peer{
-			Address:  peer[0],
-			AS:       uint32(ASNumber),
-			Password: peer[2],
-			MultiHop: multiHop,
-		}
+		//nolint
+		mpAttr, _ := anypb.New(&api.MpReachNLRIAttribute{
+			Family:   v6Family,
+			NextHops: []string{"::"}, // gobgp will fill this
+			Nlris:    []*anypb.Any{nlri},
+		})
 
-		bgpPeers = append(bgpPeers, peerConfig)
+		path = &api.Path{
+			Family: v6Family,
+			Nlri:   nlri,
+			Pattrs: []*anypb.Any{originAttr, mpAttr},
+		}
 	}
 	return
 }
+
+func insertPolicy(s *server.BgpServer, address string, p *api.Peer, family api.Family_Afi) error {
+	familyType := "v4"
+	if family == api.Family_AFI_IP6 {
+		familyType = "v6"
+	}
+
+	setName := fmt.Sprintf("peer-%s", p.Conf.NeighborAddress)
+	policyName := fmt.Sprintf("%s-%s", setName, familyType)
+
+	policy := &api.Policy{
+		Name: policyName,
+		Statements: []*api.Statement{
+			{
+				Conditions: &api.Conditions{
+					AfiSafiIn: []*api.Family{
+						{
+							Afi:  family,
+							Safi: api.Family_SAFI_UNICAST,
+						},
+					},
+					NeighborSet: &api.MatchSet{
+						Type: api.MatchSet_ANY,
+						Name: setName,
+					},
+				},
+				Actions: &api.Actions{
+					RouteAction: api.RouteAction_ACCEPT,
+					Nexthop: &api.NexthopAction{
+						Address: address,
+					},
+				},
+			},
+			{
+				Conditions: &api.Conditions{
+					NeighborSet: &api.MatchSet{
+						Type: api.MatchSet_ANY,
+						Name: setName,
+					},
+				},
+				Actions: &api.Actions{
+					RouteAction: api.RouteAction_ACCEPT,
+				},
+			},
+		},
+	}
+
+	err := s.AddPolicy(context.Background(), &api.AddPolicyRequest{
+		Policy: policy,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to add policy: %w", err)
+	}
+
+	err = s.AddPolicyAssignment(context.Background(), &api.AddPolicyAssignmentRequest{
+		Assignment: &api.PolicyAssignment{
+			Name:      "global",
+			Direction: api.PolicyDirection_EXPORT,
+			Policies: []*api.Policy{
+				{
+					Name: policy.Name,
+				},
+			},
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("failed to add policy assignment: %v", err)
+	}
+
+	return nil
+}

pkg/bgp/peers_test.go

@@ -0,0 +1,56 @@
+package bgp
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+)
+
+func TestParseBGPPeerConfig(t *testing.T) {
+	type args struct {
+		config string
+	}
+	tests := []struct {
+		name         string
+		args         args
+		wantBgpPeers []kubevip.BGPPeer
+		wantErr      bool
+	}{
+		{
+			name: "IPv4, default port",
+			args: args{config: "192.168.0.10:65000::false,192.168.0.11:65000::false"},
+			wantBgpPeers: []kubevip.BGPPeer{
+				{Address: "192.168.0.10", Port: 179, AS: 65000, MultiHop: false},
+				{Address: "192.168.0.11", Port: 179, AS: 65000, MultiHop: false},
+			},
+		},
+		{
+			name: "IPv4, different port",
+			args: args{config: "192.168.0.10:65000::false:180,192.168.0.11:65000::false:190"},
+			wantBgpPeers: []kubevip.BGPPeer{
+				{Address: "192.168.0.10", Port: 180, AS: 65000, MultiHop: false},
+				{Address: "192.168.0.11", Port: 190, AS: 65000, MultiHop: false},
+			},
+		},
+		{
+			name: "IPv6, multi-protocol",
+			args: args{config: "[fd00:1111:2222:3333:c7d9:7235:6bf7:5d52]:65501::false/mpbgp_nexthop=auto_sourceif"},
+			wantBgpPeers: []kubevip.BGPPeer{
+				{Address: "fd00:1111:2222:3333:c7d9:7235:6bf7:5d52", Port: 179, AS: 65501, MultiHop: false, MpbgpNexthop: "auto_sourceif"},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotBgpPeers, err := kubevip.ParseBGPPeerConfig(tt.args.config)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseBGPPeerConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotBgpPeers, tt.wantBgpPeers) {
+				t.Errorf("ParseBGPPeerConfig() = %v, want %v", gotBgpPeers, tt.wantBgpPeers)
+			}
+		})
+	}
+}

pkg/bgp/server.go

@@ -3,53 +3,96 @@ package bgp
 import (
 	"context"
 	"fmt"
-	"log"
 	"time"
 
-	api "github.com/osrg/gobgp/api"
-	gobgp "github.com/osrg/gobgp/pkg/server"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	api "github.com/osrg/gobgp/v3/api"
+	gobgp "github.com/osrg/gobgp/v3/pkg/server"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
+// Server manages a server object
+type Server struct {
+	s *gobgp.BgpServer
+	c *kubevip.BGPConfig
+
+	// This is a prometheus gauge indicating the state of the sessions.
+	// 1 means "ESTABLISHED", 0 means "NOT ESTABLISHED"
+	BGPSessionInfoGauge *prometheus.GaugeVec
+}
+
 // NewBGPServer takes a configuration and returns a running BGP server instance
-func NewBGPServer(c *Config) (b *Server, err error) {
+func NewBGPServer(c kubevip.BGPConfig) (b *Server, err error) {
 	if c.AS == 0 {
-		return nil, fmt.Errorf("You need to provide AS")
+		return nil, fmt.Errorf("you need to provide AS")
 	}
 
-	// if c.SourceIP != "" && c.SourceIF != "" {
-	// 	return nil, fmt.Errorf("SourceIP and SourceIF are mutually exclusive")
-	// }
+	if c.SourceIP != "" && c.SourceIF != "" {
+		return nil, fmt.Errorf("sourceIP and SourceIF are mutually exclusive")
+	}
 
 	if len(c.Peers) == 0 {
-		return nil, fmt.Errorf("You need to provide at least one peer")
+		return nil, fmt.Errorf("you need to provide at least one peer")
 	}
 
 	b = &Server{
 		s: gobgp.NewBgpServer(),
-		c: c,
+		c: &c,
+
+		BGPSessionInfoGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "kube_vip",
+			Subsystem: "manager",
+			Name:      "bgp_session_info",
+			Help:      "Display state of session by setting metric for label value with current state to 1",
+		}, []string{"state", "peer"}),
 	}
+	return
+}
+
+// Start starts the BGP server
+func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (err error) {
 	go b.s.Serve()
 
 	if err = b.s.StartBgp(context.Background(), &api.StartBgpRequest{
 		Global: &api.Global{
-			As:         c.AS,
-			RouterId:   c.RouterID,
+			Asn:        b.c.AS,
+			RouterId:   b.c.RouterID,
 			ListenPort: -1,
 		},
 	}); err != nil {
 		return
 	}
 
-	if err = b.s.MonitorPeer(context.Background(), &api.MonitorPeerRequest{}, func(p *api.Peer) { log.Println(p) }); err != nil {
+	if err = b.s.WatchEvent(context.Background(), &api.WatchEventRequest{Peer: &api.WatchEventRequest_Peer{}}, func(r *api.WatchEventResponse) {
+		if p := r.GetPeer(); p != nil && p.Type == api.WatchEventResponse_PeerEvent_STATE {
+			log.Info("[BGP]", "peer", p.String())
+			if peerStateChangeCallback != nil {
+				peerStateChangeCallback(p)
+			}
+		}
+	}); err != nil {
 		return
 	}
 
-	for _, p := range c.Peers {
+	for _, p := range b.c.Peers {
 		if err = b.AddPeer(p); err != nil {
 			return
 		}
 	}
 
+	if b.c.Zebra.Enabled {
+		if err = b.s.EnableZebra(context.Background(), &api.EnableZebraRequest{
+			Url:          b.c.Zebra.URL,
+			Version:      b.c.Zebra.Version,
+			SoftwareName: b.c.Zebra.SoftwareName,
+		}); err != nil {
+			log.Error(err.Error())
+			return
+		}
+	}
+
 	return
 }
 

pkg/bgp/types.go

@@ -1,29 +0,0 @@
-package bgp
-
-import gobgp "github.com/osrg/gobgp/pkg/server"
-
-// Peer defines a BGP Peer
-type Peer struct {
-	Address  string
-	AS       uint32
-	Password string
-	MultiHop bool
-}
-
-// Config defines the BGP server configuration
-type Config struct {
-	AS       uint32
-	RouterID string
-	NextHop  string
-	SourceIP string
-	SourceIF string
-
-	Peers []Peer
-	IPv6  bool
-}
-
-// Server manages a server object
-type Server struct {
-	s *gobgp.BgpServer
-	c *Config
-}

pkg/cluster/cluster.go

@@ -1,52 +1,80 @@
 package cluster
 
 import (
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	"github.com/plunder-app/kube-vip/pkg/vip"
-)
+	"sync"
 
-const leaderLogcount = 5
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/networkinterface"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
 
 // Cluster - The Cluster object manages the state of the cluster for a particular node
 type Cluster struct {
-	stateMachine FSM
-	stop         chan bool
-	completed    chan bool
-	Network      vip.Network
+	stop      chan bool
+	completed chan bool
+	once      sync.Once
+	Network   []vip.Network
+	arpMgr    *arp.Manager
 }
 
 // InitCluster - Will attempt to initialise all of the required settings for the cluster
-func InitCluster(c *kubevip.Config, disableVIP bool) (*Cluster, error) {
-
-	// TODO - Check for root (needed to netlink)
-	var network vip.Network
+func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Cluster, error) {
+	var networks []vip.Network
 	var err error
 
 	if !disableVIP {
 		// Start the Virtual IP Networking configuration
-		network, err = startNetworking(c)
+		networks, err = startNetworking(c, intfMgr)
 		if err != nil {
 			return nil, err
 		}
 	}
 	// Initialise the Cluster structure
 	newCluster := &Cluster{
-		Network: network,
+		Network: networks,
+		arpMgr:  arpMgr,
 	}
 
+	log.Debug("service security", "enabled", c.EnableServiceSecurity)
+
 	return newCluster, nil
 }
 
-func startNetworking(c *kubevip.Config) (vip.Network, error) {
+func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vip.Network, error) {
 	address := c.VIP
 
 	if c.Address != "" {
 		address = c.Address
 	}
 
-	network, err := vip.NewConfig(address, c.Interface, c.DDNS)
-	if err != nil {
-		return nil, err
+	addresses := vip.Split(address)
+
+	networks := []vip.Network{}
+	for _, addr := range addresses {
+		network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.RoutingTableID,
+			c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend,
+			c.EnableLoadBalancer, c.EnableServiceSecurity, intfMgr)
+		if err != nil {
+			return nil, err
+		}
+		networks = append(networks, network...)
 	}
-	return network, nil
+
+	return networks, nil
+}
+
+// Stop - Will stop the Cluster and release VIP if needed
+func (cluster *Cluster) Stop() {
+	// Close the stop channel, which will shut down the VIP (if needed)
+	if cluster.stop != nil {
+		cluster.once.Do(func() { // Ensure that the close channel can only ever be called once
+			close(cluster.stop)
+		})
+	}
+
+	// Wait until the completed channel is closed, signallign all shutdown tasks completed
+	<-cluster.completed
 }

pkg/cluster/clusterDDNS.go

@@ -2,7 +2,8 @@ package cluster
 
 import (
 	"context"
-	"github.com/plunder-app/kube-vip/pkg/vip"
+
+	"github.com/kube-vip/kube-vip/pkg/vip"
 )
 
 // StartDDNS should start go routine for dhclient to hold the lease for the IP
@@ -11,14 +12,13 @@ import (
 // during runtime if IP changes, startDDNS don't have to do reconfigure because
 // dnsUpdater already have the functionality to keep trying resolve the IP
 // and update the VIP configuration if it changes
-func (cluster *Cluster) StartDDNS(ctx context.Context) error {
-	ddnsMgr := vip.NewDDNSManager(ctx, cluster.Network)
+func (cluster *Cluster) StartDDNS(ctx context.Context, network vip.Network) error {
+	ddnsMgr := vip.NewDDNSManager(ctx, network)
 	ip, err := ddnsMgr.Start()
 	if err != nil {
 		return err
 	}
-
-	if err = cluster.Network.SetIP(ip); err != nil {
+	if err = network.SetIP(ip); err != nil {
 		return err
 	}
 

pkg/cluster/clusterLeader.go

@@ -1,456 +0,0 @@
-package cluster
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/signal"
-	"path/filepath"
-	"syscall"
-	"time"
-
-	"github.com/plunder-app/kube-vip/pkg/bgp"
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	leaderelection "github.com/plunder-app/kube-vip/pkg/leaderElection"
-	"github.com/plunder-app/kube-vip/pkg/loadbalancer"
-	"github.com/plunder-app/kube-vip/pkg/packet"
-
-	"github.com/plunder-app/kube-vip/pkg/vip"
-
-	"github.com/packethost/packngo"
-
-	log "github.com/sirupsen/logrus"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/tools/leaderelection/resourcelock"
-)
-
-const plunderLock = "plndr-cp-lock"
-
-// Manager degines the manager of the load-balancing services
-type Manager struct {
-	KubernetesClient *kubernetes.Clientset
-}
-
-// NewManager will create a new managing object
-func NewManager(path string, inCluster bool, port int) (*Manager, error) {
-	var clientset *kubernetes.Clientset
-	if inCluster {
-		// This will attempt to load the configuration when running within a POD
-		cfg, err := rest.InClusterConfig()
-		if err != nil {
-			return nil, fmt.Errorf("error creating kubernetes client config: %s", err.Error())
-		}
-		clientset, err = kubernetes.NewForConfig(cfg)
-
-		if err != nil {
-			return nil, fmt.Errorf("error creating kubernetes client: %s", err.Error())
-		}
-		// use the current context in kubeconfig
-	} else {
-		if path == "" {
-			path = filepath.Join(os.Getenv("HOME"), ".kube", "config")
-		}
-		config, err := clientcmd.BuildConfigFromFlags("", path)
-		if err != nil {
-			panic(err.Error())
-		}
-
-		// We modify the config so that we can always speak to the correct host
-		id, err := os.Hostname()
-		if err != nil {
-			return nil, err
-		}
-
-		config.Host = fmt.Sprintf("%s:%v", id, port)
-		clientset, err = kubernetes.NewForConfig(config)
-
-		if err != nil {
-			return nil, fmt.Errorf("error creating kubernetes client: %s", err.Error())
-		}
-	}
-
-	return &Manager{
-		KubernetesClient: clientset,
-	}, nil
-}
-
-// StartLeaderCluster - Begins a running instance of the Raft cluster
-func (cluster *Cluster) StartLeaderCluster(c *kubevip.Config, sm *Manager, bgpServer *bgp.Server) error {
-
-	id, err := os.Hostname()
-	if err != nil {
-		return err
-	}
-
-	log.Infof("Beginning cluster membership, namespace [%s], lock name [%s], id [%s]", c.Namespace, plunderLock, id)
-
-	// we use the Lease lock type since edits to Leases are less common
-	// and fewer objects in the cluster watch "all Leases".
-	lock := &resourcelock.LeaseLock{
-		LeaseMeta: metav1.ObjectMeta{
-			Name:      plunderLock,
-			Namespace: c.Namespace,
-		},
-		Client: sm.KubernetesClient.CoordinationV1(),
-		LockConfig: resourcelock.ResourceLockConfig{
-			Identity: id,
-		},
-	}
-
-	// use a Go context so we can tell the leaderelection code when we
-	// want to step down
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	// use a Go context so we can tell the arp loop code when we
-	// want to step down
-	ctxArp, cancelArp := context.WithCancel(context.Background())
-	defer cancelArp()
-
-	// use a Go context so we can tell the dns loop code when we
-	// want to step down
-	ctxDNS, cancelDNS := context.WithCancel(context.Background())
-	defer cancelDNS()
-
-	// listen for interrupts or the Linux SIGTERM signal and cancel
-	// our context, which the leader election code will observe and
-	// step down
-	signalChan := make(chan os.Signal, 1)
-	// Add Notification for Userland interrupt
-	signal.Notify(signalChan, syscall.SIGINT)
-
-	// Add Notification for SIGTERM (sent from Kubernetes)
-	signal.Notify(signalChan, syscall.SIGTERM)
-
-	// Add Notification for SIGKILL (sent from Kubernetes)
-	signal.Notify(signalChan, syscall.SIGKILL)
-
-	go func() {
-		<-signalChan
-		log.Info("Received termination, signaling shutdown")
-		// Cancel the context, which will in turn cancel the leadership
-		cancel()
-		// Cancel the arp context, which will in turn stop any broadcasts
-	}()
-
-	// (attempt to) Remove the virtual IP, incase it already exists
-	cluster.Network.DeleteIP()
-
-	// Managers for Vip load balancers and none-vip loadbalancers
-	nonVipLB := loadbalancer.LBManager{}
-	VipLB := loadbalancer.LBManager{}
-
-	// Defer a function to check if the bgpServer has been created and if so attempt to close it
-	defer func() {
-		if bgpServer != nil {
-			bgpServer.Close()
-		}
-	}()
-
-	// If Packet is enabled then we can begin our preperation work
-	var packetClient *packngo.Client
-	if c.EnableMetal {
-		packetClient, err = packngo.NewClient()
-		if err != nil {
-			log.Error(err)
-		}
-
-		// We're using Packet with BGP, popuplate the Peer information from the API
-		if c.EnableBGP {
-			log.Infoln("Looking up the BGP configuration from packet")
-			err = packet.BGPLookup(packetClient, c)
-			if err != nil {
-				log.Error(err)
-			}
-		}
-	}
-
-	if c.EnableBGP {
-		// Lets start BGP
-		log.Info("Starting the BGP server to adverise VIP routes to VGP peers")
-		bgpServer, err = bgp.NewBGPServer(&c.BGPConfig)
-		if err != nil {
-			log.Error(err)
-		}
-	}
-
-	if c.EnableLoadBalancer {
-
-		// Iterate through all Configurations
-		if len(c.LoadBalancers) != 0 {
-			for x := range c.LoadBalancers {
-				// If the load balancer doesn't bind to the VIP
-				if c.LoadBalancers[x].BindToVip == false {
-					err = nonVipLB.Add("", &c.LoadBalancers[x])
-					if err != nil {
-						log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-					}
-				}
-			}
-		}
-	}
-	// start the leader election code loop
-	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
-		Lock: lock,
-		// IMPORTANT: you MUST ensure that any code you have that
-		// is protected by the lease must terminate **before**
-		// you call cancel. Otherwise, you could have a background
-		// loop still running and another process could
-		// get elected before your background loop finished, violating
-		// the stated goal of the lease.
-		ReleaseOnCancel: true,
-		LeaseDuration:   time.Duration(c.LeaseDuration) * time.Second,
-		RenewDeadline:   time.Duration(c.RenewDeadline) * time.Second,
-		RetryPeriod:     time.Duration(c.RetryPeriod) * time.Second,
-		Callbacks: leaderelection.LeaderCallbacks{
-			OnStartedLeading: func(ctx context.Context) {
-				// we're notified when we start
-				log.Info("This node is starting with leadership of the cluster")
-				// setup ddns first
-				// for first time, need to wait until IP is allocated from DHCP
-				if cluster.Network.IsDDNS() {
-					if err := cluster.StartDDNS(ctxDNS); err != nil {
-						log.Error(err)
-					}
-				}
-
-				// start the dns updater if address is dns
-				if cluster.Network.IsDNS() {
-					log.Infof("starting the DNS updater for the address %s", cluster.Network.DNSName())
-					ipUpdater := vip.NewIPUpdater(cluster.Network)
-					ipUpdater.Run(ctxDNS)
-				}
-
-				err = cluster.Network.AddIP()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				if c.EnableMetal {
-					// We're not using Packet with BGP
-					if !c.EnableBGP {
-						// Attempt to attach the EIP in the standard manner
-						log.Debugf("Attaching the Packet EIP through the API to this host")
-						err = packet.AttachEIP(packetClient, c, id)
-						if err != nil {
-							log.Error(err)
-						}
-					}
-				}
-
-				if c.EnableBGP {
-					// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
-					cidrVip := fmt.Sprintf("%s/%s", cluster.Network.IP(), c.VIPCIDR)
-					log.Debugf("Attempting to advertise the address [%s] over BGP", cidrVip)
-
-					err = bgpServer.AddHost(cidrVip)
-					if err != nil {
-						log.Error(err)
-					}
-				}
-
-				if c.EnableLoadBalancer {
-					// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-					for x := range c.LoadBalancers {
-
-						if c.LoadBalancers[x].BindToVip == true {
-							err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-							if err != nil {
-								log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-
-								// Stop all load balancers associated with the VIP
-								err = VipLB.StopAll()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-
-								err = cluster.Network.DeleteIP()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-							}
-						}
-					}
-				}
-
-				if c.EnableARP == true {
-					ctxArp, cancelArp = context.WithCancel(context.Background())
-
-					ipString := cluster.Network.IP()
-
-					var ndp *vip.NdpResponder
-					if vip.IsIPv6(ipString) {
-						ndp, err = vip.NewNDPResponder(c.Interface)
-						if err != nil {
-							log.Fatalf("failed to create new NDP Responder")
-						}
-					}
-
-					go func(ctx context.Context) {
-						if ndp != nil {
-							defer ndp.Close()
-						}
-
-						for {
-							select {
-							case <-ctx.Done(): // if cancel() execute
-								return
-							default:
-								if vip.IsIPv4(ipString) {
-									// Gratuitous ARP, will broadcast to new MAC <-> IP
-									err := vip.ARPSendGratuitous(ipString, c.Interface)
-									if err != nil {
-										log.Warnf("%v", err)
-									}
-								} else {
-									// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
-									err := ndp.SendGratuitous(ipString)
-									if err != nil {
-										log.Warnf("%v", err)
-									}
-								}
-							}
-							time.Sleep(3 * time.Second)
-						}
-					}(ctxArp)
-				}
-			},
-			OnStoppedLeading: func() {
-				// we can do cleanup here
-				log.Info("This node is becoming a follower within the cluster")
-
-				// Stop the dns context
-				cancelDNS()
-				// Stop the Arp context if it is running
-				cancelArp()
-
-				// Stop the BGP server
-				if bgpServer != nil {
-					err = bgpServer.Close()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-
-				// Stop all load balancers associated with the VIP
-				err = VipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				err = cluster.Network.DeleteIP()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				log.Fatal("lost leadership, restarting kube-vip")
-			},
-			OnNewLeader: func(identity string) {
-				// we're notified when new leader elected
-				log.Infof("Node [%s] is assuming leadership of the cluster", identity)
-
-				if identity == id {
-					// We have the lock
-				}
-			},
-		},
-	})
-
-	return nil
-}
-
-// TODO - refactor an active machine func(), this will replace the singleNode code and have a single code block
-
-// func (cluster *Cluster) active(c *kubevip.Config) error {
-// 	// we're notified when we start
-// 	log.Info("This node is starting with leadership of the cluster")
-// 	// setup ddns first
-// 	// for first time, need to wait until IP is allocated from DHCP
-// 	if cluster.Network.IsDDNS() {
-// 		if err := cluster.StartDDNS(ctxDns); err != nil {
-// 			log.Error(err)
-// 		}
-// 	}
-
-// 	// start the dns updater if address is dns
-// 	if cluster.Network.IsDNS() {
-// 		log.Infof("starting the DNS updater for the address %s", cluster.Network.DNSName())
-// 		ipUpdater := vip.NewIPUpdater(cluster.Network)
-// 		ipUpdater.Run(ctxDns)
-// 	}
-
-// 	err := cluster.Network.AddIP()
-// 	if err != nil {
-// 		log.Warnf("%v", err)
-// 	}
-
-// 	if c.EnablePacket {
-// 		// We're not using Packet with BGP
-// 		if !c.EnableBGP {
-// 			// Attempt to attach the EIP in the standard manner
-// 			log.Debugf("Attaching the Packet EIP through the API to this host")
-// 			err = packet.AttachEIP(packetClient, c, id)
-// 			if err != nil {
-// 				log.Error(err)
-// 			}
-// 		}
-// 	}
-
-// 	if c.EnableBGP {
-// 		// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
-// 		cidrVip := fmt.Sprintf("%s/%s", cluster.Network.IP(), c.VIPCIDR)
-// 		log.Debugf("Attempting to advertise the address [%s] over BGP", cidrVip)
-
-// 		err = bgpServer.AddHost(cidrVip)
-// 		if err != nil {
-// 			log.Error(err)
-// 		}
-// 	}
-
-// 	if c.EnableLoadBalancer {
-// 		// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-// 		for x := range c.LoadBalancers {
-
-// 			if c.LoadBalancers[x].BindToVip == true {
-// 				err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-// 				if err != nil {
-// 					log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-
-// 					// Stop all load balancers associated with the VIP
-// 					err = VipLB.StopAll()
-// 					if err != nil {
-// 						log.Warnf("%v", err)
-// 					}
-
-// 					err = cluster.Network.DeleteIP()
-// 					if err != nil {
-// 						log.Warnf("%v", err)
-// 					}
-// 				}
-// 			}
-// 		}
-// 	}
-
-// 	if c.EnableARP == true {
-// 		ctxArp, cancelArp = context.WithCancel(context.Background())
-
-// 		go func(ctx context.Context) {
-// 			for {
-// 				select {
-// 				case <-ctx.Done(): // if cancel() execute
-// 					return
-// 				default:
-// 					// Gratuitous ARP, will broadcast to new MAC <-> IP
-// 					err = vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-// 					if err != nil {
-// 						log.Warnf("%v", err)
-// 					}
-// 				}
-// 				time.Sleep(3 * time.Second)
-// 			}
-// 		}(ctxArp)
-// 	}
-// }

pkg/cluster/clusterLeaderElection.go

@@ -0,0 +1,451 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/etcd"
+	"github.com/kube-vip/kube-vip/pkg/k8s"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+
+	log "log/slog"
+
+	clientv3 "go.etcd.io/etcd/client/v3"
+	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	watchtools "k8s.io/client-go/tools/watch"
+)
+
+// Manager degines the manager of the load-balancing services
+type Manager struct {
+	KubernetesClient   *kubernetes.Clientset
+	RetryWatcherClient *kubernetes.Clientset
+	// This channel is used to signal a shutdown
+	SignalChan chan os.Signal
+
+	EtcdClient *clientv3.Client
+}
+
+// NewManager will create a new managing object
+func NewManager(path string, inCluster bool, port int) (*Manager, error) {
+	var hostname string
+
+	// If inCluster is set then it will likely have started as a static pod or won't have the
+	// VIP up before trying to connect to the API server, we set the API endpoint to this machine to
+	// ensure connectivity. Else if the path passed is empty and not running in the cluster,
+	// attempt to look for a kubeconfig in the default HOME dir.
+
+	hostname = fmt.Sprintf("kubernetes:%v", port)
+
+	if len(path) == 0 && !inCluster {
+		path = filepath.Join(os.Getenv("HOME"), ".kube", "config")
+
+		// We modify the config so that we can always speak to the correct host
+		id, err := os.Hostname()
+		if err != nil {
+			return nil, err
+		}
+
+		hostname = fmt.Sprintf("%s:%v", id, port)
+	}
+
+	config, err := k8s.NewRestConfig(path, inCluster, hostname)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create k8s REST config: %w", err)
+	}
+
+	clientset, err := k8s.NewClientset(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating a new k8s clientset: %v", err)
+	}
+
+	rwConfig, err := k8s.NewRestConfig(path, inCluster, hostname)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create k8s REST config for retryClientSet: %w", err)
+	}
+
+	rwConfig.Timeout = 0 // empty value to disable the timeout
+	rwClientSet, err := k8s.NewClientset(rwConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create k8s client for retry watcher: %w", err)
+	}
+
+	return &Manager{
+		KubernetesClient:   clientset,
+		RetryWatcherClient: rwClientSet,
+	}, nil
+}
+
+// StartCluster - Begins a running instance of the Leader Election cluster
+func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *bgp.Server) error {
+	var err error
+
+	log.Info("cluster membership", "namespace", c.Namespace, "lock", c.LeaseName, "id", c.NodeName)
+
+	// use a Go context so we can tell the leaderelection code when we
+	// want to step down
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// use a Go context so we can tell the arp loop code when we
+	// want to step down
+	ctxArp, cancelArp := context.WithCancel(context.Background())
+	defer cancelArp()
+
+	// use a Go context so we can tell the dns loop code when we
+	// want to step down
+	ctxDNS, cancelDNS := context.WithCancel(context.Background())
+	defer cancelDNS()
+
+	// listen for interrupts or the Linux SIGTERM signal and cancel
+	// our context, which the leader election code will observe and
+	// step down
+	signalChan := make(chan os.Signal, 1)
+	// Add Notification for Userland interrupt
+	signal.Notify(signalChan, syscall.SIGINT)
+
+	// Add Notification for SIGTERM (sent from Kubernetes)
+	signal.Notify(signalChan, syscall.SIGTERM)
+
+	go func() {
+		<-signalChan
+		log.Info("Received termination, signaling cluster shutdown")
+		// Cancel the context, which will in turn cancel the leadership
+		cancel()
+		// Cancel the arp context, which will in turn stop any broadcasts
+	}()
+
+	// (attempt to) Remove the virtual IP, in case it already exists
+
+	for i := range cluster.Network {
+		deleted, err := cluster.Network[i].DeleteIP()
+		if err != nil {
+			log.Error("could not delete virtualIP", "err", err)
+		}
+		if deleted {
+			log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+		}
+	}
+
+	// Defer a function to check if the bgpServer has been created and if so attempt to close it
+	defer func() {
+		if bgpServer != nil {
+			bgpServer.Close()
+		}
+	}()
+
+	if c.EnableBGP && bgpServer == nil {
+		// Lets start BGP
+		log.Info("Starting the BGP server to advertise VIP routes to VGP peers")
+		bgpServer, err = bgp.NewBGPServer(c.BGPConfig)
+		if err != nil {
+			log.Error("new BGP server", "err", err)
+		}
+		if err := bgpServer.Start(nil); err != nil {
+			log.Error("starting BGP server", "err", err)
+		}
+	}
+
+	run := &runConfig{
+		config:  c,
+		leaseID: c.NodeName,
+		sm:      sm,
+		onStartedLeading: func(ctx context.Context) { //nolint TODO: potential clean code
+			// When we become leader, ensure we can take over VIPs even if they're preserved on other nodes
+			if c.PreserveVIPOnLeadershipLoss {
+				log.Info("Becoming leader with VIP preservation enabled - ensuring VIP takeover")
+				// Force add the VIPs (this will work even if they exist due to the precheck logic)
+				for i := range cluster.Network {
+					added, err := cluster.Network[i].AddIP(true, false)
+					if err != nil {
+						log.Error("failed to ensure VIP on leader takeover", "vip", cluster.Network[i].IP(), "err", err)
+					} else if added {
+						log.Info("took over VIP as new leader", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+					} else {
+						log.Info("VIP already configured on interface", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+					}
+				}
+			}
+
+			// Start ARP advertisements now that we have leadership
+			log.Info("Start ARP/NDP advertisement")
+			go cluster.arpMgr.StartAdvertisement(ctxArp)
+
+			// As we're leading lets start the vip service
+			err := cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, cancel)
+			if err != nil {
+				log.Error("starting VIP service on leader", "err", err)
+			}
+		},
+		onStoppedLeading: func() {
+			// we can do cleanup here
+			log.Info("This node is becoming a follower within the cluster")
+
+			// Stop the dns context
+			cancelDNS()
+			// Stop the Arp context if it is running
+			cancelArp()
+
+			// Stop the BGP server
+			if bgpServer != nil {
+				err := bgpServer.Close()
+				if err != nil {
+					log.Warn("close BGP server", "err", err)
+				}
+			}
+
+			// Handle VIP cleanup based on configuration
+			if c.PreserveVIPOnLeadershipLoss {
+				// For IPv6, we must remove VIPs immediately to avoid DAD failures on the new leader
+				// IPv6 Duplicate Address Detection will fail if the new leader tries to add an IP that is still present on this node's interface
+				// We need to check each VIP individually and only remove IPv6 VIPs
+				for i := range cluster.Network {
+					if utils.IsIPv6(cluster.Network[i].IP()) {
+						log.Info("Removing IPv6 VIP immediately (required to prevent DAD failures on new leader)", "ip", cluster.Network[i].IP())
+						deleted, err := cluster.Network[i].DeleteIP()
+						if err != nil {
+							log.Warn("delete VIP", "err", err)
+						}
+						if deleted {
+							log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+						}
+					} else {
+						log.Info("Preserving IPv4 VIP address on interface, only stopped ARP broadcasting", "ip", cluster.Network[i].IP())
+					}
+				}
+			} else {
+				// Legacy behavior: delete VIP addresses on leadership loss
+				log.Info("Deleting VIP addresses on leadership loss (legacy behavior)")
+				for i := range cluster.Network {
+					deleted, err := cluster.Network[i].DeleteIP()
+					if err != nil {
+						log.Warn("delete VIP", "err", err)
+					}
+					if deleted {
+						log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+					}
+				}
+			}
+
+			log.Error("lost leadership, restarting kube-vip")
+			panic("") // TODO - we could also return here
+		},
+		onNewLeader: func(identity string) {
+			// we're notified when new leader elected
+			log.Info("New leader", "leader", identity)
+
+			// If we're not the new leader and we have VIPs preserved from previous leadership,
+			// we need to clean them up to avoid conflicts.
+			if identity != c.NodeName && c.PreserveVIPOnLeadershipLoss {
+				log.Info("Cleaning up preserved VIPs as another node became leader", "new_leader", identity)
+				for i := range cluster.Network {
+					deleted, err := cluster.Network[i].DeleteIP()
+					if err != nil {
+						log.Warn("failed to cleanup preserved VIP", "vip", cluster.Network[i].IP(), "err", err)
+					}
+					if deleted {
+						log.Info("cleaned up preserved VIP to avoid conflict", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface(), "new_leader", identity)
+					} else {
+						log.Debug("VIP was not present on this node", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+					}
+				}
+			}
+		},
+	}
+
+	switch c.LeaderElectionType {
+	case "kubernetes", "":
+		cluster.runKubernetesLeaderElectionOrDie(ctx, run)
+	case "etcd":
+		cluster.runEtcdLeaderElectionOrDie(ctx, run)
+	default:
+		log.Info(fmt.Sprintf("LeaderElectionMode %s not supported, exiting", c.LeaderElectionType))
+	}
+
+	return nil
+}
+
+type runConfig struct {
+	config  *kubevip.Config
+	leaseID string
+	sm      *Manager
+
+	// onStartedLeading is called when this member starts leading.
+	onStartedLeading func(context.Context)
+	// onStoppedLeading is called when this member stops leading.
+	onStoppedLeading func()
+	// onNewLeader is called when the client observes a leader that is
+	// not the previously observed leader. This includes the first observed
+	// leader when the client starts.
+	onNewLeader func(identity string)
+}
+
+func (cluster *Cluster) runKubernetesLeaderElectionOrDie(ctx context.Context, run *runConfig) {
+	// we use the Lease lock type since edits to Leases are less common
+	// and fewer objects in the cluster watch "all Leases".
+	lock := &resourcelock.LeaseLock{
+		LeaseMeta: metav1.ObjectMeta{
+			Name:        run.config.LeaseName,
+			Namespace:   run.config.Namespace,
+			Annotations: run.config.LeaseAnnotations,
+		},
+		Client: run.sm.KubernetesClient.CoordinationV1(),
+		LockConfig: resourcelock.ResourceLockConfig{
+			Identity: run.leaseID,
+		},
+	}
+
+	// start the leader election code loop
+	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
+		Lock: lock,
+		// IMPORTANT: you MUST ensure that any code you have that
+		// is protected by the lease must terminate **before**
+		// you call cancel. Otherwise, you could have a background
+		// loop still running and another process could
+		// get elected before your background loop finished, violating
+		// the stated goal of the lease.
+		ReleaseOnCancel: true,
+		LeaseDuration:   time.Duration(run.config.LeaseDuration) * time.Second,
+		RenewDeadline:   time.Duration(run.config.RenewDeadline) * time.Second,
+		RetryPeriod:     time.Duration(run.config.RetryPeriod) * time.Second,
+		Callbacks: leaderelection.LeaderCallbacks{
+			OnStartedLeading: run.onStartedLeading,
+			OnStoppedLeading: run.onStoppedLeading,
+			OnNewLeader:      run.onNewLeader,
+		},
+	})
+}
+
+func (cluster *Cluster) runEtcdLeaderElectionOrDie(ctx context.Context, run *runConfig) {
+	etcd.RunElectionOrDie(ctx, &etcd.LeaderElectionConfig{
+		EtcdConfig:           etcd.ClientConfig{Client: run.sm.EtcdClient},
+		Name:                 run.config.LeaseName,
+		MemberID:             run.leaseID,
+		LeaseDurationSeconds: int64(run.config.LeaseDuration),
+		Callbacks: etcd.LeaderCallbacks{
+			OnStartedLeading: run.onStartedLeading,
+			OnStoppedLeading: run.onStoppedLeading,
+			OnNewLeader:      run.onNewLeader,
+		},
+	})
+}
+
+func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
+	// Use a restartable watcher, as this should help in the event of etcd or timeout issues
+	log.Info("Kube-Vip is watching nodes for control-plane labels")
+
+	listOptions := metav1.ListOptions{
+		LabelSelector: "node-role.kubernetes.io/control-plane",
+	}
+
+	rw, err := watchtools.NewRetryWatcherWithContext(ctxArp, "1", &cache.ListWatch{
+		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
+			return sm.RetryWatcherClient.CoreV1().Nodes().Watch(context.Background(), listOptions)
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("error creating label watcher: %s", err.Error())
+	}
+
+	go func() {
+		<-sm.SignalChan
+		log.Info("Received termination, signaling shutdown")
+		// Cancel the context
+		rw.Stop()
+	}()
+
+	ch := rw.ResultChan()
+	// defer rw.Stop()
+
+	for event := range ch {
+		// We need to inspect the event and get ResourceVersion out of it
+		switch event.Type {
+		case watch.Added, watch.Modified:
+			node, ok := event.Object.(*v1.Node)
+			if !ok {
+				return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
+			}
+			// Find the node IP address (this isn't foolproof)
+			for x := range node.Status.Addresses {
+				if node.Status.Addresses[x].Type == v1.NodeInternalIP {
+					if checkIfNodeIsReady(node) {
+						err = lb.AddBackend(node.Status.Addresses[x].Address, port)
+						if err != nil {
+							log.Error("add IPVS backend", "err", err)
+						}
+					} else {
+						err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
+						if err != nil {
+							log.Error("remove IPVS backend", "err", err)
+						}
+					}
+				}
+			}
+		case watch.Deleted:
+			node, ok := event.Object.(*v1.Node)
+			if !ok {
+				return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
+			}
+
+			// Find the node IP address (this isn't foolproof)
+			for x := range node.Status.Addresses {
+				if node.Status.Addresses[x].Type == v1.NodeInternalIP {
+					err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
+					if err != nil {
+						log.Error("Del IPVS backend", "err", err)
+					}
+				}
+			}
+
+			log.Info("Node deleted", "name", node.Name)
+
+		case watch.Bookmark:
+			// Un-used
+		case watch.Error:
+			log.Error("Error attempting to watch Kubernetes Nodes")
+
+			// This round trip allows us to handle unstructured status
+			errObject := apierrors.FromObject(event.Object)
+			statusErr, ok := errObject.(*apierrors.StatusError)
+			if !ok {
+				log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
+			}
+
+			status := statusErr.ErrStatus
+			log.Error("watcher", "status", status)
+		default:
+		}
+	}
+
+	log.Info("Exiting Node watcher")
+	return nil
+}
+
+func checkIfNodeIsReady(node *v1.Node) bool {
+	if node == nil {
+		return false
+	}
+	for _, condition := range node.Status.Conditions {
+		if condition.Type == v1.NodeReady {
+			if condition.Status == v1.ConditionTrue {
+				return true
+			}
+		}
+	}
+	return false
+}

pkg/cluster/clusterRaft.go

@@ -1,296 +0,0 @@
-package cluster
-
-import (
-	"fmt"
-	"net"
-	"time"
-
-	"github.com/hashicorp/raft"
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	"github.com/plunder-app/kube-vip/pkg/loadbalancer"
-	"github.com/plunder-app/kube-vip/pkg/vip"
-	log "github.com/sirupsen/logrus"
-)
-
-// StartRaftCluster - Begins a running instance of the Raft cluster
-func (cluster *Cluster) StartRaftCluster(c *kubevip.Config) error {
-
-	// Create local configuration address
-	localAddress := fmt.Sprintf("%s:%d", c.LocalPeer.Address, c.LocalPeer.Port)
-
-	// Begin the Raft configuration
-	config := raft.DefaultConfig()
-	config.LocalID = raft.ServerID(c.LocalPeer.ID)
-	logger := log.StandardLogger().Writer()
-	config.LogOutput = logger
-
-	// Initialize communication
-	address, err := net.ResolveTCPAddr("tcp", localAddress)
-	if err != nil {
-		return err
-	}
-
-	// Create transport
-	transport, err := raft.NewTCPTransport(localAddress, address, 3, 10*time.Second, logger)
-	if err != nil {
-		return err
-	}
-
-	// Create Raft structures
-	snapshots := raft.NewInmemSnapshotStore()
-	logStore := raft.NewInmemStore()
-	stableStore := raft.NewInmemStore()
-
-	// Cluster configuration
-	configuration := raft.Configuration{}
-
-	// Add Local Peer
-	configuration.Servers = append(configuration.Servers, raft.Server{
-		ID:      raft.ServerID(c.LocalPeer.ID),
-		Address: raft.ServerAddress(fmt.Sprintf("%s:%d", c.LocalPeer.Address, c.LocalPeer.Port))})
-
-	// If we want to start a node as leader then we will not add any remote peers, this will leave this as a cluster of one
-	// The remotePeers will add themselves to the cluster as they're added
-	if c.StartAsLeader != true {
-		for x := range c.RemotePeers {
-			// Make sure that we don't add in this server twice
-			if c.LocalPeer.Address != c.RemotePeers[x].Address {
-
-				// Build the address from the peer configuration
-				peerAddress := fmt.Sprintf("%s:%d", c.RemotePeers[x].Address, c.RemotePeers[x].Port)
-
-				// Set this peer into the raft configuration
-				configuration.Servers = append(configuration.Servers, raft.Server{
-					ID:      raft.ServerID(c.RemotePeers[x].ID),
-					Address: raft.ServerAddress(peerAddress)})
-			}
-		}
-		log.Info("This node will attempt to start as Follower")
-	} else {
-		log.Info("This node will attempt to start as Leader")
-	}
-
-	// Bootstrap cluster
-	if err := raft.BootstrapCluster(config, logStore, stableStore, snapshots, transport, configuration); err != nil {
-		return err
-	}
-
-	// Create RAFT instance
-	raftServer, err := raft.NewRaft(config, cluster.stateMachine, logStore, stableStore, snapshots, transport)
-	if err != nil {
-		return err
-	}
-
-	cluster.stop = make(chan bool, 1)
-	cluster.completed = make(chan bool, 1)
-	ticker := time.NewTicker(time.Second)
-	isLeader := c.StartAsLeader
-
-	// (attempt to) Remove the virtual IP, incase it already exists
-	cluster.Network.DeleteIP()
-
-	// leader log broadcast - this counter is used to stop flooding STDOUT with leader log entries
-	var leaderbroadcast int
-	// Managers for Vip load balancers and none-vip loadbalancers
-	nonVipLB := loadbalancer.LBManager{}
-	VipLB := loadbalancer.LBManager{}
-
-	// Iterate through all Configurations
-	for x := range c.LoadBalancers {
-		// If the load balancer doesn't bind to the VIP
-		if c.LoadBalancers[x].BindToVip == false {
-			err = nonVipLB.Add("", &c.LoadBalancers[x])
-			if err != nil {
-				log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-			}
-		}
-	}
-
-	// On a cold start the node will sleep for 5 seconds to ensure that leader elections are complete
-	log.Infoln("This instance will wait approximately 5 seconds, from cold start to ensure cluster elections are complete")
-	time.Sleep(time.Second * 5)
-
-	go func() {
-		for {
-			if c.AddPeersAsBackends == true {
-				// Get addresses and change backends
-
-				// c.LoadBalancers[0].Backends
-				// for x := range raftServer.GetConfiguration().Configuration().Servers {
-				// 	raftServer.GetConfiguration().Configuration().Servers[x].Address
-				// }
-
-			}
-			// Broadcast the current leader on this node if it's the correct time (every leaderLogcount * time.Second)
-			if leaderbroadcast == leaderLogcount {
-				log.Infof("The Node [%s] is leading", raftServer.Leader())
-				// Reset the timer
-				leaderbroadcast = 0
-
-				// ensure that if this node is the leader, it is set as the leader
-				if localAddress == string(raftServer.Leader()) {
-					// Re-broadcast arp to ensure network stays up to date
-					if c.EnableARP == true {
-						// Gratuitous ARP, will broadcast to new MAC <-> IP
-						err = vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-					}
-					if !isLeader {
-						log.Infoln("This node is leading, but isnt the leader (correcting)")
-						isLeader = true
-					}
-				} else {
-					// (attempt to) Remove the virtual IP, incase it already exists to keep nodes clean
-					cluster.Network.DeleteIP()
-					isLeader = false
-				}
-
-			}
-			leaderbroadcast++
-
-			select {
-			case leader := <-raftServer.LeaderCh():
-				log.Infoln("New Election event")
-				if leader {
-					isLeader = true
-
-					log.Info("This node is assuming leadership of the cluster")
-					err = cluster.Network.AddIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-
-					// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-
-					for x := range c.LoadBalancers {
-
-						if c.LoadBalancers[x].BindToVip == true {
-							err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-							if err != nil {
-								log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-								log.Errorf("Dropping Leadership to another node in the cluster")
-								raftServer.LeadershipTransfer()
-
-								// Stop all load balancers associated with the VIP
-								err = VipLB.StopAll()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-
-								err = cluster.Network.DeleteIP()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-							}
-						}
-					}
-
-					if c.EnableARP == true {
-						// Gratuitous ARP, will broadcast to new MAC <-> IP
-						err = vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-					}
-				} else {
-					isLeader = false
-
-					log.Info("This node is becoming a follower within the cluster")
-
-					// Stop all load balancers associated with the VIP
-					err = VipLB.StopAll()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-
-					err = cluster.Network.DeleteIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-
-			case <-ticker.C:
-
-				if isLeader {
-
-					result, err := cluster.Network.IsSet()
-					if err != nil {
-						log.WithFields(log.Fields{"error": err, "ip": cluster.Network.IP(), "interface": cluster.Network.Interface()}).Error("Could not check ip")
-					}
-
-					if result == false {
-						log.Error("This node is leader and is adopting the virtual IP")
-
-						err = cluster.Network.AddIP()
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-						// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-
-						for x := range c.LoadBalancers {
-
-							if c.LoadBalancers[x].BindToVip == true {
-								err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-								if err != nil {
-									log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-								}
-							}
-						}
-						if c.EnableARP == true {
-							// Gratuitous ARP, will broadcast to new MAC <-> IP
-							err = vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-							if err != nil {
-								log.Warnf("%v", err)
-							}
-						}
-					}
-				}
-
-			case <-cluster.stop:
-				log.Info("[RAFT] Stopping this node")
-				log.Info("[LOADBALANCER] Stopping load balancers")
-
-				// Stop all load balancers associated with the VIP
-				err = VipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				// Stop all load balancers associated with the Host
-				err = nonVipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				if isLeader {
-					log.Info("[VIP] Releasing the Virtual IP")
-					err = cluster.Network.DeleteIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-
-				close(cluster.completed)
-
-				return
-			}
-		}
-	}()
-
-	log.Info("Started")
-
-	return nil
-}
-
-// Stop - Will stop the Cluster and release VIP if needed
-func (cluster *Cluster) Stop() {
-	// Close the stop chanel, which will shut down the VIP (if needed)
-	close(cluster.stop)
-
-	// Wait until the completed channel is closed, signallign all shutdown tasks completed
-	<-cluster.completed
-
-	log.Info("Stopped")
-}

pkg/cluster/service.go

@@ -0,0 +1,418 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"net"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
+	"github.com/kube-vip/kube-vip/pkg/backend"
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	"github.com/vishvananda/netlink"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Config, sm *Manager, bgpServer *bgp.Server, cancelLeaderElection context.CancelFunc) error {
+	var err error
+
+	// listen for interrupts or the Linux SIGTERM signal and cancel
+	// our context, which the leader election code will observe and
+	// step down
+	signalChan := make(chan os.Signal, 1)
+	// Add Notification for Userland interrupt
+	signal.Notify(signalChan, syscall.SIGINT)
+
+	// Add Notification for SIGTERM (sent from Kubernetes)
+	signal.Notify(signalChan, syscall.SIGTERM)
+
+	loadbalancers := []*loadbalancer.IPVSLoadBalancer{}
+
+	var arpWG sync.WaitGroup
+
+	for i := range cluster.Network {
+		network := cluster.Network[i]
+
+		if network.IsDDNS() {
+			if err := cluster.StartDDNS(ctxDNS, cluster.Network[i]); err != nil {
+				log.Error("failed to start DDNS", "err", err)
+			}
+		}
+
+		if err := network.SetMask(c.VIPSubnet); err != nil {
+			log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
+			panic("")
+		}
+
+		// start the dns updater if address is dns
+		if network.IsDNS() {
+			log.Info("starting the DNS updater", "address", network.DNSName())
+			ipUpdater := vip.NewIPUpdater(network)
+			ipUpdater.Run(ctxDNS)
+		}
+
+		if !c.EnableRoutingTable {
+			// Normal VIP addition, use skipDAD=false for normal DAD process
+			if _, err = network.AddIP(false, false); err != nil {
+				log.Error(err.Error())
+			}
+		}
+
+		if c.EnableBGP {
+			// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
+			log.Debug("Attempting to advertise over BGP", "address", network.CIDR())
+			err = bgpServer.AddHost(network.CIDR())
+			if err != nil {
+				log.Error(err.Error())
+			}
+		}
+
+		if c.EnableLoadBalancer {
+			lb, err := loadbalancer.NewIPVSLB(network.IP(), c.LoadBalancerPort, c.LoadBalancerForwardingMethod, c.BackendHealthCheckInterval, c.Interface, cancelLeaderElection, signalChan)
+			if err != nil {
+				log.Error("Error creating IPVS LoadBalancer", "err", err)
+			}
+
+			go func() {
+				err = sm.NodeWatcher(ctxArp, lb, c.Port) //TODO: We're using the ctxARP as the context this will change when rkatz finishes his change
+				if err != nil {
+					log.Error("Error watching node labels", "err", err)
+				}
+			}()
+
+			loadbalancers = append(loadbalancers, lb)
+		}
+
+		if c.EnableARP {
+			arpWG.Add(1)
+			go cluster.layer2Update(ctxArp, network, c, &arpWG)
+		}
+	}
+
+	if c.EnableLoadBalancer {
+		// Shutdown function that will wait on this signal, unless we call it ourselves
+		<-signalChan
+		for _, lb := range loadbalancers {
+			err = lb.RemoveIPVSLB()
+			if err != nil {
+				log.Error("Error stopping IPVS LoadBalancer", "err", err)
+			}
+		}
+	}
+
+	if c.EnableRoutingTable {
+		backendMapV4 := backend.Map{}
+		backendMapV6 := backend.Map{}
+		// only check localhost
+
+		nodename := ""
+		if c.NodeName != "" {
+			nodename = c.NodeName
+		} else {
+			nodename = os.Getenv("HOSTNAME")
+		}
+
+		ips := []string{}
+		if nodename != "" {
+			if ips, err = getNodeIPs(ctxArp, nodename, sm.KubernetesClient); err != nil && !apierrors.IsNotFound(err) {
+				log.Error("failed to get IP of control-plane nod", "err", err)
+			}
+		}
+
+		if len(ips) == 0 {
+			isV6, err := isV6(cluster.Network[0].IP())
+			if err != nil {
+				return fmt.Errorf("failed to parse IP '%s'", cluster.Network[0].IP())
+			}
+			if !isV6 {
+				ips = append(ips, "127.0.0.1")
+			} else {
+				ips = append(ips, "::1")
+			}
+
+			log.Info("no IP address found for node - will fallback to use localhost address", "addresses", ips)
+		}
+
+		for _, ip := range ips {
+			entry := backend.Entry{Addr: ip, Port: c.Port}
+			ipv6, err := isV6(ip)
+			if err != nil {
+				log.Error("failed to check IP type", "IP", ip, "error", err)
+			}
+			if !ipv6 {
+				backendMapV4[entry] = false
+			} else {
+				backendMapV6[entry] = false
+			}
+		}
+
+		stop := make(chan struct{})
+
+		// will wait for system interrupt and will send stop signal to backend watch
+		go func() {
+			<-signalChan
+			stop <- struct{}{}
+		}()
+
+		backend.Watch(func() {
+			for i := range cluster.Network {
+				network := cluster.Network[i]
+				networkIP := network.IP()
+				isNetworkV6, err := isV6(networkIP)
+				if err != nil {
+					log.Error("failed to check IP type", "IP", networkIP, "error", err)
+					continue
+				}
+				log.Debug("current ip to process", "ip", networkIP)
+
+				backendMap := &backendMapV4
+				if isNetworkV6 {
+					backendMap = &backendMapV6
+				}
+
+				for entry := range *backendMap {
+					log.Debug("entry.Check() for entry", "entry", entry)
+					if entry.Check() {
+						log.Debug("entry.Check() true")
+						// Normal VIP addition with precheck, use skipDAD=false for normal DAD process
+						_, err = network.AddIP(true, false)
+						if err != nil {
+							log.Error("error adding address", "err", err)
+						}
+						if !(*backendMap)[entry] {
+							log.Info("added backend", "ip", network.IP())
+						}
+
+						err = network.AddRoute(true)
+						if err != nil && !errors.Is(err, fs.ErrExist) && !errors.Is(err, syscall.ESRCH) {
+							log.Warn(err.Error())
+						} else if err == nil && !(*backendMap)[entry] {
+							log.Info("added route", "route", network.PrepareRoute().String())
+						}
+
+						(*backendMap)[entry] = true
+						break
+					}
+					(*backendMap)[entry] = false
+				}
+
+				deleteAddress := true
+				for entry := range *backendMap {
+					if (*backendMap)[entry] {
+						deleteAddress = false
+						break
+					}
+				}
+
+				if deleteAddress {
+					err = network.DeleteRoute()
+					if err != nil && !errors.Is(err, fs.ErrNotExist) && !errors.Is(err, syscall.ESRCH) {
+						log.Warn("deleting route", "err", err)
+					} else if err == nil {
+						log.Info("deleted route", "route", network.PrepareRoute().String())
+					}
+
+					deleted, err := network.DeleteIP()
+					if err != nil {
+						log.Error("error deleting IP", "err", err)
+						panic("")
+					}
+					if deleted {
+						log.Info("deleted address", "IP", network.IP(), "interface", network.Interface())
+					}
+				}
+			}
+		}, c.BackendHealthCheckInterval, stop)
+	}
+
+	return nil
+}
+
+func isV6(ip string) (bool, error) {
+	ipaddr := net.ParseIP(ip)
+	if ipaddr == nil {
+		return false, fmt.Errorf("failed to parse IP '%s'", ip)
+	}
+	return ipaddr.To4() == nil, nil
+}
+
+func getNodeIPs(ctx context.Context, nodename string, client *kubernetes.Clientset) ([]string, error) {
+	node, err := client.CoreV1().Nodes().Get(ctx, nodename, metav1.GetOptions{})
+	if err != nil && !apierrors.IsNotFound(err) {
+		return []string{}, fmt.Errorf("failed to get data about '%s' node: %w", nodename, err)
+	}
+	ips := []string{}
+	for _, addr := range node.Status.Addresses {
+		if addr.Type == corev1.NodeInternalIP {
+			ips = append(ips, addr.Address)
+		}
+	}
+	return ips, nil
+}
+
+// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
+func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp *bgp.Server, name string, CountRouteReferences func(*netlink.Route) int) {
+	// use a Go context so we can tell the arp loop code when we
+	// want to step down
+	//nolint
+	ctxArp, cancelArp := context.WithCancel(ctx)
+
+	cluster.stop = make(chan bool, 1)
+	cluster.completed = make(chan bool, 1)
+
+	var arpWG sync.WaitGroup
+
+	log.Debug("StartLoadBalancerService")
+	for i := range cluster.Network {
+		network := cluster.Network[i]
+		log.Debug("current ip to process", "ip", network.IP(), "mask", c.VIPSubnet)
+		if err := network.SetMask(c.VIPSubnet); err != nil {
+			log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
+			panic("")
+		}
+		_, err := network.DeleteIP()
+		if err != nil {
+			log.Warn("attempted to clean existing VIP", "err", err)
+		}
+		log.Debug("config flags", "enable_routing_table", c.EnableRoutingTable, "enable_leader_election", c.EnableLeaderElection, "enable_services_election", c.EnableServicesElection)
+
+		if c.EnableRoutingTable && (c.EnableLeaderElection || c.EnableServicesElection) {
+			err = network.AddRoute(false)
+			if err != nil {
+				log.Warn(err.Error())
+			} else {
+				log.Info("successful add Route")
+			}
+		}
+
+		// Normal VIP addition, use skipDAD=false for normal DAD process
+		if _, err = network.AddIP(false, false); err != nil {
+			log.Warn(err.Error())
+		} else {
+			log.Info("successful add IP")
+		}
+
+		if c.EnableARP {
+			arpWG.Add(1)
+			go cluster.layer2Update(ctxArp, network, c, &arpWG)
+		}
+
+		if c.EnableBGP && (c.EnableLeaderElection || c.EnableServicesElection) {
+			// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
+			log.Debug("(svcs) attempting to advertise over BGP", "address", network.CIDR())
+			err = bgp.AddHost(network.CIDR())
+			if err != nil {
+				log.Error(err.Error())
+			}
+		}
+	}
+
+	go func() {
+		<-cluster.stop
+		// Stop the Arp context if it is running
+		cancelArp()
+
+		arpWG.Wait() // wait for all cluster ARP/NDP to be finished
+
+		log.Info("[LOADBALANCER] Stopping load balancers", "name", name)
+
+		if c.EnableRoutingTable {
+			for i := range cluster.Network {
+				// chek if route is not  referenced by another service
+				r := cluster.Network[i].PrepareRoute()
+				if CountRouteReferences(r) < 1 {
+					log.Info("[VIP] Deleting Route for VIP", "IP", cluster.Network[i].IP())
+					if err := cluster.Network[i].DeleteRoute(); err != nil {
+						log.Warn(err.Error())
+					}
+				}
+			}
+
+			close(cluster.completed)
+			return
+		}
+		for i := range cluster.Network {
+			if c.EnableARP && cluster.arpMgr.Count(cluster.Network[i].ARPName()) > 1 {
+				continue
+			}
+
+			// Handle VIP cleanup based on configuration
+			if c.PreserveVIPOnLeadershipLoss {
+				// For IPv6, we must remove VIPs immediately to avoid DAD failures on the new leader
+				// IPv6 Duplicate Address Detection will fail if the new leader tries to add an IP
+				// that is still present on this node's interface
+				if utils.IsIPv6(cluster.Network[i].IP()) {
+					log.Info("[VIP] Removing IPv6 VIP immediately (required to prevent DAD failures on new leader)", "ip", cluster.Network[i].IP())
+					deleted, err := cluster.Network[i].DeleteIP()
+					if err != nil {
+						log.Warn(err.Error())
+					}
+					if deleted {
+						log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+					}
+				} else {
+					log.Info("[VIP] Preserving IPv4 VIP address on interface, only stopped ARP broadcasting", "ip", cluster.Network[i].IP())
+				}
+			} else {
+				// Legacy behavior: delete VIP addresses on leadership loss
+				log.Info("[VIP] Deleting VIP", "ip", cluster.Network[i].IP())
+				deleted, err := cluster.Network[i].DeleteIP()
+				if err != nil {
+					log.Warn(err.Error())
+				}
+				if deleted {
+					log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+				}
+			}
+		}
+
+		close(cluster.completed)
+	}()
+}
+
+// Layer2Update, handles the creation of the
+func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config, arpWG *sync.WaitGroup) {
+	defer arpWG.Done()
+	log.Info("layer 2 broadcaster starting")
+	var ndp *vip.NdpResponder
+	var err error
+	ipString := network.IP()
+	if utils.IsIPv6(ipString) {
+		if network.IPisLinkLocal() {
+			log.Error("layer2 is link-local can't use NDP", "address", ipString)
+
+		} else {
+			ndp, err = vip.NewNDPResponder(network.Interface())
+			if err != nil {
+				log.Error("failed to create new NDP Responder", "error", err)
+			} else {
+				if ndp != nil {
+					defer ndp.Close()
+				}
+			}
+		}
+	}
+
+	log.Debug("layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
+
+	arpInstance := arp.NewInstance(network, ndp)
+	cluster.arpMgr.Insert(arpInstance)
+
+	<-ctx.Done() // if cancel() execute
+	log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
+	cluster.arpMgr.RemoveOnLeadershipLoss(arpInstance)
+}

pkg/cluster/singleNode.go

@@ -1,161 +1,85 @@
 package cluster
 
 import (
-	"fmt"
+	"context"
 
-	log "github.com/sirupsen/logrus"
+	log "log/slog"
 
-	"github.com/plunder-app/kube-vip/pkg/bgp"
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	"github.com/plunder-app/kube-vip/pkg/loadbalancer"
-	"github.com/plunder-app/kube-vip/pkg/vip"
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/vip"
 )
 
 // StartSingleNode will start a single node cluster
 func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) error {
 	// Start kube-vip as a single node server
 
-	// TODO - Split all this code out as a seperate function
-	log.Infoln("Starting kube-vip as a single node cluster")
+	// TODO - Split all this code out as a separate function
+	log.Info("Starting kube-vip as a single node cluster")
 
 	log.Info("This node is assuming leadership of the cluster")
 
 	cluster.stop = make(chan bool, 1)
 	cluster.completed = make(chan bool, 1)
 
-	// Managers for Vip load balancers and none-vip loadbalancers
-	nonVipLB := loadbalancer.LBManager{}
-	VipLB := loadbalancer.LBManager{}
-
-	// Iterate through all Configurations
-	for x := range c.LoadBalancers {
-		// If the load balancer doesn't bind to the VIP
-		if c.LoadBalancers[x].BindToVip == false {
-			err := nonVipLB.Add("", &c.LoadBalancers[x])
+	for i := range cluster.Network {
+		if !disableVIP {
+			deleted, err := cluster.Network[i].DeleteIP()
 			if err != nil {
-				log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
+				log.Warn("Attempted to clean existing VIP", "err", err)
+			}
+			if deleted {
+				log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+			}
+
+			// Normal VIP addition for single node, use skipDAD=false for normal DAD process
+			_, err = cluster.Network[i].AddIP(false, false)
+			if err != nil {
+				log.Warn(err.Error())
 			}
 
 		}
-	}
 
-	if !disableVIP {
-		err := cluster.Network.DeleteIP()
-		if err != nil {
-			log.Warnf("Attempted to clean existing VIP => %v", err)
-		}
-
-		err = cluster.Network.AddIP()
-		if err != nil {
-			log.Warnf("%v", err)
-		}
-
-		// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-		for x := range c.LoadBalancers {
-
-			if c.LoadBalancers[x].BindToVip == true {
-				err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-				if err != nil {
-					log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-				}
+		if c.EnableARP {
+			// Gratuitous ARP, will broadcast to new MAC <-> IP
+			err := vip.ARPSendGratuitous(cluster.Network[i].IP(), c.Interface)
+			if err != nil {
+				log.Warn(err.Error())
 			}
 		}
 	}
 
-	if c.EnableARP == true {
-		// Gratuitous ARP, will broadcast to new MAC <-> IP
-		err := vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-		if err != nil {
-			log.Warnf("%v", err)
-		}
-	}
-
 	go func() {
-		for {
-			select {
-			case <-cluster.stop:
-				log.Info("[LOADBALANCER] Stopping load balancers")
+		<-cluster.stop
 
-				// Stop all load balancers associated with the VIP
-				err := VipLB.StopAll()
+		if !disableVIP {
+			for i := range cluster.Network {
+				log.Info("[VIP] Releasing the VIP", "address", cluster.Network[i].IP())
+				deleted, err := cluster.Network[i].DeleteIP()
 				if err != nil {
-					log.Warnf("%v", err)
+					log.Warn(err.Error())
 				}
-
-				// Stop all load balancers associated with the Host
-				err = nonVipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
+				if deleted {
+					log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
 				}
-
-				if !disableVIP {
-
-					log.Info("[VIP] Releasing the Virtual IP")
-					err = cluster.Network.DeleteIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-				close(cluster.completed)
-				return
 			}
 		}
+		close(cluster.completed)
 	}()
-	log.Infoln("Started Load Balancer and Virtual IP")
+	log.Info("Started Load Balancer and Virtual IP")
 	return nil
 }
 
-// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
-func (cluster *Cluster) StartLoadBalancerService(c *kubevip.Config, bgp *bgp.Server) error {
-	// Start a kube-vip loadbalancer service
-	log.Infof("Starting advertising address [%s] with kube-vip", c.VIP)
+func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp.Server) error {
+	// use a Go context so we can tell the arp loop code when we
+	// want to step down
+	ctxArp, cancelArp := context.WithCancel(context.Background())
+	defer cancelArp()
 
-	cluster.stop = make(chan bool, 1)
-	cluster.completed = make(chan bool, 1)
+	// use a Go context so we can tell the dns loop code when we
+	// want to step down
+	ctxDNS, cancelDNS := context.WithCancel(context.Background())
+	defer cancelDNS()
 
-	err := cluster.Network.DeleteIP()
-	if err != nil {
-		log.Warnf("Attempted to clean existing VIP => %v", err)
-	}
-
-	err = cluster.Network.AddIP()
-	if err != nil {
-		log.Warnf("%v", err)
-	}
-
-	if c.EnableARP == true {
-		// Gratuitous ARP, will broadcast to new MAC <-> IP
-		err := vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-		if err != nil {
-			log.Warnf("%v", err)
-		}
-	}
-
-	if c.EnableBGP {
-		// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
-		cidrVip := fmt.Sprintf("%s/%s", cluster.Network.IP(), c.VIPCIDR)
-		log.Debugf("Attempting to advertise the address [%s] over BGP", cidrVip)
-		err = bgp.AddHost(cidrVip)
-		if err != nil {
-			log.Error(err)
-		}
-	}
-
-	go func() {
-		for {
-			select {
-			case <-cluster.stop:
-				log.Info("[LOADBALANCER] Stopping load balancers")
-				log.Info("[VIP] Releasing the Virtual IP")
-				err = cluster.Network.DeleteIP()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-				close(cluster.completed)
-				return
-			}
-		}
-	}()
-	log.Infoln("Started Load Balancer and Virtual IP")
-	return nil
+	return cluster.vipService(ctxArp, ctxDNS, c, sm, bgp, nil)
 }

pkg/cluster/state.go

@@ -1,39 +0,0 @@
-package cluster
-
-import (
-	"io"
-
-	"github.com/hashicorp/raft"
-)
-
-// FSM - Finite State Machine for Raft
-type FSM struct {
-}
-
-// Apply - TODO
-func (fsm FSM) Apply(log *raft.Log) interface{} {
-	return nil
-}
-
-// Restore - TODO
-func (fsm FSM) Restore(snap io.ReadCloser) error {
-	return nil
-}
-
-// Snapshot - TODO, returns an empty snapshot
-func (fsm FSM) Snapshot() (raft.FSMSnapshot, error) {
-	return Snapshot{}, nil
-}
-
-// Snapshot -
-type Snapshot struct {
-}
-
-// Persist -
-func (snapshot Snapshot) Persist(sink raft.SnapshotSink) error {
-	return nil
-}
-
-// Release -
-func (snapshot Snapshot) Release() {
-}

pkg/egress/egress.go

@@ -0,0 +1,100 @@
+package egress
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/kube-vip/kube-vip/pkg/iptables"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/nftables"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
+
+func Teardown(podIP, vipIP, namespace, serviceUUID string, annotations map[string]string, useNftables bool) error {
+	// Look up the destination ports from the annotations on the service
+	destinationPorts := annotations[kubevip.EgressDestinationPorts]
+	deniedNetworks := annotations[kubevip.EgressDeniedNetworks]
+	allowedNetworks := annotations[kubevip.EgressAllowedNetworks]
+	internalEgress := annotations[kubevip.EgressInternal]
+
+	protocol := iptables.ProtocolIPv4
+	IPv6 := false
+	if utils.IsIPv6(podIP) {
+		protocol = iptables.ProtocolIPv6
+		IPv6 = true
+	}
+
+	// Use the internal egress implementation
+	if internalEgress != "" {
+		return nftables.DeleteSNAT(IPv6, serviceUUID)
+	}
+
+	i, err := vip.CreateIptablesClient(useNftables, namespace, protocol)
+	if err != nil {
+		return fmt.Errorf("error Creating iptables client [%s]", err)
+	}
+
+	if deniedNetworks != "" {
+		networks := strings.Split(deniedNetworks, ",")
+		for x := range networks {
+			err = i.DeleteMangleReturnForNetwork(vip.MangleChainName, networks[x])
+			if err != nil {
+				return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
+			}
+		}
+	}
+
+	if allowedNetworks != "" {
+		networks := strings.Split(allowedNetworks, ",")
+		for x := range networks {
+			err = i.DeleteMangleMarkingForNetwork(podIP, vip.MangleChainName, networks[x])
+			if err != nil {
+				return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
+			}
+		}
+	} else {
+		// Remove the marking of egress packets
+		err = i.DeleteMangleMarking(podIP, vip.MangleChainName)
+		if err != nil {
+			return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+		}
+	}
+
+	// Clear up SNAT rules
+	if destinationPorts != "" {
+		fixedPorts := strings.Split(destinationPorts, ",")
+
+		for _, fixedPort := range fixedPorts {
+			var proto, port string
+
+			data := strings.Split(fixedPort, ":")
+			if len(data) == 0 {
+				continue
+			} else if len(data) == 1 {
+				proto = "tcp"
+				port = data[0]
+			} else {
+				proto = data[0]
+				port = data[1]
+			}
+
+			err = i.DeleteSourceNatForDestinationPort(podIP, vipIP, port, proto)
+			if err != nil {
+				return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+			}
+
+		}
+	} else {
+		err = i.DeleteSourceNat(podIP, vipIP)
+		if err != nil {
+			return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+		}
+	}
+
+	err = vip.DeleteExistingSessions(podIP, false, destinationPorts, "")
+	if err != nil {
+		return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+	}
+	return nil
+}

pkg/endpoints/endpoints.go

@@ -0,0 +1,165 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	"net"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	v1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	"k8s.io/apimachinery/pkg/watch"
+)
+
+type Processor struct {
+	config    *kubevip.Config
+	provider  providers.Provider
+	bgpServer *bgp.Server
+	worker    endpointWorker
+	instances *[]*instance.Instance
+}
+
+func NewEndpointProcessor(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
+	instances *[]*instance.Instance) *Processor {
+	return &Processor{
+		config:    config,
+		provider:  provider,
+		bgpServer: bgpServer,
+		instances: instances,
+		worker:    newEndpointWorker(config, provider, bgpServer, instances),
+	}
+}
+
+func (p *Processor) AddOrModify(ctx *servicecontext.Context, event watch.Event,
+	lastKnownGoodEndpoint *string, service *v1.Service, id string, leaderElectionActive *bool,
+	serviceFunc func(context.Context, *v1.Service) error,
+	leaderCtx *context.Context, cancel *context.CancelFunc) (bool, error) {
+
+	var err error
+	if err = p.provider.LoadObject(event.Object, *cancel); err != nil {
+		return false, fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
+	}
+
+	endpoints, err := p.worker.getEndpoints(service, id)
+	if err != nil {
+		return false, err
+	}
+
+	if err := p.worker.setInstanceEndpointsStatus(service, endpoints); err != nil {
+		log.Error("updating instance", "err", err)
+	}
+
+	// Find out if we have any local endpoints
+	// if out endpoint is empty then populate it
+	// if not, go through the endpoints and see if ours still exists
+	// If we have a local endpoint then begin the leader Election, unless it's already running
+	//
+
+	// Check that we have local endpoints
+	if len(endpoints) != 0 {
+		// Ignore IPv4
+		if service.Annotations[kubevip.EgressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
+			return true, nil
+		}
+
+		p.updateLastKnownGoodEndpoint(lastKnownGoodEndpoint, endpoints, service, leaderElectionActive, *cancel)
+		// start leader election if it's enabled and not already started
+		if !*leaderElectionActive && p.config.EnableServicesElection {
+			go func() {
+				*leaderCtx, *cancel = context.WithCancel(ctx.Ctx)
+				startLeaderElection(*leaderCtx, leaderElectionActive, service, serviceFunc)
+			}()
+		}
+
+		// There are local endpoints available on the node
+		if !p.config.EnableServicesElection && !p.config.EnableLeaderElection {
+			if err := p.worker.processInstance(ctx, service, leaderElectionActive); err != nil {
+				return false, fmt.Errorf("failed to process non-empty instance: %w", err)
+			}
+		}
+	} else {
+		// There are no local endpoints
+		p.worker.clear(ctx, lastKnownGoodEndpoint, service, *cancel, leaderElectionActive)
+	}
+
+	// Set the service accordingly
+	p.updateAnnotations(service, lastKnownGoodEndpoint)
+
+	log.Debug("watcher", "provider",
+		p.provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", *lastKnownGoodEndpoint, "active leader election", *leaderElectionActive)
+
+	return false, nil
+}
+
+func (p *Processor) Delete(service *v1.Service, id string) error {
+	if err := p.worker.delete(service, id); err != nil {
+		return fmt.Errorf("[%s] error deleting service: %w", p.provider.GetLabel(), err)
+	}
+	return nil
+}
+
+func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service, leaderElectionActive *bool, cancel context.CancelFunc) {
+	// if we haven't populated one, then do so
+	if *lastKnownGoodEndpoint == "" {
+		*lastKnownGoodEndpoint = endpoints[0]
+		return
+	}
+
+	// check out previous endpoint exists
+	stillExists := false
+
+	for x := range endpoints {
+		if endpoints[x] == *lastKnownGoodEndpoint {
+			stillExists = true
+		}
+	}
+	// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
+	if !stillExists {
+		p.worker.removeEgress(service, lastKnownGoodEndpoint)
+		if *leaderElectionActive && (p.config.EnableServicesElection || p.config.EnableLeaderElection) {
+			log.Warn("existing endpoint has been removed, restarting leaderElection", "provider", p.provider.GetLabel(), "endpoint", *lastKnownGoodEndpoint)
+			// Stop the existing leaderElection
+			cancel()
+			// disable last leaderElection flag
+			*leaderElectionActive = false
+		}
+		// Set our active endpoint to an existing one
+		*lastKnownGoodEndpoint = endpoints[0]
+	}
+}
+
+func (p *Processor) updateAnnotations(service *v1.Service, lastKnownGoodEndpoint *string) {
+	// Set the service accordingly
+	if service.Annotations[kubevip.Egress] == "true" {
+		activeEndpointAnnotation := kubevip.ActiveEndpoint
+
+		if !p.config.EnableEndpoints && p.provider.GetProtocol() == string(discoveryv1.AddressTypeIPv6) {
+			activeEndpointAnnotation = kubevip.ActiveEndpointIPv6
+		}
+		service.Annotations[activeEndpointAnnotation] = *lastKnownGoodEndpoint
+	}
+}
+
+func startLeaderElection(ctx context.Context, leaderElectionActive *bool, service *v1.Service, serviceFunc func(context.Context, *v1.Service) error) {
+	// This is a blocking function, that will restart (in the event of failure)
+	for {
+		// if the context isn't cancelled restart
+		if ctx.Err() != context.Canceled {
+			*leaderElectionActive = true
+			err := serviceFunc(ctx, service)
+			if err != nil {
+				log.Error(err.Error())
+			}
+			*leaderElectionActive = false
+		} else {
+			*leaderElectionActive = false
+			break
+		}
+	}
+}

pkg/endpoints/endpoints_bgp.go

@@ -0,0 +1,129 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	v1 "k8s.io/api/core/v1"
+)
+
+type BGP struct {
+	generic
+	bgpServer *bgp.Server
+}
+
+func newBGP(generic generic, bgpServer *bgp.Server) endpointWorker {
+	return &BGP{
+		generic:   generic,
+		bgpServer: bgpServer,
+	}
+}
+
+func (b *BGP) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
+	if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
+		for _, cluster := range instance.Clusters {
+			for i := range cluster.Network {
+				if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) {
+					log.Debug("attempting to advertise BGP service", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP())
+					err := b.bgpServer.AddHost(cluster.Network[i].CIDR())
+					if err != nil {
+						log.Error("error adding BGP host", "provider", b.provider.GetLabel(), "err", err)
+					} else {
+						log.Info("added BGP host", "provider",
+							b.provider.GetLabel(), "ip", cluster.Network[i].CIDR(), "service name", service.Name, "namespace", service.Namespace)
+						ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
+						*leaderElectionActive = true
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (b *BGP) clear(ctx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
+	if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
+		// If BGP mode is enabled - routes should be deleted
+		if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
+			for _, cluster := range instance.Clusters {
+				for i := range cluster.Network {
+					err := b.bgpServer.DelHost(cluster.Network[i].CIDR())
+					if err != nil {
+						log.Error("deleting BGP host", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "err", err)
+					} else {
+						log.Info("deleted BGP host", "provider",
+							b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace)
+						ctx.ConfiguredNetworks.Delete(cluster.Network[i].IP())
+						*leaderElectionActive = false
+					}
+				}
+			}
+
+		}
+	}
+
+	b.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
+}
+
+func (b *BGP) getEndpoints(service *v1.Service, id string) ([]string, error) {
+	return b.getAllEndpoints(service, id)
+}
+
+func (b *BGP) delete(service *v1.Service, id string) error {
+	// When no-leader-elecition mode
+	if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
+		// find all existing local endpoints
+		endpoints, err := b.getEndpoints(service, id)
+		if err != nil {
+			return fmt.Errorf("[%s] error getting endpoints: %w", b.provider.GetLabel(), err)
+		}
+
+		// If there were local endpoints deleted
+		if len(endpoints) > 0 {
+			b.deleteAction(service)
+		}
+	}
+
+	return nil
+}
+
+func (b *BGP) deleteAction(service *v1.Service) {
+	b.clearBGPHosts(service)
+}
+
+func (b *BGP) clearBGPHosts(service *v1.Service) {
+	ClearBGPHosts(service, b.instances, b.bgpServer)
+}
+
+func (b *BGP) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
+	return nil
+}
+
+func ClearBGPHosts(service *v1.Service, instances *[]*instance.Instance, bgpServer *bgp.Server) {
+	if instance := instance.FindServiceInstance(service, *instances); instance != nil {
+		ClearBGPHostsByInstance(instance, bgpServer)
+	}
+}
+
+func ClearBGPHostsByInstance(instance *instance.Instance, bgpServer *bgp.Server) {
+	if instance == nil {
+		log.Error("failed to clear BGP host for nil instance")
+		return
+	}
+	for _, cluster := range instance.Clusters {
+		for i := range cluster.Network {
+			network := cluster.Network[i]
+			err := bgpServer.DelHost(network.CIDR())
+			if err != nil {
+				log.Error("[endpoint] error deleting BGP host", "err", err)
+			} else {
+				log.Debug("[endpoint] deleted BGP host", "ip",
+					network.CIDR(), "service name", instance.ServiceSnapshot.Name, "namespace", instance.ServiceSnapshot.Namespace)
+			}
+		}
+	}
+}

pkg/endpoints/endpoints_generic.go

@@ -0,0 +1,118 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/egress"
+	"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	v1 "k8s.io/api/core/v1"
+)
+
+type endpointWorker interface {
+	processInstance(svcCtx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error
+	clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool)
+	getEndpoints(service *v1.Service, id string) ([]string, error)
+	removeEgress(service *v1.Service, lastKnownGoodEndpoint *string)
+	delete(service *v1.Service, id string) error
+	setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error
+}
+
+func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server, instances *[]*instance.Instance) endpointWorker {
+	generic := newGeneric(config, provider, instances)
+
+	if config.EnableRoutingTable {
+		return newRoutingTable(generic)
+	}
+	if config.EnableBGP {
+		return newBGP(generic, bgpServer)
+	}
+
+	return &generic
+}
+
+type generic struct {
+	config    *kubevip.Config
+	provider  providers.Provider
+	instances *[]*instance.Instance
+}
+
+func newGeneric(config *kubevip.Config, provider providers.Provider, instances *[]*instance.Instance) generic {
+	return generic{
+		config:    config,
+		provider:  provider,
+		instances: instances,
+	}
+}
+
+func (g *generic) processInstance(_ *servicecontext.Context, _ *v1.Service, _ *bool) error {
+	return nil
+}
+
+func (g *generic) clear(_ *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
+	g.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
+}
+
+func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
+	if *lastKnownGoodEndpoint != "" {
+		log.Warn("existing  endpoint has been removed, no remaining endpoints for leaderElection", "provider", g.provider.GetLabel(), "endpoint", lastKnownGoodEndpoint)
+		if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP, service.Namespace, string(service.UID), service.Annotations, g.config.EgressWithNftables); err != nil {
+			log.Error("error removing redundant egress rules", "err", err)
+		}
+
+		*lastKnownGoodEndpoint = "" // reset endpoint
+		if g.config.EnableServicesElection || g.config.EnableLeaderElection {
+			cancel() // stop services watcher
+		}
+		*leaderElectionActive = false
+	}
+}
+
+func (g *generic) getEndpoints(_ *v1.Service, id string) ([]string, error) {
+	return g.getLocalEndpoints(id)
+}
+
+func (g *generic) getLocalEndpoints(id string) ([]string, error) {
+	// Build endpoints
+	var endpoints []string
+	var err error
+	if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
+		return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
+	}
+
+	return endpoints, nil
+}
+
+func (g *generic) getAllEndpoints(service *v1.Service, id string) ([]string, error) {
+	// Build endpoints
+	var err error
+	var endpoints []string
+	if !g.config.EnableLeaderElection && !g.config.EnableServicesElection &&
+		service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
+		if endpoints, err = g.provider.GetAllEndpoints(); err != nil {
+			return nil, fmt.Errorf("[%s] error getting all endpoints: %w", g.provider.GetLabel(), err)
+		}
+	} else {
+		if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
+			return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
+		}
+	}
+
+	return endpoints, nil
+}
+
+func (g *generic) removeEgress(_ *v1.Service, _ *string) {
+}
+
+func (g *generic) delete(_ *v1.Service, _ string) error {
+	return nil
+}
+
+func (g *generic) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
+	return nil
+}

pkg/endpoints/endpoints_routing_table.go

@@ -0,0 +1,189 @@
+package endpoints
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"syscall"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/egress"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	"github.com/vishvananda/netlink"
+	v1 "k8s.io/api/core/v1"
+)
+
+type RoutingTable struct {
+	generic
+}
+
+func newRoutingTable(generic generic) endpointWorker {
+	return &RoutingTable{
+		generic: generic,
+	}
+}
+
+func (rt *RoutingTable) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
+	instance := instance.FindServiceInstance(service, *rt.instances)
+	if instance != nil {
+		for _, cluster := range instance.Clusters {
+			for i := range cluster.Network {
+				if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) && cluster.Network[i].HasEndpoints() {
+					err := cluster.Network[i].AddRoute(false)
+					if err != nil {
+						if errors.Is(err, syscall.EEXIST) {
+							// If route exists, but protocol is not set (e.g. the route was created by the older version
+							// of kube-vip) try to update it if necessary
+							isUpdated, err := cluster.Network[i].UpdateRoutes()
+							if err != nil {
+								return fmt.Errorf("[%s] error updating existing routes: %w", rt.provider.GetLabel(), err)
+							}
+							if isUpdated {
+								log.Info("updated route", "provider",
+									rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
+									service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
+							} else {
+								log.Info("route already present", "provider",
+									rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
+									service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
+							}
+						} else {
+							// If other error occurs, return error
+							return fmt.Errorf("[%s] error adding route: %s", rt.provider.GetLabel(), err.Error())
+						}
+					} else {
+						log.Info("added route", "provider",
+							rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
+							service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
+						ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
+						*leaderElectionActive = true
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
+	if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
+		if errs := ClearRoutes(service, rt.instances); len(errs) == 0 {
+			svcCtx.ConfiguredNetworks.Clear()
+		} else {
+			for _, err := range errs {
+				log.Error("error while clearing routes", "err", err)
+			}
+		}
+	}
+
+	rt.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
+}
+
+func (rt *RoutingTable) getEndpoints(service *v1.Service, id string) ([]string, error) {
+	return rt.getAllEndpoints(service, id)
+}
+
+func (rt *RoutingTable) removeEgress(service *v1.Service, lastKnownGoodEndpoint *string) {
+	if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
+		service.Namespace, string(service.UID), service.Annotations, rt.config.EgressWithNftables); err != nil {
+		log.Warn("removing redundant egress rules", "err", err)
+	}
+}
+
+func (rt *RoutingTable) delete(service *v1.Service, id string) error {
+	// When no-leader-elecition mode
+	if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
+		// find all existing local endpoints
+		endpoints, err := rt.getEndpoints(service, id)
+		if err != nil {
+			return fmt.Errorf("[%s] error getting endpoints: %w", rt.provider.GetLabel(), err)
+		}
+
+		// If there were local endpoints deleted
+		if len(endpoints) > 0 {
+			rt.deleteAction(service)
+		}
+	}
+
+	return nil
+}
+
+func (rt *RoutingTable) deleteAction(service *v1.Service) {
+	ClearRoutes(service, rt.instances)
+}
+
+func (rt *RoutingTable) setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error {
+	instance := instance.FindServiceInstanceWithTimeout(service, *rt.instances)
+	if instance == nil {
+		log.Error("failed to find the instance", "namespace", service.Namespace, "name", service.Name, "uid", service.UID, "provider", rt.provider.GetLabel())
+	} else {
+		for _, c := range instance.Clusters {
+			for n := range c.Network {
+				// if there are no endpoints set HasEndpoints false just in case
+				if len(endpoints) < 1 {
+					c.Network[n].SetHasEndpoints(false)
+				}
+				// check if endpoint are available and are of same IP family as service
+				if len(endpoints) > 0 && ((net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(endpoints[0]).To4() == nil)) {
+					c.Network[n].SetHasEndpoints(true)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func ClearRoutes(service *v1.Service, instances *[]*instance.Instance) []error {
+	errs := []error{}
+	if svcInst := instance.FindServiceInstance(service, *instances); svcInst != nil {
+		clearErrs := ClearRoutesByInstance(service, svcInst, instances)
+		errs = append(errs, clearErrs...)
+	}
+	return errs
+}
+
+func ClearRoutesByInstance(service *v1.Service, svcInst *instance.Instance, instances *[]*instance.Instance) []error {
+	if svcInst == nil {
+		return []error{fmt.Errorf("failed to remove routes for nil instance of service %s/%s, uid: %s", service.Namespace, service.Name, service.UID)}
+	}
+	errs := []error{}
+	for _, cluster := range svcInst.Clusters {
+		for i := range cluster.Network {
+			route := cluster.Network[i].PrepareRoute()
+			// check if route we are about to delete is not referenced by more than one service
+			if CountRouteReferences(route, instances) <= 1 {
+				err := cluster.Network[i].DeleteRoute()
+				if err != nil && !errors.Is(err, syscall.ESRCH) {
+					log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
+					errs = append(errs, err)
+				}
+				log.Debug("deleted route", "ip",
+					cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface())
+			}
+		}
+	}
+
+	return errs
+}
+
+func CountRouteReferences(route *netlink.Route, instances *[]*instance.Instance) int {
+	cnt := 0
+	for _, instance := range *instances {
+		for _, cluster := range instance.Clusters {
+			for n := range cluster.Network {
+				if cluster.Network[n].HasEndpoints() {
+					r := cluster.Network[n].PrepareRoute()
+					if r.Dst.String() == route.Dst.String() {
+						cnt++
+					}
+				}
+			}
+		}
+	}
+	return cnt
+}

pkg/endpoints/providers/endpoints.go

@@ -0,0 +1,137 @@
+package providers
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	watchtools "k8s.io/client-go/tools/watch"
+	"k8s.io/client-go/util/retry"
+
+	log "log/slog"
+)
+
+type Endpoints struct {
+	label string
+	//nolint:staticcheck // SA1019 endpoints are moving to an opt-in only
+	endpoints *v1.Endpoints
+}
+
+func NewEndpoints() Provider {
+	return &Endpoints{
+		label: "endpoints",
+	}
+}
+
+func (ep *Endpoints) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
+	service *v1.Service) (*watchtools.RetryWatcher, error) {
+	opts := metav1.ListOptions{
+		FieldSelector: fields.OneTermEqualSelector("metadata.name", service.Name).String(),
+	}
+
+	rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
+		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
+			return clientSet.CoreV1().Endpoints(service.Namespace).Watch(ctx, opts)
+		},
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("error creating endpoint watcher: %s", err.Error())
+	}
+
+	return rw, nil
+}
+
+func (ep *Endpoints) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
+	//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
+	eps, ok := endpoints.(*v1.Endpoints)
+	if !ok {
+		cancel()
+		return fmt.Errorf("[%s] unable to parse Kubernetes services from API watcher", ep.GetLabel())
+	}
+	ep.endpoints = eps
+	return nil
+}
+
+func (ep *Endpoints) GetAllEndpoints() ([]string, error) {
+	result := []string{}
+	for subset := range ep.endpoints.Subsets {
+		for address := range ep.endpoints.Subsets[subset].Addresses {
+			addr := strings.Split(ep.endpoints.Subsets[subset].Addresses[address].IP, "/")
+			result = append(result, addr[0])
+		}
+	}
+
+	return result, nil
+}
+
+func (ep *Endpoints) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
+	var localEndpoints []string
+
+	for _, subset := range ep.endpoints.Subsets {
+		for _, address := range subset.Addresses {
+			log.Debug("processing endpoint", "label", ep.label, "ip", address.IP)
+
+			// 1. Compare the Nodename
+			if address.NodeName != nil && id == *address.NodeName {
+				log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname, "nodename", *address.NodeName)
+				localEndpoints = append(localEndpoints, address.IP)
+				continue
+			}
+			// 2. Compare the Hostname (only useful if address.NodeName is not available)
+			if id == address.Hostname {
+				log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname)
+				localEndpoints = append(localEndpoints, address.IP)
+				continue
+			}
+		}
+	}
+	return localEndpoints, nil
+}
+
+func (ep *Endpoints) UpdateServiceAnnotation(endpoint string, _ string, service *v1.Service, clientSet *kubernetes.Clientset) error {
+	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		// Retrieve the latest version of Deployment before attempting update
+		// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
+		currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		currentServiceCopy := currentService.DeepCopy()
+		if currentServiceCopy.Annotations == nil {
+			currentServiceCopy.Annotations = make(map[string]string)
+		}
+
+		currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
+
+		_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
+		if err != nil {
+			log.Error("error updating Service Spec", "label", ep.GetLabel(), "name", currentServiceCopy.Name, "err", err)
+			return err
+		}
+		return nil
+	})
+
+	if retryErr != nil {
+		log.Error("failed to set Services", "label", ep.GetLabel(), "err", retryErr)
+		return retryErr
+	}
+	return nil
+}
+
+func (ep *Endpoints) GetLabel() string {
+	return ep.label
+}
+
+func (ep *Endpoints) GetProtocol() string {
+	return ""
+}

pkg/endpoints/providers/endpointslices.go

@@ -0,0 +1,139 @@
+package providers
+
+import (
+	"context"
+	"fmt"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	v1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	watchtools "k8s.io/client-go/tools/watch"
+	"k8s.io/client-go/util/retry"
+)
+
+type Endpointslices struct {
+	label     string
+	endpoints *discoveryv1.EndpointSlice
+}
+
+func NewEndpointslices() Provider {
+	return &Endpointslices{
+		label: "endpointslices",
+	}
+}
+
+func (ep *Endpointslices) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
+	service *v1.Service) (*watchtools.RetryWatcher, error) {
+	labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/service-name": service.Name}}
+
+	opts := metav1.ListOptions{
+		LabelSelector: labels.Set(labelSelector.MatchLabels).String(),
+	}
+
+	rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
+		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
+			return clientSet.DiscoveryV1().EndpointSlices(service.Namespace).Watch(ctx, opts)
+		},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("[%s] error creating endpointslices watcher: %s", ep.label, err.Error())
+	}
+
+	return rw, nil
+}
+
+func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
+	eps, ok := endpoints.(*discoveryv1.EndpointSlice)
+	if !ok {
+		cancel()
+		return fmt.Errorf("[%s] error casting endpoints to v1.Endpoints struct", ep.label)
+	}
+	ep.endpoints = eps
+	return nil
+}
+
+func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
+	result := []string{}
+	for _, ep := range ep.endpoints.Endpoints {
+		result = append(result, ep.Addresses...)
+	}
+	return result, nil
+}
+
+func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
+	var localEndpoints []string
+	for _, endpoint := range ep.endpoints.Endpoints {
+		if endpoint.Conditions.Serving == nil || !*endpoint.Conditions.Serving {
+			continue
+		}
+		for _, address := range endpoint.Addresses {
+			log.Debug("processing endpoint", "provider", ep.label, "ip", address)
+
+			// 1. Compare the Nodename
+			if endpoint.NodeName != nil && id == *endpoint.NodeName {
+				if endpoint.Hostname != nil {
+					log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname, "nodename", *endpoint.NodeName)
+				} else {
+					log.Debug("found endpoint", "provider", ep.label, "ip", address, "nodename", *endpoint.NodeName)
+				}
+				localEndpoints = append(localEndpoints, address)
+				continue
+			}
+
+			// 2. Compare the Hostname (only useful if endpoint.NodeName is not available)
+			if endpoint.Hostname != nil && id == *endpoint.Hostname {
+				log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname)
+				localEndpoints = append(localEndpoints, address)
+			}
+		}
+	}
+	return localEndpoints, nil
+}
+
+func (ep *Endpointslices) UpdateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, clientSet *kubernetes.Clientset) error {
+	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		// Retrieve the latest version of Deployment before attempting update
+		// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
+		currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		currentServiceCopy := currentService.DeepCopy()
+		if currentServiceCopy.Annotations == nil {
+			currentServiceCopy.Annotations = make(map[string]string)
+		}
+
+		currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
+		currentServiceCopy.Annotations[kubevip.ActiveEndpointIPv6] = endpointIPv6
+
+		_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
+		if err != nil {
+			log.Error("error updating Service Spec", "provider", ep.label, "service name", currentServiceCopy.Name, "err", err)
+			return err
+		}
+		return nil
+	})
+
+	if retryErr != nil {
+		log.Error("failed to set Services", "provider", ep.label, "err", retryErr)
+		return retryErr
+	}
+	return nil
+}
+
+func (ep *Endpointslices) GetLabel() string {
+	return ep.label
+}
+
+func (ep *Endpointslices) GetProtocol() string {
+	return string(ep.endpoints.AddressType)
+}

pkg/endpoints/providers/interface.go

@@ -0,0 +1,22 @@
+package providers
+
+import (
+	"context"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
+	watchtools "k8s.io/client-go/tools/watch"
+)
+
+type Provider interface {
+	CreateRetryWatcher(context.Context, *kubernetes.Clientset,
+		*v1.Service) (*watchtools.RetryWatcher, error)
+	GetAllEndpoints() ([]string, error)
+	GetLocalEndpoints(string, *kubevip.Config) ([]string, error)
+	GetLabel() string
+	UpdateServiceAnnotation(string, string, *v1.Service, *kubernetes.Clientset) error
+	LoadObject(runtime.Object, context.CancelFunc) error
+	GetProtocol() string
+}

pkg/etcd/client.go

@@ -0,0 +1,26 @@
+package etcd
+
+import (
+	"go.etcd.io/etcd/client/pkg/v3/transport"
+	clientv3 "go.etcd.io/etcd/client/v3"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+)
+
+func NewClient(c *kubevip.Config) (*clientv3.Client, error) {
+	tlsInfo := transport.TLSInfo{
+		TrustedCAFile: c.Etcd.CAFile,
+		CertFile:      c.Etcd.ClientCertFile,
+		KeyFile:       c.Etcd.ClientKeyFile,
+	}
+
+	clientTLS, err := tlsInfo.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	return clientv3.New(clientv3.Config{
+		Endpoints: c.Etcd.Endpoints,
+		TLS:       clientTLS,
+	})
+}

pkg/etcd/election.go

@@ -0,0 +1,227 @@
+package etcd
+
+import (
+	"context"
+	"hash/fnv"
+	"time"
+
+	log "log/slog"
+
+	"github.com/pkg/errors"
+	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/concurrency"
+)
+
+// LeaderElectionConfig allows to configure the leader election params.
+type LeaderElectionConfig struct {
+	// EtcdConfig contains the client to connect to the etcd cluster.
+	EtcdConfig ClientConfig
+
+	// Name uniquely identifies this leader election. All members of the same election
+	// should use the same value here.
+	Name string
+
+	// MemberID identifies uniquely this contestant from other in the leader election.
+	// It will be converted to an int64 using a hash, so theoretically collisions are possible
+	// when using a string. If you want to guarantee safety, us MemberUniqueID to specify a unique
+	// int64 directly.
+	// If two processes start a leader election using the same MemberID, one of them will
+	// fail.
+	MemberID string
+
+	// MemberUniqueID is the int equivalent to MemberID that allows to override the default conversion
+	// from string to int using hashing.
+	MemberUniqueID *uint64
+
+	// LeaseDurationSeconds is the duration that non-leader candidates will
+	// wait to force acquire leadership.
+	// This is just a request to the etcd server but it's not guaranteed, the server
+	// might decide to make the duration longer.
+	LeaseDurationSeconds int64
+
+	// Callbacks are callbacks that are triggered during certain lifecycle
+	// events of the LeaderElector
+	Callbacks LeaderCallbacks
+}
+
+// LeaderCallbacks are callbacks that are triggered during certain
+// lifecycle events of the election.
+type LeaderCallbacks struct {
+	// OnStartedLeading is called when this member starts leading.
+	OnStartedLeading func(context.Context)
+	// OnStoppedLeading is called when this member stops leading.
+	OnStoppedLeading func()
+	// OnNewLeader is called when the client observes a leader that is
+	// not the previously observed leader. This includes the first observed
+	// leader when the client starts.
+	OnNewLeader func(identity string)
+}
+
+// ClientConfig contains the client to connect to the etcd cluster.
+type ClientConfig struct {
+	Client *clientv3.Client
+}
+
+// RunElectionOrDie behaves the same way as RunElection but panics if there is an error.
+func RunElectionOrDie(ctx context.Context, config *LeaderElectionConfig) {
+	if err := RunElection(ctx, config); err != nil {
+		panic(err)
+	}
+}
+
+// RunElection starts a client with the provided config or panics.
+// RunElection blocks until leader election loop is
+// stopped by ctx or it has stopped holding the leader lease.
+func RunElection(ctx context.Context, config *LeaderElectionConfig) error {
+	var memberID uint64
+	if config.MemberUniqueID != nil {
+		memberID = *config.MemberUniqueID
+	} else {
+		h := fnv.New64a()
+		if _, err := h.Write(append([]byte(config.Name), []byte(config.MemberID)...)); err != nil {
+			return err
+		}
+		memberID = h.Sum64()
+	}
+
+	ttl := config.LeaseDurationSeconds
+	r := &pb.LeaseGrantRequest{TTL: ttl, ID: int64(memberID)} //nolint
+	lease, err := clientv3.RetryLeaseClient(
+		config.EtcdConfig.Client,
+	).LeaseGrant(ctx, r)
+	if err != nil {
+		return errors.Wrap(err, "creating lease")
+	}
+
+	leaseID := clientv3.LeaseID(lease.ID)
+
+	s, err := concurrency.NewSession(
+		config.EtcdConfig.Client,
+		concurrency.WithTTL(int(lease.TTL)),
+		concurrency.WithLease(leaseID),
+	)
+	if err != nil {
+		return err
+	}
+
+	election := concurrency.NewElection(s, config.Name)
+
+	m := &member{
+		client:         config.EtcdConfig.Client,
+		election:       election,
+		callbacks:      config.Callbacks,
+		memberID:       config.MemberID,
+		weAreTheLeader: make(chan struct{}, 1),
+		leaseTTL:       lease.TTL,
+	}
+
+	go m.tryToBeLeader(ctx)
+	m.watchLeaderChanges(ctx)
+
+	return nil
+}
+
+type member struct {
+	key              string
+	client           *clientv3.Client
+	election         *concurrency.Election
+	isLeader         bool
+	currentLeaderKey string
+	callbacks        LeaderCallbacks
+	memberID         string
+	weAreTheLeader   chan struct{}
+	leaseTTL         int64
+}
+
+func (m *member) watchLeaderChanges(ctx context.Context) {
+	observeCtx, observeCancel := context.WithCancel(ctx)
+	defer observeCancel()
+	changes := m.election.Observe(observeCtx)
+
+watcher:
+	for {
+		select {
+		case <-ctx.Done():
+			break watcher
+		case <-m.weAreTheLeader:
+
+			m.isLeader = true
+			m.key = m.election.Key() // by this time, this should already be set, since Campaign has already returned
+			log.Debug("Marking self as leader with key", "id", m.memberID, "key", m.key)
+		case response := <-changes:
+			log.Debug("Leader Changes", "id", m.memberID, "response", response)
+			if len(response.Kvs) == 0 {
+				// There is a race condition where just after we stop being the leader
+				// if there are no more leaders, we might get a response with no key-values
+				// just before the response channel is closed or the context is cancel
+				// In that case, just continue and let one of those two things happen
+				continue
+			}
+			newLeaderKey := response.Kvs[0].Key
+			if m.isLeader && m.key != string(newLeaderKey) {
+				// We stopped being leaders
+
+				// exit the loop, so we cancel the observe context so we stop watching
+				// for new leaders. That will close the channel and make this function exit,
+				// which also makes the routine to finish and RunElection returns
+				break watcher
+			}
+
+			if m.currentLeaderKey != string(newLeaderKey) {
+				// we observed a leader, this could be us or someone else
+				m.currentLeaderKey = string(newLeaderKey)
+				m.callbacks.OnNewLeader(string(response.Kvs[0].Value))
+			}
+		}
+	}
+
+	// If we are here, either we have stopped being leaders or we lost the watcher
+	// Make sure we call OnStoppedLeading if we were the leader.
+	if m.isLeader {
+		m.callbacks.OnStoppedLeading()
+	}
+
+	log.Debug("Exiting watcher", "id", m.memberID)
+}
+
+func (m *member) tryToBeLeader(ctx context.Context) {
+	if err := m.election.Campaign(ctx, m.memberID); err != nil {
+		log.Error("Failed trying to become the leader", "err", err)
+		// Resign just in case we acquired leadership just before failing
+		if err := m.election.Resign(m.client.Ctx()); err != nil {
+			log.Warn("Failed to resign after we failed becoming the leader, this might not be a problem if we were never the leader", "err", err)
+		}
+		return
+		// TODO: what to do here?
+		// We probably want watchLeaderChanges to exit as well, since Run
+		// is expecting us to try to become the leader, but if we are here,
+		// we won't. So if we don't panic, we need to signal it somehow
+	}
+
+	// Inform the observer that we are the leader as soon as possible,
+	// so it can detect if we stop being it
+	m.weAreTheLeader <- struct{}{}
+
+	// Once we are the leader, start the routine to resign if context is canceled
+	go m.resignOnCancel(ctx)
+
+	// After becoming the leader, we wait for at least a lease TTL to wait for
+	// the previous leader to detect the new leadership (if there was one) and
+	// stop its processes
+	// TODO: is this too cautious?
+	log.Debug("timeout before OnStartedLeading", "id", m.memberID, "timeout", m.leaseTTL)
+	time.Sleep(time.Second * time.Duration(m.leaseTTL))
+
+	// We are the leader, execute our code
+	m.callbacks.OnStartedLeading(ctx)
+
+	// Here the routine dies if OnStartedLeading doesn't block, there is nothing else to do
+}
+
+func (m *member) resignOnCancel(ctx context.Context) {
+	<-ctx.Done()
+	if err := m.election.Resign(m.client.Ctx()); err != nil && !errors.Is(err, context.Canceled) {
+		log.Error("Failed to resign after the context was canceled", "err", err)
+	}
+}

pkg/etcd/election_test.go

@@ -0,0 +1,171 @@
+//go:build integration
+// +build integration
+
+package etcd_test
+
+import (
+	"context"
+	"log"
+	"math/rand"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/etcd"
+	. "github.com/onsi/gomega"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.uber.org/zap"
+)
+
+func TestRunElectionWithMemberIDCollision(t *testing.T) {
+	t.Parallel()
+	g := NewWithT(t)
+	ctx := context.Background()
+	cli := client(g)
+	defer cli.Close()
+
+	electionName := randomElectionNameForTest("memberIDConflict")
+	log.Printf("Election name %s\n", electionName)
+	memberCtx, cancelMember1 := context.WithCancel(ctx)
+	config := &etcd.LeaderElectionConfig{
+		EtcdConfig: etcd.ClientConfig{
+			Client: cli,
+		},
+		Name:                 electionName,
+		MemberID:             randomElectionNameForTest("my-host"),
+		LeaseDurationSeconds: 1,
+		Callbacks: etcd.LeaderCallbacks{
+			OnStartedLeading: func(ctx context.Context) {
+				log.Println("I'm the leader!!!!")
+				log.Println("Renouncing as leader by canceling context")
+				cancelMember1()
+			},
+			OnNewLeader: func(identity string) {
+				log.Printf("New leader: %s\n", identity)
+			},
+			OnStoppedLeading: func() {
+				log.Println("I'm not the leader anymore")
+			},
+		},
+	}
+
+	wg := &sync.WaitGroup{}
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		g.Expect(etcd.RunElection(memberCtx, config)).To(Succeed())
+	}()
+
+	go func() {
+		defer wg.Done()
+		time.Sleep(time.Millisecond * 50) // make sure the first one becomes leader
+		g.Expect(etcd.RunElection(ctx, config)).Should(MatchError(ContainSubstring("creating lease")))
+	}()
+
+	wg.Wait()
+}
+
+func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
+	t.Parallel()
+	g := NewWithT(t)
+	ctx := context.Background()
+	cli := client(g)
+	defer cli.Close()
+
+	cliMember1 := client(g)
+	defer cliMember1.Close()
+
+	electionName := randomElectionNameForTest("steppingDown")
+	configBase := etcd.LeaderElectionConfig{
+		EtcdConfig: etcd.ClientConfig{
+			Client: cli,
+		},
+		Name:                 electionName,
+		LeaseDurationSeconds: 1,
+	}
+
+	member1Ctx, _ := context.WithCancel(ctx)
+	member2Ctx, cancelMember2 := context.WithCancel(ctx)
+
+	config1 := configBase
+	config1.EtcdConfig.Client = cliMember1
+	config1.MemberID = randomElectionNameForTest("my-host")
+	uniqueID := rand.Uint64()
+	config1.MemberUniqueID = &uniqueID
+	config1.Callbacks = baseCallbacksForName(config1.MemberID)
+	syncMembers := make(chan (any))
+	config1.Callbacks.OnStartedLeading = func(_ context.Context) {
+		log.Println("I'm my-host, the new leader!!!!")
+		close(syncMembers)
+		log.Println("Losing the leadership on purpose by stopping renewing the lease")
+		g.Expect(cliMember1.Lease.Close()).To(Succeed())
+		log.Println("Member1 leases closed")
+	}
+
+	config2 := configBase
+	config2.MemberID = randomElectionNameForTest("my-other-host")
+	config2.Callbacks = baseCallbacksForName(config2.MemberID)
+	config2.Callbacks.OnStartedLeading = func(_ context.Context) {
+		log.Println("I'm my-other-host, the new leader!!!!")
+		log.Println("Renouncing as leader by canceling context")
+		cancelMember2()
+	}
+
+	wg := &sync.WaitGroup{}
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		g.Expect(etcd.RunElection(member1Ctx, &config1)).To(Succeed())
+		log.Printf("%s routine done\n", config1.MemberID)
+	}()
+
+	go func() {
+		defer wg.Done()
+		<-syncMembers
+		g.Expect(etcd.RunElection(member2Ctx, &config2)).To(Succeed())
+		log.Printf("%s routine done\n", config2.MemberID)
+	}()
+
+	wg.Wait()
+}
+
+func baseCallbacksForName(name string) etcd.LeaderCallbacks {
+	return etcd.LeaderCallbacks{
+		OnStartedLeading: func(ctx context.Context) {
+			log.Printf("[%s] I'm the new leader!!!!\n", name)
+		},
+		OnNewLeader: func(identity string) {
+			log.Printf("[%s] New leader: %s\n", name, identity)
+		},
+		OnStoppedLeading: func() {
+			log.Printf("[%s] I'm not the leader anymore\n", name)
+		},
+	}
+}
+
+func randomElectionNameForTest(name string) string {
+	return name + "-" + randomString(6)
+}
+
+const charSet = "0123456789abcdefghijklmnopqrstuvwxyz"
+
+var rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
+
+func randomString(n int) string {
+	result := make([]byte, n)
+	for i := range result {
+		result[i] = charSet[rnd.Intn(len(charSet))]
+	}
+	return string(result)
+}
+
+func client(g Gomega) *clientv3.Client {
+	c, err := clientv3.New(clientv3.Config{
+		Endpoints: []string{"localhost:2379"},
+		Logger:    zap.NewNop(),
+	})
+	g.Expect(err).NotTo(HaveOccurred())
+	return c
+}

pkg/etcd/etcd_suite_test.go

@@ -0,0 +1,153 @@
+//go:build integration
+// +build integration
+
+package etcd_test
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	etcdVersion     = "v3.5.9"
+	etcdBinDir      = "bin"
+	etcdBinPath     = etcdBinDir + "/etcd"
+	downloadURL     = "https://storage.googleapis.com/etcd"
+	tmpDownloadFile = "etcd.tar.gz"
+	pidFile         = "etcd.pid"
+)
+
+func TestMain(m *testing.M) {
+	logrus.SetLevel(logrus.DebugLevel)
+	ctx := context.Background()
+	expectSuccess(startEtcd(ctx), "starting etcd")
+
+	os.Exit(runTestsWithCleanup(m, func() {
+		expectSuccess(stopEtcd(), "stopping etcd")
+	}))
+}
+
+func runTestsWithCleanup(m *testing.M, cleanup func()) int {
+	defer cleanup()
+	return m.Run()
+}
+
+func expectSuccess(err error, msg string) {
+	if err != nil {
+		log.Fatalf("%s: %s\n", msg, err)
+	}
+}
+
+func startEtcd(ctx context.Context) error {
+	if _, err := os.Stat(pidFile); err == nil {
+		log.Println("Etcd already running, reusing")
+		return nil
+	}
+
+	etcdPath, err := installEtcd(ctx)
+	if err != nil {
+		errors.Wrap(err, "installing etcd for tests")
+	}
+
+	etcdCmd := exec.Command(etcdPath, "--data-dir", "./etcd-data")
+	if os.Getenv("ETCD_SERVER_LOGS") == "true" {
+		log.Println("Enabling etcd server logs")
+		etcdCmd.Stdout = os.Stdout
+		etcdCmd.Stderr = os.Stderr
+	}
+	log.Println("Starting etcd")
+	if err := etcdCmd.Start(); err != nil {
+		errors.Wrap(err, "starting etcd for tests")
+	}
+
+	if err := os.WriteFile(pidFile, []byte(strconv.Itoa(etcdCmd.Process.Pid)), 0o600); err != nil {
+		return err
+	}
+
+	log.Println("Waiting for etcd to be up")
+	time.Sleep(time.Second)
+
+	return nil
+}
+
+func installEtcd(ctx context.Context) (string, error) {
+	projectRoot, err := filepath.Abs("../../")
+	if err != nil {
+		return "", err
+	}
+	binDir := filepath.Join(projectRoot, etcdBinDir)
+	etcdPath := filepath.Join(projectRoot, etcdBinPath)
+
+	if _, err := os.Stat(etcdPath); err == nil {
+		log.Println("Etcd already installed, skipping")
+		return etcdPath, nil
+	}
+
+	if err := os.MkdirAll(binDir, 0o755); err != nil {
+		return "", err
+	}
+
+	download := fmt.Sprintf("%s/%s/etcd-%s-linux-amd64.tar.gz", downloadURL, etcdVersion, etcdVersion)
+
+	// Hacky to run bash, but simplifies this code a lot
+	cmd := fmt.Sprintf("curl -sL %s | tar -xzvf - -C %s --strip-components=1", download, binDir)
+	out, err := exec.CommandContext(ctx, "bash", "-c", cmd).CombinedOutput()
+	if err != nil {
+		return "", errors.Wrapf(err, "downloading etcd: %s", string(out))
+	}
+
+	return etcdPath, nil
+}
+
+func stopEtcd() error {
+	if os.Getenv("REUSE_ETCD") == "true" {
+		log.Println("REUSE_ETCD=true, leaving etcd running")
+		return nil
+	}
+
+	if _, err := os.Stat(pidFile); os.IsNotExist(err) {
+		log.Println("Etcd pid file doesn't exit, skipping cleanup")
+		return nil
+	}
+
+	dat, err := os.ReadFile(pidFile)
+	if err != nil {
+		return err
+	}
+	pid, err := strconv.Atoi(string(dat))
+	if err != nil {
+		return err
+	}
+
+	etcdProcess, err := os.FindProcess(pid)
+	if err != nil {
+		return err
+	}
+
+	log.Println("Stopping etcd")
+	if err := etcdProcess.Kill(); err != nil {
+		return errors.Wrap(err, "Failed stopping etcd")
+	}
+
+	log.Println("Deleting etcd data")
+	if err := os.RemoveAll("./etcd-data"); err != nil {
+		return errors.Wrap(err, "deleting etcd data")
+	}
+
+	log.Println("Deleting etcd pid file")
+	if err := os.RemoveAll(pidFile); err != nil {
+		return errors.Wrap(err, "deleting pid file")
+	}
+
+	return nil
+}

pkg/instance/instance.go

@@ -0,0 +1,516 @@
+package instance
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+
+	"log/slog"
+	log "log/slog"
+
+	"github.com/vishvananda/netlink"
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
+	"github.com/kube-vip/kube-vip/pkg/cluster"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/networkinterface"
+	"github.com/kube-vip/kube-vip/pkg/sysctl"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
+
+// Instance defines an instance of everything needed to manage vips
+type Instance struct {
+	// Virtual IP / Load Balancer configuration
+	VIPConfigs []*kubevip.Config
+
+	// cluster instances
+	Clusters []*cluster.Cluster
+
+	// Service uses DHCP
+	IsDHCP              bool
+	DHCPInterface       string
+	DHCPInterfaceHwaddr string
+	DHCPInterfaceIP     string
+	DHCPHostname        string
+	DHCPClient          *vip.DHCPClient
+
+	// External Gateway IP the service is forwarded from
+	UPNPGatewayIPs []string
+
+	// Kubernetes service mapping
+	ServiceSnapshot *v1.Service
+}
+
+type Port struct {
+	Port uint16
+	Type string
+}
+
+func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Instance, error) {
+	instanceAddresses, _ := FetchServiceAddresses(svc)
+
+	var newVips []*kubevip.Config
+	var link netlink.Link
+	var err error
+
+	for _, address := range instanceAddresses {
+		// Detect if we're using a specific interface for services
+		var svcInterface string
+		svcInterface = svc.Annotations[kubevip.ServiceInterface] // If the service has a specific interface defined, then use it
+		if svcInterface == kubevip.Auto {
+			link, err = autoFindInterface(address)
+			if err != nil {
+				log.Error("automatically discover network interface for annotated IP", "address", address, "err", err)
+			} else {
+				if link == nil {
+					log.Error("automatically discover network interface for annotated IP address", "address", address)
+				}
+			}
+			if link == nil {
+				svcInterface = ""
+			} else {
+				svcInterface = getAutoInterfaceName(link, config.Interface)
+			}
+		}
+		// If it is still blank then use the
+		if svcInterface == "" {
+			switch config.ServicesInterface {
+			case kubevip.Auto:
+				link, err = autoFindInterface(address)
+				if err != nil {
+					log.Error("failed to automatically discover network interface for address", "ip", address, "err", err, "interface", config.Interface)
+				} else if link == nil {
+					log.Error("failed to automatically discover network interface for address", "ip", address, "defaulting to", config.Interface)
+				}
+				svcInterface = getAutoInterfaceName(link, config.Interface)
+			case "":
+				svcInterface = config.Interface
+			default:
+				svcInterface = config.ServicesInterface
+			}
+		}
+
+		if link == nil {
+			if link, err = netlink.LinkByName(svcInterface); err != nil {
+				return nil, fmt.Errorf("failed to get interface %s: %w", svcInterface, err)
+			}
+			if link == nil {
+				return nil, fmt.Errorf("failed to get interface %s", svcInterface)
+			}
+		}
+
+		cidrs := vip.Split(config.VIPSubnet)
+
+		ipv4AutoSubnet := false
+		ipv6AutoSubnet := false
+		if cidrs[0] == kubevip.Auto {
+			ipv4AutoSubnet = true
+		}
+
+		if len(cidrs) > 1 && cidrs[1] == kubevip.Auto {
+			ipv6AutoSubnet = true
+		}
+
+		if (config.Address != "" || config.VIP != "") && (ipv4AutoSubnet || ipv6AutoSubnet) {
+			return nil, fmt.Errorf("auto subnet discovery cannot be used if VIP address was provided")
+		}
+
+		subnet := ""
+		var err error
+		if utils.IsIPv4(address) {
+			if ipv4AutoSubnet {
+				subnet, err = autoFindSubnet(link, address)
+				if err != nil {
+					return nil, fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
+				}
+			} else {
+				if cidrs[0] != "" && cidrs[0] != kubevip.Auto {
+					subnet = cidrs[0]
+				} else {
+					subnet = "32"
+				}
+			}
+		} else {
+			if ipv6AutoSubnet {
+				subnet, err = autoFindSubnet(link, address)
+				if err != nil {
+					return nil, fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
+				}
+			} else {
+				if len(cidrs) > 1 && cidrs[1] != "" && cidrs[1] != kubevip.Auto {
+					subnet = cidrs[1]
+				} else {
+					subnet = "128"
+				}
+			}
+		}
+
+		// Generate new Virtual IP configuration
+		newVips = append(newVips, &kubevip.Config{
+			VIP:                         address,
+			Interface:                   svcInterface,
+			SingleNode:                  true,
+			EnableARP:                   config.EnableARP,
+			EnableBGP:                   config.EnableBGP,
+			VIPSubnet:                   subnet,
+			EnableRoutingTable:          config.EnableRoutingTable,
+			RoutingTableID:              config.RoutingTableID,
+			RoutingTableType:            config.RoutingTableType,
+			RoutingProtocol:             config.RoutingProtocol,
+			ArpBroadcastRate:            config.ArpBroadcastRate,
+			EnableServiceSecurity:       config.EnableServiceSecurity,
+			DNSMode:                     config.DNSMode,
+			DisableServiceUpdates:       config.DisableServiceUpdates,
+			EnableServicesElection:      config.EnableServicesElection,
+			PreserveVIPOnLeadershipLoss: config.PreserveVIPOnLeadershipLoss,
+			KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
+				EnableLeaderElection: config.EnableLeaderElection,
+			},
+		})
+	}
+
+	// Create new service
+	instance := &Instance{
+		//UID:             instanceUID,
+		//VIPs:            instanceAddresses,
+		ServiceSnapshot: svc,
+	}
+	// for _, port := range svc.Spec.Ports {
+	// 	instance.ExternalPorts = append(instance.ExternalPorts, Port{
+	// 		Port: uint16(port.Port), //nolint
+	// 		Type: string(port.Protocol),
+	// 	})
+	// }
+
+	if svc.Annotations != nil {
+		instance.DHCPInterfaceHwaddr = svc.Annotations[kubevip.HwAddrKey]
+		instance.DHCPInterfaceIP = svc.Annotations[kubevip.RequestedIP]
+		instance.DHCPHostname = svc.Annotations[kubevip.LoadbalancerHostname]
+	}
+
+	configPorts := make([]kubevip.Port, 0)
+	for _, p := range svc.Spec.Ports {
+		configPorts = append(configPorts, kubevip.Port{
+			Type: string(p.Protocol),
+			Port: int(p.Port),
+		})
+	}
+	// Generate Load Balancer config
+	newLB := kubevip.LoadBalancer{
+		Name:      fmt.Sprintf("%s-load-balancer", svc.Name),
+		Ports:     configPorts,
+		BindToVip: true,
+	}
+	for _, vip := range newVips {
+		// Add Load Balancer Configuration
+		vip.LoadBalancers = append(vip.LoadBalancers, newLB)
+	}
+	// Create Add configuration to the new service
+	instance.VIPConfigs = newVips
+
+	// If this was purposely created with the address 0.0.0.0,
+	// we will create a macvlan on the main interface and a DHCP client
+	// TODO: Consider how best to handle DHCP with multiple addresses
+	if len(instanceAddresses) == 1 && instanceAddresses[0] == "0.0.0.0" {
+		err := instance.startDHCP()
+		if err != nil {
+			return nil, err
+		}
+		select {
+		case err := <-instance.DHCPClient.ErrorChannel():
+			return nil, fmt.Errorf("error starting DHCP for %s/%s: error: %s",
+				instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
+		case ip := <-instance.DHCPClient.IPChannel():
+			instance.VIPConfigs[0].Interface = instance.DHCPInterface
+			instance.VIPConfigs[0].VIP = ip
+			instance.DHCPInterfaceIP = ip
+		}
+	}
+
+	for _, vipConfig := range instance.VIPConfigs {
+		c, err := cluster.InitCluster(vipConfig, false, intfMgr, arpMgr)
+		if err != nil {
+			log.Error("Failed to add Service %s/%s", svc.Namespace, svc.Name)
+			return nil, err
+		}
+
+		for i := range c.Network {
+			c.Network[i].SetServicePorts(svc)
+		}
+
+		instance.Clusters = append(instance.Clusters, c)
+		log.Info("(svcs) adding VIP", "ip", vipConfig.VIP, "interface", vipConfig.Interface, "namespace", svc.Namespace, "name", svc.Name)
+	}
+
+	return instance, nil
+}
+
+func autoFindInterface(ip string) (netlink.Link, error) {
+	links, err := netlink.LinkList()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list network interfaces: %w", err)
+	}
+
+	address := net.ParseIP(ip)
+
+	family := netlink.FAMILY_V4
+
+	if address.To4() == nil {
+		family = netlink.FAMILY_V6
+	}
+
+	for _, link := range links {
+		addr, err := netlink.AddrList(link, family)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get IP addresses for interface %s: %w", link.Attrs().Name, err)
+		}
+		for _, a := range addr {
+			if a.IPNet.Contains(address) {
+				return link, nil
+			}
+		}
+	}
+
+	return nil, nil
+}
+
+func autoFindSubnet(link netlink.Link, ip string) (string, error) {
+	address := net.ParseIP(ip)
+
+	family := netlink.FAMILY_V4
+	if address.To4() == nil {
+		family = netlink.FAMILY_V6
+	}
+
+	addr, err := netlink.AddrList(link, family)
+	if err != nil {
+		return "", fmt.Errorf("failed to get IP addresses for interface %s: %w", link.Attrs().Name, err)
+	}
+	for _, a := range addr {
+		if a.IPNet.Contains(address) {
+			m, _ := a.IPNet.Mask.Size()
+			return strconv.Itoa(m), nil
+		}
+	}
+	return "", fmt.Errorf("failed to find suitable subnet for address %s", ip)
+}
+
+func getAutoInterfaceName(link netlink.Link, defaultInterface string) string {
+	if link == nil {
+		return defaultInterface
+	}
+	return link.Attrs().Name
+}
+
+func (i *Instance) startDHCP() error {
+	if len(i.VIPConfigs) != 1 {
+		return fmt.Errorf("DHCP requires exactly 1 VIP config, got: %v", len(i.VIPConfigs))
+	}
+	parent, err := netlink.LinkByName(i.VIPConfigs[0].Interface)
+	if err != nil {
+		return fmt.Errorf("error finding VIP Interface, for building DHCP Link : %v", err)
+	}
+
+	// Generate name from UID
+	interfaceName := fmt.Sprintf("vip-%s", i.ServiceSnapshot.UID[0:8])
+
+	// Check if the interface doesn't exist first
+	iface, err := net.InterfaceByName(interfaceName)
+	if err != nil {
+		log.Info("creating new macvlan interface for DHCP", "interface", interfaceName)
+
+		hwaddr, err := net.ParseMAC(i.DHCPInterfaceHwaddr)
+		if i.DHCPInterfaceHwaddr != "" && err != nil {
+			return err
+		} else if hwaddr == nil {
+			hwaddr, err = net.ParseMAC(vip.GenerateMac())
+			if err != nil {
+				return err
+			}
+		}
+
+		log.Info("new macvlan interface", "interface", interfaceName, "hardware address", hwaddr)
+		mac := &netlink.Macvlan{
+			LinkAttrs: netlink.LinkAttrs{
+				Name:         interfaceName,
+				ParentIndex:  parent.Attrs().Index,
+				HardwareAddr: hwaddr,
+			},
+			Mode: netlink.MACVLAN_MODE_DEFAULT,
+		}
+
+		err = netlink.LinkAdd(mac)
+		if err != nil {
+			return fmt.Errorf("could not add %s: %v", interfaceName, err)
+		}
+
+		err = netlink.LinkSetUp(mac)
+		if err != nil {
+			return fmt.Errorf("could not bring up interface [%s] : %v", interfaceName, err)
+		}
+
+		iface, err = net.InterfaceByName(interfaceName)
+		if err != nil {
+			return fmt.Errorf("error finding new DHCP interface by name [%v]", err)
+		}
+	} else {
+		log.Info("Using existing macvlan interface for DHCP", "interface", interfaceName)
+	}
+
+	// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
+	rpfilterSetting := "0"
+
+	// Check if we need to set an override rp_filter value for the interface
+	if i.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
+		// Check the rp_filter value
+		rpFilter, err := strconv.Atoi(i.ServiceSnapshot.Annotations[kubevip.RPFilter])
+		if err != nil {
+			slog.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
+		} else {
+			if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
+				rpfilterSetting = i.ServiceSnapshot.Annotations[kubevip.RPFilter]
+			} else {
+				slog.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
+			}
+		}
+	}
+
+	err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
+	if err != nil {
+		slog.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
+	}
+	var initRebootFlag bool
+	if i.DHCPInterfaceIP != "" {
+		initRebootFlag = true
+	}
+
+	client := vip.NewDHCPClient(iface, initRebootFlag, i.DHCPInterfaceIP)
+
+	// Add hostname to dhcp client if annotated
+	if i.DHCPHostname != "" {
+		log.Info("Hostname specified for dhcp lease", "interface", interfaceName, "hostname", i.DHCPHostname)
+		client.WithHostName(i.DHCPHostname)
+	}
+
+	go client.Start()
+
+	// Set that DHCP is enabled
+	i.IsDHCP = true
+	// Set the name of the interface so that it can be removed on Service deletion
+	i.DHCPInterface = interfaceName
+	i.DHCPInterfaceHwaddr = iface.HardwareAddr.String()
+	// Add the client so that we can call it to stop function
+	i.DHCPClient = client
+
+	return nil
+}
+
+// FetchLoadBalancerIngressAddresses tries to get the addresses from status.loadBalancerIP
+func FetchLoadBalancerIngress(s *v1.Service) ([]string, []string) {
+	// If the service has no status, return empty
+	lbStatusAddresses := []string{}
+	lbStatusHostnames := []string{}
+	if len(s.Status.LoadBalancer.Ingress) == 0 {
+		return lbStatusAddresses, lbStatusHostnames
+	}
+
+	for _, ingress := range s.Status.LoadBalancer.Ingress {
+		if ingress.IP != "" {
+			lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
+		}
+		if ingress.Hostname != "" {
+			lbStatusHostnames = append(lbStatusHostnames, ingress.Hostname)
+		}
+	}
+	return lbStatusAddresses, lbStatusHostnames
+}
+
+// FetchServiceAddresses tries to get the addresses from annotations
+// kube-vip.io/loadbalancerIPs, then from spec.loadbalancerIP
+func FetchServiceAddresses(s *v1.Service) ([]string, []string) {
+	annotationAvailable := false
+	if s.Annotations != nil {
+
+		if v, annotationAvailable := s.Annotations[kubevip.LoadbalancerIPAnnotation]; annotationAvailable {
+			ips := strings.Split(v, ",")
+			var trimmedIPs []string
+			var trimmedHostnames []string
+			for _, a := range ips {
+				a = strings.TrimSpace(a)
+				ip := net.ParseIP(a)
+				if ip == nil {
+					// this is probably a DNS name
+					trimmedHostnames = append(trimmedHostnames, a)
+				} else {
+					trimmedIPs = append(trimmedIPs, ip.String())
+				}
+			}
+			return trimmedIPs, trimmedHostnames
+		}
+	}
+
+	lbStatusAddresses := []string{}
+	lbStatusHostnames := []string{}
+	if !annotationAvailable {
+		lbStatusAddresses, lbStatusHostnames = FetchLoadBalancerIngress(s)
+	}
+
+	// Spec.LoadBalancerIP legacy handling
+	// if the loadBalancerIP is different from Status.LoadBalancer.Ingress IPs
+	// return the legacy LB as spec wins over status.
+	if lbIP := net.ParseIP(s.Spec.LoadBalancerIP); lbIP != nil && len(lbStatusAddresses) > 0 {
+		isLbIPv4 := utils.IsIPv4(s.Spec.LoadBalancerIP)
+		for _, a := range lbStatusAddresses {
+			if lbStatusIP := net.ParseIP(a); lbStatusIP != nil && utils.IsIPv4(a) == isLbIPv4 && !lbIP.Equal(lbStatusIP) {
+				return []string{s.Spec.LoadBalancerIP}, []string{}
+			}
+		}
+	}
+
+	if len(lbStatusAddresses) > 0 || len(lbStatusHostnames) > 0 {
+		return lbStatusAddresses, lbStatusHostnames
+	}
+
+	if s.Spec.LoadBalancerIP != "" {
+		return []string{s.Spec.LoadBalancerIP}, []string{}
+	}
+
+	return []string{}, []string{}
+}
+
+func FindServiceInstance(svc *v1.Service, instances []*Instance) *Instance {
+	log.Debug("finding service", "namespace", svc.Namespace, "name", svc.Name, "UID", svc.UID)
+	for i := range instances {
+		log.Debug("saved service", "instance", i, "UID", instances[i].ServiceSnapshot.UID)
+		if instances[i].ServiceSnapshot.UID == svc.UID {
+			return instances[i]
+		}
+	}
+	log.Debug("instance not found", "namespace", svc.Namespace, "name", svc.Name, "UID", svc.UID)
+	return nil
+}
+
+func FindServiceInstanceWithTimeout(svc *v1.Service, instances []*Instance) *Instance {
+	log.Debug("finding service with timeout", "namespace", svc.Namespace, "name", svc.Name, "UID", svc.UID)
+	ticker := time.NewTicker(time.Millisecond * 200)
+	defer ticker.Stop()
+	to := time.NewTimer(time.Second * 60)
+	defer to.Stop()
+	for {
+		select {
+		case <-to.C:
+			return nil
+		case <-ticker.C:
+			for i := range instances {
+				log.Debug("saved service", "instance", i, "UID", instances[i].ServiceSnapshot.UID)
+				if instances[i].ServiceSnapshot.UID == svc.UID {
+					return instances[i]
+				}
+			}
+		}
+	}
+}

pkg/iptables/iptables.go

@@ -0,0 +1,741 @@
+// Copyright 2015 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iptables
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"syscall"
+)
+
+// Adds the output of stderr to exec.ExitError
+type Error struct {
+	exec.ExitError
+	cmd        exec.Cmd
+	msg        string
+	exitStatus *int //for overriding
+}
+
+func (e *Error) ExitStatus() int {
+	if e.exitStatus != nil {
+		return *e.exitStatus
+	}
+	return e.Sys().(syscall.WaitStatus).ExitStatus()
+}
+
+func (e *Error) Error() string {
+	return fmt.Sprintf("running %v: exit status %v: %v", e.cmd.Args, e.ExitStatus(), e.msg)
+}
+
+// IsNotExist returns true if the error is due to the chain or rule not existing
+func (e *Error) IsNotExist() bool {
+	if e.ExitStatus() != 1 {
+		return false
+	}
+	msgNoRuleExist := "Bad rule (does a matching rule exist in that chain?).\n"
+	msgNoChainExist := "No chain/target/match by that name.\n"
+	return strings.Contains(e.msg, msgNoRuleExist) || strings.Contains(e.msg, msgNoChainExist)
+}
+
+// Protocol to differentiate between IPv4 and IPv6
+type Protocol byte
+
+const (
+	ProtocolIPv4 Protocol = iota
+	ProtocolIPv6
+)
+
+const (
+	TableFilter      = "filter"
+	TableMangle      = "mangle"
+	TableNat         = "nat"
+	ChainInput       = "INPUT"
+	ChainPREROUTING  = "PREROUTING"
+	ChainPOSTROUTING = "POSTROUTING"
+)
+
+type IPTables struct {
+	path              string
+	proto             Protocol
+	hasCheck          bool
+	hasWait           bool
+	waitSupportSecond bool
+	hasRandomFully    bool
+	v1                int
+	v2                int
+	v3                int
+	mode              string // the underlying iptables operating mode, e.g. nf_tables
+	timeout           int    // time to wait for the iptables lock, default waits forever
+
+	nftables bool
+}
+
+// Stat represents a structured statistic entry.
+type Stat struct {
+	Packets     uint64     `json:"pkts"`
+	Bytes       uint64     `json:"bytes"`
+	Target      string     `json:"target"`
+	Protocol    string     `json:"prot"`
+	Opt         string     `json:"opt"`
+	Input       string     `json:"in"`
+	Output      string     `json:"out"`
+	Source      *net.IPNet `json:"source"`
+	Destination *net.IPNet `json:"destination"`
+	Options     string     `json:"options"`
+}
+
+type Option func(*IPTables)
+
+func IPFamily(proto Protocol) Option {
+	return func(ipt *IPTables) {
+		ipt.proto = proto
+	}
+}
+
+func Timeout(timeout int) Option {
+	return func(ipt *IPTables) {
+		ipt.timeout = timeout
+	}
+}
+
+func EnableNFTables(enable bool) Option {
+	return func(ipt *IPTables) {
+		ipt.nftables = enable
+	}
+}
+
+// New creates a new IPTables configured with the options passed as parameter.
+// For backwards compatibility, by default always uses IPv4 and timeout 0.
+// i.e. you can create an IPv6 IPTables using a timeout of 5 seconds passing
+// the IPFamily and Timeout options as follow:
+//
+//	ip6t := New(IPFamily(ProtocolIPv6), Timeout(5))
+func New(opts ...Option) (*IPTables, error) {
+
+	ipt := &IPTables{
+		proto:   ProtocolIPv4,
+		timeout: 0,
+	}
+
+	for _, opt := range opts {
+		opt(ipt)
+	}
+
+	path, err := exec.LookPath(getIptablesCommand(ipt.proto, ipt.nftables))
+	if err != nil {
+		return nil, err
+	}
+	ipt.path = path
+
+	vstring, err := getIptablesVersionString(path)
+	if err != nil {
+		return nil, fmt.Errorf("could not get iptables version: %v", err)
+	}
+	v1, v2, v3, mode, err := extractIptablesVersion(vstring)
+	if err != nil {
+		return nil, fmt.Errorf("failed to extract iptables version from [%s]: %v", vstring, err)
+	}
+	ipt.v1 = v1
+	ipt.v2 = v2
+	ipt.v3 = v3
+	ipt.mode = mode
+
+	checkPresent, waitPresent, waitSupportSecond, randomFullyPresent := getIptablesCommandSupport(v1, v2, v3)
+	ipt.hasCheck = checkPresent
+	ipt.hasWait = waitPresent
+	ipt.waitSupportSecond = waitSupportSecond
+	ipt.hasRandomFully = randomFullyPresent
+
+	return ipt, nil
+}
+
+// New creates a new IPTables for the given proto.
+// The proto will determine which command is used, either "iptables" or "ip6tables".
+func NewWithProtocol(proto Protocol) (*IPTables, error) {
+	return New(IPFamily(proto), Timeout(0))
+}
+
+// Proto returns the protocol used by this IPTables.
+func (ipt *IPTables) Proto() Protocol {
+	return ipt.proto
+}
+
+// Exists checks if given rulespec in specified table/chain exists
+func (ipt *IPTables) Exists(table, chain string, rulespec ...string) (bool, error) {
+	if !ipt.hasCheck {
+		return ipt.existsForOldIptables(table, chain, rulespec)
+
+	}
+	cmd := append([]string{"-t", table, "-C", chain}, rulespec...)
+	err := ipt.run(cmd...)
+	eerr, eok := err.(*Error)
+	switch {
+	case err == nil:
+		return true, nil
+	case eok && eerr.ExitStatus() == 1:
+		return false, nil
+	default:
+		return false, err
+	}
+}
+
+// Insert inserts rulespec to specified table/chain (in specified pos)
+func (ipt *IPTables) Insert(table, chain string, pos int, rulespec ...string) error {
+	cmd := append([]string{"-t", table, "-I", chain, strconv.Itoa(pos)}, rulespec...)
+	return ipt.run(cmd...)
+}
+
+// InsertUnique acts like Insert except that it won't insert a duplicate (no matter the position in the chain)
+func (ipt *IPTables) InsertUnique(table, chain string, pos int, rulespec ...string) error {
+	exists, err := ipt.Exists(table, chain, rulespec...)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		return ipt.Insert(table, chain, pos, rulespec...)
+	}
+
+	return nil
+}
+
+// Append appends rulespec to specified table/chain
+func (ipt *IPTables) Append(table, chain string, rulespec ...string) error {
+	cmd := append([]string{"-t", table, "-A", chain}, rulespec...)
+	return ipt.run(cmd...)
+}
+
+// AppendUnique acts like Append except that it won't add a duplicate
+func (ipt *IPTables) AppendUnique(table, chain string, rulespec ...string) error {
+	exists, err := ipt.Exists(table, chain, rulespec...)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		return ipt.Append(table, chain, rulespec...)
+	}
+
+	return nil
+}
+
+// Delete removes rulespec in specified table/chain
+func (ipt *IPTables) Delete(table, chain string, rulespec ...string) error {
+	cmd := append([]string{"-t", table, "-D", chain}, rulespec...)
+	return ipt.run(cmd...)
+}
+
+func (ipt *IPTables) DeleteIfExists(table, chain string, rulespec ...string) error {
+	exists, err := ipt.Exists(table, chain, rulespec...)
+	if err == nil && exists {
+		err = ipt.Delete(table, chain, rulespec...)
+	}
+	return err
+}
+
+// List rules in specified table/chain
+func (ipt *IPTables) ListByID(table, chain string, id int) (string, error) {
+	args := []string{"-t", table, "-S", chain, strconv.Itoa(id)}
+	rule, err := ipt.executeList(args)
+	if err != nil {
+		return "", err
+	}
+	return rule[0], nil
+}
+
+// List rules in specified table/chain
+func (ipt *IPTables) List(table, chain string) ([]string, error) {
+	args := []string{"-t", table, "-S", chain}
+	return ipt.executeList(args)
+}
+
+// List rules (with counters) in specified table/chain
+func (ipt *IPTables) ListWithCounters(table, chain string) ([]string, error) {
+	args := []string{"-t", table, "-v", "-S", chain}
+	return ipt.executeList(args)
+}
+
+// ListChains returns a slice containing the name of each chain in the specified table.
+func (ipt *IPTables) ListChains(table string) ([]string, error) {
+	args := []string{"-t", table, "-S"}
+
+	result, err := ipt.executeList(args)
+	if err != nil {
+		return nil, err
+	}
+
+	// Iterate over rules to find all default (-P) and user-specified (-N) chains.
+	// Chains definition always come before rules.
+	// Format is the following:
+	// -P OUTPUT ACCEPT
+	// -N Custom
+	var chains []string
+	for _, val := range result {
+		if strings.HasPrefix(val, "-P") || strings.HasPrefix(val, "-N") {
+			chains = append(chains, strings.Fields(val)[1])
+		} else {
+			break
+		}
+	}
+	return chains, nil
+}
+
+// '-S' is fine with non existing rule index as long as the chain exists
+// therefore pass index 1 to reduce overhead for large chains
+func (ipt *IPTables) ChainExists(table, chain string) (bool, error) {
+	err := ipt.run("-t", table, "-S", chain, "1")
+	eerr, eok := err.(*Error)
+	switch {
+	case err == nil:
+		return true, nil
+	case eok && eerr.ExitStatus() == 1:
+		return false, nil
+	default:
+		return false, err
+	}
+}
+
+// Stats lists rules including the byte and packet counts
+func (ipt *IPTables) Stats(table, chain string) ([][]string, error) {
+	args := []string{"-t", table, "-L", chain, "-n", "-v", "-x"}
+	lines, err := ipt.executeList(args)
+	if err != nil {
+		return nil, err
+	}
+
+	appendSubnet := func(addr string) string {
+		if strings.IndexByte(addr, byte('/')) < 0 {
+			if strings.IndexByte(addr, '.') < 0 {
+				return addr + "/128"
+			}
+			return addr + "/32"
+		}
+		return addr
+	}
+
+	ipv6 := ipt.proto == ProtocolIPv6
+
+	rows := [][]string{}
+	for i, line := range lines {
+		// Skip over chain name and field header
+		if i < 2 {
+			continue
+		}
+
+		// Fields:
+		// 0=pkts 1=bytes 2=target 3=prot 4=opt 5=in 6=out 7=source 8=destination 9=options
+		line = strings.TrimSpace(line)
+		fields := strings.Fields(line)
+
+		// The ip6tables verbose output cannot be naively split due to the default "opt"
+		// field containing 2 single spaces.
+		if ipv6 {
+			// Check if field 6 is "opt" or "source" address
+			dest := fields[6]
+			ip, _, _ := net.ParseCIDR(dest)
+			if ip == nil {
+				ip = net.ParseIP(dest)
+			}
+
+			// If we detected a CIDR or IP, the "opt" field is empty.. insert it.
+			if ip != nil {
+				f := []string{}
+				f = append(f, fields[:4]...)
+				f = append(f, "  ") // Empty "opt" field for ip6tables
+				f = append(f, fields[4:]...)
+				fields = f
+			}
+		}
+
+		// Adjust "source" and "destination" to include netmask, to match regular
+		// List output
+		fields[7] = appendSubnet(fields[7])
+		fields[8] = appendSubnet(fields[8])
+
+		// Combine "options" fields 9... into a single space-delimited field.
+		options := fields[9:]
+		fields = fields[:9]
+		fields = append(fields, strings.Join(options, " "))
+		rows = append(rows, fields)
+	}
+	return rows, nil
+}
+
+// ParseStat parses a single statistic row into a Stat struct. The input should
+// be a string slice that is returned from calling the Stat method.
+func (ipt *IPTables) ParseStat(stat []string) (parsed Stat, err error) {
+	// For forward-compatibility, expect at least 10 fields in the stat
+	if len(stat) < 10 {
+		return parsed, fmt.Errorf("stat contained fewer fields than expected")
+	}
+
+	// Convert the fields that are not plain strings
+	parsed.Packets, err = strconv.ParseUint(stat[0], 0, 64)
+	if err != nil {
+		return parsed, fmt.Errorf(err.Error(), "could not parse packets")
+	}
+	parsed.Bytes, err = strconv.ParseUint(stat[1], 0, 64)
+	if err != nil {
+		return parsed, fmt.Errorf(err.Error(), "could not parse bytes")
+	}
+	_, parsed.Source, err = net.ParseCIDR(stat[7])
+	if err != nil {
+		return parsed, fmt.Errorf(err.Error(), "could not parse source")
+	}
+	_, parsed.Destination, err = net.ParseCIDR(stat[8])
+	if err != nil {
+		return parsed, fmt.Errorf(err.Error(), "could not parse destination")
+	}
+
+	// Put the fields that are strings
+	parsed.Target = stat[2]
+	parsed.Protocol = stat[3]
+	parsed.Opt = stat[4]
+	parsed.Input = stat[5]
+	parsed.Output = stat[6]
+	parsed.Options = stat[9]
+
+	return parsed, nil
+}
+
+// StructuredStats returns statistics as structured data which may be further
+// parsed and marshaled.
+func (ipt *IPTables) StructuredStats(table, chain string) ([]Stat, error) {
+	rawStats, err := ipt.Stats(table, chain)
+	if err != nil {
+		return nil, err
+	}
+
+	structStats := []Stat{}
+	for _, rawStat := range rawStats {
+		stat, err := ipt.ParseStat(rawStat)
+		if err != nil {
+			return nil, err
+		}
+		structStats = append(structStats, stat)
+	}
+
+	return structStats, nil
+}
+
+func (ipt *IPTables) executeList(args []string) ([]string, error) {
+	var stdout bytes.Buffer
+	if err := ipt.runWithOutput(args, &stdout); err != nil {
+		return nil, err
+	}
+
+	rules := strings.Split(stdout.String(), "\n")
+
+	// strip trailing newline
+	if len(rules) > 0 && rules[len(rules)-1] == "" {
+		rules = rules[:len(rules)-1]
+	}
+
+	for i, rule := range rules {
+		rules[i] = filterRuleOutput(rule)
+	}
+
+	return rules, nil
+}
+
+// NewChain creates a new chain in the specified table.
+// If the chain already exists, it will result in an error.
+func (ipt *IPTables) NewChain(table, chain string) error {
+	return ipt.run("-t", table, "-N", chain)
+}
+
+const existsErr = 1
+
+// ClearChain flushed (deletes all rules) in the specified table/chain.
+// If the chain does not exist, a new one will be created
+func (ipt *IPTables) ClearChain(table, chain string) error {
+	err := ipt.NewChain(table, chain)
+
+	eerr, eok := err.(*Error)
+	switch {
+	case err == nil:
+		return nil
+	case eok && eerr.ExitStatus() == existsErr:
+		// chain already exists. Flush (clear) it.
+		return ipt.run("-t", table, "-F", chain)
+	default:
+		return err
+	}
+}
+
+// RenameChain renames the old chain to the new one.
+func (ipt *IPTables) RenameChain(table, oldChain, newChain string) error {
+	return ipt.run("-t", table, "-E", oldChain, newChain)
+}
+
+// DeleteChain deletes the chain in the specified table.
+// The chain must be empty
+func (ipt *IPTables) DeleteChain(table, chain string) error {
+	return ipt.run("-t", table, "-X", chain)
+}
+
+func (ipt *IPTables) ClearAndDeleteChain(table, chain string) error {
+	exists, err := ipt.ChainExists(table, chain)
+	if err != nil || !exists {
+		return err
+	}
+	err = ipt.run("-t", table, "-F", chain)
+	if err == nil {
+		err = ipt.run("-t", table, "-X", chain)
+	}
+	return err
+}
+
+func (ipt *IPTables) ClearAll() error {
+	return ipt.run("-F")
+}
+
+func (ipt *IPTables) DeleteAll() error {
+	return ipt.run("-X")
+}
+
+// ChangePolicy changes policy on chain to target
+func (ipt *IPTables) ChangePolicy(table, chain, target string) error {
+	return ipt.run("-t", table, "-P", chain, target)
+}
+
+// Check if the underlying iptables command supports the --random-fully flag
+func (ipt *IPTables) HasRandomFully() bool {
+	return ipt.hasRandomFully
+}
+
+// Return version components of the underlying iptables command
+func (ipt *IPTables) GetIptablesVersion() (int, int, int) {
+	return ipt.v1, ipt.v2, ipt.v3
+}
+
+// run runs an iptables command with the given arguments, ignoring
+// any stdout output
+func (ipt *IPTables) run(args ...string) error {
+	return ipt.runWithOutput(args, nil)
+}
+
+// runWithOutput runs an iptables command with the given arguments,
+// writing any stdout output to the given writer
+func (ipt *IPTables) runWithOutput(args []string, stdout io.Writer) error {
+	args = append([]string{ipt.path}, args...)
+	if ipt.hasWait {
+		args = append(args, "--wait")
+		if ipt.timeout != 0 && ipt.waitSupportSecond {
+			args = append(args, strconv.Itoa(ipt.timeout))
+		}
+	} else {
+		fmu, err := newXtablesFileLock()
+		if err != nil {
+			return err
+		}
+		ul, err := fmu.tryLock()
+		if err != nil {
+			syscall.Close(fmu.fd)
+			return err
+		}
+		defer func() {
+			_ = ul.Unlock()
+		}()
+	}
+
+	var stderr bytes.Buffer
+	cmd := exec.Cmd{
+		Path:   ipt.path,
+		Args:   args,
+		Stdout: stdout,
+		Stderr: &stderr,
+	}
+
+	if err := cmd.Run(); err != nil {
+		switch e := err.(type) {
+		case *exec.ExitError:
+			return &Error{*e, cmd, stderr.String(), nil}
+		default:
+			return err
+		}
+	}
+
+	return nil
+}
+
+// getIptablesCommand returns the correct command for the given protocol, either "iptables" or "ip6tables".
+func getIptablesCommand(proto Protocol, nftables bool) string {
+	if proto == ProtocolIPv6 {
+		if nftables {
+			return "ip6tables-nft"
+		}
+		return "ip6tables-legacy"
+	}
+
+	if nftables {
+		return "iptables-nft"
+	}
+	return "iptables-legacy"
+}
+
+// Checks if iptables has the "-C" and "--wait" flag
+func getIptablesCommandSupport(v1 int, v2 int, v3 int) (bool, bool, bool, bool) {
+	return iptablesHasCheckCommand(v1, v2, v3), iptablesHasWaitCommand(v1, v2, v3), iptablesWaitSupportSecond(v1, v2), iptablesHasRandomFully(v1, v2, v3)
+}
+
+// getIptablesVersion returns the first three components of the iptables version
+// and the operating mode (e.g. nf_tables or legacy)
+// e.g. "iptables v1.3.66" would return (1, 3, 66, legacy, nil)
+func extractIptablesVersion(str string) (int, int, int, string, error) {
+	versionMatcher := regexp.MustCompile(`v([0-9]+)\.([0-9]+)\.([0-9]+)(?:\s+\((\w+))?`)
+	result := versionMatcher.FindStringSubmatch(str)
+	if result == nil {
+		return 0, 0, 0, "", fmt.Errorf("no iptables version found in string: %s", str)
+	}
+
+	v1, err := strconv.Atoi(result[1])
+	if err != nil {
+		return 0, 0, 0, "", err
+	}
+
+	v2, err := strconv.Atoi(result[2])
+	if err != nil {
+		return 0, 0, 0, "", err
+	}
+
+	v3, err := strconv.Atoi(result[3])
+	if err != nil {
+		return 0, 0, 0, "", err
+	}
+
+	mode := "legacy"
+	if result[4] != "" {
+		mode = result[4]
+	}
+	return v1, v2, v3, mode, nil
+}
+
+// Runs "iptables --version" to get the version string
+func getIptablesVersionString(path string) (string, error) {
+	cmd := exec.Command(path, "--version")
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err := cmd.Run()
+	if err != nil {
+		return "", err
+	}
+	return out.String(), nil
+}
+
+// Checks if an iptables version is after 1.4.11, when --check was added
+func iptablesHasCheckCommand(v1 int, v2 int, v3 int) bool {
+	if v1 > 1 {
+		return true
+	}
+	if v1 == 1 && v2 > 4 {
+		return true
+	}
+	if v1 == 1 && v2 == 4 && v3 >= 11 {
+		return true
+	}
+	return false
+}
+
+// Checks if an iptables version is after 1.4.20, when --wait was added
+func iptablesHasWaitCommand(v1 int, v2 int, v3 int) bool { //nolint
+	if v1 > 1 {
+		return true
+	}
+	if v1 == 1 && v2 > 4 {
+		return true
+	}
+	if v1 == 1 && v2 == 4 && v3 >= 20 {
+		return true
+	}
+	return false
+}
+
+// Checks if an iptablse version is after 1.6.0, when --wait support second
+func iptablesWaitSupportSecond(v1 int, v2 int) bool {
+	if v1 > 1 {
+		return true
+	}
+	if v1 == 1 && v2 >= 6 {
+		return true
+	}
+	return false
+}
+
+// Checks if an iptables version is after 1.6.2, when --random-fully was added
+func iptablesHasRandomFully(v1 int, v2 int, v3 int) bool {
+	if v1 > 1 {
+		return true
+	}
+	if v1 == 1 && v2 > 6 {
+		return true
+	}
+	if v1 == 1 && v2 == 6 && v3 >= 2 {
+		return true
+	}
+	return false
+}
+
+// Checks if a rule specification exists for a table
+func (ipt *IPTables) existsForOldIptables(table, chain string, rulespec []string) (bool, error) {
+	rs := strings.Join(append([]string{"-A", chain}, rulespec...), " ")
+	args := []string{"-t", table, "-S"}
+	var stdout bytes.Buffer
+	err := ipt.runWithOutput(args, &stdout)
+	if err != nil {
+		return false, err
+	}
+	return strings.Contains(stdout.String(), rs), nil
+}
+
+// counterRegex is the regex used to detect nftables counter format
+var counterRegex = regexp.MustCompile(`^\[([0-9]+):([0-9]+)\] `)
+
+// filterRuleOutput works around some inconsistencies in output.
+// For example, when iptables is in legacy vs. nftables mode, it produces
+// different results.
+func filterRuleOutput(rule string) string {
+	out := rule
+
+	// work around an output difference in nftables mode where counters
+	// are output in iptables-save format, rather than iptables -S format
+	// The string begins with "[0:0]"
+	//
+	// Fixes #49
+	if groups := counterRegex.FindStringSubmatch(out); groups != nil {
+		// drop the brackets
+		out = out[len(groups[0]):]
+		out = fmt.Sprintf("%s -c %s %s", out, groups[1], groups[2])
+	}
+
+	return out
+}
+
+func GetIPTablesRuleSpecification(rule, specification string) string {
+	parts := strings.Split(rule, " ")
+	for i, part := range parts {
+		if part == specification && i+1 < len(parts) {
+			return parts[i+1]
+		}
+	}
+
+	return ""
+}

pkg/iptables/lock.go

@@ -0,0 +1,84 @@
+// Copyright 2015 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iptables
+
+import (
+	"os"
+	"sync"
+	"syscall"
+)
+
+const (
+	// In earlier versions of iptables, the xtables lock was implemented
+	// via a Unix socket, but now flock is used via this lockfile:
+	// http://git.netfilter.org/iptables/commit/?id=aa562a660d1555b13cffbac1e744033e91f82707
+	// Note the LSB-conforming "/run" directory does not exist on old
+	// distributions, so assume "/var" is symlinked
+	xtablesLockFilePath = "/var/run/xtables.lock"
+
+	defaultFilePerm = 0600
+)
+
+type Unlocker interface {
+	Unlock() error
+}
+
+type nopUnlocker struct{}
+
+func (n nopUnlocker) Unlock() error { return nil }
+
+type fileLock struct {
+	// mu is used to protect against concurrent invocations from within this process
+	mu sync.Mutex
+	fd int
+}
+
+// tryLock takes an exclusive lock on the xtables lock file without blocking.
+// This is best-effort only: if the exclusive lock would block (i.e. because
+// another process already holds it), no error is returned. Otherwise, any
+// error encountered during the locking operation is returned.
+// The returned Unlocker should be used to release the lock when the caller is
+// done invoking iptables commands.
+func (l *fileLock) tryLock() (Unlocker, error) {
+	l.mu.Lock()
+	err := syscall.Flock(l.fd, syscall.LOCK_EX|syscall.LOCK_NB)
+	switch err {
+	case syscall.EWOULDBLOCK:
+		l.mu.Unlock()
+		return nopUnlocker{}, nil
+	case nil:
+		return l, nil
+	default:
+		l.mu.Unlock()
+		return nil, err
+	}
+}
+
+// Unlock closes the underlying file, which implicitly unlocks it as well. It
+// also unlocks the associated mutex.
+func (l *fileLock) Unlock() error {
+	defer l.mu.Unlock()
+	return syscall.Close(l.fd)
+}
+
+// newXtablesFileLock opens a new lock on the xtables lockfile without
+// acquiring the lock
+func newXtablesFileLock() (*fileLock, error) {
+	fd, err := syscall.Open(xtablesLockFilePath, os.O_CREATE, defaultFilePerm)
+	if err != nil {
+		return nil, err
+	}
+	return &fileLock{fd: fd}, nil
+}

pkg/iptables/version.go

@@ -0,0 +1,91 @@
+package iptables
+
+import (
+	"fmt"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+type Version struct {
+	Major       int
+	Minor       int
+	Patch       int
+	BackendMode string
+}
+
+func (v Version) String() string {
+	return fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)
+}
+
+func (v Version) Compare(other Version) int {
+	if v.Major != other.Major {
+		return v.Major - other.Major
+	}
+	if v.Minor != other.Minor {
+		return v.Minor - other.Minor
+	}
+	return v.Patch - other.Patch
+}
+
+func ParseVersion(versionString string) (Version, error) {
+	re := regexp.MustCompile(`v([0-9]+)\.([0-9]+)\.([0-9]+)`)
+	match := re.FindStringSubmatch(versionString)
+	if len(match) != 4 {
+		return Version{}, fmt.Errorf("invalid version string: %s", versionString)
+	}
+	major, _ := strconv.Atoi(match[1])
+	minor, _ := strconv.Atoi(match[2])
+	patch, _ := strconv.Atoi(match[3])
+	return Version{Major: major, Minor: minor, Patch: patch}, nil
+}
+
+func GetVersion() (Version, error) {
+	ver := Version{}
+	cmd := exec.Command("iptables", "--version")
+	out, err := cmd.Output()
+	if err != nil {
+		return ver, fmt.Errorf("run cmd 'iptables --version' wtith error: %v", err)
+	}
+
+	ver, err = ParseVersion(string(out))
+	if err != nil {
+		return ver, err
+	}
+
+	nft4 := getOutput("iptables-nft-save")
+	legacy4 := getOutput("iptables-legacy-save")
+
+	nft6 := getOutput("ip6tables-nft-save")
+	legacy6 := getOutput("ip6tables-legacy-save")
+
+	if strings.Contains(nft4, "KUBE-IPTABLES") ||
+		strings.Contains(nft6, "KUBE-IPTABLES") ||
+		strings.Contains(nft4, "KUBE-KUBELET") ||
+		strings.Contains(nft6, "KUBE-KUBELET") {
+		ver.BackendMode = "nft"
+	} else if strings.Contains(legacy4, "KUBE-IPTABLES") ||
+		strings.Contains(legacy6, "KUBE-IPTABLES") ||
+		strings.Contains(legacy4, "KUBE-KUBELET") ||
+		strings.Contains(legacy6, "KUBE-KUBELET") {
+		ver.BackendMode = "legacy"
+	} else {
+		nftCount := strings.Count(nft4, "\n") + strings.Count(nft6, "\n")
+		legacyCount := strings.Count(legacy4, "\n") + strings.Count(legacy6, "\n")
+
+		if nftCount >= legacyCount {
+			ver.BackendMode = "nft"
+		} else {
+			ver.BackendMode = "legacy"
+		}
+	}
+
+	return ver, nil
+}
+
+func getOutput(name string) string {
+	cmd := exec.Command(name)
+	out, _ := cmd.Output()
+	return string(out)
+}

pkg/k8s/client.go

@@ -0,0 +1,123 @@
+package k8s
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net"
+	"time"
+
+	log "log/slog"
+
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+const (
+	defaultTimeout = 10 * time.Second
+)
+
+// NewClientset takes REST config and returns k8s clientest.
+func NewClientset(config *rest.Config) (*kubernetes.Clientset, error) {
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating kubernetes client: %s", err.Error())
+	}
+	return clientset, nil
+}
+
+// NewRestConfig takes an optional configPath and creates a new REST config for clientset.
+// If the configPath is not specified, and inCluster is true, then an
+// InClusterConfig is used.
+// Also takes a hostname which allow for overriding the config's hostname.
+func NewRestConfig(configPath string, inCluster bool, hostname string) (*rest.Config, error) {
+	config, err := restConfig(configPath, inCluster, defaultTimeout)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create rest config: %w", err)
+	}
+
+	if len(hostname) > 0 {
+		config.Host = hostname
+	}
+
+	return config, nil
+}
+
+func restConfig(kubeconfig string, inCluster bool, timeout time.Duration) (*rest.Config, error) {
+	var cfg *rest.Config
+	var err error
+	if inCluster {
+		if cfg, err = rest.InClusterConfig(); err != nil {
+			return nil, fmt.Errorf("failed to get incluster config: %w", err)
+		}
+	} else if kubeconfig != "" {
+		if cfg, err = clientcmd.BuildConfigFromFlags("", kubeconfig); err != nil {
+			return nil, fmt.Errorf("failed to build config from file '%s': %w", kubeconfig, err)
+		}
+	} else {
+		return nil, fmt.Errorf("failed to build config from file: path to KubeConfig not specified")
+	}
+
+	// Override some of the defaults allowing a little bit more flexibility speaking with the API server
+	// these should hopefully be redundant, however issues will still be logged.
+	cfg.QPS = 100
+	cfg.Burst = 250
+	cfg.Timeout = timeout
+	return cfg, nil
+}
+
+func findAddressFromRemoteCert(address string) ([]net.IP, error) {
+
+	// TODO: should we care at this point, probably not as we just want the certificates
+	conf := &tls.Config{
+		MinVersion:         tls.VersionTLS12,
+		InsecureSkipVerify: true, //nolint
+	}
+	d := &net.Dialer{
+		Timeout: time.Duration(3) * time.Second,
+	}
+
+	// Create the TCP connection
+	conn, err := tls.DialWithDialer(d, "tcp", address, conf)
+	if err != nil {
+		return nil, err
+	}
+
+	defer conn.Close()
+	// Grab the certificactes
+	certs := conn.ConnectionState().PeerCertificates
+	if len(certs) > 1 {
+		return nil, fmt.Errorf("[k8s client] not designed to recive multiple certs from API server")
+	}
+
+	return certs[0].IPAddresses, nil
+}
+
+func FindWorkingKubernetesAddress(configPath string, inCluster bool) (*rest.Config, error) {
+	// check with loopback, and retrieve its certificate
+	ips, err := findAddressFromRemoteCert("127.0.0.1:6443")
+	if err != nil {
+		return nil, err
+	}
+	for x := range ips {
+		log.Debug("[k8s client] testing", "address", ips[x].String())
+		c, err := NewRestConfig(configPath, inCluster, net.JoinHostPort(ips[x].String(), "6443"))
+		if err != nil {
+			log.Error("failed to create k8s REST config", "err", err)
+		}
+
+		c.Timeout = 2 * time.Second
+		k, err := NewClientset(c)
+		if err != nil {
+			log.Error("failed to create k8s clientset", "err", err)
+		}
+
+		_, err = k.DiscoveryClient.ServerVersion()
+		if err == nil {
+			log.Info("[k8s client] working", "address", ips[x].String())
+			c.Timeout = defaultTimeout
+			return c, nil
+		}
+	}
+	return nil, fmt.Errorf("unable to find a working address for the local API server [%v]", err)
+}

pkg/k8s/client_test.go

@@ -0,0 +1,70 @@
+package k8s
+
+//"192.168.0.174:6443"
+
+// func Test_findAddressFromRemoteCert(t *testing.T) {
+// 	type args struct {
+// 		address string
+// 	}
+// 	tests := []struct {
+// 		name    string
+// 		args    args
+// 		want    []net.IP
+// 		wantErr bool
+// 	}{
+// 		{
+// 			name:    "test server",
+// 			args:    args{address: "192.168.0.174:6443"},
+// 			want:    []net.IP{net.IPv4(10, 96, 0, 1), net.IPv4(192, 168, 0, 174)},
+// 			wantErr: false,
+// 		},
+// 	}
+// 	for _, tt := range tests {
+// 		t.Run(tt.name, func(t *testing.T) {
+// 			got, err := findAddressFromRemoteCert(tt.args.address)
+// 			if (err != nil) != tt.wantErr {
+// 				t.Errorf("findAddressFromRemoteCert() error = %v, wantErr %v", err, tt.wantErr)
+// 				return
+// 			}
+// 			if !reflect.DeepEqual(got, tt.want) {
+// 				t.Errorf("findAddressFromRemoteCert() = %v, want %v", got, tt.want)
+// 			}
+// 		})
+// 	}
+// }
+
+// //findAddressFromRemoteCert() =
+// //[10.96.0.1 192.168.0.174]
+// //[10.96.0.1 192.168.0.174]
+
+// func Test_findWorkingKubernetesAddress(t *testing.T) {
+// 	type args struct {
+// 		configPath string
+// 		inCluster  bool
+// 	}
+// 	tests := []struct {
+// 		name    string
+// 		args    args
+// 		want    *kubernetes.Clientset
+// 		wantErr bool
+// 	}{
+// 		{
+// 			name: "test",
+// 			args: args{
+// 				configPath: "/home/dan/super-admin.conf",
+// 				inCluster:  false,
+// 			},
+// 			wantErr: false,
+// 		},
+// 	}
+// 	for _, tt := range tests {
+// 		t.Run(tt.name, func(t *testing.T) {
+// 			_, err := FindWorkingKubernetesAddress(tt.args.configPath, tt.args.inCluster)
+// 			if (err != nil) != tt.wantErr {
+// 				t.Errorf("findWorkingKubernetesAddress() error = %v, wantErr %v", err, tt.wantErr)
+// 				return
+// 			}
+
+// 		})
+// 	}
+// }