Unintentionally created duplicate functions that can't be used

Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# Enable GitHub funding
+
+github: [kube-vip]
+

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,35 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Environment (please complete the following information):**
+ - OS/Distro: [e.g. Ubuntu 1804]
+ - Kubernetes Version: [e.g. v.1.18]
+ - Kube-vip Version: [e.g. 0.2.3]
+
+**`Kube-vip.yaml`:**
+If Possible add in your kube-vip manifest (please remove anything that is confidential)
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: gomod
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: /
+    schedule:
+      interval: weekly

.github/workflows/anchore-syft.yml

@@ -0,0 +1,31 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# This workflow checks out code, builds an image, performs a container image
+# scan with Anchore's Syft tool, and uploads the results to the GitHub Dependency
+# submission API.
+
+# For more information on the Anchore sbom-action usage
+# and parameters, see https://github.com/anchore/sbom-action. For more
+# information about the Anchore SBOM tool, Syft, see
+# https://github.com/anchore/syft
+name: Anchore Syft SBOM scan
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  sbom:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref_name }}
+      - name: Anchore SBOM Action
+        uses: anchore/sbom-action@v0.20.4
+        with:
+          format: cyclonedx-json

.github/workflows/ci-pull-request.yaml

@@ -0,0 +1,51 @@
+name: For each PR
+on:
+  pull_request:
+jobs:
+  e2e-tests:
+    runs-on: ubuntu-latest
+    name: E2E tests
+    strategy:
+      matrix:
+        mode: ["arp", "rt", "bgp"]
+      fail-fast: true
+      max-parallel: 1
+    steps:
+      - name: Ensure fs wont cause issues
+        run: sudo sysctl fs.inotify.max_user_instances=8192 &&  sudo sysctl fs.inotify.max_user_watches=524288
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Build image locally
+        run: make dockerx86Local
+      - name: Run Manifest generation tests
+        run: make manifest-test
+      - name: Run ARP mode tests v1.29.0 onwards
+        run: make e2e-tests129-arp
+        if: matrix.mode== 'arp'
+      - name: Run RT mode tests v1.29.0 onwards
+        run: make e2e-tests129-rt
+        if: matrix.mode== 'rt'
+      - name: Get GoBGP binaries
+        run: make get-gobgp
+        if: matrix.mode== 'bgp'
+      - name: Run BGP mode tests v1.29.0 onwards
+        run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 make e2e-tests129-bgp
+        if: matrix.mode== 'bgp'
+  service-e2e-tests:
+    runs-on: ubuntu-latest
+    name: E2E service tests
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Build image with iptables
+        run: make dockerx86ActionIPTables
+      - name: Run tests
+        run: DOCKERTAG=action make service-tests

.github/workflows/ci.yaml

@@ -0,0 +1,62 @@
+name: For each commit
+on:
+  push:
+  pull_request:
+jobs:
+  validation:
+    runs-on: ubuntu-latest
+    name: Checks and linters
+    steps:
+      - name: Init
+        run: sudo apt-get update && sudo apt-get install -y build-essential &&  sudo sysctl fs.inotify.max_user_instances=8192 &&  sudo sysctl fs.inotify.max_user_watches=524288
+      - name: Install golangci-lint
+        run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.64.8
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: All checks
+        run: make check
+  unit-tests:
+    runs-on: ubuntu-latest
+    name: Unit tests
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Run tests
+        run: make unit-tests
+  integration-tests:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Run tests
+        run: make integration-tests
+  image-vul-check:
+    runs-on: ubuntu-latest
+    name: Image vulnerability scan
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Build image with iptables
+        run: make dockerx86ActionIPTables
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: "plndr/kube-vip:action"
+          format: "table"
+          exit-code: "1"
+          ignore-unfixed: true
+          vuln-type: "os,library"
+          severity: "CRITICAL,HIGH"

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,75 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '17 10 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      
+    - name: Install Go
+      uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+        
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v3
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3

.github/workflows/main.yaml

@@ -0,0 +1,52 @@
+name: Build and publish main image regularly
+
+on:
+  schedule:
+    - cron: '25 0 * * *'
+  workflow_dispatch:
+
+
+jobs:
+  nightly_build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to Github Packages
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}   
+      - name: Build standard version
+        id: docker_build
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: >-
+            plndr/kube-vip:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip:${{ github.ref_name }}
+      - name: Build iptables version
+        id: docker_build_iptables
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile_iptables
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: >-
+            plndr/kube-vip-iptables:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip-iptables:${{ github.ref_name }}
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/release.yaml

@@ -0,0 +1,64 @@
+name: Publish Releases to Docker Hub and GitHub Container Registry
+
+on:
+  push:
+    tags:
+      - '*'
+  workflow_dispatch:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to Github Packages
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Generate Metadata
+        uses: docker/metadata-action@v5.7.0
+        id: metadata
+        with:
+          labels: |
+            org.opencontainers.image.documentation=https://kube-vip.io/docs/
+      - name: Build and push main branch
+        id: docker_build
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          labels: ${{ steps.metadata.outputs.labels }}
+          tags: >-
+            plndr/kube-vip:${{ github.ref_name }},
+            plndr/kube-vip:latest,
+            ghcr.io/kube-vip/kube-vip:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip:latest
+      - name: Build iptables version and push main branch
+        id: docker_build_iptables
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile_iptables
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          labels: ${{ steps.metadata.outputs.labels }}
+          tags: >-
+            plndr/kube-vip-iptables:${{ github.ref_name }},
+            plndr/kube-vip-iptables:latest,
+            ghcr.io/kube-vip/kube-vip-iptables:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip-iptables:latest
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

.gitignore

@@ -0,0 +1,7 @@
+.idea
+kube-vip
+.vscode
+bin
+testing/e2e/etcd/certs
+pkg/etcd/etcd.pid
+pkg/etcd/etcd-data

.golangci.yml

@@ -0,0 +1,13 @@
+run:
+  timeout: 10m
+  
+linters:
+  enable:
+  - bodyclose
+  - gofmt
+  - goimports
+  - revive
+  - gosec
+  - misspell
+  - unconvert
+  - unparam

CODEOWNERS

@@ -0,0 +1,10 @@
+
+# For more information on the syntax of the CODEOWNERS file, see:
+# https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners
+
+# The Kube-Vip maintainers team
+*   @thebsdbox @yastij
+
+# Emeritus Maintainers
+#
+# N/A

CODE_OF_CONDUCT.md

@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+conduct@kube-vip.io.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

CONTRIBUTING.md

@@ -0,0 +1,289 @@
+# Developer Guide
+
+Thank you for taking the time out to contribute to the **kube-vip** project!
+
+This guide will walk you through the process of making your first commit and how
+to effectively get it merged upstream.
+
+<!-- toc -->
+- [Getting Started](#getting-started)
+- [Contribute](#contribute)
+  - [GitHub Workflow](#github-workflow)
+  - [Getting reviewers](#getting-reviewers)
+  - [Inclusive Naming](#inclusive-naming)
+  - [Building and testing your change](#building-and-testing-your-change)
+  - [Reverting a commit](#reverting-a-commit)
+  - [Sign-off Your Work](#sign-off-your-work)
+- [Issue and PR Management](#issue-and-pr-management)
+  - [Filing An Issue](#filing-an-issue)
+  - [Issue Triage](#issue-triage)
+<!-- /toc -->
+
+## Getting Started
+
+To get started, let's ensure you have completed the following prerequisites for
+contributing to project **kube-vip**:
+
+1. Read and observe the [code of conduct](CODE_OF_CONDUCT.md).
+2. Check out the [Architecture documentation](https://kube-vip.io) for the **kube-vip**
+   architecture and design.
+3. Set up your [development environment](docs/contributors/manual-installation.md)
+
+Now that you're setup, skip ahead to learn how to [contribute](#contribute).
+
+**Also**, A GitHub account will be required in order to submit code changes and
+interact with the project. For committing any changes in  **Github** it is required for
+you to have a [github account](https://github.com/join).
+
+## Contribute
+
+There are multiple ways in which you can contribute, either by contributing
+code in the form of new features or bug-fixes or non-code contributions like
+helping with code reviews, triaging of bugs, documentation updates, filing
+[new issues](#filing-an-issue) or writing blogs/manuals etc.
+
+### GitHub Workflow
+
+Developers work in their own forked copy of the repository and when ready,
+submit pull requests to have their changes considered and merged into the
+project's repository.
+
+1. Fork your own copy of the repository to your GitHub account by clicking on
+   `Fork` button on [kube-vip's GitHub repository](https://github.com/kube-vip/kube-vip).
+2. Clone the forked repository on your local setup.
+
+    ```bash
+    git clone https://github.com/$user/kube-vip
+    ```
+
+    Add a remote upstream to track upstream kube-vip repository.
+
+    ```bash
+    git remote add upstream https://github.com/kube-vip/kube-vip
+    ```
+
+    Never push to upstream remote
+
+    ```bash
+    git remote set-url --push upstream no_push
+    ```
+
+3. Create a topic branch.
+
+    ```bash
+    git checkout -b branchName
+    ```
+
+4. Make changes and commit it locally. Make sure that your commit is
+   [signed](#sign-off-your-work).
+
+    ```bash
+    git add <modifiedFile>
+    git commit -s
+    ```
+
+5. Update the "Unreleased" section of the [CHANGELOG](CHANGELOG.md) for any
+   significant change that impacts users.
+6. Keeping branch in sync with upstream.
+
+    ```bash
+    git checkout branchName
+    git fetch upstream
+    git rebase upstream/main
+    ```
+
+7. Push local branch to your forked repository.
+
+    ```bash
+    git push -f $remoteBranchName branchName
+    ```
+
+8. Create a Pull request on GitHub.
+   Visit your fork at `https://github.com/kube-vip/kube-vip` and click
+   `Compare & Pull Request` button next to your `remoteBranchName` branch.
+
+### Getting reviewers
+
+Once you have opened a Pull Request (PR), reviewers will be assigned to your
+PR and they may provide review comments which you need to address.
+Commit changes made in response to review comments to the same branch on your
+fork. Once a PR is ready to merge, squash any *fix review feedback, typo*
+and *merged* sorts of commits.
+
+To make it easier for reviewers to review your PR, consider the following:
+
+1. Follow the golang [coding conventions](https://github.com/golang/go/wiki/CodeReviewComments).
+2. Format your code with `make golangci-fix`; if the [linters](ci/README.md) flag an issue that
+   cannot be fixed automatically, an error message will be displayed so you can address the issue.
+3. Follow [git commit](https://chris.beams.io/posts/git-commit/) guidelines.
+4. Follow [logging](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md) guidelines.
+
+If your PR fixes a bug or implements a new feature, add the appropriate test
+cases to our [automated test suite](ci/README.md) to guarantee enough
+coverage. A PR that makes significant code changes without contributing new test
+cases will be flagged by reviewers and will not be accepted.
+
+### Inclusive Naming
+
+For symbol names and documentation, do not introduce new usage of harmful
+language such as 'master / slave' (or 'slave' independent of 'master') and
+'blacklist / whitelist'. For more information about what constitutes harmful
+language and for a reference word replacement list, please refer to the
+[Inclusive Naming Initiative](https://inclusivenaming.org/).
+
+We are committed to removing all harmful language from the project. If you
+detect existing usage of harmful language in code or documentation, please
+report the issue to us or open a Pull Request to address it directly. Thanks!
+
+### Building and testing your change
+
+To build the **kube-vip** Docker image together with all **kube-vip** bits, you can simply
+do:
+
+1. Checkout your feature branch and `cd` into it.
+2. Run `make dockerx86`
+
+The second step will compile the **kube-vip** code in a `golang` container, and build
+a `Ubuntu 20.04` Docker image that includes all the generated binaries. [`Docker`](https://docs.docker.com/install)
+must be installed on your local machine in advance.
+
+Alternatively, you can build the **kube-vip** code in your local Go environment. The
+**kube-vip** project uses the [Go modules support](https://github.com/golang/go/wiki/Modules) which was introduced in Go 1.11. It
+facilitates dependency tracking and no longer requires projects to live inside
+the `$GOPATH`.
+
+To develop locally, you can follow these steps:
+
+ 1. [Install Go 1.19](https://golang.org/doc/install)
+ 2. Checkout your feature branch and `cd` into it.
+ 3. To build all Go files and install them under `bin`, run `make bin`
+ 4. To run all Go unit tests, run `make test-unit`
+ 5. To build the **kube-vip** Ubuntu Docker image separately with the binaries generated in step 2, run `make ubuntu`
+
+### CI testing
+
+For more information about the tests we run as part of CI, please refer to
+[ci/README.md](ci/README.md).
+
+### Reverting a commit
+
+1. Create a branch in your forked repo
+
+    ```bash
+    git checkout -b revertName
+    ```
+
+2. Sync the branch with upstream
+
+    ```bash
+    git fetch upstream
+    git rebase upstream/main
+    ```
+
+3. Create a revert based on the SHA of the commit. The commit needs to be
+   [signed](#sign-off-your-work).
+
+    ```bash
+    git revert -s SHA
+    ```
+
+4. Push this new commit.
+
+    ```bash
+    git push $remoteRevertName revertName
+    ```
+
+5. Create a Pull Request on GitHub.
+   Visit your fork at `https://github.com/kube-vip/kube-vip` and click
+   `Compare & Pull Request` button next to your `remoteRevertName` branch.
+
+### Sign-off Your Work
+
+It is recommended to sign your work when contributing to the **kube-vip**
+repository. 
+
+Git provides the `-s` command-line option to append the required line
+automatically to the commit message:
+
+```bash
+git commit -s -m 'This is my commit message'
+```
+
+For an existing commit, you can also use this option with `--amend`:
+
+```bash
+git commit -s --amend
+```
+
+If more than one person works on something it's possible for more than one
+person to sign-off on it. For example:
+
+```bash
+Signed-off-by: Some Developer somedev@example.com
+Signed-off-by: Another Developer anotherdev@example.com
+```
+
+We use the [DCO Github App](https://github.com/apps/dco) to enforce that all
+commits in a Pull Request include the required `Signed-off-by` line. If this is
+not the case, the app will report a failed status for the Pull Request and it
+will be blocked from being merged.
+
+Compared to our earlier CLA, DCO tends to make the experience simpler for new
+contributors. If you are contributing as an employee, there is no need for your
+employer to sign anything; the DCO assumes you are authorized to submit
+contributions (it's your responsibility to check with your employer).
+
+## Issue and PR Management
+
+We use labels and workflows (some manual, some automated with GitHub Actions) to
+help us manage triage, prioritize, and track issue progress. For a detailed
+discussion, see [docs/issue-management.md](docs/contributors/issue-management.md).
+
+### Filing An Issue
+
+Help is always appreciated. If you find something that needs fixing, please file
+an issue [here](https://github.com/kube-vip/kube-vip/issues). Please ensure
+that the issue is self explanatory and has enough information for an assignee to
+get started.
+
+Before picking up a task, go through the existing
+[issues](https://github.com/kube-vip/kube-vip/issues) and make sure that your
+change is not already being worked on. If it does not exist, please create a new
+issue and discuss it with other members.
+
+For simple contributions to **kube-vip**, please ensure that this minimum set of
+labels are included on your issue:
+
+* **kind** -- common ones are `kind/feature`, `kind/support`, `kind/bug`,
+  `kind/documentation`, or `kind/design`. For an overview of the different types
+  of issues that can be submitted, see [Issue and PR
+  Kinds](#issue-and-pr-kinds).
+  The kind of issue will determine the issue workflow.
+* **area** (optional) -- if you know the area the issue belongs in, you can assign it.
+  Otherwise, another community member will label the issue during triage. The
+  area label will identify the area of interest an issue or PR belongs in and
+  will ensure the appropriate reviewers shepherd the issue or PR through to its
+  closure. For an overview of areas, see the
+  [`docs/github-labels.md`](docs/contributors/github-labels.md).
+* **size** (optional) -- if you have an idea of the size (lines of code, complexity,
+  effort) of the issue, you can label it using a [size label](#size). The size
+  can be updated during backlog grooming by contributors. This estimate is used
+  to guide the number of features selected for a milestone.
+
+All other labels will be assigned during issue triage.
+
+### Issue Triage
+
+Once an issue has been submitted, the CI (GitHub actions) or a human will
+automatically review the submitted issue or PR to ensure that it has all relevant
+information. If information is lacking or there is another problem with the
+submitted issue, an appropriate `triage/<?>` label will be applied.
+
+After an issue has been triaged, the maintainers can prioritize the issue with
+an appropriate `priority/<?>` label.
+
+Once an issue has been submitted, categorized, triaged, and prioritized it
+is marked as `ready-to-work`. A ready-to-work issue should have labels
+indicating assigned areas, prioritization, and should not have any remaining
+triage labels.
+

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:experimental
 
-FROM golang:1.13-alpine as dev
+FROM golang:1.24.5-alpine3.22 as dev
 RUN apk add --no-cache git ca-certificates make
 RUN adduser -D appuser
 COPY . /src/
@@ -12,5 +12,8 @@ RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
     CGO_ENABLED=0 GOOS=linux make build
 
 FROM scratch
+# Add Certificates into the image, for anything that does API calls
+COPY --from=dev /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+# Add kube-vip binary
 COPY --from=dev /src/kube-vip /
-ENTRYPOINT ["/kube-vip"]
+ENTRYPOINT ["/kube-vip"]

Dockerfile_iptables

@@ -0,0 +1,21 @@
+# syntax=docker/dockerfile:experimental
+
+FROM golang:1.24.5-alpine3.22 as dev
+RUN apk add --no-cache git make
+RUN adduser -D appuser
+COPY . /src/
+WORKDIR /src
+
+ENV GO111MODULE=on
+RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
+    --mount=type=cache,sharing=locked,id=goroot,target=/root/.cache/go-build \
+    CGO_ENABLED=0 GOOS=linux make build
+
+FROM alpine:3.22.1
+# Update pkgs and add iptables
+RUN apk upgrade && \
+    apk add --no-cache iptables iptables-legacy
+
+# Add kube-vip binary
+COPY --from=dev /src/kube-vip /
+ENTRYPOINT ["/kube-vip"]

Makefile

@@ -1,12 +1,12 @@
-
 SHELL := /bin/sh
 
 # The name of the executable (default is current directory name)
 TARGET := kube-vip
-.DEFAULT_GOAL: $(TARGET)
+.DEFAULT_GOAL := $(TARGET)
 
 # These will be provided to the target
-VERSION := 0.1.8
+VERSION := v1.0.0
+
 BUILD := `git rev-parse HEAD`
 
 # Operating System Default (LINUX)
@@ -14,18 +14,14 @@ TARGETOS=linux
 
 # Use linker flags to provide version/build settings to the target
 LDFLAGS=-ldflags "-s -w -X=main.Version=$(VERSION) -X=main.Build=$(BUILD) -extldflags -static"
-
-# go source files, ignore vendor directory
-SRC = $(shell find . -type f -name '*.go' -not -path "./vendor/*")
-
 DOCKERTAG ?= $(VERSION)
-REPOSITORY = plndr
+REPOSITORY ?= plndr
 
-.PHONY: all build clean install uninstall fmt simplify check run
+.PHONY: all build clean install uninstall simplify check run e2e-tests
 
 all: check install
 
-$(TARGET): $(SRC)
+$(TARGET):
 	@go build $(LDFLAGS) -o $(TARGET)
 
 build: $(TARGET)
@@ -41,44 +37,143 @@ install:
 uninstall: clean
 	@rm -f $$(which ${TARGET})
 
-fmt:
-	@gofmt -l -w $(SRC)
-
 demo:
 	@cd demo
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 	@cd ..
 
 ## Remote (push of images)
-
 # This build a local docker image (x86 only) for quick testing
+
+dockerx86Dev:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 --push -t $(REPOSITORY)/$(TARGET):dev .
+	@echo New single x86 Architecture Docker image created
+
+dockerx86Iptables:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 -f ./Dockerfile_iptables --push -t $(REPOSITORY)/$(TARGET):dev .
+	@echo New single x86 Architecture Docker image created
+
+dockerx86IptablesLocal:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 -f ./Dockerfile_iptables -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@echo New single x86 Architecture Docker image created
+
 dockerx86:
+	@-rm ./kube-vip
 	@docker buildx build  --platform linux/amd64 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
-	@echo New Multi Architecture Docker image created
+	@echo New single x86 Architecture Docker image created
 
 docker:
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 ## Local (docker load of images)
-
 # This will build a local docker image (x86 only), use make dockerLocal for all architectures
 dockerx86Local:
+	@-rm ./kube-vip
 	@docker buildx build  --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
+dockerx86Action:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):action .
+	@echo New Multi Architecture Docker image created
+
+dockerx86ActionIPTables:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 -f ./Dockerfile_iptables --load -t $(REPOSITORY)/$(TARGET):action .
+	@echo New Multi Architecture Docker image created
+
 dockerLocal:
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7 --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 simplify:
-	@gofmt -s -l -w $(SRC)
+	@gofmt -s -l -w *.go pkg cmd
 
 check:
-	@test -z $(shell gofmt -l main.go | tee /dev/stderr) || echo "[WARN] Fix formatting issues with 'make fmt'"
-	@for d in $$(go list ./... | grep -v /vendor/); do golint $${d}; done
-	@go tool vet ${SRC}
+	go mod tidy
+	test -z "$(git status --porcelain)"
+	test -z $(shell gofmt -l *.go pkg cmd) || echo "[WARN] Fix formatting issues with 'make simplify'"
+	golangci-lint run
+	go vet ./...
 
 run: install
 	@$(TARGET)
+
+manifests:
+	@make build
+	@mkdir -p ./docs/manifests/$(VERSION)/
+	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-arp.yaml
+	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-lb.yaml
+	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --bgp --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-bgp.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
+	@-rm ./kube-vip
+
+manifest-test: 
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --enableLoadBalancer
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --bgp --controlplane --services 
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster
+
+unit-tests:
+	go test ./...
+
+integration-tests:
+	go test -tags=integration,e2e -v ./pkg/etcd
+
+e2e-tests:
+	GOMAXPROCS=4 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd
+
+e2e-tests129-arp:
+	GOMAXPROCS=4 TEST_MODE=arp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
+
+e2e-tests129-rt:
+	GOMAXPROCS=4 TEST_MODE=rt V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
+
+e2e-tests129-bgp:
+	GOMAXPROCS=4 TEST_MODE=bgp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
+
+e2e-tests129: e2e-tests129-arp e2e-tests129-rt e2e-tests129-bgp
+
+service-tests:
+	E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/services -Services -simple -deployments -leaderActive -leaderFailover -localDeploy -egress -egressIPv6 -dualStack
+
+trivy: dockerx86ActionIPTables
+	docker run -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy:0.47.0 \
+		image  \
+		--format table \
+		--exit-code  1 \
+		--ignore-unfixed \
+		--vuln-type  'os,library' \
+		--severity  'CRITICAL,HIGH'  \
+		$(REPOSITORY)/$(TARGET):action
+
+kind-quick:
+	echo "Standing up your cluster"
+	kind create cluster --config ./testing/kind/kind.yaml --name kube-vip
+	kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
+	kubectl create configmap --namespace kube-system kubevip --from-literal range-global=172.18.100.10-172.18.100.30
+	kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
+	kind load docker-image --name kube-vip $(REPOSITORY)/$(TARGET):$(DOCKERTAG)
+	docker run --network host --rm $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --services --inCluster --arp --servicesElection --interface  eth0 | kubectl apply -f -
+
+kind-reload:
+	kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name kube-vip
+	kubectl rollout restart -n kube-system daemonset/kube-vip-ds
+
+get-gobgp:
+	mkdir -p bin
+	wget -nc --directory-prefix=bin https://github.com/osrg/gobgp/releases/download/v3.37.0/gobgp_3.37.0_linux_amd64.tar.gz
+	tar -xvzf bin/gobgp_3.37.0_linux_amd64.tar.gz -C bin
+

README.md

@@ -1,25 +1,39 @@
 # kube-vip
 
+High Availability and Load-Balancing
 
-**NOTE** All documentation of both usage and architecture are now available at [https://kube-vip.io](https://kube-vip.io)
+![](https://github.com/kube-vip/kube-vip/raw/main/kube-vip.png)
 
+[![Build and publish main image regularly](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml/badge.svg)](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml) [![LFX Health Score](https://img.shields.io/static/v1?label=Health%20Score&message=Healthy&color=A7F3D0&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip) [![LFX Active Contributors](https://img.shields.io/static/v1?label=Active%20contributors%20(1Y)&message=212&color=0094FF&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip)
 
 ## Overview
-Kubernetes Virtual IP and Load-Balancer for both control pane and Kubernetes services
+Kubernetes Virtual IP and Load-Balancer for both control plane and Kubernetes services
 
 The idea behind `kube-vip` is a small self-contained Highly-Available option for all environments, especially:
 
 - Bare-Metal
 - Edge (arm / Raspberry PI)
+- Virtualisation
 - Pretty much anywhere else :)
 
-![](/overview.png)
+**NOTE:** All documentation of both usage and architecture are now available at [https://kube-vip.io](https://kube-vip.io).
 
-The `kube-vip` application builds a multi-node or multi-pod cluster to provide High-Availability. When a leader is elected, this node will inherit the Virtual IP and become the leader of the load-balancing within the cluster. 
+## Features
 
-When running **out of cluster** it will use [raft](https://en.wikipedia.org/wiki/Raft_(computer_science)) clustering technology 
+Kube-Vip was originally created to provide a HA solution for the Kubernetes control plane, over time it has evolved to incorporate that same functionality into Kubernetes service type [load-balancers](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer).
 
-When running **in cluster** it will use [leader election](https://godoc.org/k8s.io/client-go/tools/leaderelection)
+- VIP addresses can be both IPv4 or IPv6
+- Control Plane with ARP (Layer 2) or BGP (Layer 3)
+- Control Plane using either [leader election](https://godoc.org/k8s.io/client-go/tools/leaderelection) or [raft](https://en.wikipedia.org/wiki/Raft_(computer_science))
+- Control Plane HA with kubeadm (static Pods)
+- Control Plane HA with K3s/and others (daemonsets)
+- Service LoadBalancer using [leader election](https://godoc.org/k8s.io/client-go/tools/leaderelection) for ARP (Layer 2)
+- Service LoadBalancer using multiple nodes with BGP
+- Service LoadBalancer address pools per namespace or global
+- Service LoadBalancer address via (existing network DHCP)
+- Service LoadBalancer address exposure to gateway via UPNP
+- Egress! Kube-vip will utilise a service loadbalancer as both the ingress and **egress** for a pod. 
+- ... manifest generation, vendor API integrations and many more...
 
 ## Why?
 
@@ -31,53 +45,27 @@ The purpose of `kube-vip` is to simplify the building of HA Kubernetes clusters,
 
 **VIP**:
 - [Keepalived](https://www.keepalived.org/)
-- [Ucarp](https://ucarp.wordpress.com/)
+- [UCARP](https://ucarp.wordpress.com/)
 - Hardware Load-balancer (functionality differs per vendor)
 
 
 **LoadBalancing**:
 - [HAProxy](http://www.haproxy.org/)
 - [Nginx](http://nginx.com)
-- Hardware Load-balancer(functionality differs per vendor)
+- Hardware Load-balancer (functionality differs per vendor)
 
 All of these would require a separate level of configuration and in some infrastructures multiple teams in order to implement. Also when considering the software components, they may require packaging into containers or if they’re pre-packaged then security and transparency may be an issue. Finally, in edge environments we may have limited room for hardware (no HW load-balancer) or packages solutions in the correct architectures might not exist (e.g. ARM). Luckily with `kube-vip` being written in GO, it’s small(ish) and easy to build for multiple architectures, with the added security benefit of being the only thing needed in the container.
 
+## Troubleshooting and Feedback
 
-## Standalone Usage
+Please raise issues on the GitHub repository and as mentioned check the documentation at [https://kube-vip.io](https://kube-vip.io/).
 
-The usage of `kube-vip` can either be directly by taking the binary / building yourself (`make build`), or alternatively through a pre-built docker container which can be found in the plunder Docker Hub repository [https://hub.docker.com/r/plndr/kube-vip](https://hub.docker.com/r/plndr/kube-vip). For further 
+## Contributing
 
-### Configuration
+Thanks for taking the time to join our community and start contributing! We welcome pull requests. Feel free to dig through the [issues](https://github.com/kube-vip/kube-vip/issues) and jump in.
 
-To generate the basic `yaml` configuration:
+:warning: This project has issue compiling on MacOS, please compile it on linux distribution
 
-```
-kube-vip sample config > config.yaml
-```
+## Star History
 
-Modify the `localPeer` section to match this particular instance (local IP address/port etc..) and ensure that the `remotePeers` section is correct for the current instance and all other instances in the cluster. Also ensure that the `interface` is the correct interface that the `vip` will bind to.
-
-
-## Starting a simple cluster
-
-To start `kube-vip` ensure the configuration for the `localPeers` and `remotePeers` is correct for each instance and the cluster as a whole and start:
-
-```
-kube-vip start -c /config.yaml
-INFO[0000] Reading configuration from [config.yaml]       
-INFO[0000] 2020-02-01T15:41:04.287Z [INFO]  raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server1 Address:192.168.0.70:10000} {Suffrage:Voter ID:server2 Address:192.168.0.71:10000} {Suffrage:Voter ID:server3 Address:192.168.0.72:10000}]" 
-INFO[0000] 2020-02-01T15:41:04.287Z [INFO]  raft: entering follower state: follower="Node at 192.168.0.70:10000 [Follower]" leader= 
-INFO[0000] Started                                      
-INFO[0000] The Node [] is leading                       
-INFO[0001] The Node [] is leading                       
-INFO[0001] 2020-02-01T15:41:05.522Z [WARN]  raft: heartbeat timeout reached, starting election: last-leader= 
-INFO[0001] 2020-02-01T15:41:05.522Z [INFO]  raft: entering candidate state: node="Node at 192.168.0.70:10000 [Candidate]" term=2 
-INFO[0001] 2020-02-01T15:41:05.522Z [DEBUG] raft: votes: needed=2 
-INFO[0001] 2020-02-01T15:41:05.522Z [DEBUG] raft: vote granted: from=server1 term=2 tally=1 
-INFO[0001] 2020-02-01T15:41:05.523Z [DEBUG] raft: newer term discovered, fallback to follower 
-INFO[0001] 2020-02-01T15:41:05.523Z [INFO]  raft: entering follower state: follower="Node at 192.168.0.70:10000 [Follower]" leader= 
-INFO[0001] 2020-02-01T15:41:05.838Z [WARN]  raft: failed to get previous log: previous-index=2 last-index=1 error="log not found" 
-INFO[0002] The Node [192.168.0.72:10000] is leading    
-```
-
-After a few seconds with additional nodes started a leader election will take place and the leader will assume the **vip**.
+[![Star History Chart](https://api.star-history.com/svg?repos=kube-vip/kube-vip&type=Date)](https://star-history.com/#kube-vip/kube-vip&Date)

ROADMAP.md

@@ -0,0 +1,35 @@
+# Kube-Vip Roadmap
+
+This document outlines the roadmap for the **kube-vip** project and only covers the technologies within this particular project, other projects that augment or provide additional functionality (such as cloud-providers) may have their own roadmaps in future. The functionality for **kube-vip** has grown either been developed organically or through real-world needs, and this is the first attempt to put into words a plan for the future of **kube-vip** and will additional evolve over time. This means that items listed or detailed here are not necessarily set in stone and the roadmap can grow/shrink as the project matures. We definitely welcome suggestions and ideas from everyone about the roadmap and **kube-vip** features. Reach us through Issues, Slack or email <catch-all>@kube-vip.io.
+
+## Release methodology
+
+The **kube-vip** project attempts to follow a tick-tock release cycle, this typically means that one release will come **packed** with new features where the following release will come with fixes, code sanitation and performance enhancements.
+
+## Roadmap
+
+The **kube-vip** project offers two main areas of functionality:
+
+- HA Kubernetes clusters through a control-plane VIP
+- Kubernetes `service type:LoadBalancer`
+
+Whilst both of these functions share underlying technologies and code they will have slightly differing roadmaps.
+
+### HA Kubernetes Control Plane
+
+- **Re-implement LoadBalancing** - due to a previous request the HTTP loadbalancing was removed leaving just HA for the control plane. This functionality will be re-implemented either through the original round-robin HTTP requests or utilising IPVS.
+- **Utilise the Kubernetes API to determine additional Control Plane members** - Once a single node cluster is running **kube-vip** could use the API to determine the additional members, at this time a Cluster-API provider needs to drop a static manifest per CP node.
+- **Re-evaluate raft** - **kube-vip** is mainly designed to run within a Kubernetes cluster, however it's original design was a raft cluster external to Kubernetes. Unfortunately given some of the upgrade paths identified in things like CAPV moving to leaderElection within Kubernetes became a better idea.
+
+## Kubernetes `service type:LoadBalancer`
+
+- **`ARP` LeaderElection per loadBalancer** - Currently only one pod that is elected leader will field all traffic for a VIP.. extending this to generate a leaderElection token per service would allow services to proliferate across all pods across the cluster
+- **Aligning of `service` and `manager`** - The move to allow hybrid (be both HA control plane and offer load-balancer services at the same time) introduced a duplicate code path.. these need to converge as it's currently confusing for contributors.
+
+## Global **Kube-Vip** items
+
+- **Improved metrics** - At this time the scaffolding for monitoring exists, however this needs drastically extending to provide greater observability to what is happening within **kube-vip**
+- **Windows support** - The Go SDK didn't support the capability for low-levels sockets for ARP originally, this should be revisited.
+- **Additional BGP features** :
+  - Communities
+  - BFD

cmd/kube-vip-config.go

@@ -1,146 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-
-	"github.com/ghodss/yaml"
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	appv1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-// [sample configuration] - flags
-var cliConfig kubevip.Config
-var cliConfigLB kubevip.LoadBalancer
-var cliLocalPeer string
-var cliRemotePeers, cliBackends []string
-
-func init() {
-	kubeVipSampleConfig.Flags().StringVar(&cliConfig.Interface, "interface", "eth0", "Name of the interface to bind to")
-	kubeVipSampleConfig.Flags().StringVar(&cliConfig.VIP, "vip", "192.168.0.1", "The Virtual IP address")
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfig.SingleNode, "singleNode", false, "Start this instance as a single node")
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfig.StartAsLeader, "startAsLeader", false, "Start this instance as the cluster leader")
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfig.GratuitousARP, "arp", true, "Use ARP broadcasts to improve VIP re-allocations")
-	kubeVipSampleConfig.Flags().StringVar(&cliLocalPeer, "localPeer", "server1:192.168.0.1:10000", "Settings for this peer, format: id:address:port")
-	kubeVipSampleConfig.Flags().StringSliceVar(&cliRemotePeers, "remotePeers", []string{"server2:192.168.0.2:10000", "server3:192.168.0.3:10000"}, "Comma seperated remotePeers, format: id:address:port")
-	// Load Balancer flags
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfigLB.BindToVip, "lbBindToVip", false, "Bind example load balancer to VIP")
-	kubeVipSampleConfig.Flags().StringVar(&cliConfigLB.Type, "lbType", "tcp", "Type of load balancer instance (TCP/HTTP)")
-	kubeVipSampleConfig.Flags().StringVar(&cliConfigLB.Name, "lbName", "Example Load Balancer", "The name of a load balancer instance")
-	kubeVipSampleConfig.Flags().IntVar(&cliConfigLB.Port, "lbPort", 8080, "Port that load balancer will expose on")
-	kubeVipSampleConfig.Flags().StringSliceVar(&cliBackends, "lbBackends", []string{"192.168.0.1:8080", "192.168.0.2:8080"}, "Comma seperated backends, format: address:port")
-}
-
-var kubeVipSampleConfig = &cobra.Command{
-	Use:   "config",
-	Short: "Generate a Sample configuration",
-	Run: func(cmd *cobra.Command, args []string) {
-
-		// // Parse localPeer
-		// p, err := kubevip.ParsePeerConfig(cliLocalPeer)
-		// if err != nil {
-		// 	cmd.Help()
-		// 	log.Fatalln(err)
-		// }
-		// cliConfig.LocalPeer = *p
-
-		// // Parse remotePeers
-		// //Iterate backends
-		// for i := range cliRemotePeers {
-		// 	p, err := kubevip.ParsePeerConfig(cliRemotePeers[i])
-		// 	if err != nil {
-		// 		cmd.Help()
-		// 		log.Fatalln(err)
-		// 	}
-		// 	cliConfig.RemotePeers = append(cliConfig.RemotePeers, *p)
-		// }
-
-		// //Iterate backends
-		// for i := range cliBackends {
-		// 	b, err := kubevip.ParseBackendConfig(cliBackends[i])
-		// 	if err != nil {
-		// 		cmd.Help()
-		// 		log.Fatalln(err)
-		// 	}
-		// 	cliConfigLB.Backends = append(cliConfigLB.Backends, *b)
-		// }
-
-		// Add the basic Load-Balancer to the configuration
-		cliConfig.LoadBalancers = append(cliConfig.LoadBalancers, cliConfigLB)
-
-		err := cliConfig.ParseFlags(cliLocalPeer, cliRemotePeers, cliBackends)
-		if err != nil {
-			cmd.Help()
-			log.Fatalln(err)
-		}
-
-		err = kubevip.ParseEnvironment(&cliConfig)
-		if err != nil {
-			cmd.Help()
-			log.Fatalln(err)
-		}
-
-		cliConfig.PrintConfig()
-	},
-}
-
-var kubeVipSampleManifest = &cobra.Command{
-	Use:   "manifest",
-	Short: "Generate a Sample kubernetes manifest",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Generate the sample manifest specification
-		p := &appv1.Pod{
-			TypeMeta: metav1.TypeMeta{
-				Kind:       "Pod",
-				APIVersion: "v1",
-			},
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      "kube-vip",
-				Namespace: "kube-system",
-			},
-			Spec: appv1.PodSpec{
-				Containers: []appv1.Container{
-					{
-						Name:  "kube-vip",
-						Image: fmt.Sprintf("docker.io/plndr/kube-vip:%s", Release.Version),
-						SecurityContext: &appv1.SecurityContext{
-							Capabilities: &appv1.Capabilities{
-								Add: []appv1.Capability{
-									"NET_ADMIN",
-									"SYS_TIME",
-								},
-							},
-						},
-						Args: []string{
-							"start",
-							"-c",
-							"/etc/kube-vip/config.yaml",
-						},
-						VolumeMounts: []appv1.VolumeMount{
-							{
-								Name:      "config",
-								MountPath: "/etc/kube-vip/",
-							},
-						},
-					},
-				},
-				Volumes: []appv1.Volume{
-					{
-						Name: "config",
-						VolumeSource: appv1.VolumeSource{
-							HostPath: &appv1.HostPathVolumeSource{
-								Path: "/etc/kube-vip/",
-							},
-						},
-					},
-				},
-				HostNetwork: true,
-			},
-		}
-
-		b, _ := yaml.Marshal(p)
-		fmt.Printf(string(b))
-	},
-}

cmd/kube-vip-kubeadm.go

@@ -1,72 +1,19 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
-	"net"
 	"os"
 
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
 	"github.com/spf13/cobra"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
 )
 
 // kubeadm adds two subcommands for managing a vip during a kubeadm init/join
 // It is designed to operate "light" and take minimal input to start
 
-var initConfig kubevip.Config
-var initLoadBalancer kubevip.LoadBalancer
-
-// Points to a kubernetes configuration file
-var kubeConfigPath string
-
 func init() {
-
-	localpeer, err := autoGenLocalPeer()
-	if err != nil {
-		log.Fatalln(err)
-	}
-	initConfig.LocalPeer = *localpeer
-	//initConfig.Peers = append(initConfig.Peers, *localpeer)
-	kubeKubeadm.PersistentFlags().StringVar(&initConfig.Interface, "interface", "", "Name of the interface to bind to")
-	kubeKubeadm.PersistentFlags().StringVar(&initConfig.VIP, "vip", "", "The Virtual IP address")
-	kubeKubeadm.PersistentFlags().StringVar(&initConfig.VIPCIDR, "cidr", "", "The CIDR range for the virtual IP address")
-	kubeKubeadm.PersistentFlags().BoolVar(&initConfig.GratuitousARP, "arp", true, "Enable Arp for Vip changes")
-
-	// Clustering type (leaderElection)
-	kubeKubeadm.PersistentFlags().BoolVar(&initConfig.EnableLeaderElection, "leaderElection", false, "Use the Kubernetes leader election mechanism for clustering")
-	kubeKubeadm.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 5, "Length of time a Kubernetes leader lease can be held for")
-	kubeKubeadm.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time a Kubernetes leader can attempt to renew its lease")
-	kubeKubeadm.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Number of times the host will retry to hold a lease")
-
-	// Clustering type (raft)
-	kubeKubeadm.PersistentFlags().BoolVar(&initConfig.StartAsLeader, "startAsLeader", false, "Start this instance as the cluster leader")
-	kubeKubeadm.PersistentFlags().BoolVar(&initConfig.AddPeersAsBackends, "addPeersToLB", true, "Add raft peers to the load-balancer")
-
-	// Packet flags
-	kubeKubeadm.PersistentFlags().BoolVar(&initConfig.EnablePacket, "packet", false, "This will use the Packet API (requires the token ENV) to update the EIP <-> VIP")
-	kubeKubeadm.PersistentFlags().StringVar(&initConfig.PacketAPIKey, "packetKey", "", "The API token for authenticating with the Packet API")
-	kubeKubeadm.PersistentFlags().StringVar(&initConfig.PacketProject, "packetProject", "", "The name of project already created within Packet")
-
-	// Load Balancer flags
-	kubeKubeadm.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "lbEnable", false, "Enable a load-balancer on the VIP")
-	kubeKubeadm.PersistentFlags().BoolVar(&initLoadBalancer.BindToVip, "lbBindToVip", true, "Bind example load balancer to VIP")
-	kubeKubeadm.PersistentFlags().StringVar(&initLoadBalancer.Type, "lbType", "tcp", "Type of load balancer instance (TCP/HTTP)")
-	kubeKubeadm.PersistentFlags().StringVar(&initLoadBalancer.Name, "lbName", "Kubeadm Load Balancer", "The name of a load balancer instance")
-	kubeKubeadm.PersistentFlags().IntVar(&initLoadBalancer.Port, "lbPort", 6443, "Port that load balancer will expose on")
-	kubeKubeadm.PersistentFlags().IntVar(&initLoadBalancer.BackendPort, "lbBackEndPort", 6444, "A port that all backends may be using (optional)")
-
-	// BGP flags
-	kubeKubeadm.PersistentFlags().BoolVar(&initConfig.EnableBGP, "bgp", false, "This will enable BGP support within kube-vip")
-	kubeKubeadm.PersistentFlags().StringVar(&initConfig.BGPConfig.RouterID, "bgpRouterID", "", "The routerID for the bgp server")
-	kubeKubeadm.PersistentFlags().Uint32Var(&initConfig.BGPConfig.AS, "localAS", 65000, "The local AS number for the bgp server")
-	kubeKubeadm.PersistentFlags().StringVar(&initConfig.BGPPeerConfig.Address, "peerAddress", "", "The address of a BGP peer")
-	kubeKubeadm.PersistentFlags().Uint32Var(&initConfig.BGPPeerConfig.AS, "peerAS", 65000, "The AS number for a BGP peer")
-
 	kubeKubeadmJoin.Flags().StringVar(&kubeConfigPath, "config", "/etc/kubernetes/admin.conf", "The path of a Kubernetes configuration file")
 
 	kubeKubeadm.AddCommand(kubeKubeadmInit)
@@ -76,8 +23,8 @@ func init() {
 var kubeKubeadm = &cobra.Command{
 	Use:   "kubeadm",
 	Short: "Kubeadm functions",
-	Run: func(cmd *cobra.Command, args []string) {
-		cmd.Help()
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		_ = cmd.Help()
 		// TODO - A load of text detailing what's actually happening
 	},
 }
@@ -86,124 +33,84 @@ var kubeKubeadmInit = &cobra.Command{
 	Use:   "init",
 	Short: "kube-vip init",
 	Long:  "The \"init\" subcommand will generate the Kubernetes manifest that will be started by kubeadm through the kubeadm init process",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
 		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
+		err := kubevip.ParseEnvironment(&initConfig)
+		if err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+
 		// TODO - check for certain things VIP/interfaces
 		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+			_ = cmd.Help()
+			log.Error("No interface is specified for kube-vip to bind to")
+			return
 		}
 
-		if initConfig.VIP == "" {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+		if initConfig.VIP == "" && initConfig.Address == "" {
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
-		cfg := kubevip.GenerateManifestFromConfig(&initConfig, Release.Version)
 
-		fmt.Println(cfg)
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
+			if err != nil {
+				log.Error("generating VIPSubnet", "err", err)
+				return
+			}
+		}
+
+		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
+		fmt.Println(cfg) // output manifest to stdout
 	},
 }
 
 var kubeKubeadmJoin = &cobra.Command{
 	Use:   "join",
 	Short: "kube-vip join",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
 		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
-		// TODO - check for certain things VIP/interfaces
-		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+		err := kubevip.ParseEnvironment(&initConfig)
+		if err != nil {
+			log.Error("parsing environment", "err", err)
+			return
 		}
 
-		if initConfig.VIP == "" {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+		// TODO - check for certain things VIP/interfaces
+		if initConfig.Interface == "" {
+			_ = cmd.Help()
+			log.Error("No interface is specified for kube-vip to bind to")
+			return
+		}
+
+		if initConfig.VIP == "" && initConfig.Address == "" {
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
 
 		if _, err := os.Stat(kubeConfigPath); os.IsNotExist(err) {
-			log.Fatalf("Unable to find file [%s]", kubeConfigPath)
+			log.Error("kubeConfig not found", "Path", kubeConfigPath)
+			return
 		}
 
-		// We will use kubeconfig in order to find all the master nodes
-		// use the current context in kubeconfig
-		config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
-		if err != nil {
-			log.Fatal(err.Error())
-		}
-
-		// create the clientset
-		clientset, err := kubernetes.NewForConfig(config)
-		if err != nil {
-			log.Fatal(err.Error())
-		}
-
-		opts := metav1.ListOptions{}
-		opts.LabelSelector = "node-role.kubernetes.io/master"
-		nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), opts)
-
-		// Iterate over all nodes that are masters and find the details to build a peer list
-		for x := range nodes.Items {
-			// Get hostname and address
-			var nodeAddress, nodeHostname string
-			for y := range nodes.Items[x].Status.Addresses {
-				switch nodes.Items[x].Status.Addresses[y].Type {
-				case corev1.NodeHostName:
-					nodeHostname = nodes.Items[x].Status.Addresses[y].Address
-				case corev1.NodeInternalIP:
-					nodeAddress = nodes.Items[x].Status.Addresses[y].Address
-				}
-			}
-
-			newPeer, err := kubevip.ParsePeerConfig(fmt.Sprintf("%s:%s:%d", nodeHostname, nodeAddress, 10000))
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
 			if err != nil {
-				panic(err.Error())
+				log.Error("generating VIPSubnet", "err", err)
+				return
 			}
-			initConfig.RemotePeers = append(initConfig.RemotePeers, *newPeer)
-
 		}
-		// Generate manifest and print
-		cfg := kubevip.GenerateManifestFromConfig(&initConfig, Release.Version)
-		fmt.Println(cfg)
+
+		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
+		fmt.Println(cfg) // output manifest to stdout
 	},
 }
-
-func autoGenLocalPeer() (*kubevip.RaftPeer, error) {
-	// hostname // address // defaultport
-	h, err := os.Hostname()
-	if err != nil {
-		return nil, err
-	}
-
-	var a string
-	addrs, err := net.InterfaceAddrs()
-	if err != nil {
-		return nil, err
-	}
-	for _, address := range addrs {
-		// check the address type and if it is not a loopback the display it
-		if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
-			if ipnet.IP.To4() != nil {
-				a = ipnet.IP.String()
-				break
-			}
-		}
-	}
-	if a == "" {
-		return nil, fmt.Errorf("Unable to find local address")
-	}
-	return &kubevip.RaftPeer{
-		ID:      h,
-		Address: a,
-		Port:    10000,
-	}, nil
-
-}

cmd/kube-vip-manifests.go

@@ -0,0 +1,150 @@
+package cmd
+
+import (
+	"fmt"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/spf13/cobra"
+)
+
+// manifests will eventually deprecate the kubeadm set of subcommands
+// manifests will be used to generate:
+// - Pod spec manifest, mainly used for a static pod (kubeadm)
+// - Daemonset manifest, mainly used to run kube-vip as a deamonset within Kubernetes (k3s/rke)
+// - RBAC manifest, used to generate the RBAC permissions for kube-vip
+
+var taint, role, rolebinding bool
+
+func init() {
+	kubeManifest.PersistentFlags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
+	kubeManifestDaemon.PersistentFlags().BoolVar(&taint, "taint", false, "Taint the manifest for only running on control planes")
+	kubeManifestRbac.PersistentFlags().BoolVar(&role, "role", false, "Generate only a Role inside the serviceNamespace access")
+	kubeManifestRbac.PersistentFlags().BoolVar(&rolebinding, "rolebinding", false, "Generate only a RoleBinding for namespaced access")
+
+	kubeManifest.AddCommand(kubeManifestPod)
+	kubeManifest.AddCommand(kubeManifestDaemon)
+	kubeManifest.AddCommand(kubeManifestRbac)
+}
+
+var kubeManifest = &cobra.Command{
+	Use:   "manifest",
+	Short: "Manifest functions",
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		_ = cmd.Help()
+		// TODO - A load of text detailing what's actually happening
+	},
+}
+
+var kubeManifestPod = &cobra.Command{
+	Use:   "pod",
+	Short: "Generate a Pod Manifest",
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		var err error
+
+		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
+		// TODO - A load of text detailing what's actually happening
+		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+
+		// The control plane has a requirement for a VIP being specified
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
+		}
+
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
+			if err != nil {
+				log.Error("config parse", "err", err)
+				return
+			}
+		}
+
+		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
+		fmt.Println(cfg) // output manifest to stdout
+	},
+}
+
+var kubeManifestDaemon = &cobra.Command{
+	Use:   "daemonset",
+	Short: "Generate a Daemonset Manifest",
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		var err error
+
+		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
+		// TODO - A load of text detailing what's actually happening
+		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+		// The control plane has a requirement for a VIP being specified
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
+		}
+
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
+			if err != nil {
+				log.Error("config parse", "err", err)
+				return
+			}
+		}
+
+		cfg := kubevip.GenerateDaemonsetManifestFromConfig(&initConfig, Release.Version, inCluster, taint)
+		fmt.Println(cfg) // output manifest to stdout
+	},
+}
+
+var kubeManifestRbac = &cobra.Command{
+	Use:   "rbac",
+	Short: "Generate an RBAC Manifest",
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		var err error
+
+		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
+		// TODO - A load of text detailing what's actually happening
+		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+
+		// The control plane has a requirement for a VIP being specified
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
+		}
+
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
+			if err != nil {
+				log.Error("generating VIPSubnet", "err", err)
+				return
+			}
+		}
+		saCfg := kubevip.GenerateSA(&initConfig)
+		roleCfg := kubevip.GenerateRole(&initConfig, role)
+		if role {
+			rolebinding = true
+		}
+		roleBindingCfg := kubevip.GenerateRoleBinding(rolebinding, saCfg, roleCfg)
+
+		// Output the YAML manifests to stdout
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(saCfg))
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(roleCfg))
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(roleBindingCfg))
+	},
+}

cmd/kube-vip-start.go

@@ -1,122 +0,0 @@
-package cmd
-
-import (
-	"os"
-	"os/signal"
-
-	"github.com/plunder-app/kube-vip/pkg/cluster"
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-)
-
-// Start as a single node (no cluster), start as a leader in the cluster
-var startConfig kubevip.Config
-var startConfigLB kubevip.LoadBalancer
-var startLocalPeer, startKubeConfigPath string
-var startRemotePeers, startBackends []string
-var inCluster bool
-
-func init() {
-	// Get the configuration file
-	kubeVipStart.Flags().StringVarP(&configPath, "config", "c", "", "Path to a kube-vip configuration")
-	kubeVipStart.Flags().BoolVarP(&disableVIP, "disableVIP", "d", false, "Disable the VIP functionality")
-
-	// Pointers so we can see if they're nil (and not called)
-	kubeVipStart.Flags().StringVar(&startConfig.Interface, "interface", "eth0", "Name of the interface to bind to")
-	kubeVipStart.Flags().StringVar(&startConfig.VIP, "vip", "192.168.0.1", "The Virtual IP address")
-	kubeVipStart.Flags().BoolVar(&startConfig.SingleNode, "singleNode", false, "Start this instance as a single node")
-	kubeVipStart.Flags().BoolVar(&startConfig.StartAsLeader, "startAsLeader", false, "Start this instance as the cluster leader")
-	kubeVipStart.Flags().BoolVar(&startConfig.GratuitousARP, "arp", false, "Use ARP broadcasts to improve VIP re-allocations")
-	kubeVipStart.Flags().StringVar(&startLocalPeer, "localPeer", "server1:192.168.0.1:10000", "Settings for this peer, format: id:address:port")
-	kubeVipStart.Flags().StringSliceVar(&startRemotePeers, "remotePeers", []string{"server2:192.168.0.2:10000", "server3:192.168.0.3:10000"}, "Comma seperated remotePeers, format: id:address:port")
-	// Load Balancer flags
-	kubeVipStart.Flags().BoolVar(&startConfigLB.BindToVip, "lbBindToVip", false, "Bind example load balancer to VIP")
-	kubeVipStart.Flags().StringVar(&startConfigLB.Type, "lbType", "tcp", "Type of load balancer instance (TCP/HTTP)")
-	kubeVipStart.Flags().StringVar(&startConfigLB.Name, "lbName", "Example Load Balancer", "The name of a load balancer instance")
-	kubeVipStart.Flags().IntVar(&startConfigLB.Port, "lbPort", 8080, "Port that load balancer will expose on")
-	kubeVipStart.Flags().IntVar(&startConfigLB.BackendPort, "lbBackEndPort", 6443, "A port that all backends may be using (optional)")
-	kubeVipStart.Flags().StringSliceVar(&startBackends, "lbBackends", []string{"192.168.0.1:8080", "192.168.0.2:8080"}, "Comma seperated backends, format: address:port")
-
-	// Cluster configuration
-	kubeVipStart.Flags().StringVar(&startKubeConfigPath, "kubeConfig", "/etc/kubernetes/admin.conf", "The path of a kubernetes configuration file")
-	kubeVipStart.Flags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
-	kubeVipStart.Flags().BoolVar(&startConfig.EnableLeaderElection, "leaderElection", false, "Use the Kubernetes leader election mechanism for clustering")
-
-}
-
-var kubeVipStart = &cobra.Command{
-	Use:   "start",
-	Short: "Start the Virtual IP / Load balancer",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
-		var err error
-
-		// If a configuration file is loaded, then it will overwrite flags
-
-		if configPath != "" {
-			c, err := kubevip.OpenConfig(configPath)
-			if err != nil {
-				log.Fatalf("%v", err)
-			}
-			startConfig = *c
-		}
-
-		// parse environment variables, these will overwrite anything loaded or flags
-		err = kubevip.ParseEnvironment(&startConfig)
-		if err != nil {
-			log.Fatalln(err)
-		}
-
-		var newCluster *cluster.Cluster
-
-		if startConfig.SingleNode {
-			// If the Virtual IP isn't disabled then create the netlink configuration
-			newCluster, err = cluster.InitCluster(&startConfig, disableVIP)
-			if err != nil {
-				log.Fatalf("%v", err)
-			}
-			// Start a single node cluster
-			newCluster.StartSingleNode(&startConfig, disableVIP)
-		} else {
-			if disableVIP {
-				log.Fatalln("Cluster mode requires the Virtual IP to be enabled, use single node with no VIP")
-			}
-
-			// If the Virtual IP isn't disabled then create the netlink configuration
-			newCluster, err = cluster.InitCluster(&startConfig, disableVIP)
-			if err != nil {
-				log.Fatalf("%v", err)
-			}
-
-			if startConfig.EnableLeaderElection {
-				cm, err := cluster.NewManager(startKubeConfigPath, inCluster)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-
-				// Leader Cluster will block
-				err = newCluster.StartLeaderCluster(&startConfig, cm)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-			} else {
-
-				// // Start a multi-node (raft) cluster, this doesn't block so will wait on signal
-				err = newCluster.StartRaftCluster(&startConfig)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-				signalChan := make(chan os.Signal, 1)
-				signal.Notify(signalChan, os.Interrupt)
-
-				<-signalChan
-
-				newCluster.Stop()
-			}
-
-		}
-
-	},
-}

cmd/kube-vip.go

@@ -1,29 +1,35 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
+	"net"
+	"net/http"
 	"os"
-	"strconv"
+	"strings"
+	"time"
 
-	"github.com/plunder-app/kube-vip/pkg/service"
-	log "github.com/sirupsen/logrus"
+	log "log/slog"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/manager"
+	"github.com/kube-vip/kube-vip/pkg/vip"
 )
 
-// Path to the configuration file
-var configPath string
-
-// Disable the Virtual IP (bind to the existing network stack)
-var disableVIP bool
-
-// Run as a load balancer service (within a pod / kubernetes)
-var serviceArp bool
+// Is kube-vip running within cluster
+var inCluster bool
 
 // ConfigMap name within a Kubernetes cluster
 var configMap string
 
-// Configure the level of loggin
-var logLevel uint32
+// Points to a kubernetes configuration file
+var kubeConfigPath string
 
 // Release - this struct contains the release information populated when building kube-vip
 var Release struct {
@@ -31,32 +37,122 @@ var Release struct {
 	Build   string
 }
 
+// Structs used via the various subcommands
+var (
+	initConfig       kubevip.Config
+	initLoadBalancer kubevip.LoadBalancer
+)
+
 var kubeVipCmd = &cobra.Command{
 	Use:   "kube-vip",
 	Short: "This is a server for providing a Virtual IP and load-balancer for the Kubernetes control-plane",
 }
 
 func init() {
+	// Basic flags
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Interface, "interface", "", "Name of the interface to bind to")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesInterface, "serviceInterface", "", "Name of the interface to bind to (for services)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIP, "vip", "", "The Virtual IP address")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPSubnet, "vipSubnet", "", "The Virtual IP address subnet e.g. /32 /24 /8 etc.. (Default to 32 for IPv4 and 128 for IPv6)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.NodeName, "nodeName", "", "Name to be used for lease holder. Must be unique for each node/instance")
+
+	// VIP flags
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Address, "address", "", "an address (IP or DNS name) to use as a VIP")
+	kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.Port, "port", 6443, "Port for the VIP")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for VIP changes")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableWireguard, "wireguard", false, "Enable Wireguard for services VIPs")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableRoutingTable, "table", false, "Enable Routing Table for services VIPs")
+
+	// LoadBalancer flags
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "enableLoadBalancer", false, "enable loadbalancing on the VIP with IPVS")
+	kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.LoadBalancerPort, "lbPort", 6443, "loadbalancer port for the VIP")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerForwardingMethod, "lbForwardingMethod", "local", "loadbalancer forwarding method")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DDNS, "ddns", false, "use Dynamic DNS + DHCP to allocate VIP for address")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MirrorDestInterface, "mirrorDestInterface", "", "network interface where all traffic that traverses the service interface will be mirrored to. Source interface will use default interface is servicesInterface is not set.")
+
+	// Clustering type (leaderElection)
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLeaderElection, "leaderElection", false, "Use the Kubernetes leader election mechanism for clustering")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaderElectionType, "leaderElectionType", "kubernetes", "Defines the backend to run the leader election: kubernetes or etcd. Defaults to kubernetes.")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaseName, "leaseName", "plndr-cp-lock", "Name of the lease that is used for leader election")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 5, "Length of time (in seconds) a Kubernetes leader lease can be held for")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time (in seconds) a Kubernetes leader can attempt to renew its lease")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Length of time (in seconds) the LeaderElector clients should wait between tries of actions")
+
+	// BGP flags
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableBGP, "bgp", false, "This will enable BGP support within kube-vip")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.RouterID, "bgpRouterID", "", "The routerID for the bgp server")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.SourceIF, "sourceIF", "", "The source interface for bgp peering (not to be used with sourceIP)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.SourceIP, "sourceIP", "", "The source address for bgp peering (not to be used with sourceIF)")
+	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.AS, "localAS", 65000, "The local AS number for the bgp server")
+	kubeVipCmd.PersistentFlags().Uint64Var(&initConfig.BGPConfig.HoldTime, "bgpHoldTimer", 30, "The hold timer for all bgp peers (it defines the time a session is held)")
+	kubeVipCmd.PersistentFlags().Uint64Var(&initConfig.BGPConfig.KeepaliveInterval, "bgpKeepAliveInterval", 10, "The keepalive interval for all bgp peers (it defines the heartbeat of keepalive messages)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPPeerConfig.Address, "peerAddress", "", "The address of a BGP peer")
+	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPPeerConfig.AS, "peerAS", 65000, "The AS number for a BGP peer")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPPeerConfig.Password, "peerPass", "", "The md5 password for a BGP peer")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPPeerConfig.MultiHop, "multihop", false, "This will enable BGP multihop support")
+	kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.BGPPeers, "bgppeers", []string{}, "Comma separated BGP Peer, format: address:as:password:multihop")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Annotations, "annotations", "", "Set Node annotations prefix for parsing")
+
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPConfig.Zebra.Enabled, "zebra", false, "This will enable Zebra support within kube-vip")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.URL, "zebraUrl", "unix:/var/run/frr/zserv.api", "Path to the unix domain socket for connecting to Zebra daemon")
+	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.Zebra.Version, "zebraVersion", 6, "Zebra API Version")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.SoftwareName, "zebraSoftwareName", "frr8.3", "Software Name for Zebra")
+
+	// Namespace for kube-vip
+	kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The namespace for the configmap defined within the cluster")
 
 	// Manage logging
-	kubeVipCmd.PersistentFlags().Uint32Var(&logLevel, "log", 4, "Set the level of logging")
+	kubeVipCmd.PersistentFlags().Int32Var(&initConfig.Logging, "log", 0, "Set the level of logging")
 
 	// Service flags
-	kubeVipService.Flags().StringVarP(&configMap, "configMap", "c", "kube-vip", "The configuration map defined within the cluster")
-	kubeVipService.Flags().StringVarP(&service.Interface, "interface", "i", "eth0", "Name of the interface to bind to")
-	kubeVipService.Flags().BoolVar(&service.OutSideCluster, "OutSideCluster", false, "Start Controller outside of cluster")
-	kubeVipService.Flags().BoolVar(&service.EnableArp, "arp", false, "Use ARP broadcasts to improve VIP re-allocations")
+	kubeVipService.Flags().StringVarP(&configMap, "configMap", "c", "plndr", "The configuration map defined within the cluster")
+
+	// Routing Table flags
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableID, "tableID", 198, "The routing table used for all table entries")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableType, "tableType", unix.RTN_UNICAST, "The type of route that will be added to the routing table")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingProtocol, "routingProtocol", 248, "The routing protocol value used to create routes")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.CleanRoutingTable, "cleanRoutingTable", false, "Clean routing table of redundant routes on start")
+
+	// Behaviour flags
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableControlPlane, "controlplane", false, "Enable HA for control plane")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DetectControlPlane, "autodetectcp", false, "Determine working address for control plane (from loopback)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServices, "services", false, "Enable Kubernetes services")
+
+	// Extended behaviour flags
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServicesElection, "servicesElection", false, "Enable leader election per kubernetes service")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassOnly, "lbClassOnly", false, "Enable load balancing only for services with LoadBalancerClass \"kube-vip.io/kube-vip-class\"")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerClassName, "lbClassName", "kube-vip.io/kube-vip-class", "Name of load balancer class for kube-VIP, defaults to \"kube-vip.io/kube-vip-class\"")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassLegacyHandling, "lbClassNameLegacyHandling", true, "Use legacy LoadBalancer class name handling (e.g. accepting services both with empty and non-empty class)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServiceSecurity, "onlyAllowTrafficServicePorts", false, "Only allow traffic to service ports, others will be dropped, defaults to false")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableNodeLabeling, "enableNodeLabeling", false, "Enable leader node labeling with \"kube-vip.io/has-ip=<VIP address>\", defaults to false")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesLeaseName, "servicesLeaseName", "plndr-svcs-lock", "Name of the lease that is used for leader election for services (in arp mode)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.DNSMode, "dnsMode", "first", "Name of the mode that DNS lookup will be performed (first, ipv4, ipv6, dual)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DisableServiceUpdates, "disableServiceUpdates", false, "If true, kube-vip will process services as usual, but will not update service's Status.LoadBalancer.Ingress slice")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpoints, "enableEndpoints", false, "If enabled, kube-vip will only advertise services, but will use the (deprecated since v1.33) endpoints for IP addresses")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoInterfaceGlobalScope, "loInterfaceGlobalScope", false, "If true, kube-vip will set global scope when using the lo interface, otherwise a host scope will be used by default")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.HealthCheckPort, "healthCheckPort", 0, "If set to non-zero (> 1024), then this is the port that the healthcheck will listen on")
+
+	// Prometheus HTTP Server
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.PrometheusHTTPServer, "prometheusHTTPServer", ":2112", "Host and port used to expose Prometheus metrics via an HTTP server")
+
+	// Etcd
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.CAFile, "etcdCACert", "", "Verify certificates of TLS-enabled secure servers using this CA bundle file")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.ClientCertFile, "etcdCert", "", "Identify secure client using this TLS certificate file")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.ClientKeyFile, "etcdKey", "", "Identify secure client using this TLS key file")
+	kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.Etcd.Endpoints, "etcdEndpoints", nil, "Etcd member endpoints")
+
+	// Kubernetes client specific flags
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.K8sConfigFile, "k8sConfigPath", "/etc/kubernetes/admin.conf", "Path to the configuration file used with the Kubernetes client")
 
 	kubeVipCmd.AddCommand(kubeKubeadm)
+	kubeVipCmd.AddCommand(kubeManifest)
+	kubeVipCmd.AddCommand(kubeVipManager)
 	kubeVipCmd.AddCommand(kubeVipSample)
 	kubeVipCmd.AddCommand(kubeVipService)
-	kubeVipCmd.AddCommand(kubeVipStart)
 	kubeVipCmd.AddCommand(kubeVipVersion)
 
-	// Sample commands
-	kubeVipSample.AddCommand(kubeVipSampleConfig)
-	kubeVipSample.AddCommand(kubeVipSampleManifest)
-
+	// Set the logging level for all subsequent functions
+	log.SetLogLoggerLevel(log.Level(initConfig.Logging))
 }
 
 // Execute - starts the command parsing process
@@ -70,7 +166,7 @@ func Execute() {
 var kubeVipVersion = &cobra.Command{
 	Use:   "version",
 	Short: "Version and Release information about the Kubernetes Virtual IP Server",
-	Run: func(cmd *cobra.Command, args []string) {
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 		fmt.Printf("Kube-VIP Release Information\n")
 		fmt.Printf("Version:  %s\n", Release.Version)
 		fmt.Printf("Build:    %s\n", Release.Build)
@@ -80,56 +176,298 @@ var kubeVipVersion = &cobra.Command{
 var kubeVipSample = &cobra.Command{
 	Use:   "sample",
 	Short: "Generate a Sample configuration",
-	Run: func(cmd *cobra.Command, args []string) {
-		cmd.Help()
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		_ = cmd.Help()
 	},
 }
 
 var kubeVipService = &cobra.Command{
 	Use:   "service",
 	Short: "Start the Virtual IP / Load balancer as a service within a Kubernetes cluster",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 
-		// User Environment variables as an option to make manifest clearer
-		envInterface := os.Getenv("vip_interface")
-		if envInterface != "" {
-			service.Interface = envInterface
+		// parse environment variables, these will overwrite anything loaded or flags
+		err := kubevip.ParseEnvironment(&initConfig)
+		if err != nil {
+			log.Error("parsing env", "err", err)
+			return
 		}
 
+		// Change RTN_UNSPEC to default type
+		if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
+			initConfig.RoutingProtocol = unix.RTN_UNICAST
+		}
+
+		// Set the logging level for all subsequent functions
+		log.SetLogLoggerLevel(log.Level(initConfig.Logging))
+
+		if err := initConfig.CheckInterface(); err != nil {
+			log.Error("checking interface", "err", err)
+			return
+		}
+
+		// User Environment variables as an option to make manifest clearer
 		envConfigMap := os.Getenv("vip_configmap")
-		if envInterface != "" {
+		if envConfigMap != "" {
 			configMap = envConfigMap
 		}
 
-		envLog := os.Getenv("vip_loglevel")
-		if envLog != "" {
-			logLevel, err := strconv.Atoi(envLog)
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
 			if err != nil {
-				panic(fmt.Sprintf("Unable to parse environment variable [vip_loglevel], should be int"))
+				log.Error("generating CIDR", "err", err)
+				return
 			}
-			log.SetLevel(log.Level(logLevel))
-		}
-
-		envArp := os.Getenv("vip_arp")
-		if envArp != "" {
-			arpBool, err := strconv.ParseBool(envArp)
-			if err != nil {
-				panic(fmt.Sprintf("Unable to parse environment variable [arp], should be bool (true/false)"))
-			}
-			service.EnableArp = arpBool
 		}
 
 		// Define the new service manager
-		mgr, err := service.NewManager(configMap)
+		mgr, err := manager.New(configMap, &initConfig)
 		if err != nil {
-			log.Fatalf("%v", err)
+			log.Error("new manager", "err", err)
+			return
 		}
+
 		// Start the service manager, this will watch the config Map and construct kube-vip services for it
 		err = mgr.Start()
 		if err != nil {
-			log.Fatalf("%v", err)
+			log.Error("manager start", "err", err)
+			return
 		}
 	},
 }
+
+var kubeVipManager = &cobra.Command{
+	Use:   "manager",
+	Short: "Start the kube-vip manager",
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		// parse environment variables, these will overwrite anything loaded or flags
+		err := kubevip.ParseEnvironment(&initConfig)
+		if err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+
+		// Change RTN_UNSPEC to default type
+		if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
+			initConfig.RoutingProtocol = unix.RTN_UNICAST
+		}
+
+		// Set the logging level for all subsequent functions
+		log.SetLogLoggerLevel(log.Level(initConfig.Logging))
+
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
+			if err != nil {
+				log.Error("No interface is specified for kube-vip to bind to")
+				return
+			}
+		}
+
+		// Welome messages
+		log.Info("kube-vip.io", "version", Release.Version, "build", Release.Build)
+
+		// start prometheus server
+		if initConfig.PrometheusHTTPServer != "" {
+			go servePrometheusHTTPServer(cmd.Context(), PrometheusHTTPServerConfig{
+				Addr: initConfig.PrometheusHTTPServer,
+			})
+		}
+
+		// Determine the kube-vip mode
+		var mode string
+		if initConfig.EnableARP {
+			mode = "ARP"
+		}
+
+		if initConfig.EnableBGP {
+			mode = "BGP"
+		}
+
+		if initConfig.EnableWireguard {
+			mode = "Wireguard"
+		}
+
+		if initConfig.EnableRoutingTable {
+			mode = "Routing Table"
+		}
+
+		// Provide configuration to output/logging
+		log.Info("starting", "namespace", initConfig.Namespace, "Mode", mode, "Control Plane", initConfig.EnableControlPlane, "Services", initConfig.EnableServices)
+
+		// End if nothing is enabled
+		if !initConfig.EnableServices && !initConfig.EnableControlPlane {
+			log.Error("no features are enabled")
+			return
+		}
+
+		if !initConfig.EnableARP && strings.Contains(initConfig.VIPSubnet, kubevip.Auto) {
+			log.Error("auto subnet discovery cannot be used outside ARP mode")
+			return
+		}
+
+		if strings.Contains(initConfig.VIPSubnet, kubevip.Auto) && initConfig.Address != "" {
+			log.Error("auto subnet discovery cannot be used if VIP address was provided")
+			return
+		}
+
+		// If we're using wireguard then all traffic goes through the wg0 interface
+		if initConfig.EnableWireguard {
+			if initConfig.Interface == "" {
+				// Set the vip interface to the wireguard interface
+				initConfig.Interface = "wg0"
+			}
+
+			log.Info("configuring Wireguard networking")
+			l, err := netlink.LinkByName(initConfig.Interface)
+			if err != nil {
+				if strings.Contains(err.Error(), "Link not found") {
+					log.Warn("attempting to create wireguard interface", "interface not found", initConfig.Interface)
+					err = netlink.LinkAdd(&netlink.Wireguard{LinkAttrs: netlink.LinkAttrs{Name: initConfig.Interface}})
+					if err != nil {
+						log.Error("adding link", "err", err)
+						return
+					}
+					l, err = netlink.LinkByName(initConfig.Interface)
+					if err != nil {
+						log.Error("finding link", "err", err)
+						return
+					}
+				}
+			}
+			err = netlink.LinkSetUp(l)
+			if err != nil {
+				log.Error("setting link UP", "err", err)
+				return
+			}
+
+		} else { // if we're not using Wireguard then we'll need to use an actual interface
+			// Check if the interface needs auto-detecting
+			if initConfig.Interface == "" {
+				log.Info("No interface is specified for VIP in config, auto-detecting default Interface")
+				defaultIF, err := vip.GetDefaultGatewayInterface()
+				if err != nil {
+					_ = cmd.Help()
+					log.Error("detecting interface", "err", err)
+					return
+				}
+				initConfig.Interface = defaultIF.Name
+				log.Info("kube-vip bind", "interface", initConfig.Interface)
+
+				go func() {
+					if err := vip.MonitorDefaultInterface(context.TODO(), defaultIF); err != nil {
+
+						log.Error("interface monitor", "err", err)
+						return
+					}
+				}()
+			}
+		}
+		// Perform a check on the state of the interface
+		if err := initConfig.CheckInterface(); err != nil {
+			log.Error("checking interface", "err", err)
+			return
+		}
+
+		// User Environment variables as an option to make manifest clearer
+		envConfigMap := os.Getenv("vip_configmap")
+		if envConfigMap != "" {
+			configMap = envConfigMap
+		}
+
+		// Define the new service manager
+		mgr, err := manager.New(configMap, &initConfig)
+		if err != nil {
+			log.Error("new manager", "err", err)
+			return
+		}
+
+		prometheus.MustRegister(mgr.PrometheusCollector()...)
+
+		// Start the service manager, this will watch the config Map and construct kube-vip services for it
+		err = mgr.Start()
+		if err != nil {
+			log.Error("start manager", "err", err)
+			return
+		}
+	},
+}
+
+// PrometheusHTTPServerConfig defines the Prometheus server configuration.
+type PrometheusHTTPServerConfig struct {
+	// Addr sets the http server address used to expose the metric endpoint
+	Addr string
+}
+
+func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerConfig) {
+	var err error
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.Handler())
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { //nolint TODO
+		_, _ = w.Write([]byte(`<html>
+			<head><title>kube-vip</title></head>
+			<body>
+			<h1>kube-vip Metrics</h1>
+			<p><a href="` + "/metrics" + `">Metrics</a></p>
+			</body>
+			</html>`))
+	})
+
+	srv := &http.Server{
+		Addr:              config.Addr,
+		Handler:           mux,
+		ReadHeaderTimeout: 2 * time.Second,
+	}
+
+	go func() {
+		if err = srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			log.Error("prometheus HTTP server", "err", err)
+			return
+		}
+	}()
+
+	log.Info("prometheus HTTP server started")
+
+	<-ctx.Done()
+
+	log.Info("prometheus HTTP server stopped")
+
+	ctxShutDown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer func() {
+		cancel()
+	}()
+
+	if err = srv.Shutdown(ctxShutDown); err != nil {
+		log.Error("shutting down prometheus HTTP server", "err", err)
+		return
+	}
+
+	if err == http.ErrServerClosed {
+		err = nil
+	}
+}
+
+func GenerateCidrRange(address string) (string, error) {
+	var cidrs []string
+
+	addresses := strings.Split(address, ",")
+	for _, a := range addresses {
+		ip := net.ParseIP(a)
+		if ip == nil {
+			ips, err := net.LookupIP(a)
+			if len(ips) == 0 || err != nil {
+				return "", fmt.Errorf("invalid IP address: %s from [%s], %v", a, address, err)
+			}
+			ip = ips[0]
+		}
+
+		if ip.To4() != nil {
+			cidrs = append(cidrs, "32")
+		} else {
+			cidrs = append(cidrs, "128")
+		}
+	}
+
+	return strings.Join(cidrs, ","), nil
+}

demo/README.md

@@ -0,0 +1,46 @@
+# Demo client-server
+
+This contains some example code to determine how long "failovers" are taking within kube-vip, the server component should live within the cluster and the client should be externally.
+
+## Deploy the server
+
+Simply apply the manifest to a working cluster that has kube-vip deployed:
+
+```
+kubectl apply -f ./demo/server/deploy.yaml
+```
+
+Retrieve the loadBalancer IP that is fronting the service:
+
+```
+kubectl get svc demo-service
+NAME           TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)           AGE
+demo-service   LoadBalancer   10.104.18.147   192.168.0.217   10002:32529/UDP   117m
+```
+
+## Connect the client
+
+From elsewhere, clone the kube-vip repository and connect the client to the server endpoint (loadBalancer IP) with the following command:
+
+```
+go run ./demo/client/main.go -address=<vip>
+```
+
+You will only see output when the client has reconcilled the connection to a pod beneath the service, where it will print the timestamp to reconnection along with the time in milliseconds it took:
+
+```
+15:58:35.916952 3008
+15:58:45.947506 2005
+15:58:57.983151 3007
+15:59:08.013450 2005
+15:59:20.046491 3008
+15:59:30.076341 2507
+15:59:42.110747 3008
+```
+
+## Kill some pods to test
+
+On a machine or control plane that has `kubectl` and has the credentials to speak to the cluster we will run a command to find the demo pod and kill it every 10 seconds:
+
+`while true ; do kubectl delete pod $(kubectl get pods | grep -v NAME | grep vip| awk '{ print $1 }'); sleep 10; done`
+

demo/client/main.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"net"
+	"time"
+)
+
+const udpdata = "a3ViZS12aXAK=kube-vip"
+
+func main() {
+	address := flag.String("address", "127.0.0.1", "The address of the server")
+	port := flag.Int("port", 10002, "the port of the server")
+	interval := flag.Float64("interval", 1000, "Interval in milliseconds")
+	flag.Parse()
+	var errorTime time.Time
+	var errorOccurred bool
+	for {
+		p := make([]byte, 2048)
+		conn, err := net.Dial("udp", fmt.Sprintf("%s:%d", *address, *port))
+		if err != nil {
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			continue
+		}
+
+		err = conn.SetDeadline(time.Now().Add(time.Duration(*interval) * time.Millisecond))
+		if err != nil {
+			//fmt.Printf("Connectivity error [%v]", err)
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			if err = conn.Close(); err != nil {
+				fmt.Printf("Error closing connection [%v]", err)
+			}
+			continue
+		}
+
+		_, err = fmt.Fprint(conn, udpdata)
+		if err != nil {
+			fmt.Printf("Error writing data [%v]", err)
+		}
+
+		_, err = bufio.NewReader(conn).Read(p)
+		if err != nil {
+			//fmt.Printf("read error %v\n", err)
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			if err = conn.Close(); err != nil {
+				fmt.Printf("Error closing connection [%v]", err)
+			}
+			continue
+		}
+		time.Sleep(time.Duration(*interval) * time.Millisecond)
+		if errorOccurred {
+			finishTime := time.Since(errorTime)
+			//fmt.Printf("connectivity reconciled in %dms\n", finishTime.Milliseconds())
+			//t :=time.Now().Format("15:04:05.000000")
+			fmt.Printf("%s %d\n", time.Now().Format("15:04:05.000000"), finishTime.Milliseconds())
+
+			errorOccurred = false
+		}
+	}
+}

demo/go.mod

@@ -1,3 +0,0 @@
-module github.com/plunder-app/kube-vip/demo
-
-go 1.13

demo/server/Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:experimental
 
-FROM golang:1.13-alpine as dev
+FROM golang:1.19-alpine as dev
 RUN apk add --no-cache git ca-certificates
 RUN adduser -D appuser
 COPY main.go /src/
@@ -13,4 +13,4 @@ RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
 
 FROM scratch
 COPY --from=dev /src/demo /
-CMD ["/demo"]
+CMD ["/demo"]

demo/server/Makefile

@@ -45,7 +45,7 @@ fmt:
 	@gofmt -l -w $(SRC)
 
 docker:
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 simplify:

demo/server/deploy.yaml

@@ -6,7 +6,7 @@ metadata:
     app: kube-vip-demo
   name: kube-vip-demo
 spec:
-  replicas: 3
+  replicas: 1
   selector:
     matchLabels:
       app: kube-vip-demo
@@ -30,4 +30,25 @@ spec:
         ports:
             - containerPort: 10001
             - containerPort: 10002
-status: {}
+status: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: demo-service
+  namespace: default
+  labels:
+    app: demo-service
+  annotations:
+    kube-vip.io/egress: "true"
+spec:
+  type: LoadBalancer
+  # "Local" preserves the client source IP and avoids a second hop for
+  # LoadBalancer and NodePort
+  externalTrafficPolicy: Local
+  ports:
+  - name: demo-udp
+    port: 10002
+    protocol: UDP
+  selector:
+    app: kube-vip-demo

demo/server/go.mod

@@ -0,0 +1,3 @@
+module github.com/kube-vip/kube-vip/demo
+
+go 1.13

demo/server/main.go

@@ -32,7 +32,7 @@ func main() {
 		}
 	}
 
-	if serverType == strings.ToLower("udp") {
+	if serverType == "udp" {
 		// Start the UDP echo server
 
 		ServerAddr, err := net.ResolveUDPAddr("udp", ":10002")

example/deploy/0.1.2.yaml

@@ -28,7 +28,7 @@ spec:
                   - kube-vip-cluster
             topologyKey: "kubernetes.io/hostname"
       containers:
-      - image: plndr/kube-vip:0.1.2
+      - image: ghcr.io/kube-vip/kube-vip:0.3.7
         imagePullPolicy: Always
         name: kube-vip
         command:

example/deploy/0.1.3.yaml

@@ -58,7 +58,7 @@ spec:
                   - kube-vip-cluster
             topologyKey: "kubernetes.io/hostname"
       containers:
-      - image: plndr/kube-vip:0.1.3
+      - image: ghcr.io/kube-vip/kube-vip:0.3.7
         imagePullPolicy: Always
         name: kube-vip
         command:

example/deploy/0.3.5.yaml

@@ -0,0 +1,55 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  creationTimestamp: null
+  name: kube-vip-ds
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      name: kube-vip-ds
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        name: kube-vip-ds
+    spec:
+      containers:
+      - args:
+        - manager
+        env:
+        - name: vip_arp
+          value: "true"
+        - name: vip_interface
+          value: eth0
+        - name: port
+          value: "6443"
+        - name: vip_cidr
+          value: "32"
+        - name: svc_enable
+          value: "true"
+        - name: vip_startleader
+          value: "false"
+        - name: vip_addpeerstolb
+          value: "true"
+        - name: vip_localpeer
+          value: ip-172-20-40-207:172.20.40.207:10000
+        - name: vip_address
+        image: plndr/kube-vip:v0.3.5
+        imagePullPolicy: Always
+        name: kube-vip
+        resources: {}
+        securityContext:
+          capabilities:
+            add:
+            - NET_ADMIN
+            - NET_RAW
+            - SYS_TIME
+      hostNetwork: true
+      serviceAccountName: kube-vip
+  updateStrategy: {}
+status:
+  currentNumberScheduled: 0
+  desiredNumberScheduled: 0
+  numberMisscheduled: 0
+  numberReady: 0

example/deployment.yaml

@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:latest
+        ports:
+        - containerPort: 80

example/service-interface-annotation.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-interface-ens192-service
+  annotations:
+    kube-vip.io/serviceInterface: ens192
+spec:
+  selector:
+    app: nginx
+  type: LoadBalancer
+  ports:
+  - name: http
+    port: 80
+    targetPort: 80
+    protocol: TCP

example/service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-service
+spec:
+  selector:
+    app: nginx
+  type: LoadBalancer
+  ports:
+  - name: http
+    port: 80
+    targetPort: 80
+    protocol: TCP

go.mod

@@ -1,27 +1,165 @@
-module github.com/plunder-app/kube-vip
+module github.com/kube-vip/kube-vip
 
-go 1.14
+go 1.24.4
 
 require (
-	github.com/davecgh/go-spew v1.1.1
-	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
-	github.com/digineo/go-dhclient v1.0.2
-	github.com/eapache/queue v1.1.0 // indirect
-	github.com/ghodss/yaml v1.0.0
-	github.com/golang/protobuf v1.3.2
-	github.com/hashicorp/raft v1.1.2
-	github.com/imdario/mergo v0.3.8 // indirect
-	github.com/osrg/gobgp v2.0.0+incompatible
-	github.com/packethost/packngo v0.2.1-0.20200424110205-36917dbc292f
+	github.com/cloudflare/ipvs v0.11.0
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
+	github.com/docker/docker v28.3.3+incompatible
+	github.com/florianl/go-conntrack v0.4.0
+	github.com/google/go-cmp v0.7.0
+	github.com/google/nftables v0.3.0
+	github.com/gookit/slog v0.5.8
+	github.com/huin/goupnp v1.3.0
+	github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d
+	github.com/jpillora/backoff v1.0.0
+	github.com/mdlayher/ndp v1.1.0
+	github.com/onsi/ginkgo/v2 v2.23.4
+	github.com/onsi/gomega v1.37.0
+	github.com/osrg/gobgp/v3 v3.37.0
 	github.com/pkg/errors v0.9.1
-	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/cobra v0.0.6
-	github.com/vishvananda/netlink v1.1.0
-	golang.org/x/net v0.0.0-20191004110552-13f9640d40b9
-	k8s.io/api v0.18.0
-	k8s.io/apimachinery v0.18.0
-	k8s.io/client-go v0.18.0
-	k8s.io/klog v1.0.0
+	github.com/prometheus/client_golang v1.22.0
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cobra v1.9.1
+	github.com/stretchr/testify v1.10.0
+	github.com/vishvananda/netlink v1.3.1
+	go.etcd.io/etcd/api/v3 v3.6.1
+	go.etcd.io/etcd/client/pkg/v3 v3.6.1
+	go.etcd.io/etcd/client/v3 v3.6.1
+	go.uber.org/zap v1.27.0
+	golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
+	golang.org/x/sync v0.15.0
+	golang.org/x/sys v0.33.0
+	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
+	google.golang.org/grpc v1.73.0
+	google.golang.org/protobuf v1.36.6
+	k8s.io/api v0.33.3
+	k8s.io/apimachinery v0.33.3
+	k8s.io/client-go v0.33.3
+	k8s.io/klog/v2 v2.130.1
+	sigs.k8s.io/kind v0.29.0
+	sigs.k8s.io/yaml v1.4.0
 )
 
-replace github.com/osrg/gobgp v2.0.0+incompatible => github.com/osrg/gobgp v0.0.0-20191101114856-a42a1a5f6bf0
+require (
+	al.essio.dev/pkg/shellescape v1.5.1 // indirect
+	github.com/BurntSushi/toml v1.4.0 // indirect
+	github.com/Microsoft/go-winio v0.4.14 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/coreos/go-semver v0.3.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/dlclark/regexp2 v1.11.4 // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/eapache/channels v1.1.0 // indirect
+	github.com/eapache/queue v1.1.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.12.1 // indirect
+	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/gnostic-models v0.6.9 // indirect
+	github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gookit/color v1.5.4 // indirect
+	github.com/gookit/goutil v0.6.18 // indirect
+	github.com/gookit/gsr v0.1.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/josharian/native v1.1.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/k-sone/critbitgo v1.4.0 // indirect
+	github.com/magiconair/properties v1.8.9 // indirect
+	github.com/mailru/easyjson v0.9.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mdlayher/genetlink v1.3.2 // indirect
+	github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect
+	github.com/mdlayher/packet v1.1.2 // indirect
+	github.com/mdlayher/socket v0.5.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/sys/atomicwriter v0.1.0 // indirect
+	github.com/moby/term v0.5.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/pelletier/go-toml v1.9.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.62.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/sagikazarmark/locafero v0.6.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/spf13/viper v1.19.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tj/go-spin v1.1.0 // indirect
+	github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/vishvananda/netns v0.0.5 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xlab/c-for-go v1.3.0 // indirect
+	github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.37.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 // indirect
+	go.opentelemetry.io/otel/metric v1.37.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.37.0 // indirect
+	go.opentelemetry.io/otel/trace v1.37.0 // indirect
+	go.uber.org/automaxprocs v1.6.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/crypto v0.39.0 // indirect
+	golang.org/x/mod v0.25.0 // indirect
+	golang.org/x/net v0.41.0 // indirect
+	golang.org/x/oauth2 v0.30.0 // indirect
+	golang.org/x/term v0.32.0 // indirect
+	golang.org/x/text v0.26.0 // indirect
+	golang.org/x/time v0.9.0 // indirect
+	golang.org/x/tools v0.33.0 // indirect
+	golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
+	k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect
+	modernc.org/cc/v4 v4.24.4 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/opt v0.1.4 // indirect
+	modernc.org/sortutil v1.2.1 // indirect
+	modernc.org/strutil v1.2.1 // indirect
+	modernc.org/token v1.1.0 // indirect
+	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
+)

go.sum

@@ -1,415 +1,509 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
-github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
-github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
-github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
-github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
-github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
-github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+al.essio.dev/pkg/shellescape v1.5.1 h1:86HrALUujYS/h+GtqoB26SBEdkWfmMI6FubjXlsXyho=
+al.essio.dev/pkg/shellescape v1.5.1/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
+github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
+github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
-github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
-github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
-github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
-github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
-github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
+github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
+github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8=
+github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
+github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
+github.com/cloudflare/ipvs v0.11.0 h1:niLcbqfv8+RSYk+yI+jhiCIHobEdEhO++mcwCX0Znsw=
+github.com/cloudflare/ipvs v0.11.0/go.mod h1:XsbuKcQpqb3rpjYPtsFsM8BNZlgLw+Z0iC6RhocqDd8=
+github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
+github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
+github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
+github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
+github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
+github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
+github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
+github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgryski/go-farm v0.0.0-20171119141306-ac7624ea8da3/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
-github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
-github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
-github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/digineo/go-dhclient v1.0.2 h1:69ZRY+AZnAx+BjO7UWTYWCGgnxW6oBOGwJXgciuLSEU=
-github.com/digineo/go-dhclient v1.0.2/go.mod h1:DPvyqGEW8irJvp2lrnGfQWpjj6VidXX9STLBTfNing4=
-github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
+github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
+github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
+github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
+github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
+github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
+github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k=
 github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0=
-github.com/eapache/queue v1.0.2/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
-github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
-github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I=
-github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/fsnotify/fsnotify v1.4.2/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
-github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
-github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
-github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
-github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
-github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU=
+github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
+github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/florianl/go-conntrack v0.4.0 h1:TlYkxytdwgVayfU0cKwkHurQA0Rd1ZSEBRckRYDUu18=
+github.com/florianl/go-conntrack v0.4.0/go.mod h1:iPDx4oIats2T7X7Jm3PFyRCJM1GfZhJaSHOWROYOrE8=
+github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
+github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
+github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
+github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
+github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ=
+github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4=
+github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
+github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
+github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
+github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw=
+github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
-github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY=
-github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
-github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI=
-github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
-github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
-github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
-github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
-github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-hclog v0.9.1 h1:9PZfAcVEvez4yhLH2TBU64/h/z4xlFI80cWXRrxuKuM=
-github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
-github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
-github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
-github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
-github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
-github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
-github.com/hashicorp/go-retryablehttp v0.6.6 h1:HJunrbHTDDbBb/ay4kxa1n+dLmttUlnP3V9oNE4hmsM=
-github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
-github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM=
-github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/hcl v0.0.0-20170509225359-392dba7d905e/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
+github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg=
+github.com/google/nftables v0.3.0/go.mod h1:BCp9FsrbF1Fn/Yu6CLUc9GGZFw/+hsxfluNXXmxBfRM=
+github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
+github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
+github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
+github.com/gookit/goutil v0.6.18 h1:MUVj0G16flubWT8zYVicIuisUiHdgirPAkmnfD2kKgw=
+github.com/gookit/goutil v0.6.18/go.mod h1:AY/5sAwKe7Xck+mEbuxj0n/bc3qwrGNe3Oeulln7zBA=
+github.com/gookit/gsr v0.1.0 h1:0gadWaYGU4phMs0bma38t+Do5OZowRMEVlHv31p0Zig=
+github.com/gookit/gsr v0.1.0/go.mod h1:7wv4Y4WCnil8+DlDYHBjidzrEzfHhXEoFjEA0pPPWpI=
+github.com/gookit/slog v0.5.8 h1:XZCeHLQvvOZWcSUDZcqxXITsL9+d1ESsKZoASBmK1lI=
+github.com/gookit/slog v0.5.8/go.mod h1:s0ViFOY/IgUuT4MDPF0l9x5/npcciy8pL4xwWZadnoc=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/raft v1.1.2 h1:oxEL5DDeurYxLd3UbcY/hccgSPhLLpiBZ1YxtWEq59c=
-github.com/hashicorp/raft v1.1.2/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8=
-github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
-github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jessevdk/go-flags v1.3.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok=
-github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/k-sone/critbitgo v1.3.1-0.20191024122315-48c9e1530131 h1:2bjzgZk4GiWAFkj15/SkmxIO30u69RyPiSS+F0d+Kzs=
-github.com/k-sone/critbitgo v1.3.1-0.20191024122315-48c9e1530131/go.mod h1:7E6pyoyADnFxlUBEKcnfS49b7SUAQGMK+OAp/UQvo0s=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
-github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8CrSJVmaolDVOxTfS9kc36uB6H40kdbQq8=
+github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Goc3h8EklBH5mspfHFxBnEoURQCGzQQH1ga9Myjvis=
+github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc=
+github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d h1:VkCNWh6tuQLgDBc6KrUOz/L1mCUQGnR1Ujj8uTgpwwk=
+github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d/go.mod h1:VvGYjkZoJyKqlmT1yzakUs4mfKMNB0XdODP0+rdml6k=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
+github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
+github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
+github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
+github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ=
+github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok=
+github.com/jsimonetti/rtnetlink v0.0.0-20201216134343-bde56ed16391/go.mod h1:cR77jAZG3Y3bsb8hF6fHJbFoyFukLFOkQ98S0pQz3xw=
+github.com/jsimonetti/rtnetlink v0.0.0-20201220180245-69540ac93943/go.mod h1:z4c53zj6Eex712ROyh8WI0ihysb5j2ROyV42iNogmAs=
+github.com/jsimonetti/rtnetlink v0.0.0-20210122163228-8d122574c736/go.mod h1:ZXpIyOK59ZnN7J0BV99cZUPmsqDRZ3eq5X+st7u/oSA=
+github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b/go.mod h1:8w9Rh8m+aHZIG69YPGGem1i5VzoyRC8nw2kA8B+ik5U=
+github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190/go.mod h1:NmKSdU4VGSiv1bMsdqNALI4RSvvjtz65tTMCnD05qLo=
+github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/k-sone/critbitgo v1.4.0 h1:l71cTyBGeh6X5ATh6Fibgw3+rtNT80BA0uNNWgkPrbE=
+github.com/k-sone/critbitgo v1.4.0/go.mod h1:7E6pyoyADnFxlUBEKcnfS49b7SUAQGMK+OAp/UQvo0s=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/magiconair/properties v1.7.3/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
-github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/mdlayher/raw v0.0.0-20191004140158-e1402808046b h1:8Oryv4wHvBHAxi9Swzu1zyN4BFrJKvv5pOnN0scSTw8=
-github.com/mdlayher/raw v0.0.0-20191004140158-e1402808046b/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/mapstructure v0.0.0-20170523030023-d0303fe80992/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a8RkpQM=
+github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
+github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo=
+github.com/mdlayher/ethtool v0.0.0-20211028163843-288d040e9d60/go.mod h1:aYbhishWc4Ai3I2U4Gaa2n3kHWSwzme6EsG/46HRQbE=
+github.com/mdlayher/genetlink v1.0.0/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc=
+github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw=
+github.com/mdlayher/genetlink v1.3.2/go.mod h1:tcC3pkCrPUGIKKsCsp0B3AdaaKuHtaxoJRz3cc+528o=
+github.com/mdlayher/ndp v1.1.0 h1:QylGKGVtH60sKZUE88+IW5ila1Z/M9/OXhWdsVKuscs=
+github.com/mdlayher/ndp v1.1.0/go.mod h1:FmgESgemgjl38vuOIyAHWUUL6vQKA/pQNkvXdWsdQFM=
+github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA=
+github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M=
+github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY=
+github.com/mdlayher/netlink v1.1.1/go.mod h1:WTYpFb/WTvlRJAyKhZL5/uy69TDDpHHu2VZmb2XgV7o=
+github.com/mdlayher/netlink v1.2.0/go.mod h1:kwVW1io0AZy9A1E2YYgaD4Cj+C+GPkU6klXCMzIJ9p8=
+github.com/mdlayher/netlink v1.2.1/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU=
+github.com/mdlayher/netlink v1.2.2-0.20210123213345-5cc92139ae3e/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU=
+github.com/mdlayher/netlink v1.3.0/go.mod h1:xK/BssKuwcRXHrtN04UBkwQ6dY9VviGGuriDdoPSWys=
+github.com/mdlayher/netlink v1.4.0/go.mod h1:dRJi5IABcZpBD2A3D0Mv/AiX8I9uDEu5oGkAVrekmf8=
+github.com/mdlayher/netlink v1.4.1/go.mod h1:e4/KuJ+s8UhfUpO9z00/fDZZmhSrs+oxyqAS9cNgn6Q=
+github.com/mdlayher/netlink v1.5.0/go.mod h1:1Kr8BBFxGyUyNmztC9WLOayqYVAd2wsgOZm18nqGuzQ=
+github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 h1:A1Cq6Ysb0GM0tpKMbdCXCIfBclan4oHk1Jb+Hrejirg=
+github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42/go.mod h1:BB4YCPDOzfy7FniQ/lxuYQ3dgmM2cZumHbK8RpTjN2o=
+github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY=
+github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4=
+github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00/go.mod h1:GAFlyu4/XV68LkQKYzKhIo/WW7j3Zi0YRAz/BOoanUc=
+github.com/mdlayher/socket v0.0.0-20211007213009-516dcbdf0267/go.mod h1:nFZ1EtZYK8Gi/k6QNu7z7CgO20i/4ExeQswwWuPmG/g=
+github.com/mdlayher/socket v0.1.0/go.mod h1:mYV5YIZAfHh4dzDVzI8x8tWLWCliuX8Mon5Awbj+qDs=
+github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos=
+github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ=
+github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws=
+github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
+github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
+github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw=
+github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs=
+github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
+github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
+github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
+github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=
-github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
-github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/osrg/gobgp v0.0.0-20191101114856-a42a1a5f6bf0 h1:sqWev+JAbQevLXYorfyTzf1c5VubkPi+Q83O9oBCNgA=
-github.com/osrg/gobgp v0.0.0-20191101114856-a42a1a5f6bf0/go.mod h1:IVw8wEHROhX0qrmI8c6j3N8EDXZSC4YkktSzkX/JZ8Q=
-github.com/packethost/packngo v0.2.1-0.20200424110205-36917dbc292f h1:9JqXevAco/bD2ldBRC33NfFIDRZ3Xn1n40M/LS7/vDw=
-github.com/packethost/packngo v0.2.1-0.20200424110205-36917dbc292f/go.mod h1:erURcsqYzwc9wSb04TX4so+s6F3uZtbXUil0W1LCGHA=
-github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
-github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
-github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
-github.com/pelletier/go-toml v1.0.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
-github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
+github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus=
+github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8=
+github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y=
+github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
+github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
+github.com/osrg/gobgp/v3 v3.37.0 h1:+ObuOdvj7G7nxrT0fKFta+EAupdWf/q1WzbXydr8IOY=
+github.com/osrg/gobgp/v3 v3.37.0/go.mod h1:kVHVFy1/fyZHJ8P32+ctvPeJogn9qKwa1YCeMRXXrP0=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
+github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
+github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
+github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
+github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
+github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
-github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/sirupsen/logrus v0.0.0-20170713114250-a3f95b5c4235/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
-github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v0.0.0-20170217164146-9be650865eab/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cast v1.1.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
-github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.0-20170731170427-b26b538f6930/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/cobra v0.0.6 h1:breEStsVwemnKh2/s6gMvSdMEkwW0sK8vGStnlVBMCs=
-github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
-github.com/spf13/jwalterweatherman v0.0.0-20170523133247-0efa5202c046/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
-github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.0/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.0.0/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
-github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
-github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
+github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
+github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
+github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
+github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk=
+github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0=
+github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
+github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
+github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
+github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
+github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
+github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
+github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
+github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
+github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
+github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
+github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
-github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
-github.com/vishvananda/netlink v0.0.0-20170802012344-a95659537721/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
-github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
-github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
-github.com/vishvananda/netns v0.0.0-20170707011535-86bef332bfc3/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
-github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
-github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
-github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
+github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds=
+github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
+github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 h1:pyC9PaHYZFgEKFdlp3G8RaCKgVpHZnecvArXvPXcFkM=
+github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
+github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
+github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
+github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+github.com/xlab/c-for-go v1.3.0 h1:WDc+/E59g1OcZ4dYB4K9V4Doh4NnVBMYxaZFhNi77Lg=
+github.com/xlab/c-for-go v1.3.0/go.mod h1:0qVnGIfoNg7c5gABptM38Y7nnXgAMo5KYWVQPycu63Y=
+github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 h1:Sw125DKxZhPUI4JLlWugkzsrlB50jR9v2khiD9FxuSo=
+github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245/go.mod h1:C+diUUz7pxhNY6KAoLgrTYARGWnt82zWTylZlxT92vk=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
+github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.etcd.io/etcd/api/v3 v3.6.1 h1:yJ9WlDih9HT457QPuHt/TH/XtsdN2tubyxyQHSHPsEo=
+go.etcd.io/etcd/api/v3 v3.6.1/go.mod h1:lnfuqoGsXMlZdTJlact3IB56o3bWp1DIlXPIGKRArto=
+go.etcd.io/etcd/client/pkg/v3 v3.6.1 h1:CxDVv8ggphmamrXM4Of8aCC8QHzDM4tGcVr9p2BSoGk=
+go.etcd.io/etcd/client/pkg/v3 v3.6.1/go.mod h1:aTkCp+6ixcVTZmrJGa7/Mc5nMNs59PEgBbq+HCmWyMc=
+go.etcd.io/etcd/client/v3 v3.6.1 h1:KelkcizJGsskUXlsxjVrSmINvMMga0VWwFF0tSPGEP0=
+go.etcd.io/etcd/client/v3 v3.6.1/go.mod h1:fCbPUdjWNLfx1A6ATo9syUmFVxqHH9bCnPLBZmnLmMY=
+go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
+go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
+go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
+go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 h1:bDMKF3RUSxshZ5OjOTi8rsHGaPKsAt76FaqgvIUySLc=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0/go.mod h1:dDT67G/IkA46Mr2l9Uj7HsQVwsjASyV9SjGofsiUZDA=
+go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
+go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
+go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
+go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
+go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
+go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
+go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
+go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
+go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
+go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
+go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
+go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a h1:y6sBfNd1b9Wy08a6K1Z1DZc4aXABUN5TKjkYhz7UKmo=
-golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
+golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
+golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329 h1:9kj3STMvgqy3YA4VQXBrN7925ICMxD5wzMRcgA30588=
+golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
+golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
+golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190419010253-1f3472d942ba/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
-golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
+golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
+golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
+golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
+golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9 h1:L2auWcuQIvxz9xSEqzESnV/QN/gNRXNApHi3fYwl2w0=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU=
-golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210110051926-789bb1bd4061/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210123111255-9b0068b26619/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
+golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
+golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
+golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
+golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
+golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190826182127-07722704da13/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191026034945-b2104f82a97d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
+golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
+golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20170731182057-09f6ed296fc6/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/grpc v1.5.1/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.21.0 h1:G+97AoqBnmZIT91cLG/EkCoK9NSelj64P8bOHHNmGn0=
-google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uIfPMv78iAJGcPKDeqAFnaLBropIC4=
+golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA=
+golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10 h1:3GDAcqdIg1ozBNLgPy4SLT84nfcBjr6rhGtXYtrkWLU=
+golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10/go.mod h1:T97yPqesLiNrOYxkwmhMI0ZIlJDm+p0PMR8eRVeR5tQ=
+google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY=
+google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
+google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok=
+google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc=
+google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
+google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4=
+gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.0.0-20170721122051-25c4ec802a7d/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
-gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20191022112108-ee025456fe28/go.mod h1:YZLKf07TTEX58hlaDFZRbZEdP4uwSiqhU91o1aN3EvM=
-k8s.io/api v0.18.0 h1:lwYk8Vt7rsVTwjRU6pzEsa9YNhThbmbocQlKvNBB4EQ=
-k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8=
-k8s.io/apimachinery v0.18.0 h1:fuPfYpk3cs1Okp/515pAf0dNhL66+8zk8RLbSX+EgAE=
-k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA=
-k8s.io/client-go v0.18.0 h1:yqKw4cTUQraZK3fcVCMeSa+lqKwcjZ5wtcOIPnxQno4=
-k8s.io/client-go v0.18.0/go.mod h1:uQSYDYs4WhVZ9i6AIoEZuwUggLVEF64HOD37boKAtF8=
-k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
-k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
-k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
-k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
-k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
-k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c h1:/KUFqjjqAcY4Us6luF5RDNZ16KJtb49HfR3ZHB9qYXM=
-k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E=
-k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU=
-k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
-sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
-sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E=
-sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
-sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
-sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
-sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
-sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
+gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
+honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
+honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
+k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8=
+k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE=
+k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA=
+k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM=
+k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA=
+k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg=
+k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
+k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
+k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4=
+k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8=
+k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0=
+k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+modernc.org/cc/v4 v4.24.4 h1:TFkx1s6dCkQpd6dKurBNmpo+G8Zl4Sq/ztJ+2+DEsh0=
+modernc.org/cc/v4 v4.24.4/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
+modernc.org/ccorpus2 v1.5.2 h1:Ui+4tc58mf/W+2arcYCJR903y3zl3ecsI7Fpaaqozyw=
+modernc.org/ccorpus2 v1.5.2/go.mod h1:Wifvo4Q/qS/h1aRoC2TffcHsnxwTikmi1AuLANuucJQ=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
+modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
+pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
+sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=
+sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
+sigs.k8s.io/kind v0.29.0 h1:3TpCsyh908IkXXpcSnsMjWdwdWjIl7o9IMZImZCWFnI=
+sigs.k8s.io/kind v0.29.0/go.mod h1:ldWQisw2NYyM6k64o/tkZng/1qQW7OlzcN5a8geJX3o=
+sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
+sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
+sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
+sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc=
+sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

kubernetes-control-plane.md

@@ -1,147 +0,0 @@
-# Load Balancing a Kubernetes Cluster (Control-Plane)
-
-This document covers all of the details for using `kube-vip` to build a HA Kubernetes cluster
-
-`tl;dr version`
-- Generate/modify first node `kube-vip` config/manifest
-- `init` first node
-- `join` remaining nodes
-- Add remaining config/manifests
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.75 |
-| controlPlane01 | 10.0.0.70 |
-| controlPlane02 | 10.0.0.71 |
-| controlPlane03 | 10.0.0.72 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.17.0.
-
-### Generate the `kube-vip` configuration
-
-Make sure that the config directory exists: `sudo mkdir -p /etc/kube-vip/`, this directory can be any directory however the `hostPath` in the manifest will need modifying to point to the correct path.
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1.5 sample config | sudo tee /etc/kube-vip/config.yaml
-```
-
-### Modify the configuration
-
-**Cluster Configuration**
-Modify the `remotePeers` to point to the correct addresses of the other two nodes, ensure that their `id` is unique otherwise this will confuse the raft algorithm. The `localPeer` should be the configuration of the current node (`controlPlane01`), which is where this instance of the cluster will run. 
-
-As this node will be the first node, it will need to elect itself leader as until this occurs the VIP won’t be activated!
-
-`startAsLeader: true`
-
-**VIP Config**
-We will need to set our VIP address to `192.168.0.75` and to ensure all hosts are updated when the VIP moves we will enable ARP broadcasts `gratuitousARP: true`
-
-**Load Balancer**
-We will configure the load balancer to sit on the standard API-Server port `6443` and we will configure the backends to point to the API-servers that will be configured to run on port `6444`. Also for the Kubernetes Control Plane we will configure the load balancer to be of `type: tcp`.
-
-We can also use `6443` for both the VIP and the API-Servers, in order to do this we need to specify that the api-server is bound to it's local IP. To do this we use the `--apiserver-advertise-address` flag as part of the `init`, this means that we can then bind the same port to the VIP and we wont have a port conflict.
-
-**config.yaml**
-
-`user@controlPlane01:/etc/kube-vip$ cat config.yaml`
-
-...
-
-``` 
-remotePeers:
-- id: server2
-  address: 192.168.0.71
-  port: 10000
-- id: server3
-  address: 192.168.0.72
-  port: 10000
-localPeer:
-  id: server1
-  address: 192.168.0.70
-  port: 10000
-vip: 192.168.0.75
-gratuitousARP: true
-singleNode: false
-startAsLeader: true
-interface: ens192
-loadBalancers:
-- name: Kubernetes Control Plane
-  type: tcp
-  port: 6443
-  bindToVip: true
-  backends:
-  - port: 6444
-    address: 192.168.0.70
-  - port: 6444
-    address: 192.168.0.71
-  - port: 6444
-    address: 192.168.0.72
-```
-
-### First Node
-
-To generate the basic Kubernetes static pod `yaml` configuration:
-
-Make sure that the manifest directory exists: `sudo mkdir -p /etc/kubernetes/manifests/`
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1.5 sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Ensure that `image: plndr/kube-vip:<x>` is modified to point to a specific version (`0.1.5` at the time of writing), refer to [docker hub](https://hub.docker.com/r/plndr/kube-vip/tags) for details. Also ensure that the `hostPath` points to the correct `kube-vip` configuration, if it isn’t the above path.
-
-The **vip** is set to `192.168.0.75` and this first node will elect itself as leader, and as part of the `kubeadm init` it will use the VIP in order to speak back to the initialising api-server.
-
-`sudo kubeadm init --control-plane-endpoint “192.168.0.75:6443” --apiserver-bind-port 6444 --upload-certs --kubernetes-version “v1.17.0”`
-
-Once this node is up and running we will be able to see the control-plane pods, including the `kube-vip` pod:
-
-```
-$ kubectl get pods -A
-NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
-<...>
-kube-system   kube-vip-controlplane01                  1/1     Running   0          10m
-```
-
-### Remaining Nodes
-
-We first will need to create the `kube-vip` configuration that resides in `/etc/kube-vip/config.yaml` or we can regenerate it from scratch using the above example. Ensure that the configuration is almost identical with the `localPeer` and `remotePeers` sections are updated for each node. Finally, ensure that the remaining nodes will behave as standard cluster nodes by setting `startAsLeader: false`.
-
-At this point **DON’T** generate the manifests, this is due to some bizarre `kubeadm/kubelet` behaviour.
-
-```
-  kubeadm join 192.168.0.75:6443 --token <tkn> \
-    --discovery-token-ca-cert-hash sha256:<hash> \
-    --control-plane --certificate-key <key> 
-
-```
-
-**After** this node has been added to the cluster, we can add the manifest to also add this node as a `kube-vip` member. (Adding the manifest afterwards doesn’t interfere with `kubeadm`). 
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1.5 sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Once this node is added we will be able to see that the `kube-vip` pod is up and running as expected:
-
-```
-user@controlPlane01:~$ kubectl get pods -A | grep vip
-kube-system   kube-vip-controlplane01                  1/1     Running             1          16m
-kube-system   kube-vip-controlplane02                  1/1     Running             0          18m
-kube-system   kube-vip-controlplane03                  1/1     Running             0          20m
-
-```
-
-If we look at the logs, we can see that the VIP is running on the second node and we’re waiting for our third node to join the cluster:
-
-```
-$ kubectl logs kube-vip-controlplane02  -n kube-system
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-
-```

main.go

@@ -1,6 +1,8 @@
 package main
 
-import "github.com/plunder-app/kube-vip/cmd"
+import (
+	"github.com/kube-vip/kube-vip/cmd"
+)
 
 // Version is populated from the Makefile and is tied to the release TAG
 var Version string
@@ -9,6 +11,7 @@ var Version string
 var Build string
 
 func main() {
+
 	cmd.Release.Version = Version
 	cmd.Release.Build = Build
 	cmd.Execute()

pkg/arp/arp.go

@@ -0,0 +1,149 @@
+package arp
+
+import (
+	"context"
+	log "log/slog"
+	"sync"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
+
+type Manager struct {
+	instances map[string]*Instance
+	config    *kubevip.Config
+}
+
+type Instance struct {
+	network vip.Network
+	ndp     *vip.NdpResponder
+	mu      sync.Mutex
+	counter int
+}
+
+func NewManager(config *kubevip.Config) *Manager {
+	return &Manager{
+		instances: make(map[string]*Instance),
+		config:    config,
+	}
+}
+
+func NewInstance(network vip.Network, ndp *vip.NdpResponder) *Instance {
+	return &Instance{
+		ndp:     ndp,
+		network: network,
+		counter: 1,
+	}
+}
+
+func (i *Instance) Name() string {
+	return i.network.ARPName()
+}
+
+func (m *Manager) Insert(instance *Instance) {
+	i, ok := m.instances[instance.Name()]
+	if !ok {
+		log.Info("inserting ARP/NDP instance", "name", instance.Name())
+		m.instances[instance.Name()] = instance
+	} else {
+		i.mu.Lock()
+		defer i.mu.Unlock()
+		i.counter++
+	}
+}
+
+func (m *Manager) Remove(instance *Instance) {
+	if i, ok := m.instances[instance.Name()]; ok {
+		i.mu.Lock()
+		defer i.mu.Unlock()
+		if i.counter > 1 {
+			i.counter--
+		} else {
+			log.Info("removing ARP/NDP instance", "name", instance.Name())
+			delete(m.instances, instance.Name())
+		}
+	}
+}
+
+func (m *Manager) Count(name string) int {
+	if i, ok := m.instances[name]; ok {
+		i.mu.Lock()
+		defer i.mu.Unlock()
+		return i.counter
+	}
+	return 0
+}
+
+func (m *Manager) StartAdvertisement(ctx context.Context) {
+	log.Info("Starting ARP/NDP advertisement")
+	for {
+		select {
+		case <-ctx.Done(): // if cancel() execute
+			return
+		default:
+			for _, instance := range m.instances {
+				if instance.counter > 0 {
+					ensureIPAndSendGratuitous(instance)
+				}
+			}
+		}
+		if m.config.ArpBroadcastRate < 500 {
+			log.Error("arp broadcast rate is too low", "rate (ms)", m.config.ArpBroadcastRate, "setting to (ms)", "3000")
+			m.config.ArpBroadcastRate = 3000
+		}
+		time.Sleep(time.Duration(m.config.ArpBroadcastRate) * time.Millisecond)
+	}
+}
+
+// ensureIPAndSendGratuitous - adds IP to the interface if missing, and send
+// either a gratuitous ARP or gratuitous NDP. Re-adds the interface if it is IPv6
+// and in a dadfailed state.
+func ensureIPAndSendGratuitous(instance *Instance) {
+	iface := instance.network.Interface()
+	ipString := instance.network.IP()
+
+	// Check if IP is dadfailed
+	if instance.network.IsDADFAIL() {
+		log.Warn("IP address is in dadfailed state, removing config", "ip", ipString, "interface", iface)
+		deleted, err := instance.network.DeleteIP()
+		if err != nil {
+			log.Warn(err.Error())
+		}
+		if deleted {
+			log.Info("deleted and recreating address", "IP", ipString, "interface", iface)
+			// if _, err := instance.network.AddIP(false); err != nil {
+			// 	log.Error("failed to recreate address", "IP", ipString, "interface", iface)
+			// }
+		}
+	}
+
+	// Ensure the address exists on the interface before attempting to ARP
+	// if instance.network.HasEndpoints() {
+	if added, err := instance.network.AddIP(true); err != nil {
+		log.Warn(err.Error())
+	} else if added {
+		log.Warn("Re-applied the VIP configuration", "ip", ipString, "interface", iface)
+	}
+	// }
+
+	if vip.IsIPv6(ipString) {
+		// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
+		if instance.ndp == nil {
+			log.Error("NDP responder was not created")
+		} else {
+			err := instance.ndp.SendGratuitous(ipString)
+			if err != nil {
+				log.Warn(err.Error())
+			}
+		}
+
+	} else {
+		// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
+		err := vip.ARPSendGratuitous(ipString, iface)
+		if err != nil {
+			log.Warn(err.Error())
+		}
+	}
+
+}

pkg/backend/backend.go

@@ -0,0 +1,88 @@
+package backend
+
+import (
+	"fmt"
+	"time"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/k8s"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+type Entry struct {
+	Addr    string
+	Port    uint16
+	IsLocal bool
+}
+
+type Map map[Entry]bool
+
+func (e *Entry) Check() bool {
+	var client *kubernetes.Clientset
+	var err error
+	var config *rest.Config
+
+	adminConfigPath := "/etc/kubernetes/admin.conf"
+	// TODO: add one more switch case of homeConfigPath if there is such scenario in future
+	// homeConfigPath := filepath.Join(os.Getenv("HOME"), ".kube", "config")
+
+	var k8sAddr string
+	if vip.IsIPv4(e.Addr) {
+		k8sAddr = fmt.Sprintf("%s:%v", e.Addr, e.Port)
+	} else {
+		k8sAddr = fmt.Sprintf("[%s]:%v", e.Addr, e.Port)
+	}
+
+	switch {
+	case utils.FileExists(adminConfigPath):
+		config, err = k8s.NewRestConfig(adminConfigPath, false, k8sAddr)
+		if err != nil {
+			log.Error("create k8s REST config", "path", adminConfigPath, "err", err)
+			return false
+		}
+	default:
+		config, err = k8s.NewRestConfig("", true, k8sAddr)
+		if err != nil {
+			log.Error("create k8s REST config", "err", err)
+			return false
+		}
+	}
+
+	client, err = k8s.NewClientset(config)
+	if err != nil {
+		log.Error("create k8s client", "err", err)
+		return false
+	}
+
+	_, err = client.DiscoveryClient.ServerVersion()
+	if err != nil {
+		log.Error("discover k8s version", "err", err)
+		return false
+	}
+	return true
+}
+
+func Watch(tickAction func(), interval int, stop chan struct{}) {
+	if interval <= 0 {
+		interval = 5
+	}
+
+	ticker := time.NewTicker(time.Second * time.Duration(interval))
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-stop:
+			ticker.Stop()
+			return
+		case <-ticker.C:
+			ticker.Stop()
+			tickAction()
+			ticker.Reset(time.Second * time.Duration(interval))
+		}
+	}
+}

pkg/bgp/hosts.go

@@ -2,9 +2,10 @@ package bgp
 
 import (
 	"context"
+	"fmt"
 	"net"
 
-	api "github.com/osrg/gobgp/api"
+	api "github.com/osrg/gobgp/v3/api"
 )
 
 // AddHost will update peers of a host
@@ -13,15 +14,20 @@ func (b *Server) AddHost(addr string) (err error) {
 	if err != nil {
 		return err
 	}
+
 	p := b.getPath(ip)
 	if p == nil {
-		return
+		return fmt.Errorf("failed to get path for %v", ip)
 	}
 
 	_, err = b.s.AddPath(context.Background(), &api.AddPathRequest{
 		Path: p,
 	})
 
+	if err != nil {
+		return err
+	}
+
 	return
 }
 

pkg/bgp/peers.go

@@ -7,93 +7,335 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/golang/protobuf/ptypes"
-	"github.com/golang/protobuf/ptypes/any"
-	api "github.com/osrg/gobgp/api"
+	//nolint
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	api "github.com/osrg/gobgp/v3/api"
+	"github.com/osrg/gobgp/v3/pkg/server"
+	"google.golang.org/protobuf/types/known/anypb"
 )
 
-//AddPeer will add peers to the BGP configuration
+// AddPeer will add peers to the BGP configuration
 func (b *Server) AddPeer(peer Peer) (err error) {
-	port := 179
-
-	if t := strings.SplitN(peer.Address, ":", 2); len(t) == 2 {
-		peer.Address = t[0]
-
-		if port, err = strconv.Atoi(t[1]); err != nil {
-			return fmt.Errorf("Unable to parse port '%s' as int: %s", t[1], err)
-		}
-	}
-
 	p := &api.Peer{
 		Conf: &api.PeerConf{
 			NeighborAddress: peer.Address,
-			PeerAs:          peer.AS,
+			PeerAsn:         peer.AS,
+			AuthPassword:    peer.Password,
 		},
 
 		Timers: &api.Timers{
 			Config: &api.TimersConfig{
-				ConnectRetry: 10,
+				ConnectRetry:      10,
+				HoldTime:          b.c.HoldTime,
+				KeepaliveInterval: b.c.KeepaliveInterval,
 			},
 		},
 
+		// This enables routes to be sent to routers across multiple hops
+		EbgpMultihop: &api.EbgpMultihop{
+			Enabled:     peer.MultiHop,
+			MultihopTtl: 50,
+		},
+
 		Transport: &api.Transport{
 			MtuDiscovery:  true,
 			RemoteAddress: peer.Address,
-			RemotePort:    uint32(port),
+			RemotePort:    uint32(179),
 		},
 	}
 
-	// if b.c.SourceIP != "" {
-	// 	p.Transport.LocalAddress = b.c.SourceIP
-	// }
-
-	// if b.c.SourceIF != "" {
-	// 	p.Transport.BindInterface = b.c.SourceIF
-	// }
-
-	return b.s.AddPeer(context.Background(), &api.AddPeerRequest{
-		Peer: p,
-	})
-}
-
-func (b *Server) getPath(ip net.IP) *api.Path {
-	var pfxLen uint32 = 32
-	if ip.To4() == nil {
-		if !b.c.IPv6 {
-			return nil
+	if b.c.MpbgpNexthop != "" {
+		p.AfiSafis = []*api.AfiSafi{
+			{
+				Config: &api.AfiSafiConfig{
+					Family: &api.Family{
+						Afi:  api.Family_AFI_IP,
+						Safi: api.Family_SAFI_UNICAST,
+					},
+					Enabled: true,
+				},
+			},
+			{
+				Config: &api.AfiSafiConfig{
+					Family: &api.Family{
+						Afi:  api.Family_AFI_IP6,
+						Safi: api.Family_SAFI_UNICAST,
+					},
+					Enabled: true,
+				},
+			},
 		}
 
-		pfxLen = 128
+		peer.setMpbgpOptions(b.c)
+
+		ipv4Address, ipv6Address, err := peer.findMpbgpAddresses(p, b.c)
+		if err != nil {
+			return fmt.Errorf("failed to get MP-BGP addresses: %w", err)
+		}
+
+		mask := "128"
+		address := ipv4Address
+		family := api.Family_AFI_IP
+		if vip.IsIPv4(p.Conf.NeighborAddress) {
+			mask = "32"
+			address = ipv6Address
+			family = api.Family_AFI_IP6
+		}
+
+		err = b.s.AddDefinedSet(context.Background(), &api.AddDefinedSetRequest{
+			DefinedSet: &api.DefinedSet{
+				DefinedType: api.DefinedType_NEIGHBOR,
+				Name:        fmt.Sprintf("peer-%s", p.Conf.NeighborAddress),
+				List:        []string{fmt.Sprintf("%s/%s", p.Conf.NeighborAddress, mask)},
+			},
+		})
+		if err != nil {
+			return fmt.Errorf("failed to add defined set: %v", err)
+		}
+
+		if address != "" {
+			if err := insertPolicy(b.s, address, p, family); err != nil {
+				return fmt.Errorf("failed to add policy: %w", err)
+			}
+		}
+	} else {
+		if b.c.SourceIP != "" {
+			p.Transport.LocalAddress = b.c.SourceIP
+		}
+
+		if b.c.SourceIF != "" {
+			p.Transport.BindInterface = b.c.SourceIF
+		}
 	}
 
-	nlri, _ := ptypes.MarshalAny(&api.IPAddressPrefix{
-		Prefix:    ip.String(),
-		PrefixLen: pfxLen,
-	})
+	if err := b.s.AddPeer(context.Background(), &api.AddPeerRequest{Peer: p}); err != nil {
+		return fmt.Errorf("failed to add peer: %v", err)
+	}
 
-	a1, _ := ptypes.MarshalAny(&api.OriginAttribute{
+	return nil
+}
+
+func (b *Server) getPath(ip net.IP) (path *api.Path) {
+	isV6 := ip.To4() == nil
+
+	//nolint
+	originAttr, _ := anypb.New(&api.OriginAttribute{
 		Origin: 0,
 	})
 
-	var nh string
-	if b.c.NextHop != "" {
-		nh = b.c.NextHop
-	} else if b.c.SourceIP != "" {
-		nh = b.c.SourceIP
+	if !isV6 {
+		//nolint
+		nlri, _ := anypb.New(&api.IPAddressPrefix{
+			Prefix:    ip.String(),
+			PrefixLen: 32,
+		})
+
+		//nolint
+		nhAttr, _ := anypb.New(&api.NextHopAttribute{
+			NextHop: "0.0.0.0", // gobgp will fill this
+		})
+
+		path = &api.Path{
+			Family: &api.Family{
+				Afi:  api.Family_AFI_IP,
+				Safi: api.Family_SAFI_UNICAST,
+			},
+			Nlri:   nlri,
+			Pattrs: []*anypb.Any{originAttr, nhAttr},
+		}
 	} else {
-		nh = b.c.RouterID
-	}
+		//nolint
+		nlri, _ := anypb.New(&api.IPAddressPrefix{
+			Prefix:    ip.String(),
+			PrefixLen: 128,
+		})
 
-	a2, _ := ptypes.MarshalAny(&api.NextHopAttribute{
-		NextHop: nh,
-	})
-
-	return &api.Path{
-		Family: &api.Family{
-			Afi:  api.Family_AFI_IP,
+		v6Family := &api.Family{
+			Afi:  api.Family_AFI_IP6,
 			Safi: api.Family_SAFI_UNICAST,
-		},
-		Nlri:   nlri,
-		Pattrs: []*any.Any{a1, a2},
+		}
+
+		//nolint
+		mpAttr, _ := anypb.New(&api.MpReachNLRIAttribute{
+			Family:   v6Family,
+			NextHops: []string{"::"}, // gobgp will fill this
+			Nlris:    []*anypb.Any{nlri},
+		})
+
+		path = &api.Path{
+			Family: v6Family,
+			Nlri:   nlri,
+			Pattrs: []*anypb.Any{originAttr, mpAttr},
+		}
 	}
+	return
+}
+
+// ParseBGPPeerConfig - take a string and parses it into an array of peers
+func ParseBGPPeerConfig(config string) (bgpPeers []Peer, err error) {
+	peers := strings.Split(config, ",")
+	if len(peers) == 0 {
+		return nil, fmt.Errorf("no BGP Peer configurations found")
+	}
+
+	for x := range peers {
+		peerStr := peers[x]
+		config := strings.Split(peerStr, "/")
+		peerStr = config[0]
+		if peerStr == "" {
+			continue
+		}
+		isV6Peer := peerStr[0] == '['
+
+		address := ""
+		if isV6Peer {
+			addressEndPos := strings.IndexByte(peerStr, ']')
+			if addressEndPos == -1 {
+				return nil, fmt.Errorf("no matching ] found for IPv6 BGP Peer")
+			}
+			address = peerStr[1:addressEndPos]
+			peerStr = peerStr[addressEndPos+1:]
+		}
+
+		peer := strings.Split(peerStr, ":")
+		if len(peer) < 2 {
+			return nil, fmt.Errorf("mandatory peering params <host>:<AS> incomplete")
+		}
+
+		if !isV6Peer {
+			address = peer[0]
+		}
+
+		ASNumber, err := strconv.ParseUint(peer[1], 10, 32)
+		if err != nil {
+			return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
+		}
+
+		password := ""
+		if len(peer) >= 3 {
+			password = peer[2]
+		}
+
+		multiHop := false
+		if len(peer) >= 4 {
+			multiHop, err = strconv.ParseBool(peer[3])
+			if err != nil {
+				return nil, fmt.Errorf("BGP MultiHop format error (true/false) [%s]", peer[1])
+			}
+		}
+
+		var port uint64
+		if len(peer) >= 5 {
+			port, err = strconv.ParseUint(peer[4], 10, 16)
+			if err != nil {
+				return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
+			}
+		} else {
+			port = 179
+		}
+
+		var mpbgpNexthop, mpbgpIPv4, mpbgpIPv6 string
+
+		if len(config) > 1 {
+			configData := strings.Split(config[1], ";")
+			for _, cfg := range configData {
+				c := strings.Split(cfg, "=")
+				switch c[0] {
+				case "mpbgp_nexthop":
+					mpbgpNexthop = c[1]
+				case "mpbgp_ipv4":
+					mpbgpIPv4 = c[1]
+				case "mpbgp_ipv6":
+					mpbgpIPv6 = c[1]
+				default:
+					return nil, fmt.Errorf("peer configuration parameter '%s' is not supported", c[0])
+				}
+			}
+		}
+
+		peerConfig := Peer{
+			Address:      address,
+			AS:           uint32(ASNumber),
+			Port:         uint16(port),
+			Password:     password,
+			MultiHop:     multiHop,
+			MpbgpNexthop: mpbgpNexthop,
+			MpbgpIPv4:    mpbgpIPv4,
+			MpbgpIPv6:    mpbgpIPv6,
+		}
+
+		bgpPeers = append(bgpPeers, peerConfig)
+	}
+	return
+}
+
+func insertPolicy(s *server.BgpServer, address string, p *api.Peer, family api.Family_Afi) error {
+	familyType := "v4"
+	if family == api.Family_AFI_IP6 {
+		familyType = "v6"
+	}
+
+	setName := fmt.Sprintf("peer-%s", p.Conf.NeighborAddress)
+	policyName := fmt.Sprintf("%s-%s", setName, familyType)
+
+	policy := &api.Policy{
+		Name: policyName,
+		Statements: []*api.Statement{
+			{
+				Conditions: &api.Conditions{
+					AfiSafiIn: []*api.Family{
+						{
+							Afi:  family,
+							Safi: api.Family_SAFI_UNICAST,
+						},
+					},
+					NeighborSet: &api.MatchSet{
+						Type: api.MatchSet_ANY,
+						Name: setName,
+					},
+				},
+				Actions: &api.Actions{
+					RouteAction: api.RouteAction_ACCEPT,
+					Nexthop: &api.NexthopAction{
+						Address: address,
+					},
+				},
+			},
+			{
+				Conditions: &api.Conditions{
+					NeighborSet: &api.MatchSet{
+						Type: api.MatchSet_ANY,
+						Name: setName,
+					},
+				},
+				Actions: &api.Actions{
+					RouteAction: api.RouteAction_ACCEPT,
+				},
+			},
+		},
+	}
+
+	err := s.AddPolicy(context.Background(), &api.AddPolicyRequest{
+		Policy: policy,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to add policy: %w", err)
+	}
+
+	err = s.AddPolicyAssignment(context.Background(), &api.AddPolicyAssignmentRequest{
+		Assignment: &api.PolicyAssignment{
+			Name:      "global",
+			Direction: api.PolicyDirection_EXPORT,
+			Policies: []*api.Policy{
+				{
+					Name: policy.Name,
+				},
+			},
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("failed to add policy assignment: %v", err)
+	}
+
+	return nil
 }

pkg/bgp/peers_test.go

@@ -0,0 +1,54 @@
+package bgp
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseBGPPeerConfig(t *testing.T) {
+	type args struct {
+		config string
+	}
+	tests := []struct {
+		name         string
+		args         args
+		wantBgpPeers []Peer
+		wantErr      bool
+	}{
+		{
+			name: "IPv4, default port",
+			args: args{config: "192.168.0.10:65000::false,192.168.0.11:65000::false"},
+			wantBgpPeers: []Peer{
+				{Address: "192.168.0.10", Port: 179, AS: 65000, MultiHop: false},
+				{Address: "192.168.0.11", Port: 179, AS: 65000, MultiHop: false},
+			},
+		},
+		{
+			name: "IPv4, different port",
+			args: args{config: "192.168.0.10:65000::false:180,192.168.0.11:65000::false:190"},
+			wantBgpPeers: []Peer{
+				{Address: "192.168.0.10", Port: 180, AS: 65000, MultiHop: false},
+				{Address: "192.168.0.11", Port: 190, AS: 65000, MultiHop: false},
+			},
+		},
+		{
+			name: "IPv6, multi-protocol",
+			args: args{config: "[fd00:1111:2222:3333:c7d9:7235:6bf7:5d52]:65501::false/mpbgp_nexthop=auto_sourceif"},
+			wantBgpPeers: []Peer{
+				{Address: "fd00:1111:2222:3333:c7d9:7235:6bf7:5d52", Port: 179, AS: 65501, MultiHop: false, MpbgpNexthop: "auto_sourceif"},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotBgpPeers, err := ParseBGPPeerConfig(tt.args.config)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseBGPPeerConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotBgpPeers, tt.wantBgpPeers) {
+				t.Errorf("ParseBGPPeerConfig() = %v, want %v", gotBgpPeers, tt.wantBgpPeers)
+			}
+		})
+	}
+}

pkg/bgp/server.go

@@ -3,53 +3,85 @@ package bgp
 import (
 	"context"
 	"fmt"
-	"log"
 	"time"
 
-	api "github.com/osrg/gobgp/api"
-	gobgp "github.com/osrg/gobgp/pkg/server"
+	log "log/slog"
+
+	api "github.com/osrg/gobgp/v3/api"
+	gobgp "github.com/osrg/gobgp/v3/pkg/server"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // NewBGPServer takes a configuration and returns a running BGP server instance
 func NewBGPServer(c *Config) (b *Server, err error) {
 	if c.AS == 0 {
-		return nil, fmt.Errorf("You need to provide AS")
+		return nil, fmt.Errorf("you need to provide AS")
 	}
 
-	// if c.SourceIP != "" && c.SourceIF != "" {
-	// 	return nil, fmt.Errorf("SourceIP and SourceIF are mutually exclusive")
-	// }
+	if c.SourceIP != "" && c.SourceIF != "" {
+		return nil, fmt.Errorf("sourceIP and SourceIF are mutually exclusive")
+	}
 
 	if len(c.Peers) == 0 {
-		return nil, fmt.Errorf("You need to provide at least one peer")
+		return nil, fmt.Errorf("you need to provide at least one peer")
 	}
 
 	b = &Server{
 		s: gobgp.NewBgpServer(),
 		c: c,
+
+		BGPSessionInfoGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "kube_vip",
+			Subsystem: "manager",
+			Name:      "bgp_session_info",
+			Help:      "Display state of session by setting metric for label value with current state to 1",
+		}, []string{"state", "peer"}),
 	}
+	return
+}
+
+// Start starts the BGP server
+func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (err error) {
 	go b.s.Serve()
 
 	if err = b.s.StartBgp(context.Background(), &api.StartBgpRequest{
 		Global: &api.Global{
-			As:         c.AS,
-			RouterId:   c.RouterID,
+			Asn:        b.c.AS,
+			RouterId:   b.c.RouterID,
 			ListenPort: -1,
 		},
 	}); err != nil {
 		return
 	}
 
-	if err = b.s.MonitorPeer(context.Background(), &api.MonitorPeerRequest{}, func(p *api.Peer) { log.Println(p) }); err != nil {
+	if err = b.s.WatchEvent(context.Background(), &api.WatchEventRequest{Peer: &api.WatchEventRequest_Peer{}}, func(r *api.WatchEventResponse) {
+		if p := r.GetPeer(); p != nil && p.Type == api.WatchEventResponse_PeerEvent_STATE {
+			log.Info("[BGP]", "peer", p.String())
+			if peerStateChangeCallback != nil {
+				peerStateChangeCallback(p)
+			}
+		}
+	}); err != nil {
 		return
 	}
 
-	for _, p := range c.Peers {
+	for _, p := range b.c.Peers {
 		if err = b.AddPeer(p); err != nil {
 			return
 		}
 	}
 
+	if b.c.Zebra.Enabled {
+		if err = b.s.EnableZebra(context.Background(), &api.EnableZebraRequest{
+			Url:          b.c.Zebra.URL,
+			Version:      b.c.Zebra.Version,
+			SoftwareName: b.c.Zebra.SoftwareName,
+		}); err != nil {
+			log.Error(err.Error())
+			return
+		}
+	}
+
 	return
 }
 

pkg/bgp/types.go

@@ -1,27 +1,145 @@
 package bgp
 
-import gobgp "github.com/osrg/gobgp/pkg/server"
+import (
+	"fmt"
+	"net"
+
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	api "github.com/osrg/gobgp/v3/api"
+	gobgp "github.com/osrg/gobgp/v3/pkg/server"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/vishvananda/netlink"
+)
 
 // Peer defines a BGP Peer
 type Peer struct {
-	Address string
-	AS      uint32
+	Address      string
+	Port         uint16
+	AS           uint32
+	Password     string
+	MultiHop     bool
+	MpbgpNexthop string
+	MpbgpIPv4    string
+	MpbgpIPv6    string
+}
+
+func (p *Peer) setMpbgpOptions(server *Config) {
+	if p.MpbgpNexthop == "" {
+		p.MpbgpNexthop = server.MpbgpNexthop
+	}
+
+	if p.MpbgpIPv4 == "" {
+		p.MpbgpIPv4 = server.MpbgpIPv4
+	}
+
+	if p.MpbgpIPv6 == "" {
+		p.MpbgpIPv6 = server.MpbgpIPv6
+	}
+}
+
+func (p *Peer) findMpbgpAddresses(ap *api.Peer, server *Config) (string, string, error) {
+	var ipv4Address, ipv6Address string
+	switch p.MpbgpNexthop {
+	case "fixed":
+		ap.Transport.LocalAddress = server.SourceIP
+		if p.MpbgpIPv4 == "" && p.MpbgpIPv6 == "" {
+			return "", "", fmt.Errorf("to use MP-BGP with fixed address at least one IPv4 or IPv6 address has to be provided [current - IPv4: %s, IPv6: %s]",
+				p.MpbgpIPv4, p.MpbgpIPv6)
+		}
+
+		if p.MpbgpIPv4 != "" {
+			if net.ParseIP(p.MpbgpIPv4) == nil {
+				return "", "", fmt.Errorf("provided address '%s' is not a valid IPv4 address", p.MpbgpIPv4)
+			}
+		}
+		if p.MpbgpIPv6 != "" {
+			if net.ParseIP(p.MpbgpIPv6) == nil {
+				return "", "", fmt.Errorf("provided address '%s' is not a valid IPv6 address", p.MpbgpIPv6)
+			}
+		}
+
+		ipv4Address = p.MpbgpIPv4
+		ipv6Address = p.MpbgpIPv6
+	case "auto_sourceip":
+		ap.Transport.LocalAddress = server.SourceIP
+
+		// Resolve the local interface by SourceIP
+		iface, err := vip.GetInterfaceByIP(server.SourceIP)
+		if err != nil {
+			return "", "", fmt.Errorf("failed to get interface by IP: %v", err)
+		}
+
+		if vip.IsIPv4(server.SourceIP) {
+			// Get the non link-local IPv6 address on that interface
+			ipv6Address, err = vip.GetNonLinkLocalIP(iface, netlink.FAMILY_V6)
+			if err != nil {
+				return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
+			}
+		} else {
+			// Get the non link-local IPv4 address on that interface
+			ipv4Address, err = vip.GetNonLinkLocalIP(iface, netlink.FAMILY_V4)
+			if err != nil {
+				return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
+			}
+		}
+	case "auto_sourceif":
+		ap.Transport.BindInterface = server.SourceIF
+
+		iface, err := netlink.LinkByName(server.SourceIF)
+		if err != nil {
+			return "", "", fmt.Errorf("failed to get interface by name: %v", err)
+		}
+
+		// Get the non link-local IPv4 address on that interface
+		ipv4Address, err = vip.GetNonLinkLocalIP(&iface, netlink.FAMILY_V4)
+		if err != nil {
+			return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
+		}
+
+		// Get the non link-local IPv6 address on that interface
+		ipv6Address, err = vip.GetNonLinkLocalIP(&iface, netlink.FAMILY_V6)
+		if err != nil {
+			return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
+		}
+	default:
+		return "", "", fmt.Errorf("option %s for MP-BPG nexthop is not supported", server.MpbgpNexthop)
+	}
+
+	return ipv4Address, ipv6Address, nil
 }
 
 // Config defines the BGP server configuration
 type Config struct {
-	AS       uint32
-	RouterID string
-	NextHop  string
-	SourceIP string
-	SourceIF string
+	AS           uint32
+	RouterID     string
+	SourceIP     string
+	SourceIF     string
+	MpbgpNexthop string
+	MpbgpIPv4    string
+	MpbgpIPv6    string
+
+	HoldTime          uint64
+	KeepaliveInterval uint64
 
 	Peers []Peer
-	IPv6  bool
+
+	Zebra ZebraConfig
+}
+
+// Defines Zebra connection configuration. More on the topic - https://github.com/osrg/gobgp/blob/master/docs/sources/zebra.md#configuration
+type ZebraConfig struct {
+	Enabled      bool
+	URL          string
+	Version      uint32
+	SoftwareName string
 }
 
 // Server manages a server object
 type Server struct {
 	s *gobgp.BgpServer
 	c *Config
+
+	// This is a prometheus gauge indicating the state of the sessions.
+	// 1 means "ESTABLISHED", 0 means "NOT ESTABLISHED"
+	BGPSessionInfoGauge *prometheus.GaugeVec
 }

pkg/cluster/cluster.go

@@ -1,49 +1,80 @@
 package cluster
 
 import (
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	"github.com/plunder-app/kube-vip/pkg/vip"
-)
+	"sync"
 
-const leaderLogcount = 5
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/networkinterface"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
 
 // Cluster - The Cluster object manages the state of the cluster for a particular node
 type Cluster struct {
-	stateMachine FSM
-	stop         chan bool
-	completed    chan bool
-	network      *vip.Network
+	stop      chan bool
+	completed chan bool
+	once      sync.Once
+	Network   []vip.Network
+	arpMgr    *arp.Manager
 }
 
 // InitCluster - Will attempt to initialise all of the required settings for the cluster
-func InitCluster(c *kubevip.Config, disableVIP bool) (*Cluster, error) {
-
-	// TODO - Check for root (needed to netlink)
-	var network *vip.Network
+func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Cluster, error) {
+	var networks []vip.Network
 	var err error
 
 	if !disableVIP {
 		// Start the Virtual IP Networking configuration
-		network, err = startNetworking(c)
+		networks, err = startNetworking(c, intfMgr)
 		if err != nil {
 			return nil, err
 		}
 	}
 	// Initialise the Cluster structure
 	newCluster := &Cluster{
-		network: network,
+		Network: networks,
+		arpMgr:  arpMgr,
 	}
 
+	log.Debug("service security", "enabled", c.EnableServiceSecurity)
+
 	return newCluster, nil
 }
 
-func startNetworking(c *kubevip.Config) (*vip.Network, error) {
-	network, err := vip.NewConfig(c.VIP, c.Interface)
-	if err != nil {
-		// log.WithFields(log.Fields{"error": err}).Error("Network failure")
+func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vip.Network, error) {
+	address := c.VIP
 
-		// os.Exit(-1)
-		return nil, err
+	if c.Address != "" {
+		address = c.Address
 	}
-	return &network, nil
+
+	addresses := vip.Split(address)
+
+	networks := []vip.Network{}
+	for _, addr := range addresses {
+		network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.RoutingTableID,
+			c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend,
+			c.EnableLoadBalancer, c.EnableServiceSecurity, intfMgr)
+		if err != nil {
+			return nil, err
+		}
+		networks = append(networks, network...)
+	}
+
+	return networks, nil
+}
+
+// Stop - Will stop the Cluster and release VIP if needed
+func (cluster *Cluster) Stop() {
+	// Close the stop channel, which will shut down the VIP (if needed)
+	if cluster.stop != nil {
+		cluster.once.Do(func() { // Ensure that the close channel can only ever be called once
+			close(cluster.stop)
+		})
+	}
+
+	// Wait until the completed channel is closed, signallign all shutdown tasks completed
+	<-cluster.completed
 }

pkg/cluster/clusterDDNS.go

@@ -0,0 +1,28 @@
+package cluster
+
+import (
+	"context"
+
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
+
+// StartDDNS should start go routine for dhclient to hold the lease for the IP
+// StartDDNS should wait until IP is allocated from DHCP, set it to cluster.Network
+// so the OnStartedLeading can continue to configure the VIP initially
+// during runtime if IP changes, startDDNS don't have to do reconfigure because
+// dnsUpdater already have the functionality to keep trying resolve the IP
+// and update the VIP configuration if it changes
+func (cluster *Cluster) StartDDNS(ctx context.Context) error {
+	for i := range cluster.Network {
+		ddnsMgr := vip.NewDDNSManager(ctx, cluster.Network[i])
+		ip, err := ddnsMgr.Start()
+		if err != nil {
+			return err
+		}
+		if err = cluster.Network[i].SetIP(ip); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/cluster/clusterLeader.go

@@ -1,329 +0,0 @@
-package cluster
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/signal"
-	"path/filepath"
-	"syscall"
-	"time"
-
-	"github.com/plunder-app/kube-vip/pkg/bgp"
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	leaderelection "github.com/plunder-app/kube-vip/pkg/leaderElection"
-	"github.com/plunder-app/kube-vip/pkg/loadbalancer"
-	"github.com/plunder-app/kube-vip/pkg/packet"
-
-	"github.com/plunder-app/kube-vip/pkg/vip"
-
-	"github.com/packethost/packngo"
-
-	log "github.com/sirupsen/logrus"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/tools/leaderelection/resourcelock"
-)
-
-const plunderLock = "plunder-lock"
-const namespace = "kube-system"
-
-// Manager degines the manager of the load-balancing services
-type Manager struct {
-	clientSet *kubernetes.Clientset
-}
-
-// NewManager will create a new managing object
-func NewManager(path string, inCluster bool) (*Manager, error) {
-	var clientset *kubernetes.Clientset
-	if inCluster {
-		// This will attempt to load the configuration when running within a POD
-		cfg, err := rest.InClusterConfig()
-		if err != nil {
-			return nil, fmt.Errorf("error creating kubernetes client config: %s", err.Error())
-		}
-		clientset, err = kubernetes.NewForConfig(cfg)
-
-		if err != nil {
-			return nil, fmt.Errorf("error creating kubernetes client: %s", err.Error())
-		}
-		// use the current context in kubeconfig
-	} else {
-		if path == "" {
-			path = filepath.Join(os.Getenv("HOME"), ".kube", "config")
-		}
-		config, err := clientcmd.BuildConfigFromFlags("", path)
-		if err != nil {
-			panic(err.Error())
-		}
-
-		// We modify the config so that we can always speak to the correct host
-		id, err := os.Hostname()
-		if err != nil {
-			return nil, err
-		}
-
-		// TODO - we need to make the host/port configurable
-
-		config.Host = fmt.Sprintf("%s:6443", id)
-		clientset, err = kubernetes.NewForConfig(config)
-
-		if err != nil {
-			return nil, fmt.Errorf("error creating kubernetes client: %s", err.Error())
-		}
-	}
-
-	return &Manager{
-		clientSet: clientset,
-	}, nil
-}
-
-// StartLeaderCluster - Begins a running instance of the Raft cluster
-func (cluster *Cluster) StartLeaderCluster(c *kubevip.Config, sm *Manager) error {
-
-	id, err := os.Hostname()
-	if err != nil {
-		return err
-	}
-
-	log.Infof("Beginning cluster membership, namespace [%s], lock name [%s], id [%s]", namespace, plunderLock, id)
-
-	// we use the Lease lock type since edits to Leases are less common
-	// and fewer objects in the cluster watch "all Leases".
-	lock := &resourcelock.LeaseLock{
-		LeaseMeta: metav1.ObjectMeta{
-			Name:      plunderLock,
-			Namespace: namespace,
-		},
-		Client: sm.clientSet.CoordinationV1(),
-		LockConfig: resourcelock.ResourceLockConfig{
-			Identity: id,
-		},
-	}
-
-	// use a Go context so we can tell the leaderelection code when we
-	// want to step down
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	// use a Go context so we can tell the arp loop code when we
-	// want to step down
-	ctxArp, cancelArp := context.WithCancel(context.Background())
-	defer cancelArp()
-
-	// listen for interrupts or the Linux SIGTERM signal and cancel
-	// our context, which the leader election code will observe and
-	// step down
-	signalChan := make(chan os.Signal, 1)
-	// Add Notification for Userland interrupt
-	signal.Notify(signalChan, syscall.SIGINT)
-
-	// Add Notification for SIGTERM (sent from Kubernetes)
-	signal.Notify(signalChan, syscall.SIGTERM)
-
-	// Add Notification for SIGKILL (sent from Kubernetes)
-	signal.Notify(signalChan, syscall.SIGKILL)
-
-	go func() {
-		<-signalChan
-		log.Info("Received termination, signaling shutdown")
-		// Cancel the context, which will in turn cancel the leadership
-		cancel()
-		// Cancel the arp context, which will in turn stop any broadcasts
-	}()
-
-	// (attempt to) Remove the virtual IP, incase it already exists
-	cluster.network.DeleteIP()
-
-	// Managers for Vip load balancers and none-vip loadbalancers
-	nonVipLB := loadbalancer.LBManager{}
-	VipLB := loadbalancer.LBManager{}
-
-	// BGP server
-	var bgpServer *bgp.Server
-	// Defer a function to check if the bgpServer has been created and if so attempt to close it
-	defer func() {
-		if bgpServer != nil {
-			bgpServer.Close()
-		}
-	}()
-
-	// If Packet is enabled then we can begin our preperation work
-	var packetClient *packngo.Client
-	if c.EnablePacket {
-		packetClient, err = packngo.NewClient()
-		if err != nil {
-			log.Error(err)
-		}
-
-		// We're using Packet with BGP, popuplate the Peer information from the API
-		if c.EnableBGP {
-			log.Infoln("Looking up the BGP configuration from packet")
-			err = packet.BGPLookup(packetClient, c)
-			if err != nil {
-				log.Error(err)
-			}
-		}
-	}
-
-	if c.EnableBGP {
-		// Lets start BGP
-		log.Info("Starting the BGP server to adverise VIP routes to VGP peers")
-		bgpServer, err = bgp.NewBGPServer(&c.BGPConfig)
-		if err != nil {
-			log.Error(err)
-		}
-	}
-
-	if c.EnableLoadBalancer {
-
-		// Iterate through all Configurations
-		if len(c.LoadBalancers) != 0 {
-			for x := range c.LoadBalancers {
-				// If the load balancer doesn't bind to the VIP
-				if c.LoadBalancers[x].BindToVip == false {
-					err = nonVipLB.Add("", &c.LoadBalancers[x])
-					if err != nil {
-						log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-					}
-				}
-			}
-		}
-	}
-	// start the leader election code loop
-	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
-		Lock: lock,
-		// IMPORTANT: you MUST ensure that any code you have that
-		// is protected by the lease must terminate **before**
-		// you call cancel. Otherwise, you could have a background
-		// loop still running and another process could
-		// get elected before your background loop finished, violating
-		// the stated goal of the lease.
-		ReleaseOnCancel: true,
-		LeaseDuration:   time.Duration(c.LeaseDuration) * time.Second,
-		RenewDeadline:   time.Duration(c.RenewDeadline) * time.Second,
-		RetryPeriod:     time.Duration(c.RetryPeriod) * time.Second,
-		Callbacks: leaderelection.LeaderCallbacks{
-			OnStartedLeading: func(ctx context.Context) {
-
-				// we're notified when we start
-				log.Info("This node is starting with leadership of the cluster")
-				err = cluster.network.AddIP()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				if c.EnablePacket {
-					// We're not using Packet with BGP
-					if !c.EnableBGP {
-						// Attempt to attach the EIP in the standard manner
-						log.Debugf("Attaching the Packet EIP through the API to this host")
-						err = packet.AttachEIP(packetClient, c, id)
-						if err != nil {
-							log.Error(err)
-						}
-					}
-				}
-
-				if c.EnableBGP {
-					// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
-					cidrVip := fmt.Sprintf("%s/%s", c.VIP, c.VIPCIDR)
-					log.Debugf("Attempting to advertise the address [%s] over BGP", cidrVip)
-
-					err = bgpServer.AddHost(cidrVip)
-					if err != nil {
-						log.Error(err)
-					}
-				}
-
-				if c.EnableLoadBalancer {
-					// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-					for x := range c.LoadBalancers {
-
-						if c.LoadBalancers[x].BindToVip == true {
-							err = VipLB.Add(c.VIP, &c.LoadBalancers[x])
-							if err != nil {
-								log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-
-								// Stop all load balancers associated with the VIP
-								err = VipLB.StopAll()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-
-								err = cluster.network.DeleteIP()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-							}
-						}
-					}
-				}
-
-				if c.GratuitousARP == true {
-					ctxArp, cancelArp = context.WithCancel(context.Background())
-
-					go func(ctx context.Context) {
-						for {
-							select {
-							case <-ctx.Done(): // if cancel() execute
-								return
-							default:
-								// Gratuitous ARP, will broadcast to new MAC <-> IP
-								err = vip.ARPSendGratuitous(c.VIP, c.Interface)
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-							}
-							time.Sleep(3 * time.Second)
-						}
-					}(ctxArp)
-				}
-			},
-			OnStoppedLeading: func() {
-				// we can do cleanup here
-				log.Info("This node is becoming a follower within the cluster")
-
-				// Stop the Arp context if it is running
-				cancelArp()
-
-				// Stop the BGP server
-				if bgpServer != nil {
-					err = bgpServer.Close()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-
-				// Stop all load balancers associated with the VIP
-				err = VipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				err = cluster.network.DeleteIP()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-			},
-			OnNewLeader: func(identity string) {
-				// we're notified when new leader elected
-				log.Infof("Node [%s] is assuming leadership of the cluster", identity)
-
-				if identity == id {
-					// We have the lock
-				}
-			},
-		},
-	})
-
-	//<-signalChan
-	log.Infof("Shutting down Kube-Vip Leader Election cluster")
-
-	// Force a removal of the VIP (ignore the error if we don't have it)
-	cluster.network.DeleteIP()
-
-	return nil
-}

pkg/cluster/clusterLeaderElection.go

@@ -0,0 +1,390 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/etcd"
+	"github.com/kube-vip/kube-vip/pkg/k8s"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
+
+	log "log/slog"
+
+	clientv3 "go.etcd.io/etcd/client/v3"
+	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	watchtools "k8s.io/client-go/tools/watch"
+)
+
+// Manager degines the manager of the load-balancing services
+type Manager struct {
+	KubernetesClient   *kubernetes.Clientset
+	RetryWatcherClient *kubernetes.Clientset
+	// This channel is used to signal a shutdown
+	SignalChan chan os.Signal
+
+	EtcdClient *clientv3.Client
+}
+
+// NewManager will create a new managing object
+func NewManager(path string, inCluster bool, port int) (*Manager, error) {
+	var hostname string
+
+	// If inCluster is set then it will likely have started as a static pod or won't have the
+	// VIP up before trying to connect to the API server, we set the API endpoint to this machine to
+	// ensure connectivity. Else if the path passed is empty and not running in the cluster,
+	// attempt to look for a kubeconfig in the default HOME dir.
+
+	hostname = fmt.Sprintf("kubernetes:%v", port)
+
+	if len(path) == 0 && !inCluster {
+		path = filepath.Join(os.Getenv("HOME"), ".kube", "config")
+
+		// We modify the config so that we can always speak to the correct host
+		id, err := os.Hostname()
+		if err != nil {
+			return nil, err
+		}
+
+		hostname = fmt.Sprintf("%s:%v", id, port)
+	}
+
+	config, err := k8s.NewRestConfig(path, inCluster, hostname)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create k8s REST config: %w", err)
+	}
+
+	clientset, err := k8s.NewClientset(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating a new k8s clientset: %v", err)
+	}
+
+	rwConfig, err := k8s.NewRestConfig(path, inCluster, hostname)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create k8s REST config for retryClientSet: %w", err)
+	}
+
+	rwConfig.Timeout = 0 // empty value to disable the timeout
+	rwClientSet, err := k8s.NewClientset(rwConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create k8s client for retry watcher: %w", err)
+	}
+
+	return &Manager{
+		KubernetesClient:   clientset,
+		RetryWatcherClient: rwClientSet,
+	}, nil
+}
+
+// StartCluster - Begins a running instance of the Leader Election cluster
+func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *bgp.Server) error {
+	var err error
+
+	log.Info("cluster membership", "namespace", c.Namespace, "lock", c.LeaseName, "id", c.NodeName)
+
+	// use a Go context so we can tell the leaderelection code when we
+	// want to step down
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// use a Go context so we can tell the arp loop code when we
+	// want to step down
+	ctxArp, cancelArp := context.WithCancel(context.Background())
+	defer cancelArp()
+
+	// use a Go context so we can tell the dns loop code when we
+	// want to step down
+	ctxDNS, cancelDNS := context.WithCancel(context.Background())
+	defer cancelDNS()
+
+	// listen for interrupts or the Linux SIGTERM signal and cancel
+	// our context, which the leader election code will observe and
+	// step down
+	signalChan := make(chan os.Signal, 1)
+	// Add Notification for Userland interrupt
+	signal.Notify(signalChan, syscall.SIGINT)
+
+	// Add Notification for SIGTERM (sent from Kubernetes)
+	signal.Notify(signalChan, syscall.SIGTERM)
+
+	go func() {
+		<-signalChan
+		log.Info("Received termination, signaling cluster shutdown")
+		// Cancel the context, which will in turn cancel the leadership
+		cancel()
+		// Cancel the arp context, which will in turn stop any broadcasts
+	}()
+
+	// (attempt to) Remove the virtual IP, in case it already exists
+
+	for i := range cluster.Network {
+		deleted, err := cluster.Network[i].DeleteIP()
+		if err != nil {
+			log.Error("could not delete virtualIP", "err", err)
+		}
+		if deleted {
+			log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+		}
+	}
+
+	// Defer a function to check if the bgpServer has been created and if so attempt to close it
+	defer func() {
+		if bgpServer != nil {
+			bgpServer.Close()
+		}
+	}()
+
+	if c.EnableBGP && bgpServer == nil {
+		// Lets start BGP
+		log.Info("Starting the BGP server to advertise VIP routes to VGP peers")
+		bgpServer, err = bgp.NewBGPServer(&c.BGPConfig)
+		if err != nil {
+			log.Error("new BGP server", "err", err)
+		}
+		if err := bgpServer.Start(nil); err != nil {
+			log.Error("starting BGP server", "err", err)
+		}
+	}
+
+	run := &runConfig{
+		config:  c,
+		leaseID: c.NodeName,
+		sm:      sm,
+		onStartedLeading: func(ctx context.Context) { //nolint TODO: potential clean code
+			// As we're leading lets start the vip service
+			err := cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, cancel)
+			if err != nil {
+				log.Error("starting VIP service on leader", "err", err)
+			}
+		},
+		onStoppedLeading: func() {
+			// we can do cleanup here
+			log.Info("This node is becoming a follower within the cluster")
+
+			// Stop the dns context
+			cancelDNS()
+			// Stop the Arp context if it is running
+			cancelArp()
+
+			// Stop the BGP server
+			if bgpServer != nil {
+				err := bgpServer.Close()
+				if err != nil {
+					log.Warn("close BGP server", "err", err)
+				}
+			}
+
+			for i := range cluster.Network {
+				deleted, err := cluster.Network[i].DeleteIP()
+				if err != nil {
+					log.Warn("delete VIP", "err", err)
+				}
+				if deleted {
+					log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+				}
+			}
+
+			log.Error("lost leadership, restarting kube-vip")
+			panic("") // TODO - we could also return here
+		},
+		onNewLeader: func(identity string) {
+			// we're notified when new leader elected
+			log.Info("New leader", "leader", identity)
+		},
+	}
+
+	switch c.LeaderElectionType {
+	case "kubernetes", "":
+		cluster.runKubernetesLeaderElectionOrDie(ctx, run)
+	case "etcd":
+		cluster.runEtcdLeaderElectionOrDie(ctx, run)
+	default:
+		log.Info(fmt.Sprintf("LeaderElectionMode %s not supported, exiting", c.LeaderElectionType))
+	}
+
+	return nil
+}
+
+type runConfig struct {
+	config  *kubevip.Config
+	leaseID string
+	sm      *Manager
+
+	// onStartedLeading is called when this member starts leading.
+	onStartedLeading func(context.Context)
+	// onStoppedLeading is called when this member stops leading.
+	onStoppedLeading func()
+	// onNewLeader is called when the client observes a leader that is
+	// not the previously observed leader. This includes the first observed
+	// leader when the client starts.
+	onNewLeader func(identity string)
+}
+
+func (cluster *Cluster) runKubernetesLeaderElectionOrDie(ctx context.Context, run *runConfig) {
+	// we use the Lease lock type since edits to Leases are less common
+	// and fewer objects in the cluster watch "all Leases".
+	lock := &resourcelock.LeaseLock{
+		LeaseMeta: metav1.ObjectMeta{
+			Name:        run.config.LeaseName,
+			Namespace:   run.config.Namespace,
+			Annotations: run.config.LeaseAnnotations,
+		},
+		Client: run.sm.KubernetesClient.CoordinationV1(),
+		LockConfig: resourcelock.ResourceLockConfig{
+			Identity: run.leaseID,
+		},
+	}
+
+	// start the leader election code loop
+	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
+		Lock: lock,
+		// IMPORTANT: you MUST ensure that any code you have that
+		// is protected by the lease must terminate **before**
+		// you call cancel. Otherwise, you could have a background
+		// loop still running and another process could
+		// get elected before your background loop finished, violating
+		// the stated goal of the lease.
+		ReleaseOnCancel: true,
+		LeaseDuration:   time.Duration(run.config.LeaseDuration) * time.Second,
+		RenewDeadline:   time.Duration(run.config.RenewDeadline) * time.Second,
+		RetryPeriod:     time.Duration(run.config.RetryPeriod) * time.Second,
+		Callbacks: leaderelection.LeaderCallbacks{
+			OnStartedLeading: run.onStartedLeading,
+			OnStoppedLeading: run.onStoppedLeading,
+			OnNewLeader:      run.onNewLeader,
+		},
+	})
+}
+
+func (cluster *Cluster) runEtcdLeaderElectionOrDie(ctx context.Context, run *runConfig) {
+	etcd.RunElectionOrDie(ctx, &etcd.LeaderElectionConfig{
+		EtcdConfig:           etcd.ClientConfig{Client: run.sm.EtcdClient},
+		Name:                 run.config.LeaseName,
+		MemberID:             run.leaseID,
+		LeaseDurationSeconds: int64(run.config.LeaseDuration),
+		Callbacks: etcd.LeaderCallbacks{
+			OnStartedLeading: run.onStartedLeading,
+			OnStoppedLeading: run.onStoppedLeading,
+			OnNewLeader:      run.onNewLeader,
+		},
+	})
+}
+
+func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
+	// Use a restartable watcher, as this should help in the event of etcd or timeout issues
+	log.Info("Kube-Vip is watching nodes for control-plane labels")
+
+	listOptions := metav1.ListOptions{
+		LabelSelector: "node-role.kubernetes.io/control-plane",
+	}
+
+	rw, err := watchtools.NewRetryWatcherWithContext(ctxArp, "1", &cache.ListWatch{
+		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
+			return sm.RetryWatcherClient.CoreV1().Nodes().Watch(context.Background(), listOptions)
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("error creating label watcher: %s", err.Error())
+	}
+
+	go func() {
+		<-sm.SignalChan
+		log.Info("Received termination, signaling shutdown")
+		// Cancel the context
+		rw.Stop()
+	}()
+
+	ch := rw.ResultChan()
+	// defer rw.Stop()
+
+	for event := range ch {
+		// We need to inspect the event and get ResourceVersion out of it
+		switch event.Type {
+		case watch.Added, watch.Modified:
+			node, ok := event.Object.(*v1.Node)
+			if !ok {
+				return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
+			}
+			// Find the node IP address (this isn't foolproof)
+			for x := range node.Status.Addresses {
+				if node.Status.Addresses[x].Type == v1.NodeInternalIP {
+					if checkIfNodeIsReady(node) {
+						err = lb.AddBackend(node.Status.Addresses[x].Address, port)
+						if err != nil {
+							log.Error("add IPVS backend", "err", err)
+						}
+					} else {
+						err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
+						if err != nil {
+							log.Error("remove IPVS backend", "err", err)
+						}
+					}
+				}
+			}
+		case watch.Deleted:
+			node, ok := event.Object.(*v1.Node)
+			if !ok {
+				return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
+			}
+
+			// Find the node IP address (this isn't foolproof)
+			for x := range node.Status.Addresses {
+				if node.Status.Addresses[x].Type == v1.NodeInternalIP {
+					err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
+					if err != nil {
+						log.Error("Del IPVS backend", "err", err)
+					}
+				}
+			}
+
+			log.Info("Node deleted", "name", node.Name)
+
+		case watch.Bookmark:
+			// Un-used
+		case watch.Error:
+			log.Error("Error attempting to watch Kubernetes Nodes")
+
+			// This round trip allows us to handle unstructured status
+			errObject := apierrors.FromObject(event.Object)
+			statusErr, ok := errObject.(*apierrors.StatusError)
+			if !ok {
+				log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
+			}
+
+			status := statusErr.ErrStatus
+			log.Error("watcher", "status", status)
+		default:
+		}
+	}
+
+	log.Info("Exiting Node watcher")
+	return nil
+}
+
+func checkIfNodeIsReady(node *v1.Node) bool {
+	if node == nil {
+		return false
+	}
+	for _, condition := range node.Status.Conditions {
+		if condition.Type == v1.NodeReady {
+			if condition.Status == v1.ConditionTrue {
+				return true
+			}
+		}
+	}
+	return false
+}

pkg/cluster/clusterRaft.go

@@ -1,296 +0,0 @@
-package cluster
-
-import (
-	"fmt"
-	"net"
-	"time"
-
-	"github.com/hashicorp/raft"
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	"github.com/plunder-app/kube-vip/pkg/loadbalancer"
-	"github.com/plunder-app/kube-vip/pkg/vip"
-	log "github.com/sirupsen/logrus"
-)
-
-// StartRaftCluster - Begins a running instance of the Raft cluster
-func (cluster *Cluster) StartRaftCluster(c *kubevip.Config) error {
-
-	// Create local configuration address
-	localAddress := fmt.Sprintf("%s:%d", c.LocalPeer.Address, c.LocalPeer.Port)
-
-	// Begin the Raft configuration
-	config := raft.DefaultConfig()
-	config.LocalID = raft.ServerID(c.LocalPeer.ID)
-	logger := log.StandardLogger().Writer()
-	config.LogOutput = logger
-
-	// Initialize communication
-	address, err := net.ResolveTCPAddr("tcp", localAddress)
-	if err != nil {
-		return err
-	}
-
-	// Create transport
-	transport, err := raft.NewTCPTransport(localAddress, address, 3, 10*time.Second, logger)
-	if err != nil {
-		return err
-	}
-
-	// Create Raft structures
-	snapshots := raft.NewInmemSnapshotStore()
-	logStore := raft.NewInmemStore()
-	stableStore := raft.NewInmemStore()
-
-	// Cluster configuration
-	configuration := raft.Configuration{}
-
-	// Add Local Peer
-	configuration.Servers = append(configuration.Servers, raft.Server{
-		ID:      raft.ServerID(c.LocalPeer.ID),
-		Address: raft.ServerAddress(fmt.Sprintf("%s:%d", c.LocalPeer.Address, c.LocalPeer.Port))})
-
-	// If we want to start a node as leader then we will not add any remote peers, this will leave this as a cluster of one
-	// The remotePeers will add themselves to the cluster as they're added
-	if c.StartAsLeader != true {
-		for x := range c.RemotePeers {
-			// Make sure that we don't add in this server twice
-			if c.LocalPeer.Address != c.RemotePeers[x].Address {
-
-				// Build the address from the peer configuration
-				peerAddress := fmt.Sprintf("%s:%d", c.RemotePeers[x].Address, c.RemotePeers[x].Port)
-
-				// Set this peer into the raft configuration
-				configuration.Servers = append(configuration.Servers, raft.Server{
-					ID:      raft.ServerID(c.RemotePeers[x].ID),
-					Address: raft.ServerAddress(peerAddress)})
-			}
-		}
-		log.Info("This node will attempt to start as Follower")
-	} else {
-		log.Info("This node will attempt to start as Leader")
-	}
-
-	// Bootstrap cluster
-	if err := raft.BootstrapCluster(config, logStore, stableStore, snapshots, transport, configuration); err != nil {
-		return err
-	}
-
-	// Create RAFT instance
-	raftServer, err := raft.NewRaft(config, cluster.stateMachine, logStore, stableStore, snapshots, transport)
-	if err != nil {
-		return err
-	}
-
-	cluster.stop = make(chan bool, 1)
-	cluster.completed = make(chan bool, 1)
-	ticker := time.NewTicker(time.Second)
-	isLeader := c.StartAsLeader
-
-	// (attempt to) Remove the virtual IP, incase it already exists
-	cluster.network.DeleteIP()
-
-	// leader log broadcast - this counter is used to stop flooding STDOUT with leader log entries
-	var leaderbroadcast int
-	// Managers for Vip load balancers and none-vip loadbalancers
-	nonVipLB := loadbalancer.LBManager{}
-	VipLB := loadbalancer.LBManager{}
-
-	// Iterate through all Configurations
-	for x := range c.LoadBalancers {
-		// If the load balancer doesn't bind to the VIP
-		if c.LoadBalancers[x].BindToVip == false {
-			err = nonVipLB.Add("", &c.LoadBalancers[x])
-			if err != nil {
-				log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-			}
-		}
-	}
-
-	// On a cold start the node will sleep for 5 seconds to ensure that leader elections are complete
-	log.Infoln("This instance will wait approximately 5 seconds, from cold start to ensure cluster elections are complete")
-	time.Sleep(time.Second * 5)
-
-	go func() {
-		for {
-			if c.AddPeersAsBackends == true {
-				// Get addresses and change backends
-
-				// c.LoadBalancers[0].Backends
-				// for x := range raftServer.GetConfiguration().Configuration().Servers {
-				// 	raftServer.GetConfiguration().Configuration().Servers[x].Address
-				// }
-
-			}
-			// Broadcast the current leader on this node if it's the correct time (every leaderLogcount * time.Second)
-			if leaderbroadcast == leaderLogcount {
-				log.Infof("The Node [%s] is leading", raftServer.Leader())
-				// Reset the timer
-				leaderbroadcast = 0
-
-				// ensure that if this node is the leader, it is set as the leader
-				if localAddress == string(raftServer.Leader()) {
-					// Re-broadcast arp to ensure network stays up to date
-					if c.GratuitousARP == true {
-						// Gratuitous ARP, will broadcast to new MAC <-> IP
-						err = vip.ARPSendGratuitous(c.VIP, c.Interface)
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-					}
-					if !isLeader {
-						log.Infoln("This node is leading, but isnt the leader (correcting)")
-						isLeader = true
-					}
-				} else {
-					// (attempt to) Remove the virtual IP, incase it already exists to keep nodes clean
-					cluster.network.DeleteIP()
-					isLeader = false
-				}
-
-			}
-			leaderbroadcast++
-
-			select {
-			case leader := <-raftServer.LeaderCh():
-				log.Infoln("New Election event")
-				if leader {
-					isLeader = true
-
-					log.Info("This node is assuming leadership of the cluster")
-					err = cluster.network.AddIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-
-					// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-
-					for x := range c.LoadBalancers {
-
-						if c.LoadBalancers[x].BindToVip == true {
-							err = VipLB.Add(c.VIP, &c.LoadBalancers[x])
-							if err != nil {
-								log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-								log.Errorf("Dropping Leadership to another node in the cluster")
-								raftServer.LeadershipTransfer()
-
-								// Stop all load balancers associated with the VIP
-								err = VipLB.StopAll()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-
-								err = cluster.network.DeleteIP()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-							}
-						}
-					}
-
-					if c.GratuitousARP == true {
-						// Gratuitous ARP, will broadcast to new MAC <-> IP
-						err = vip.ARPSendGratuitous(c.VIP, c.Interface)
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-					}
-				} else {
-					isLeader = false
-
-					log.Info("This node is becoming a follower within the cluster")
-
-					// Stop all load balancers associated with the VIP
-					err = VipLB.StopAll()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-
-					err = cluster.network.DeleteIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-
-			case <-ticker.C:
-
-				if isLeader {
-
-					result, err := cluster.network.IsSet()
-					if err != nil {
-						log.WithFields(log.Fields{"error": err, "ip": cluster.network.IP(), "interface": cluster.network.Interface()}).Error("Could not check ip")
-					}
-
-					if result == false {
-						log.Error("This node is leader and is adopting the virtual IP")
-
-						err = cluster.network.AddIP()
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-						// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-
-						for x := range c.LoadBalancers {
-
-							if c.LoadBalancers[x].BindToVip == true {
-								err = VipLB.Add(c.VIP, &c.LoadBalancers[x])
-								if err != nil {
-									log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-								}
-							}
-						}
-						if c.GratuitousARP == true {
-							// Gratuitous ARP, will broadcast to new MAC <-> IP
-							err = vip.ARPSendGratuitous(c.VIP, c.Interface)
-							if err != nil {
-								log.Warnf("%v", err)
-							}
-						}
-					}
-				}
-
-			case <-cluster.stop:
-				log.Info("[RAFT] Stopping this node")
-				log.Info("[LOADBALANCER] Stopping load balancers")
-
-				// Stop all load balancers associated with the VIP
-				err = VipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				// Stop all load balancers associated with the Host
-				err = nonVipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				if isLeader {
-					log.Info("[VIP] Releasing the Virtual IP")
-					err = cluster.network.DeleteIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-
-				close(cluster.completed)
-
-				return
-			}
-		}
-	}()
-
-	log.Info("Started")
-
-	return nil
-}
-
-// Stop - Will stop the Cluster and release VIP if needed
-func (cluster *Cluster) Stop() {
-	// Close the stop chanel, which will shut down the VIP (if needed)
-	close(cluster.stop)
-
-	// Wait until the completed channel is closed, signallign all shutdown tasks completed
-	<-cluster.completed
-
-	log.Info("Stopped")
-}

pkg/cluster/service.go

@@ -0,0 +1,381 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"net"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
+	"github.com/kube-vip/kube-vip/pkg/backend"
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	"github.com/vishvananda/netlink"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Config, sm *Manager, bgpServer *bgp.Server, cancelLeaderElection context.CancelFunc) error {
+	var err error
+
+	// listen for interrupts or the Linux SIGTERM signal and cancel
+	// our context, which the leader election code will observe and
+	// step down
+	signalChan := make(chan os.Signal, 1)
+	// Add Notification for Userland interrupt
+	signal.Notify(signalChan, syscall.SIGINT)
+
+	// Add Notification for SIGTERM (sent from Kubernetes)
+	signal.Notify(signalChan, syscall.SIGTERM)
+
+	loadbalancers := []*loadbalancer.IPVSLoadBalancer{}
+
+	var arpWG sync.WaitGroup
+
+	for i := range cluster.Network {
+		network := cluster.Network[i]
+		if err := network.SetMask(c.VIPSubnet); err != nil {
+			log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
+			panic("")
+		}
+		if network.IsDDNS() {
+			if err := cluster.StartDDNS(ctxDNS); err != nil {
+				log.Error(err.Error())
+			}
+		}
+
+		// start the dns updater if address is dns
+		if network.IsDNS() {
+			log.Info("starting the DNS updater", "address", network.DNSName())
+			ipUpdater := vip.NewIPUpdater(network)
+			ipUpdater.Run(ctxDNS)
+		}
+
+		if !c.EnableRoutingTable {
+			if _, err = network.AddIP(false); err != nil {
+				log.Error(err.Error())
+			}
+		}
+
+		if c.EnableBGP {
+			// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
+			log.Debug("Attempting to advertise over BGP", "address", network.CIDR())
+			err = bgpServer.AddHost(network.CIDR())
+			if err != nil {
+				log.Error(err.Error())
+			}
+		}
+
+		if c.EnableLoadBalancer {
+			lb, err := loadbalancer.NewIPVSLB(network.IP(), c.LoadBalancerPort, c.LoadBalancerForwardingMethod, c.BackendHealthCheckInterval, c.Interface, cancelLeaderElection, signalChan)
+			if err != nil {
+				log.Error("Error creating IPVS LoadBalancer", "err", err)
+			}
+
+			go func() {
+				err = sm.NodeWatcher(ctxArp, lb, c.Port) //TODO: We're using the ctxARP as the context this will change when rkatz finishes his change
+				if err != nil {
+					log.Error("Error watching node labels", "err", err)
+				}
+			}()
+
+			loadbalancers = append(loadbalancers, lb)
+		}
+
+		if c.EnableARP {
+			arpWG.Add(1)
+			go cluster.layer2Update(ctxArp, network, c, &arpWG)
+		}
+	}
+
+	if c.EnableLoadBalancer {
+		// Shutdown function that will wait on this signal, unless we call it ourselves
+		<-signalChan
+		for _, lb := range loadbalancers {
+			err = lb.RemoveIPVSLB()
+			if err != nil {
+				log.Error("Error stopping IPVS LoadBalancer", "err", err)
+			}
+		}
+	}
+
+	if c.EnableRoutingTable {
+		backendMapV4 := backend.Map{}
+		backendMapV6 := backend.Map{}
+		// only check localhost
+
+		nodename := ""
+		if c.NodeName != "" {
+			nodename = c.NodeName
+		} else {
+			nodename = os.Getenv("HOSTNAME")
+		}
+
+		ips := []string{}
+		if nodename != "" {
+			if ips, err = getNodeIPs(ctxArp, nodename, sm.KubernetesClient); err != nil && !apierrors.IsNotFound(err) {
+				log.Error("failed to get IP of control-plane nod", "err", err)
+			}
+		}
+
+		if len(ips) == 0 {
+			isV6, err := isV6(cluster.Network[0].IP())
+			if err != nil {
+				return fmt.Errorf("failed to parse IP '%s'", cluster.Network[0].IP())
+			}
+			if !isV6 {
+				ips = append(ips, "127.0.0.1")
+			} else {
+				ips = append(ips, "::1")
+			}
+
+			log.Info("no IP address found for node - will fallback to use localhost address", "addresses", ips)
+		}
+
+		for _, ip := range ips {
+			entry := backend.Entry{Addr: ip, Port: c.Port}
+			ipv6, err := isV6(ip)
+			if err != nil {
+				log.Error("failed to check IP type", "IP", ip, "error", err)
+			}
+			if !ipv6 {
+				backendMapV4[entry] = false
+			} else {
+				backendMapV6[entry] = false
+			}
+		}
+
+		stop := make(chan struct{})
+
+		// will wait for system interrupt and will send stop signal to backend watch
+		go func() {
+			<-signalChan
+			stop <- struct{}{}
+		}()
+
+		backend.Watch(func() {
+			for i := range cluster.Network {
+				network := cluster.Network[i]
+				networkIP := network.IP()
+				isNetworkV6, err := isV6(networkIP)
+				if err != nil {
+					log.Error("failed to check IP type", "IP", networkIP, "error", err)
+					continue
+				}
+
+				backendMap := &backendMapV4
+				if isNetworkV6 {
+					backendMap = &backendMapV6
+				}
+
+				for entry := range *backendMap {
+					if entry.Check() {
+						_, err = network.AddIP(true)
+						if err != nil {
+							log.Error("error adding address", "err", err)
+						}
+						if !(*backendMap)[entry] {
+							log.Info("added backend", "ip", network.IP())
+						}
+
+						err = network.AddRoute(true)
+						if err != nil && !errors.Is(err, fs.ErrExist) && !errors.Is(err, syscall.ESRCH) {
+							log.Warn(err.Error())
+						} else if err == nil && !(*backendMap)[entry] {
+							log.Info("added route", "route", network.PrepareRoute().String())
+						}
+
+						(*backendMap)[entry] = true
+						break
+					}
+					(*backendMap)[entry] = false
+				}
+
+				deleteAddress := true
+				for entry := range *backendMap {
+					if (*backendMap)[entry] {
+						deleteAddress = false
+						break
+					}
+				}
+
+				if deleteAddress {
+					err = network.DeleteRoute()
+					if err != nil && !errors.Is(err, fs.ErrNotExist) && !errors.Is(err, syscall.ESRCH) {
+						log.Warn("deleting route", "err", err)
+					} else if err == nil {
+						log.Info("deleted route", "route", network.PrepareRoute().String())
+					}
+
+					deleted, err := network.DeleteIP()
+					if err != nil {
+						log.Error("error deleting IP", "err", err)
+						panic("")
+					}
+					if deleted {
+						log.Info("deleted address", "IP", network.IP(), "interface", network.Interface())
+					}
+				}
+			}
+		}, c.BackendHealthCheckInterval, stop)
+	}
+
+	return nil
+}
+
+func isV6(ip string) (bool, error) {
+	ipaddr := net.ParseIP(ip)
+	if ipaddr == nil {
+		return false, fmt.Errorf("failed to parse IP '%s'", ip)
+	}
+	return ipaddr.To4() == nil, nil
+}
+
+func getNodeIPs(ctx context.Context, nodename string, client *kubernetes.Clientset) ([]string, error) {
+	node, err := client.CoreV1().Nodes().Get(ctx, nodename, metav1.GetOptions{})
+	if err != nil && !apierrors.IsNotFound(err) {
+		return []string{}, fmt.Errorf("failed to get data about '%s' node: %w", nodename, err)
+	}
+	ips := []string{}
+	for _, addr := range node.Status.Addresses {
+		if addr.Type == corev1.NodeInternalIP {
+			ips = append(ips, addr.Address)
+		}
+	}
+	return ips, nil
+}
+
+// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
+func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp *bgp.Server, name string, CountRouteReferences func(*netlink.Route) int) {
+	// use a Go context so we can tell the arp loop code when we
+	// want to step down
+	//nolint
+	ctxArp, cancelArp := context.WithCancel(ctx)
+
+	cluster.stop = make(chan bool, 1)
+	cluster.completed = make(chan bool, 1)
+
+	var arpWG sync.WaitGroup
+
+	for i := range cluster.Network {
+		network := cluster.Network[i]
+		if err := network.SetMask(c.VIPSubnet); err != nil {
+			log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
+			panic("")
+		}
+		_, err := network.DeleteIP()
+		if err != nil {
+			log.Warn("attempted to clean existing VIP", "err", err)
+		}
+
+		if c.EnableRoutingTable && (c.EnableLeaderElection || c.EnableServicesElection) {
+			err = network.AddRoute(false)
+			if err != nil {
+				log.Warn(err.Error())
+			}
+		} else if !c.EnableRoutingTable {
+			if _, err = network.AddIP(false); err != nil {
+				log.Warn(err.Error())
+			}
+		}
+
+		if c.EnableARP {
+			arpWG.Add(1)
+			go cluster.layer2Update(ctxArp, network, c, &arpWG)
+		}
+
+		if c.EnableBGP && (c.EnableLeaderElection || c.EnableServicesElection) {
+			// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
+			log.Debug("(svcs) attempting to advertise over BGP", "address", network.CIDR())
+			err = bgp.AddHost(network.CIDR())
+			if err != nil {
+				log.Error(err.Error())
+			}
+		}
+	}
+
+	go func() {
+		<-cluster.stop
+		// Stop the Arp context if it is running
+		cancelArp()
+
+		arpWG.Wait() // wait for all cluster ARP/NDP to be finished
+
+		log.Info("[LOADBALANCER] Stopping load balancers", "name", name)
+
+		if c.EnableRoutingTable {
+			for i := range cluster.Network {
+				// chek if route is not  referenced by another service
+				r := cluster.Network[i].PrepareRoute()
+				if CountRouteReferences(r) < 1 {
+					log.Info("[VIP] Deleting Route for VIP", "IP", cluster.Network[i].IP())
+					if err := cluster.Network[i].DeleteRoute(); err != nil {
+						log.Warn(err.Error())
+					}
+				}
+			}
+
+			close(cluster.completed)
+			return
+		}
+		for i := range cluster.Network {
+			if c.EnableARP && cluster.arpMgr.Count(cluster.Network[i].ARPName()) > 0 {
+				continue
+			}
+			log.Info("[VIP] Deleting VIP", "ip", cluster.Network[i].IP())
+			deleted, err := cluster.Network[i].DeleteIP()
+			if err != nil {
+				log.Warn(err.Error())
+			}
+			if deleted {
+				log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+			}
+		}
+
+		close(cluster.completed)
+	}()
+}
+
+// Layer2Update, handles the creation of the
+func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config, arpWG *sync.WaitGroup) {
+	defer arpWG.Done()
+	log.Info("layer 2 broadcaster starting")
+	var ndp *vip.NdpResponder
+	var err error
+	ipString := network.IP()
+	if vip.IsIPv6(ipString) {
+		if network.IPisLinkLocal() {
+			log.Error("layer2 is link-local can't use NDP", "address", ipString)
+
+		} else {
+			ndp, err = vip.NewNDPResponder(network.Interface())
+			if err != nil {
+				log.Error("failed to create new NDP Responder", "error", err)
+			} else {
+				if ndp != nil {
+					defer ndp.Close()
+				}
+			}
+		}
+	}
+
+	log.Debug("layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
+
+	arpInstance := arp.NewInstance(network, ndp)
+	cluster.arpMgr.Insert(arpInstance)
+
+	<-ctx.Done() // if cancel() execute
+	log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
+	cluster.arpMgr.Remove(arpInstance)
+}

pkg/cluster/singleNode.go

@@ -1,103 +1,84 @@
 package cluster
 
 import (
-	log "github.com/sirupsen/logrus"
+	"context"
 
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	"github.com/plunder-app/kube-vip/pkg/loadbalancer"
-	"github.com/plunder-app/kube-vip/pkg/vip"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/vip"
 )
 
 // StartSingleNode will start a single node cluster
 func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) error {
 	// Start kube-vip as a single node server
 
-	// TODO - Split all this code out as a seperate function
-	log.Infoln("Starting kube-vip as a single node cluster")
+	// TODO - Split all this code out as a separate function
+	log.Info("Starting kube-vip as a single node cluster")
 
 	log.Info("This node is assuming leadership of the cluster")
 
 	cluster.stop = make(chan bool, 1)
 	cluster.completed = make(chan bool, 1)
 
-	// Managers for Vip load balancers and none-vip loadbalancers
-	nonVipLB := loadbalancer.LBManager{}
-	VipLB := loadbalancer.LBManager{}
-
-	// Iterate through all Configurations
-	for x := range c.LoadBalancers {
-		// If the load balancer doesn't bind to the VIP
-		if c.LoadBalancers[x].BindToVip == false {
-			err := nonVipLB.Add("", &c.LoadBalancers[x])
+	for i := range cluster.Network {
+		if !disableVIP {
+			deleted, err := cluster.Network[i].DeleteIP()
 			if err != nil {
-				log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
+				log.Warn("Attempted to clean existing VIP", "err", err)
+			}
+			if deleted {
+				log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+			}
+
+			_, err = cluster.Network[i].AddIP(false)
+			if err != nil {
+				log.Warn(err.Error())
 			}
 
 		}
-	}
 
-	if !disableVIP {
-		err := cluster.network.DeleteIP()
-		if err != nil {
-			log.Warnf("Attempted to clean existing VIP => %v", err)
-		}
-
-		err = cluster.network.AddIP()
-		if err != nil {
-			log.Warnf("%v", err)
-		}
-
-		// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-		for x := range c.LoadBalancers {
-
-			if c.LoadBalancers[x].BindToVip == true {
-				err = VipLB.Add(c.VIP, &c.LoadBalancers[x])
-				if err != nil {
-					log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-				}
+		if c.EnableARP {
+			// Gratuitous ARP, will broadcast to new MAC <-> IP
+			err := vip.ARPSendGratuitous(cluster.Network[i].IP(), c.Interface)
+			if err != nil {
+				log.Warn(err.Error())
 			}
 		}
 	}
 
-	if c.GratuitousARP == true {
-		// Gratuitous ARP, will broadcast to new MAC <-> IP
-		err := vip.ARPSendGratuitous(c.VIP, c.Interface)
-		if err != nil {
-			log.Warnf("%v", err)
-		}
-	}
-
 	go func() {
-		for {
-			select {
-			case <-cluster.stop:
-				log.Info("[LOADBALANCER] Stopping load balancers")
+		<-cluster.stop
 
-				// Stop all load balancers associated with the VIP
-				err := VipLB.StopAll()
+		if !disableVIP {
+			for i := range cluster.Network {
+				log.Info("[VIP] Releasing the VIP", "address", cluster.Network[i].IP())
+				deleted, err := cluster.Network[i].DeleteIP()
 				if err != nil {
-					log.Warnf("%v", err)
+					log.Warn(err.Error())
 				}
-
-				// Stop all load balancers associated with the Host
-				err = nonVipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
+				if deleted {
+					log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
 				}
-
-				if !disableVIP {
-
-					log.Info("[VIP] Releasing the Virtual IP")
-					err = cluster.network.DeleteIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-				close(cluster.completed)
-				return
 			}
 		}
+		close(cluster.completed)
 	}()
-	log.Infoln("Started Load Balancer and Virtual IP")
+	log.Info("Started Load Balancer and Virtual IP")
 	return nil
 }
+
+func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp.Server) error {
+	// use a Go context so we can tell the arp loop code when we
+	// want to step down
+	ctxArp, cancelArp := context.WithCancel(context.Background())
+	defer cancelArp()
+
+	// use a Go context so we can tell the dns loop code when we
+	// want to step down
+	ctxDNS, cancelDNS := context.WithCancel(context.Background())
+	defer cancelDNS()
+
+	return cluster.vipService(ctxArp, ctxDNS, c, sm, bgp, nil)
+}

pkg/cluster/state.go

@@ -1,39 +0,0 @@
-package cluster
-
-import (
-	"io"
-
-	"github.com/hashicorp/raft"
-)
-
-// FSM - Finite State Machine for Raft
-type FSM struct {
-}
-
-// Apply - TODO
-func (fsm FSM) Apply(log *raft.Log) interface{} {
-	return nil
-}
-
-// Restore - TODO
-func (fsm FSM) Restore(snap io.ReadCloser) error {
-	return nil
-}
-
-// Snapshot - TODO, returns an empty snapshot
-func (fsm FSM) Snapshot() (raft.FSMSnapshot, error) {
-	return Snapshot{}, nil
-}
-
-// Snapshot -
-type Snapshot struct {
-}
-
-// Persist -
-func (snapshot Snapshot) Persist(sink raft.SnapshotSink) error {
-	return nil
-}
-
-// Release -
-func (snapshot Snapshot) Release() {
-}

pkg/detector/interfaces.go

@@ -22,7 +22,7 @@ func FindIPAddress(addrName string) (string, string, error) {
 		}
 		for _, a := range addrs {
 			if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
-				if ipnet.IP.To4() != nil {
+				if ipnet.IP.To16() != nil {
 					address = ipnet.IP.String()
 					// If we're not searching for a specific adapter return the first one
 					if addrName == "" {

pkg/egress/egress.go

@@ -0,0 +1,99 @@
+package egress
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/kube-vip/kube-vip/pkg/iptables"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/nftables"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
+
+func Teardown(podIP, vipIP, namespace, serviceUUID string, annotations map[string]string, useNftables bool) error {
+	// Look up the destination ports from the annotations on the service
+	destinationPorts := annotations[kubevip.EgressDestinationPorts]
+	deniedNetworks := annotations[kubevip.EgressDeniedNetworks]
+	allowedNetworks := annotations[kubevip.EgressAllowedNetworks]
+	internalEgress := annotations[kubevip.EgressInternal]
+
+	protocol := iptables.ProtocolIPv4
+	IPv6 := false
+	if vip.IsIPv6(podIP) {
+		protocol = iptables.ProtocolIPv6
+		IPv6 = true
+	}
+
+	// Use the internal egress implementation
+	if internalEgress != "" {
+		return nftables.DeleteSNAT(IPv6, serviceUUID)
+	}
+
+	i, err := vip.CreateIptablesClient(useNftables, namespace, protocol)
+	if err != nil {
+		return fmt.Errorf("error Creating iptables client [%s]", err)
+	}
+
+	if deniedNetworks != "" {
+		networks := strings.Split(deniedNetworks, ",")
+		for x := range networks {
+			err = i.DeleteMangleReturnForNetwork(vip.MangleChainName, networks[x])
+			if err != nil {
+				return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
+			}
+		}
+	}
+
+	if allowedNetworks != "" {
+		networks := strings.Split(allowedNetworks, ",")
+		for x := range networks {
+			err = i.DeleteMangleMarkingForNetwork(podIP, vip.MangleChainName, networks[x])
+			if err != nil {
+				return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
+			}
+		}
+	} else {
+		// Remove the marking of egress packets
+		err = i.DeleteMangleMarking(podIP, vip.MangleChainName)
+		if err != nil {
+			return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+		}
+	}
+
+	// Clear up SNAT rules
+	if destinationPorts != "" {
+		fixedPorts := strings.Split(destinationPorts, ",")
+
+		for _, fixedPort := range fixedPorts {
+			var proto, port string
+
+			data := strings.Split(fixedPort, ":")
+			if len(data) == 0 {
+				continue
+			} else if len(data) == 1 {
+				proto = "tcp"
+				port = data[0]
+			} else {
+				proto = data[0]
+				port = data[1]
+			}
+
+			err = i.DeleteSourceNatForDestinationPort(podIP, vipIP, port, proto)
+			if err != nil {
+				return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+			}
+
+		}
+	} else {
+		err = i.DeleteSourceNat(podIP, vipIP)
+		if err != nil {
+			return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+		}
+	}
+
+	err = vip.DeleteExistingSessions(podIP, false, destinationPorts, "")
+	if err != nil {
+		return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+	}
+	return nil
+}

pkg/endpoints/endpoints.go

@@ -0,0 +1,165 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	"net"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	v1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	"k8s.io/apimachinery/pkg/watch"
+)
+
+type Processor struct {
+	config    *kubevip.Config
+	provider  providers.Provider
+	bgpServer *bgp.Server
+	worker    endpointWorker
+	instances *[]*instance.Instance
+}
+
+func NewEndpointProcessor(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
+	instances *[]*instance.Instance) *Processor {
+	return &Processor{
+		config:    config,
+		provider:  provider,
+		bgpServer: bgpServer,
+		instances: instances,
+		worker:    newEndpointWorker(config, provider, bgpServer, instances),
+	}
+}
+
+func (p *Processor) AddOrModify(ctx *servicecontext.Context, event watch.Event,
+	lastKnownGoodEndpoint *string, service *v1.Service, id string, leaderElectionActive *bool,
+	serviceFunc func(context.Context, *v1.Service) error,
+	leaderCtx *context.Context, cancel *context.CancelFunc) (bool, error) {
+
+	var err error
+	if err = p.provider.LoadObject(event.Object, *cancel); err != nil {
+		return false, fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
+	}
+
+	endpoints, err := p.worker.getEndpoints(service, id)
+	if err != nil {
+		return false, err
+	}
+
+	if err := p.worker.setInstanceEndpointsStatus(service, endpoints); err != nil {
+		log.Error("updating instance", "err", err)
+	}
+
+	// Find out if we have any local endpoints
+	// if out endpoint is empty then populate it
+	// if not, go through the endpoints and see if ours still exists
+	// If we have a local endpoint then begin the leader Election, unless it's already running
+	//
+
+	// Check that we have local endpoints
+	if len(endpoints) != 0 {
+		// Ignore IPv4
+		if service.Annotations[kubevip.EgressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
+			return true, nil
+		}
+
+		p.updateLastKnownGoodEndpoint(lastKnownGoodEndpoint, endpoints, service, leaderElectionActive, *cancel)
+		// start leader election if it's enabled and not already started
+		if !*leaderElectionActive && p.config.EnableServicesElection {
+			go func() {
+				*leaderCtx, *cancel = context.WithCancel(ctx.Ctx)
+				startLeaderElection(*leaderCtx, leaderElectionActive, service, serviceFunc)
+			}()
+		}
+
+		// There are local endpoints available on the node
+		if !p.config.EnableServicesElection && !p.config.EnableLeaderElection {
+			if err := p.worker.processInstance(ctx, service, leaderElectionActive); err != nil {
+				return false, fmt.Errorf("failed to process non-empty instance: %w", err)
+			}
+		}
+	} else {
+		// There are no local endpoints
+		p.worker.clear(ctx, lastKnownGoodEndpoint, service, *cancel, leaderElectionActive)
+	}
+
+	// Set the service accordingly
+	p.updateAnnotations(service, lastKnownGoodEndpoint)
+
+	log.Debug("watcher", "provider",
+		p.provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", *lastKnownGoodEndpoint, "active leader election", *leaderElectionActive)
+
+	return false, nil
+}
+
+func (p *Processor) Delete(service *v1.Service, id string) error {
+	if err := p.worker.delete(service, id); err != nil {
+		return fmt.Errorf("[%s] error deleting service: %w", p.provider.GetLabel(), err)
+	}
+	return nil
+}
+
+func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service, leaderElectionActive *bool, cancel context.CancelFunc) {
+	// if we haven't populated one, then do so
+	if *lastKnownGoodEndpoint == "" {
+		*lastKnownGoodEndpoint = endpoints[0]
+		return
+	}
+
+	// check out previous endpoint exists
+	stillExists := false
+
+	for x := range endpoints {
+		if endpoints[x] == *lastKnownGoodEndpoint {
+			stillExists = true
+		}
+	}
+	// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
+	if !stillExists {
+		p.worker.removeEgress(service, lastKnownGoodEndpoint)
+		if *leaderElectionActive && (p.config.EnableServicesElection || p.config.EnableLeaderElection) {
+			log.Warn("existing endpoint has been removed, restarting leaderElection", "provider", p.provider.GetLabel(), "endpoint", *lastKnownGoodEndpoint)
+			// Stop the existing leaderElection
+			cancel()
+			// disable last leaderElection flag
+			*leaderElectionActive = false
+		}
+		// Set our active endpoint to an existing one
+		*lastKnownGoodEndpoint = endpoints[0]
+	}
+}
+
+func (p *Processor) updateAnnotations(service *v1.Service, lastKnownGoodEndpoint *string) {
+	// Set the service accordingly
+	if service.Annotations[kubevip.Egress] == "true" {
+		activeEndpointAnnotation := kubevip.ActiveEndpoint
+
+		if !p.config.EnableEndpoints && p.provider.GetProtocol() == string(discoveryv1.AddressTypeIPv6) {
+			activeEndpointAnnotation = kubevip.ActiveEndpointIPv6
+		}
+		service.Annotations[activeEndpointAnnotation] = *lastKnownGoodEndpoint
+	}
+}
+
+func startLeaderElection(ctx context.Context, leaderElectionActive *bool, service *v1.Service, serviceFunc func(context.Context, *v1.Service) error) {
+	// This is a blocking function, that will restart (in the event of failure)
+	for {
+		// if the context isn't cancelled restart
+		if ctx.Err() != context.Canceled {
+			*leaderElectionActive = true
+			err := serviceFunc(ctx, service)
+			if err != nil {
+				log.Error(err.Error())
+			}
+			*leaderElectionActive = false
+		} else {
+			*leaderElectionActive = false
+			break
+		}
+	}
+}

pkg/endpoints/endpoints_bgp.go

@@ -0,0 +1,125 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	v1 "k8s.io/api/core/v1"
+)
+
+type BGP struct {
+	generic
+	bgpServer *bgp.Server
+}
+
+func newBGP(generic generic, bgpServer *bgp.Server) endpointWorker {
+	return &BGP{
+		generic:   generic,
+		bgpServer: bgpServer,
+	}
+}
+
+func (b *BGP) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
+	if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
+		for _, cluster := range instance.Clusters {
+			for i := range cluster.Network {
+				if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) {
+					log.Debug("attempting to advertise BGP service", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP())
+					err := b.bgpServer.AddHost(cluster.Network[i].CIDR())
+					if err != nil {
+						log.Error("error adding BGP host", "provider", b.provider.GetLabel(), "err", err)
+					} else {
+						log.Info("added BGP host", "provider",
+							b.provider.GetLabel(), "ip", cluster.Network[i].CIDR(), "service name", service.Name, "namespace", service.Namespace)
+						ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
+						*leaderElectionActive = true
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (b *BGP) clear(ctx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
+	if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
+		// If BGP mode is enabled - routes should be deleted
+		if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
+			for _, cluster := range instance.Clusters {
+				for i := range cluster.Network {
+					err := b.bgpServer.DelHost(cluster.Network[i].CIDR())
+					if err != nil {
+						log.Error("deleting BGP host", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "err", err)
+					} else {
+						log.Info("deleted BGP host", "provider",
+							b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace)
+						ctx.ConfiguredNetworks.Delete(cluster.Network[i])
+						*leaderElectionActive = false
+					}
+				}
+			}
+
+		}
+	}
+
+	b.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
+}
+
+func (b *BGP) getEndpoints(service *v1.Service, id string) ([]string, error) {
+	return b.getAllEndpoints(service, id)
+}
+
+func (b *BGP) delete(service *v1.Service, id string) error {
+	// When no-leader-elecition mode
+	if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
+		// find all existing local endpoints
+		endpoints, err := b.getEndpoints(service, id)
+		if err != nil {
+			return fmt.Errorf("[%s] error getting endpoints: %w", b.provider.GetLabel(), err)
+		}
+
+		// If there were local endpoints deleted
+		if len(endpoints) > 0 {
+			b.deleteAction(service)
+		}
+	}
+
+	return nil
+}
+
+func (b *BGP) deleteAction(service *v1.Service) {
+	b.clearBGPHosts(service)
+}
+
+func (b *BGP) clearBGPHosts(service *v1.Service) {
+	ClearBGPHosts(service, b.instances, b.bgpServer)
+}
+
+func (b *BGP) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
+	return nil
+}
+
+func ClearBGPHosts(service *v1.Service, instances *[]*instance.Instance, bgpServer *bgp.Server) {
+	if instance := instance.FindServiceInstance(service, *instances); instance != nil {
+		ClearBGPHostsByInstance(instance, bgpServer)
+	}
+}
+
+func ClearBGPHostsByInstance(instance *instance.Instance, bgpServer *bgp.Server) {
+	for _, cluster := range instance.Clusters {
+		for i := range cluster.Network {
+			network := cluster.Network[i]
+			err := bgpServer.DelHost(network.CIDR())
+			if err != nil {
+				log.Error("[endpoint] error deleting BGP host", "err", err)
+			} else {
+				log.Debug("[endpoint] deleted BGP host", "ip",
+					network.CIDR(), "service name", instance.ServiceSnapshot.Name, "namespace", instance.ServiceSnapshot.Namespace)
+			}
+		}
+	}
+}

pkg/endpoints/endpoints_generic.go

@@ -0,0 +1,118 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/egress"
+	"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	v1 "k8s.io/api/core/v1"
+)
+
+type endpointWorker interface {
+	processInstance(svcCtx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error
+	clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool)
+	getEndpoints(service *v1.Service, id string) ([]string, error)
+	removeEgress(service *v1.Service, lastKnownGoodEndpoint *string)
+	delete(service *v1.Service, id string) error
+	setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error
+}
+
+func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server, instances *[]*instance.Instance) endpointWorker {
+	generic := newGeneric(config, provider, instances)
+
+	if config.EnableRoutingTable {
+		return newRoutingTable(generic)
+	}
+	if config.EnableBGP {
+		return newBGP(generic, bgpServer)
+	}
+
+	return &generic
+}
+
+type generic struct {
+	config    *kubevip.Config
+	provider  providers.Provider
+	instances *[]*instance.Instance
+}
+
+func newGeneric(config *kubevip.Config, provider providers.Provider, instances *[]*instance.Instance) generic {
+	return generic{
+		config:    config,
+		provider:  provider,
+		instances: instances,
+	}
+}
+
+func (g *generic) processInstance(_ *servicecontext.Context, _ *v1.Service, _ *bool) error {
+	return nil
+}
+
+func (g *generic) clear(_ *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
+	g.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
+}
+
+func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
+	if *lastKnownGoodEndpoint != "" {
+		log.Warn("existing  endpoint has been removed, no remaining endpoints for leaderElection", "provider", g.provider.GetLabel(), "endpoint", lastKnownGoodEndpoint)
+		if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP, service.Namespace, string(service.UID), service.Annotations, g.config.EgressWithNftables); err != nil {
+			log.Error("error removing redundant egress rules", "err", err)
+		}
+
+		*lastKnownGoodEndpoint = "" // reset endpoint
+		if g.config.EnableServicesElection || g.config.EnableLeaderElection {
+			cancel() // stop services watcher
+		}
+		*leaderElectionActive = false
+	}
+}
+
+func (g *generic) getEndpoints(_ *v1.Service, id string) ([]string, error) {
+	return g.getLocalEndpoints(id)
+}
+
+func (g *generic) getLocalEndpoints(id string) ([]string, error) {
+	// Build endpoints
+	var endpoints []string
+	var err error
+	if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
+		return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
+	}
+
+	return endpoints, nil
+}
+
+func (g *generic) getAllEndpoints(service *v1.Service, id string) ([]string, error) {
+	// Build endpoints
+	var err error
+	var endpoints []string
+	if !g.config.EnableLeaderElection && !g.config.EnableServicesElection &&
+		service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
+		if endpoints, err = g.provider.GetAllEndpoints(); err != nil {
+			return nil, fmt.Errorf("[%s] error getting all endpoints: %w", g.provider.GetLabel(), err)
+		}
+	} else {
+		if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
+			return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
+		}
+	}
+
+	return endpoints, nil
+}
+
+func (g *generic) removeEgress(_ *v1.Service, _ *string) {
+}
+
+func (g *generic) delete(_ *v1.Service, _ string) error {
+	return nil
+}
+
+func (g *generic) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
+	return nil
+}

pkg/endpoints/endpoints_routing_table.go

@@ -0,0 +1,178 @@
+package endpoints
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"syscall"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/egress"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	"github.com/vishvananda/netlink"
+	v1 "k8s.io/api/core/v1"
+)
+
+type RoutingTable struct {
+	generic
+}
+
+func newRoutingTable(generic generic) endpointWorker {
+	return &RoutingTable{
+		generic: generic,
+	}
+}
+
+func (rt *RoutingTable) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
+	instance := instance.FindServiceInstance(service, *rt.instances)
+	if instance != nil {
+		for _, cluster := range instance.Clusters {
+			for i := range cluster.Network {
+				if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) && cluster.Network[i].HasEndpoints() {
+					err := cluster.Network[i].AddRoute(false)
+					if err != nil {
+						if errors.Is(err, syscall.EEXIST) {
+							// If route exists, but protocol is not set (e.g. the route was created by the older version
+							// of kube-vip) try to update it if necessary
+							isUpdated, err := cluster.Network[i].UpdateRoutes()
+							if err != nil {
+								return fmt.Errorf("[%s] error updating existing routes: %w", rt.provider.GetLabel(), err)
+							}
+							if isUpdated {
+								log.Info("updated route", "provider",
+									rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
+									service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
+							} else {
+								log.Info("route already present", "provider",
+									rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
+									service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
+							}
+						} else {
+							// If other error occurs, return error
+							return fmt.Errorf("[%s] error adding route: %s", rt.provider.GetLabel(), err.Error())
+						}
+					} else {
+						log.Info("added route", "provider",
+							rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
+							service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
+						ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
+						*leaderElectionActive = true
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
+	if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
+		if errs := ClearRoutes(service, rt.instances); len(errs) == 0 {
+			svcCtx.ConfiguredNetworks.Clear()
+		} else {
+			for _, err := range errs {
+				log.Error("error while clearing routes", "err", err)
+			}
+		}
+	}
+
+	rt.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
+}
+
+func (rt *RoutingTable) getEndpoints(service *v1.Service, id string) ([]string, error) {
+	return rt.getAllEndpoints(service, id)
+}
+
+func (rt *RoutingTable) removeEgress(service *v1.Service, lastKnownGoodEndpoint *string) {
+	if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
+		service.Namespace, string(service.UID), service.Annotations, rt.config.EgressWithNftables); err != nil {
+		log.Warn("removing redundant egress rules", "err", err)
+	}
+}
+
+func (rt *RoutingTable) delete(service *v1.Service, id string) error {
+	// When no-leader-elecition mode
+	if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
+		// find all existing local endpoints
+		endpoints, err := rt.getEndpoints(service, id)
+		if err != nil {
+			return fmt.Errorf("[%s] error getting endpoints: %w", rt.provider.GetLabel(), err)
+		}
+
+		// If there were local endpoints deleted
+		if len(endpoints) > 0 {
+			rt.deleteAction(service)
+		}
+	}
+
+	return nil
+}
+
+func (rt *RoutingTable) deleteAction(service *v1.Service) {
+	ClearRoutes(service, rt.instances)
+}
+
+func (rt *RoutingTable) setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error {
+	instance := instance.FindServiceInstance(service, *rt.instances)
+	if instance == nil {
+		log.Error("failed to find the instance", "service", service.UID, "provider", rt.provider.GetLabel())
+	} else {
+		for _, c := range instance.Clusters {
+			for n := range c.Network {
+				// if there are no endpoints set HasEndpoints false just in case
+				if len(endpoints) < 1 {
+					c.Network[n].SetHasEndpoints(false)
+				}
+				// check if endpoint are available and are of same IP family as service
+				if len(endpoints) > 0 && ((net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(endpoints[0]).To4() == nil)) {
+					c.Network[n].SetHasEndpoints(true)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func ClearRoutes(service *v1.Service, instances *[]*instance.Instance) []error {
+	errs := []error{}
+	if instance := instance.FindServiceInstance(service, *instances); instance != nil {
+		for _, cluster := range instance.Clusters {
+			for i := range cluster.Network {
+				route := cluster.Network[i].PrepareRoute()
+				// check if route we are about to delete is not referenced by more than one service
+				if CountRouteReferences(route, instances) <= 1 {
+					err := cluster.Network[i].DeleteRoute()
+					if err != nil && !errors.Is(err, syscall.ESRCH) {
+						log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
+						errs = append(errs, err)
+					}
+					log.Debug("deleted route", "ip",
+						cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface())
+				}
+			}
+		}
+	}
+	return errs
+}
+
+func CountRouteReferences(route *netlink.Route, instances *[]*instance.Instance) int {
+	cnt := 0
+	for _, instance := range *instances {
+		for _, cluster := range instance.Clusters {
+			for n := range cluster.Network {
+				if cluster.Network[n].HasEndpoints() {
+					r := cluster.Network[n].PrepareRoute()
+					if r.Dst.String() == route.Dst.String() {
+						cnt++
+					}
+				}
+			}
+		}
+	}
+	return cnt
+}

pkg/endpoints/providers/endpoints.go

@@ -0,0 +1,137 @@
+package providers
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	watchtools "k8s.io/client-go/tools/watch"
+	"k8s.io/client-go/util/retry"
+
+	log "log/slog"
+)
+
+type Endpoints struct {
+	label string
+	//nolint:staticcheck // SA1019 endpoints are moving to an opt-in only
+	endpoints *v1.Endpoints
+}
+
+func NewEndpoints() Provider {
+	return &Endpoints{
+		label: "endpoints",
+	}
+}
+
+func (ep *Endpoints) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
+	service *v1.Service) (*watchtools.RetryWatcher, error) {
+	opts := metav1.ListOptions{
+		FieldSelector: fields.OneTermEqualSelector("metadata.name", service.Name).String(),
+	}
+
+	rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
+		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
+			return clientSet.CoreV1().Endpoints(service.Namespace).Watch(ctx, opts)
+		},
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("error creating endpoint watcher: %s", err.Error())
+	}
+
+	return rw, nil
+}
+
+func (ep *Endpoints) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
+	//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
+	eps, ok := endpoints.(*v1.Endpoints)
+	if !ok {
+		cancel()
+		return fmt.Errorf("[%s] unable to parse Kubernetes services from API watcher", ep.GetLabel())
+	}
+	ep.endpoints = eps
+	return nil
+}
+
+func (ep *Endpoints) GetAllEndpoints() ([]string, error) {
+	result := []string{}
+	for subset := range ep.endpoints.Subsets {
+		for address := range ep.endpoints.Subsets[subset].Addresses {
+			addr := strings.Split(ep.endpoints.Subsets[subset].Addresses[address].IP, "/")
+			result = append(result, addr[0])
+		}
+	}
+
+	return result, nil
+}
+
+func (ep *Endpoints) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
+	var localEndpoints []string
+
+	for _, subset := range ep.endpoints.Subsets {
+		for _, address := range subset.Addresses {
+			log.Debug("processing endpoint", "label", ep.label, "ip", address.IP)
+
+			// 1. Compare the Nodename
+			if address.NodeName != nil && id == *address.NodeName {
+				log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname, "nodename", *address.NodeName)
+				localEndpoints = append(localEndpoints, address.IP)
+				continue
+			}
+			// 2. Compare the Hostname (only useful if address.NodeName is not available)
+			if id == address.Hostname {
+				log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname)
+				localEndpoints = append(localEndpoints, address.IP)
+				continue
+			}
+		}
+	}
+	return localEndpoints, nil
+}
+
+func (ep *Endpoints) UpdateServiceAnnotation(endpoint string, _ string, service *v1.Service, clientSet *kubernetes.Clientset) error {
+	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		// Retrieve the latest version of Deployment before attempting update
+		// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
+		currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		currentServiceCopy := currentService.DeepCopy()
+		if currentServiceCopy.Annotations == nil {
+			currentServiceCopy.Annotations = make(map[string]string)
+		}
+
+		currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
+
+		_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
+		if err != nil {
+			log.Error("error updating Service Spec", "label", ep.GetLabel(), "name", currentServiceCopy.Name, "err", err)
+			return err
+		}
+		return nil
+	})
+
+	if retryErr != nil {
+		log.Error("failed to set Services", "label", ep.GetLabel(), "err", retryErr)
+		return retryErr
+	}
+	return nil
+}
+
+func (ep *Endpoints) GetLabel() string {
+	return ep.label
+}
+
+func (ep *Endpoints) GetProtocol() string {
+	return ""
+}

pkg/endpoints/providers/endpointslices.go

@@ -0,0 +1,139 @@
+package providers
+
+import (
+	"context"
+	"fmt"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	v1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	watchtools "k8s.io/client-go/tools/watch"
+	"k8s.io/client-go/util/retry"
+)
+
+type Endpointslices struct {
+	label     string
+	endpoints *discoveryv1.EndpointSlice
+}
+
+func NewEndpointslices() Provider {
+	return &Endpointslices{
+		label: "endpointslices",
+	}
+}
+
+func (ep *Endpointslices) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
+	service *v1.Service) (*watchtools.RetryWatcher, error) {
+	labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/service-name": service.Name}}
+
+	opts := metav1.ListOptions{
+		LabelSelector: labels.Set(labelSelector.MatchLabels).String(),
+	}
+
+	rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
+		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
+			return clientSet.DiscoveryV1().EndpointSlices(service.Namespace).Watch(ctx, opts)
+		},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("[%s] error creating endpointslices watcher: %s", ep.label, err.Error())
+	}
+
+	return rw, nil
+}
+
+func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
+	eps, ok := endpoints.(*discoveryv1.EndpointSlice)
+	if !ok {
+		cancel()
+		return fmt.Errorf("[%s] error casting endpoints to v1.Endpoints struct", ep.label)
+	}
+	ep.endpoints = eps
+	return nil
+}
+
+func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
+	result := []string{}
+	for _, ep := range ep.endpoints.Endpoints {
+		result = append(result, ep.Addresses...)
+	}
+	return result, nil
+}
+
+func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
+	var localEndpoints []string
+	for _, endpoint := range ep.endpoints.Endpoints {
+		if !*endpoint.Conditions.Serving {
+			continue
+		}
+		for _, address := range endpoint.Addresses {
+			log.Debug("processing endpoint", "provider", ep.label, "ip", address)
+
+			// 1. Compare the Nodename
+			if endpoint.NodeName != nil && id == *endpoint.NodeName {
+				if endpoint.Hostname != nil {
+					log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname, "nodename", *endpoint.NodeName)
+				} else {
+					log.Debug("found endpoint", "provider", ep.label, "ip", address, "nodename", *endpoint.NodeName)
+				}
+				localEndpoints = append(localEndpoints, address)
+				continue
+			}
+
+			// 2. Compare the Hostname (only useful if endpoint.NodeName is not available)
+			if endpoint.Hostname != nil && id == *endpoint.Hostname {
+				log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname)
+				localEndpoints = append(localEndpoints, address)
+			}
+		}
+	}
+	return localEndpoints, nil
+}
+
+func (ep *Endpointslices) UpdateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, clientSet *kubernetes.Clientset) error {
+	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		// Retrieve the latest version of Deployment before attempting update
+		// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
+		currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		currentServiceCopy := currentService.DeepCopy()
+		if currentServiceCopy.Annotations == nil {
+			currentServiceCopy.Annotations = make(map[string]string)
+		}
+
+		currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
+		currentServiceCopy.Annotations[kubevip.ActiveEndpointIPv6] = endpointIPv6
+
+		_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
+		if err != nil {
+			log.Error("error updating Service Spec", "provider", ep.label, "service name", currentServiceCopy.Name, "err", err)
+			return err
+		}
+		return nil
+	})
+
+	if retryErr != nil {
+		log.Error("failed to set Services", "provider", ep.label, "err", retryErr)
+		return retryErr
+	}
+	return nil
+}
+
+func (ep *Endpointslices) GetLabel() string {
+	return ep.label
+}
+
+func (ep *Endpointslices) GetProtocol() string {
+	return string(ep.endpoints.AddressType)
+}

pkg/endpoints/providers/interface.go

@@ -0,0 +1,22 @@
+package providers
+
+import (
+	"context"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
+	watchtools "k8s.io/client-go/tools/watch"
+)
+
+type Provider interface {
+	CreateRetryWatcher(context.Context, *kubernetes.Clientset,
+		*v1.Service) (*watchtools.RetryWatcher, error)
+	GetAllEndpoints() ([]string, error)
+	GetLocalEndpoints(string, *kubevip.Config) ([]string, error)
+	GetLabel() string
+	UpdateServiceAnnotation(string, string, *v1.Service, *kubernetes.Clientset) error
+	LoadObject(runtime.Object, context.CancelFunc) error
+	GetProtocol() string
+}

pkg/etcd/client.go

@@ -0,0 +1,26 @@
+package etcd
+
+import (
+	"go.etcd.io/etcd/client/pkg/v3/transport"
+	clientv3 "go.etcd.io/etcd/client/v3"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+)
+
+func NewClient(c *kubevip.Config) (*clientv3.Client, error) {
+	tlsInfo := transport.TLSInfo{
+		TrustedCAFile: c.Etcd.CAFile,
+		CertFile:      c.Etcd.ClientCertFile,
+		KeyFile:       c.Etcd.ClientKeyFile,
+	}
+
+	clientTLS, err := tlsInfo.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	return clientv3.New(clientv3.Config{
+		Endpoints: c.Etcd.Endpoints,
+		TLS:       clientTLS,
+	})
+}

pkg/etcd/election.go

@@ -0,0 +1,227 @@
+package etcd
+
+import (
+	"context"
+	"hash/fnv"
+	"time"
+
+	log "log/slog"
+
+	"github.com/pkg/errors"
+	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/concurrency"
+)
+
+// LeaderElectionConfig allows to configure the leader election params.
+type LeaderElectionConfig struct {
+	// EtcdConfig contains the client to connect to the etcd cluster.
+	EtcdConfig ClientConfig
+
+	// Name uniquely identifies this leader election. All members of the same election
+	// should use the same value here.
+	Name string
+
+	// MemberID identifies uniquely this contestant from other in the leader election.
+	// It will be converted to an int64 using a hash, so theoretically collisions are possible
+	// when using a string. If you want to guarantee safety, us MemberUniqueID to specify a unique
+	// int64 directly.
+	// If two processes start a leader election using the same MemberID, one of them will
+	// fail.
+	MemberID string
+
+	// MemberUniqueID is the int equivalent to MemberID that allows to override the default conversion
+	// from string to int using hashing.
+	MemberUniqueID *uint64
+
+	// LeaseDurationSeconds is the duration that non-leader candidates will
+	// wait to force acquire leadership.
+	// This is just a request to the etcd server but it's not guaranteed, the server
+	// might decide to make the duration longer.
+	LeaseDurationSeconds int64
+
+	// Callbacks are callbacks that are triggered during certain lifecycle
+	// events of the LeaderElector
+	Callbacks LeaderCallbacks
+}
+
+// LeaderCallbacks are callbacks that are triggered during certain
+// lifecycle events of the election.
+type LeaderCallbacks struct {
+	// OnStartedLeading is called when this member starts leading.
+	OnStartedLeading func(context.Context)
+	// OnStoppedLeading is called when this member stops leading.
+	OnStoppedLeading func()
+	// OnNewLeader is called when the client observes a leader that is
+	// not the previously observed leader. This includes the first observed
+	// leader when the client starts.
+	OnNewLeader func(identity string)
+}
+
+// ClientConfig contains the client to connect to the etcd cluster.
+type ClientConfig struct {
+	Client *clientv3.Client
+}
+
+// RunElectionOrDie behaves the same way as RunElection but panics if there is an error.
+func RunElectionOrDie(ctx context.Context, config *LeaderElectionConfig) {
+	if err := RunElection(ctx, config); err != nil {
+		panic(err)
+	}
+}
+
+// RunElection starts a client with the provided config or panics.
+// RunElection blocks until leader election loop is
+// stopped by ctx or it has stopped holding the leader lease.
+func RunElection(ctx context.Context, config *LeaderElectionConfig) error {
+	var memberID uint64
+	if config.MemberUniqueID != nil {
+		memberID = *config.MemberUniqueID
+	} else {
+		h := fnv.New64a()
+		if _, err := h.Write(append([]byte(config.Name), []byte(config.MemberID)...)); err != nil {
+			return err
+		}
+		memberID = h.Sum64()
+	}
+
+	ttl := config.LeaseDurationSeconds
+	r := &pb.LeaseGrantRequest{TTL: ttl, ID: int64(memberID)} //nolint
+	lease, err := clientv3.RetryLeaseClient(
+		config.EtcdConfig.Client,
+	).LeaseGrant(ctx, r)
+	if err != nil {
+		return errors.Wrap(err, "creating lease")
+	}
+
+	leaseID := clientv3.LeaseID(lease.ID)
+
+	s, err := concurrency.NewSession(
+		config.EtcdConfig.Client,
+		concurrency.WithTTL(int(lease.TTL)),
+		concurrency.WithLease(leaseID),
+	)
+	if err != nil {
+		return err
+	}
+
+	election := concurrency.NewElection(s, config.Name)
+
+	m := &member{
+		client:         config.EtcdConfig.Client,
+		election:       election,
+		callbacks:      config.Callbacks,
+		memberID:       config.MemberID,
+		weAreTheLeader: make(chan struct{}, 1),
+		leaseTTL:       lease.TTL,
+	}
+
+	go m.tryToBeLeader(ctx)
+	m.watchLeaderChanges(ctx)
+
+	return nil
+}
+
+type member struct {
+	key              string
+	client           *clientv3.Client
+	election         *concurrency.Election
+	isLeader         bool
+	currentLeaderKey string
+	callbacks        LeaderCallbacks
+	memberID         string
+	weAreTheLeader   chan struct{}
+	leaseTTL         int64
+}
+
+func (m *member) watchLeaderChanges(ctx context.Context) {
+	observeCtx, observeCancel := context.WithCancel(ctx)
+	defer observeCancel()
+	changes := m.election.Observe(observeCtx)
+
+watcher:
+	for {
+		select {
+		case <-ctx.Done():
+			break watcher
+		case <-m.weAreTheLeader:
+
+			m.isLeader = true
+			m.key = m.election.Key() // by this time, this should already be set, since Campaign has already returned
+			log.Debug("Marking self as leader with key", "id", m.memberID, "key", m.key)
+		case response := <-changes:
+			log.Debug("Leader Changes", "id", m.memberID, "response", response)
+			if len(response.Kvs) == 0 {
+				// There is a race condition where just after we stop being the leader
+				// if there are no more leaders, we might get a response with no key-values
+				// just before the response channel is closed or the context is cancel
+				// In that case, just continue and let one of those two things happen
+				continue
+			}
+			newLeaderKey := response.Kvs[0].Key
+			if m.isLeader && m.key != string(newLeaderKey) {
+				// We stopped being leaders
+
+				// exit the loop, so we cancel the observe context so we stop watching
+				// for new leaders. That will close the channel and make this function exit,
+				// which also makes the routine to finish and RunElection returns
+				break watcher
+			}
+
+			if m.currentLeaderKey != string(newLeaderKey) {
+				// we observed a leader, this could be us or someone else
+				m.currentLeaderKey = string(newLeaderKey)
+				m.callbacks.OnNewLeader(string(response.Kvs[0].Value))
+			}
+		}
+	}
+
+	// If we are here, either we have stopped being leaders or we lost the watcher
+	// Make sure we call OnStoppedLeading if we were the leader.
+	if m.isLeader {
+		m.callbacks.OnStoppedLeading()
+	}
+
+	log.Debug("Exiting watcher", "id", m.memberID)
+}
+
+func (m *member) tryToBeLeader(ctx context.Context) {
+	if err := m.election.Campaign(ctx, m.memberID); err != nil {
+		log.Error("Failed trying to become the leader", "err", err)
+		// Resign just in case we acquired leadership just before failing
+		if err := m.election.Resign(m.client.Ctx()); err != nil {
+			log.Warn("Failed to resign after we failed becoming the leader, this might not be a problem if we were never the leader", "err", err)
+		}
+		return
+		// TODO: what to do here?
+		// We probably want watchLeaderChanges to exit as well, since Run
+		// is expecting us to try to become the leader, but if we are here,
+		// we won't. So if we don't panic, we need to signal it somehow
+	}
+
+	// Inform the observer that we are the leader as soon as possible,
+	// so it can detect if we stop being it
+	m.weAreTheLeader <- struct{}{}
+
+	// Once we are the leader, start the routine to resign if context is canceled
+	go m.resignOnCancel(ctx)
+
+	// After becoming the leader, we wait for at least a lease TTL to wait for
+	// the previous leader to detect the new leadership (if there was one) and
+	// stop its processes
+	// TODO: is this too cautious?
+	log.Debug("timeout before OnStartedLeading", "id", m.memberID, "timeout", m.leaseTTL)
+	time.Sleep(time.Second * time.Duration(m.leaseTTL))
+
+	// We are the leader, execute our code
+	m.callbacks.OnStartedLeading(ctx)
+
+	// Here the routine dies if OnStartedLeading doesn't block, there is nothing else to do
+}
+
+func (m *member) resignOnCancel(ctx context.Context) {
+	<-ctx.Done()
+	if err := m.election.Resign(m.client.Ctx()); err != nil {
+		log.Error("Failed to resign after the context was canceled", "err", err)
+	}
+}

pkg/etcd/election_test.go

@@ -0,0 +1,169 @@
+//go:build integration
+// +build integration
+
+package etcd_test
+
+import (
+	"context"
+	"log"
+	"math/rand"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/etcd"
+	. "github.com/onsi/gomega"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.uber.org/zap"
+)
+
+func TestRunElectionWithMemberIDCollision(t *testing.T) {
+	t.Parallel()
+	g := NewWithT(t)
+	ctx := context.Background()
+	cli := client(g)
+	defer cli.Close()
+
+	electionName := randomElectionNameForTest("memberIDConflict")
+	log.Printf("Election name %s\n", electionName)
+	memberCtx, cancelMember1 := context.WithCancel(ctx)
+	config := &etcd.LeaderElectionConfig{
+		EtcdConfig: etcd.ClientConfig{
+			Client: cli,
+		},
+		Name:                 electionName,
+		MemberID:             "my-host",
+		LeaseDurationSeconds: 1,
+		Callbacks: etcd.LeaderCallbacks{
+			OnStartedLeading: func(ctx context.Context) {
+				log.Println("I'm the leader!!!!")
+				log.Println("Renouncing as leader by canceling context")
+				cancelMember1()
+			},
+			OnNewLeader: func(identity string) {
+				log.Printf("New leader: %s\n", identity)
+			},
+			OnStoppedLeading: func() {
+				log.Println("I'm not the leader anymore")
+			},
+		},
+	}
+
+	wg := &sync.WaitGroup{}
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		g.Expect(etcd.RunElection(memberCtx, config)).To(Succeed())
+	}()
+
+	go func() {
+		defer wg.Done()
+		time.Sleep(time.Millisecond * 50) // make sure the first one becomes leader
+		g.Expect(etcd.RunElection(ctx, config)).Should(MatchError(ContainSubstring("creating lease")))
+	}()
+
+	wg.Wait()
+}
+
+func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
+	t.Parallel()
+	g := NewWithT(t)
+	ctx := context.Background()
+	cli := client(g)
+	defer cli.Close()
+
+	cliMember1 := client(g)
+	defer cliMember1.Close()
+
+	electionName := randomElectionNameForTest("steppingDown")
+	configBase := etcd.LeaderElectionConfig{
+		EtcdConfig: etcd.ClientConfig{
+			Client: cli,
+		},
+		Name:                 electionName,
+		LeaseDurationSeconds: 1,
+	}
+
+	member1Ctx, _ := context.WithCancel(ctx)
+	member2Ctx, cancelMember2 := context.WithCancel(ctx)
+
+	config1 := configBase
+	config1.EtcdConfig.Client = cliMember1
+	config1.MemberID = "my-host"
+	uniqueID := rand.Uint64()
+	config1.MemberUniqueID = &uniqueID
+	config1.Callbacks = baseCallbacksForName(config1.MemberID)
+	config1.Callbacks.OnStartedLeading = func(_ context.Context) {
+		log.Println("I'm my-host, the new leader!!!!")
+		log.Println("Losing the leadership on purpose by stopping renewing the lease")
+		g.Expect(cliMember1.Lease.Close()).To(Succeed())
+		log.Println("Member1 leases closed")
+	}
+
+	config2 := configBase
+	config2.MemberID = "my-other-host"
+	config2.Callbacks = baseCallbacksForName(config2.MemberID)
+	config2.Callbacks.OnStartedLeading = func(_ context.Context) {
+		log.Println("I'm my-other-host, the new leader!!!!")
+		log.Println("Renouncing as leader by canceling context")
+		cancelMember2()
+	}
+
+	wg := &sync.WaitGroup{}
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		g.Expect(etcd.RunElection(member1Ctx, &config1)).To(Succeed())
+		log.Println("Member1 routine done")
+	}()
+
+	go func() {
+		defer wg.Done()
+		time.Sleep(time.Millisecond * 50) // Make sure member1 becomes leader
+		g.Expect(etcd.RunElection(member2Ctx, &config2)).To(Succeed())
+		log.Println("Member2 routine done")
+	}()
+
+	wg.Wait()
+}
+
+func baseCallbacksForName(name string) etcd.LeaderCallbacks {
+	return etcd.LeaderCallbacks{
+		OnStartedLeading: func(ctx context.Context) {
+			log.Printf("[%s] I'm the new leader!!!!\n", name)
+		},
+		OnNewLeader: func(identity string) {
+			log.Printf("[%s] New leader: %s\n", name, identity)
+		},
+		OnStoppedLeading: func() {
+			log.Printf("[%s] I'm not the leader anymore\n", name)
+		},
+	}
+}
+
+func randomElectionNameForTest(name string) string {
+	return name + "-" + randomString(6)
+}
+
+const charSet = "0123456789abcdefghijklmnopqrstuvwxyz"
+
+var rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
+
+func randomString(n int) string {
+	result := make([]byte, n)
+	for i := range result {
+		result[i] = charSet[rnd.Intn(len(charSet))]
+	}
+	return string(result)
+}
+
+func client(g Gomega) *clientv3.Client {
+	c, err := clientv3.New(clientv3.Config{
+		Endpoints: []string{"localhost:2379"},
+		Logger:    zap.NewNop(),
+	})
+	g.Expect(err).NotTo(HaveOccurred())
+	return c
+}

pkg/etcd/etcd_suite_test.go

@@ -0,0 +1,153 @@
+//go:build integration
+// +build integration
+
+package etcd_test
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	etcdVersion     = "v3.5.9"
+	etcdBinDir      = "bin"
+	etcdBinPath     = etcdBinDir + "/etcd"
+	downloadURL     = "https://storage.googleapis.com/etcd"
+	tmpDownloadFile = "etcd.tar.gz"
+	pidFile         = "etcd.pid"
+)
+
+func TestMain(m *testing.M) {
+	logrus.SetLevel(logrus.DebugLevel)
+	ctx := context.Background()
+	expectSuccess(startEtcd(ctx), "starting etcd")
+
+	os.Exit(runTestsWithCleanup(m, func() {
+		expectSuccess(stopEtcd(), "stopping etcd")
+	}))
+}
+
+func runTestsWithCleanup(m *testing.M, cleanup func()) int {
+	defer cleanup()
+	return m.Run()
+}
+
+func expectSuccess(err error, msg string) {
+	if err != nil {
+		log.Fatalf("%s: %s\n", msg, err)
+	}
+}
+
+func startEtcd(ctx context.Context) error {
+	if _, err := os.Stat(pidFile); err == nil {
+		log.Println("Etcd already running, reusing")
+		return nil
+	}
+
+	etcdPath, err := installEtcd(ctx)
+	if err != nil {
+		errors.Wrap(err, "installing etcd for tests")
+	}
+
+	etcdCmd := exec.Command(etcdPath, "--data-dir", "./etcd-data")
+	if os.Getenv("ETCD_SERVER_LOGS") == "true" {
+		log.Println("Enabling etcd server logs")
+		etcdCmd.Stdout = os.Stdout
+		etcdCmd.Stderr = os.Stderr
+	}
+	log.Println("Starting etcd")
+	if err := etcdCmd.Start(); err != nil {
+		errors.Wrap(err, "starting etcd for tests")
+	}
+
+	if err := os.WriteFile(pidFile, []byte(strconv.Itoa(etcdCmd.Process.Pid)), 0o600); err != nil {
+		return err
+	}
+
+	log.Println("Waiting for etcd to be up")
+	time.Sleep(time.Second)
+
+	return nil
+}
+
+func installEtcd(ctx context.Context) (string, error) {
+	projectRoot, err := filepath.Abs("../../")
+	if err != nil {
+		return "", err
+	}
+	binDir := filepath.Join(projectRoot, etcdBinDir)
+	etcdPath := filepath.Join(projectRoot, etcdBinPath)
+
+	if _, err := os.Stat(etcdPath); err == nil {
+		log.Println("Etcd already installed, skipping")
+		return etcdPath, nil
+	}
+
+	if err := os.MkdirAll(binDir, 0o755); err != nil {
+		return "", err
+	}
+
+	download := fmt.Sprintf("%s/%s/etcd-%s-linux-amd64.tar.gz", downloadURL, etcdVersion, etcdVersion)
+
+	// Hacky to run bash, but simplifies this code a lot
+	cmd := fmt.Sprintf("curl -sL %s | tar -xzvf - -C %s --strip-components=1", download, binDir)
+	out, err := exec.CommandContext(ctx, "bash", "-c", cmd).CombinedOutput()
+	if err != nil {
+		return "", errors.Wrapf(err, "downloading etcd: %s", string(out))
+	}
+
+	return etcdPath, nil
+}
+
+func stopEtcd() error {
+	if os.Getenv("REUSE_ETCD") == "true" {
+		log.Println("REUSE_ETCD=true, leaving etcd running")
+		return nil
+	}
+
+	if _, err := os.Stat(pidFile); os.IsNotExist(err) {
+		log.Println("Etcd pid file doesn't exit, skipping cleanup")
+		return nil
+	}
+
+	dat, err := os.ReadFile(pidFile)
+	if err != nil {
+		return err
+	}
+	pid, err := strconv.Atoi(string(dat))
+	if err != nil {
+		return err
+	}
+
+	etcdProcess, err := os.FindProcess(pid)
+	if err != nil {
+		return err
+	}
+
+	log.Println("Stopping etcd")
+	if err := etcdProcess.Kill(); err != nil {
+		return errors.Wrap(err, "Failed stopping etcd")
+	}
+
+	log.Println("Deleting etcd data")
+	if err := os.RemoveAll("./etcd-data"); err != nil {
+		return errors.Wrap(err, "deleting etcd data")
+	}
+
+	log.Println("Deleting etcd pid file")
+	if err := os.RemoveAll(pidFile); err != nil {
+		return errors.Wrap(err, "deleting pid file")
+	}
+
+	return nil
+}

pkg/instance/instance.go

@@ -0,0 +1,482 @@
+package instance
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+
+	"log/slog"
+	log "log/slog"
+
+	"github.com/vishvananda/netlink"
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
+	"github.com/kube-vip/kube-vip/pkg/cluster"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/networkinterface"
+	"github.com/kube-vip/kube-vip/pkg/sysctl"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
+
+// Instance defines an instance of everything needed to manage vips
+type Instance struct {
+	// Virtual IP / Load Balancer configuration
+	VIPConfigs []*kubevip.Config
+
+	// cluster instances
+	Clusters []*cluster.Cluster
+
+	// Service uses DHCP
+	IsDHCP              bool
+	DHCPInterface       string
+	DHCPInterfaceHwaddr string
+	DHCPInterfaceIP     string
+	DHCPHostname        string
+	DHCPClient          *vip.DHCPClient
+
+	// External Gateway IP the service is forwarded from
+	UPNPGatewayIPs []string
+
+	// Kubernetes service mapping
+	ServiceSnapshot *v1.Service
+}
+
+type Port struct {
+	Port uint16
+	Type string
+}
+
+func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Instance, error) {
+	instanceAddresses := FetchServiceAddresses(svc)
+	//instanceUID := string(svc.UID)
+
+	var newVips []*kubevip.Config
+	var link netlink.Link
+	var err error
+
+	for _, address := range instanceAddresses {
+		// Detect if we're using a specific interface for services
+		var svcInterface string
+		svcInterface = svc.Annotations[kubevip.ServiceInterface] // If the service has a specific interface defined, then use it
+		if svcInterface == kubevip.Auto {
+			link, err = autoFindInterface(address)
+			if err != nil {
+				log.Error("automatically discover network interface for annotated IP", "address", address, "err", err)
+			} else {
+				if link == nil {
+					log.Error("automatically discover network interface for annotated IP address", "address", address)
+				}
+			}
+			if link == nil {
+				svcInterface = ""
+			} else {
+				svcInterface = getAutoInterfaceName(link, config.Interface)
+			}
+		}
+		// If it is still blank then use the
+		if svcInterface == "" {
+			switch config.ServicesInterface {
+			case kubevip.Auto:
+				link, err = autoFindInterface(address)
+				if err != nil {
+					log.Error("failed to automatically discover network interface for address", "ip", address, "err", err, "interface", config.Interface)
+				} else if link == nil {
+					log.Error("failed to automatically discover network interface for address", "ip", address, "defaulting to", config.Interface)
+				}
+				svcInterface = getAutoInterfaceName(link, config.Interface)
+			case "":
+				svcInterface = config.Interface
+			default:
+				svcInterface = config.ServicesInterface
+			}
+		}
+
+		if link == nil {
+			if link, err = netlink.LinkByName(svcInterface); err != nil {
+				return nil, fmt.Errorf("failed to get interface %s: %w", svcInterface, err)
+			}
+			if link == nil {
+				return nil, fmt.Errorf("failed to get interface %s", svcInterface)
+			}
+		}
+
+		cidrs := vip.Split(config.VIPSubnet)
+
+		ipv4AutoSubnet := false
+		ipv6AutoSubnet := false
+		if cidrs[0] == kubevip.Auto {
+			ipv4AutoSubnet = true
+		}
+
+		if len(cidrs) > 1 && cidrs[1] == kubevip.Auto {
+			ipv6AutoSubnet = true
+		}
+
+		if (config.Address != "" || config.VIP != "") && (ipv4AutoSubnet || ipv6AutoSubnet) {
+			return nil, fmt.Errorf("auto subnet discovery cannot be used if VIP address was provided")
+		}
+
+		subnet := ""
+		var err error
+		if vip.IsIPv4(address) {
+			if ipv4AutoSubnet {
+				subnet, err = autoFindSubnet(link, address)
+				if err != nil {
+					return nil, fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
+				}
+			} else {
+				if cidrs[0] != "" && cidrs[0] != kubevip.Auto {
+					subnet = cidrs[0]
+				} else {
+					subnet = "32"
+				}
+			}
+		} else {
+			if ipv6AutoSubnet {
+				subnet, err = autoFindSubnet(link, address)
+				if err != nil {
+					return nil, fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
+				}
+			} else {
+				if len(cidrs) > 1 && cidrs[1] != "" && cidrs[1] != kubevip.Auto {
+					subnet = cidrs[1]
+				} else {
+					subnet = "128"
+				}
+			}
+		}
+
+		// Generate new Virtual IP configuration
+		newVips = append(newVips, &kubevip.Config{
+			VIP:                    address,
+			Interface:              svcInterface,
+			SingleNode:             true,
+			EnableARP:              config.EnableARP,
+			EnableBGP:              config.EnableBGP,
+			VIPSubnet:              subnet,
+			EnableRoutingTable:     config.EnableRoutingTable,
+			RoutingTableID:         config.RoutingTableID,
+			RoutingTableType:       config.RoutingTableType,
+			RoutingProtocol:        config.RoutingProtocol,
+			ArpBroadcastRate:       config.ArpBroadcastRate,
+			EnableServiceSecurity:  config.EnableServiceSecurity,
+			DNSMode:                config.DNSMode,
+			DisableServiceUpdates:  config.DisableServiceUpdates,
+			EnableServicesElection: config.EnableServicesElection,
+			KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
+				EnableLeaderElection: config.EnableLeaderElection,
+			},
+		})
+	}
+
+	// Create new service
+	instance := &Instance{
+		//UID:             instanceUID,
+		//VIPs:            instanceAddresses,
+		ServiceSnapshot: svc,
+	}
+	// for _, port := range svc.Spec.Ports {
+	// 	instance.ExternalPorts = append(instance.ExternalPorts, Port{
+	// 		Port: uint16(port.Port), //nolint
+	// 		Type: string(port.Protocol),
+	// 	})
+	// }
+
+	if svc.Annotations != nil {
+		instance.DHCPInterfaceHwaddr = svc.Annotations[kubevip.HwAddrKey]
+		instance.DHCPInterfaceIP = svc.Annotations[kubevip.RequestedIP]
+		instance.DHCPHostname = svc.Annotations[kubevip.LoadbalancerHostname]
+	}
+
+	configPorts := make([]kubevip.Port, 0)
+	for _, p := range svc.Spec.Ports {
+		configPorts = append(configPorts, kubevip.Port{
+			Type: string(p.Protocol),
+			Port: int(p.Port),
+		})
+	}
+	// Generate Load Balancer config
+	newLB := kubevip.LoadBalancer{
+		Name:      fmt.Sprintf("%s-load-balancer", svc.Name),
+		Ports:     configPorts,
+		BindToVip: true,
+	}
+	for _, vip := range newVips {
+		// Add Load Balancer Configuration
+		vip.LoadBalancers = append(vip.LoadBalancers, newLB)
+	}
+	// Create Add configuration to the new service
+	instance.VIPConfigs = newVips
+
+	// If this was purposely created with the address 0.0.0.0,
+	// we will create a macvlan on the main interface and a DHCP client
+	// TODO: Consider how best to handle DHCP with multiple addresses
+	if len(instanceAddresses) == 1 && instanceAddresses[0] == "0.0.0.0" {
+		err := instance.startDHCP()
+		if err != nil {
+			return nil, err
+		}
+		select {
+		case err := <-instance.DHCPClient.ErrorChannel():
+			return nil, fmt.Errorf("error starting DHCP for %s/%s: error: %s",
+				instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
+		case ip := <-instance.DHCPClient.IPChannel():
+			instance.VIPConfigs[0].Interface = instance.DHCPInterface
+			instance.VIPConfigs[0].VIP = ip
+			instance.DHCPInterfaceIP = ip
+		}
+	}
+
+	for _, vipConfig := range instance.VIPConfigs {
+		c, err := cluster.InitCluster(vipConfig, false, intfMgr, arpMgr)
+		if err != nil {
+			log.Error("Failed to add Service %s/%s", svc.Namespace, svc.Name)
+			return nil, err
+		}
+
+		for i := range c.Network {
+			c.Network[i].SetServicePorts(svc)
+		}
+
+		instance.Clusters = append(instance.Clusters, c)
+		log.Info("(svcs) adding VIP", "ip", vipConfig.VIP, "interface", vipConfig.Interface, "namespace", svc.Namespace, "name", svc.Name)
+	}
+
+	return instance, nil
+}
+
+func autoFindInterface(ip string) (netlink.Link, error) {
+	links, err := netlink.LinkList()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list network interfaces: %w", err)
+	}
+
+	address := net.ParseIP(ip)
+
+	family := netlink.FAMILY_V4
+
+	if address.To4() == nil {
+		family = netlink.FAMILY_V6
+	}
+
+	for _, link := range links {
+		addr, err := netlink.AddrList(link, family)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get IP addresses for interface %s: %w", link.Attrs().Name, err)
+		}
+		for _, a := range addr {
+			if a.IPNet.Contains(address) {
+				return link, nil
+			}
+		}
+	}
+
+	return nil, nil
+}
+
+func autoFindSubnet(link netlink.Link, ip string) (string, error) {
+	address := net.ParseIP(ip)
+
+	family := netlink.FAMILY_V4
+	if address.To4() == nil {
+		family = netlink.FAMILY_V6
+	}
+
+	addr, err := netlink.AddrList(link, family)
+	if err != nil {
+		return "", fmt.Errorf("failed to get IP addresses for interface %s: %w", link.Attrs().Name, err)
+	}
+	for _, a := range addr {
+		if a.IPNet.Contains(address) {
+			m, _ := a.IPNet.Mask.Size()
+			return strconv.Itoa(m), nil
+		}
+	}
+	return "", fmt.Errorf("failed to find suitable subnet for address %s", ip)
+}
+
+func getAutoInterfaceName(link netlink.Link, defaultInterface string) string {
+	if link == nil {
+		return defaultInterface
+	}
+	return link.Attrs().Name
+}
+
+func (i *Instance) startDHCP() error {
+	if len(i.VIPConfigs) != 1 {
+		return fmt.Errorf("DHCP requires exactly 1 VIP config, got: %v", len(i.VIPConfigs))
+	}
+	parent, err := netlink.LinkByName(i.VIPConfigs[0].Interface)
+	if err != nil {
+		return fmt.Errorf("error finding VIP Interface, for building DHCP Link : %v", err)
+	}
+
+	// Generate name from UID
+	interfaceName := fmt.Sprintf("vip-%s", i.ServiceSnapshot.UID[0:8])
+
+	// Check if the interface doesn't exist first
+	iface, err := net.InterfaceByName(interfaceName)
+	if err != nil {
+		log.Info("creating new macvlan interface for DHCP", "interface", interfaceName)
+
+		hwaddr, err := net.ParseMAC(i.DHCPInterfaceHwaddr)
+		if i.DHCPInterfaceHwaddr != "" && err != nil {
+			return err
+		} else if hwaddr == nil {
+			hwaddr, err = net.ParseMAC(vip.GenerateMac())
+			if err != nil {
+				return err
+			}
+		}
+
+		log.Info("new macvlan interface", "interface", interfaceName, "hardware address", hwaddr)
+		mac := &netlink.Macvlan{
+			LinkAttrs: netlink.LinkAttrs{
+				Name:         interfaceName,
+				ParentIndex:  parent.Attrs().Index,
+				HardwareAddr: hwaddr,
+			},
+			Mode: netlink.MACVLAN_MODE_DEFAULT,
+		}
+
+		err = netlink.LinkAdd(mac)
+		if err != nil {
+			return fmt.Errorf("could not add %s: %v", interfaceName, err)
+		}
+
+		err = netlink.LinkSetUp(mac)
+		if err != nil {
+			return fmt.Errorf("could not bring up interface [%s] : %v", interfaceName, err)
+		}
+
+		iface, err = net.InterfaceByName(interfaceName)
+		if err != nil {
+			return fmt.Errorf("error finding new DHCP interface by name [%v]", err)
+		}
+	} else {
+		log.Info("Using existing macvlan interface for DHCP", "interface", interfaceName)
+	}
+
+	// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
+	rpfilterSetting := "0"
+
+	// Check if we need to set an override rp_filter value for the interface
+	if i.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
+		// Check the rp_filter value
+		rpFilter, err := strconv.Atoi(i.ServiceSnapshot.Annotations[kubevip.RPFilter])
+		if err != nil {
+			slog.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
+		} else {
+			if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
+				rpfilterSetting = i.ServiceSnapshot.Annotations[kubevip.RPFilter]
+			} else {
+				slog.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
+			}
+		}
+	}
+
+	err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
+	if err != nil {
+		slog.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
+	}
+	var initRebootFlag bool
+	if i.DHCPInterfaceIP != "" {
+		initRebootFlag = true
+	}
+
+	client := vip.NewDHCPClient(iface, initRebootFlag, i.DHCPInterfaceIP)
+
+	// Add hostname to dhcp client if annotated
+	if i.DHCPHostname != "" {
+		log.Info("Hostname specified for dhcp lease", "interface", interfaceName, "hostname", i.DHCPHostname)
+		client.WithHostName(i.DHCPHostname)
+	}
+
+	go client.Start()
+
+	// Set that DHCP is enabled
+	i.IsDHCP = true
+	// Set the name of the interface so that it can be removed on Service deletion
+	i.DHCPInterface = interfaceName
+	i.DHCPInterfaceHwaddr = iface.HardwareAddr.String()
+	// Add the client so that we can call it to stop function
+	i.DHCPClient = client
+
+	return nil
+}
+
+// FetchIngressAddresses tries to get the addresses from status.loadBalancerIP
+func FetchLoadBalancerIngressAddresses(s *v1.Service) []string {
+	// If the service has no status, return empty
+	lbStatusAddresses := []string{}
+
+	if len(s.Status.LoadBalancer.Ingress) == 0 {
+		return lbStatusAddresses
+	}
+
+	for _, ingress := range s.Status.LoadBalancer.Ingress {
+		if ingress.IP != "" {
+			lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
+		}
+		// TODO: Handle hostname if needed
+	}
+
+	return lbStatusAddresses
+}
+
+// FetchServiceAddresses tries to get the addresses from annotations
+// kube-vip.io/loadbalancerIPs, then from spec.loadbalancerIP
+func FetchServiceAddresses(s *v1.Service) []string {
+	annotationAvailable := false
+	if s.Annotations != nil {
+
+		if v, annotationAvailable := s.Annotations[kubevip.LoadbalancerIPAnnotation]; annotationAvailable {
+			ips := strings.Split(v, ",")
+			var trimmedIPs []string
+			for _, ip := range ips {
+				trimmedIPs = append(trimmedIPs, strings.TrimSpace(ip))
+			}
+			return trimmedIPs
+		}
+	}
+
+	lbStatusAddresses := []string{}
+	if !annotationAvailable {
+		if len(s.Status.LoadBalancer.Ingress) > 0 {
+			for _, ingress := range s.Status.LoadBalancer.Ingress {
+				lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
+			}
+		}
+	}
+
+	lbIP := net.ParseIP(s.Spec.LoadBalancerIP)
+	isLbIPv4 := vip.IsIPv4(s.Spec.LoadBalancerIP)
+
+	if len(lbStatusAddresses) > 0 {
+		for _, a := range lbStatusAddresses {
+			if lbStatusIP := net.ParseIP(a); lbStatusIP != nil && lbIP != nil && vip.IsIPv4(a) == isLbIPv4 && !lbIP.Equal(lbStatusIP) {
+				return []string{s.Spec.LoadBalancerIP}
+			}
+		}
+		return lbStatusAddresses
+	}
+
+	if s.Spec.LoadBalancerIP != "" {
+		return []string{s.Spec.LoadBalancerIP}
+	}
+
+	return []string{}
+}
+
+func FindServiceInstance(svc *v1.Service, instances []*Instance) *Instance {
+	log.Debug("finding service", "UID", svc.UID)
+	for i := range instances {
+		log.Debug("saved service", "instance", i, "UID", instances[i].ServiceSnapshot.UID)
+		if instances[i].ServiceSnapshot.UID == svc.UID {
+			return instances[i]
+		}
+	}
+	return nil
+}

pkg/iptables/iptables.go

@@ -0,0 +1,741 @@
+// Copyright 2015 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iptables
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"syscall"
+)
+
+// Adds the output of stderr to exec.ExitError
+type Error struct {
+	exec.ExitError
+	cmd        exec.Cmd
+	msg        string
+	exitStatus *int //for overriding
+}
+
+func (e *Error) ExitStatus() int {
+	if e.exitStatus != nil {
+		return *e.exitStatus
+	}
+	return e.Sys().(syscall.WaitStatus).ExitStatus()
+}
+
+func (e *Error) Error() string {
+	return fmt.Sprintf("running %v: exit status %v: %v", e.cmd.Args, e.ExitStatus(), e.msg)
+}
+
+// IsNotExist returns true if the error is due to the chain or rule not existing
+func (e *Error) IsNotExist() bool {
+	if e.ExitStatus() != 1 {
+		return false
+	}
+	msgNoRuleExist := "Bad rule (does a matching rule exist in that chain?).\n"
+	msgNoChainExist := "No chain/target/match by that name.\n"
+	return strings.Contains(e.msg, msgNoRuleExist) || strings.Contains(e.msg, msgNoChainExist)
+}
+
+// Protocol to differentiate between IPv4 and IPv6
+type Protocol byte
+
+const (
+	ProtocolIPv4 Protocol = iota
+	ProtocolIPv6
+)
+
+const (
+	TableFilter      = "filter"
+	TableMangle      = "mangle"
+	TableNat         = "nat"
+	ChainInput       = "INPUT"
+	ChainPREROUTING  = "PREROUTING"
+	ChainPOSTROUTING = "POSTROUTING"
+)
+
+type IPTables struct {
+	path              string
+	proto             Protocol
+	hasCheck          bool
+	hasWait           bool
+	waitSupportSecond bool
+	hasRandomFully    bool
+	v1                int
+	v2                int
+	v3                int
+	mode              string // the underlying iptables operating mode, e.g. nf_tables
+	timeout           int    // time to wait for the iptables lock, default waits forever
+
+	nftables bool
+}
+
+// Stat represents a structured statistic entry.
+type Stat struct {
+	Packets     uint64     `json:"pkts"`
+	Bytes       uint64     `json:"bytes"`
+	Target      string     `json:"target"`
+	Protocol    string     `json:"prot"`
+	Opt         string     `json:"opt"`
+	Input       string     `json:"in"`
+	Output      string     `json:"out"`
+	Source      *net.IPNet `json:"source"`
+	Destination *net.IPNet `json:"destination"`
+	Options     string     `json:"options"`
+}
+
+type Option func(*IPTables)
+
+func IPFamily(proto Protocol) Option {
+	return func(ipt *IPTables) {
+		ipt.proto = proto
+	}
+}
+
+func Timeout(timeout int) Option {
+	return func(ipt *IPTables) {
+		ipt.timeout = timeout
+	}
+}
+
+func EnableNFTables(enable bool) Option {
+	return func(ipt *IPTables) {
+		ipt.nftables = enable
+	}
+}
+
+// New creates a new IPTables configured with the options passed as parameter.
+// For backwards compatibility, by default always uses IPv4 and timeout 0.
+// i.e. you can create an IPv6 IPTables using a timeout of 5 seconds passing
+// the IPFamily and Timeout options as follow:
+//
+//	ip6t := New(IPFamily(ProtocolIPv6), Timeout(5))
+func New(opts ...Option) (*IPTables, error) {
+
+	ipt := &IPTables{
+		proto:   ProtocolIPv4,
+		timeout: 0,
+	}
+
+	for _, opt := range opts {
+		opt(ipt)
+	}
+
+	path, err := exec.LookPath(getIptablesCommand(ipt.proto, ipt.nftables))
+	if err != nil {
+		return nil, err
+	}
+	ipt.path = path
+
+	vstring, err := getIptablesVersionString(path)
+	if err != nil {
+		return nil, fmt.Errorf("could not get iptables version: %v", err)
+	}
+	v1, v2, v3, mode, err := extractIptablesVersion(vstring)
+	if err != nil {
+		return nil, fmt.Errorf("failed to extract iptables version from [%s]: %v", vstring, err)
+	}
+	ipt.v1 = v1
+	ipt.v2 = v2
+	ipt.v3 = v3
+	ipt.mode = mode
+
+	checkPresent, waitPresent, waitSupportSecond, randomFullyPresent := getIptablesCommandSupport(v1, v2, v3)
+	ipt.hasCheck = checkPresent
+	ipt.hasWait = waitPresent
+	ipt.waitSupportSecond = waitSupportSecond
+	ipt.hasRandomFully = randomFullyPresent
+
+	return ipt, nil
+}
+
+// New creates a new IPTables for the given proto.
+// The proto will determine which command is used, either "iptables" or "ip6tables".
+func NewWithProtocol(proto Protocol) (*IPTables, error) {
+	return New(IPFamily(proto), Timeout(0))
+}
+
+// Proto returns the protocol used by this IPTables.
+func (ipt *IPTables) Proto() Protocol {
+	return ipt.proto
+}
+
+// Exists checks if given rulespec in specified table/chain exists
+func (ipt *IPTables) Exists(table, chain string, rulespec ...string) (bool, error) {
+	if !ipt.hasCheck {
+		return ipt.existsForOldIptables(table, chain, rulespec)
+
+	}
+	cmd := append([]string{"-t", table, "-C", chain}, rulespec...)
+	err := ipt.run(cmd...)
+	eerr, eok := err.(*Error)
+	switch {
+	case err == nil:
+		return true, nil
+	case eok && eerr.ExitStatus() == 1:
+		return false, nil
+	default:
+		return false, err
+	}
+}
+
+// Insert inserts rulespec to specified table/chain (in specified pos)
+func (ipt *IPTables) Insert(table, chain string, pos int, rulespec ...string) error {
+	cmd := append([]string{"-t", table, "-I", chain, strconv.Itoa(pos)}, rulespec...)
+	return ipt.run(cmd...)
+}
+
+// InsertUnique acts like Insert except that it won't insert a duplicate (no matter the position in the chain)
+func (ipt *IPTables) InsertUnique(table, chain string, pos int, rulespec ...string) error {
+	exists, err := ipt.Exists(table, chain, rulespec...)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		return ipt.Insert(table, chain, pos, rulespec...)
+	}
+
+	return nil
+}
+
+// Append appends rulespec to specified table/chain
+func (ipt *IPTables) Append(table, chain string, rulespec ...string) error {
+	cmd := append([]string{"-t", table, "-A", chain}, rulespec...)
+	return ipt.run(cmd...)
+}
+
+// AppendUnique acts like Append except that it won't add a duplicate
+func (ipt *IPTables) AppendUnique(table, chain string, rulespec ...string) error {
+	exists, err := ipt.Exists(table, chain, rulespec...)
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		return ipt.Append(table, chain, rulespec...)
+	}
+
+	return nil
+}
+
+// Delete removes rulespec in specified table/chain
+func (ipt *IPTables) Delete(table, chain string, rulespec ...string) error {
+	cmd := append([]string{"-t", table, "-D", chain}, rulespec...)
+	return ipt.run(cmd...)
+}
+
+func (ipt *IPTables) DeleteIfExists(table, chain string, rulespec ...string) error {
+	exists, err := ipt.Exists(table, chain, rulespec...)
+	if err == nil && exists {
+		err = ipt.Delete(table, chain, rulespec...)
+	}
+	return err
+}
+
+// List rules in specified table/chain
+func (ipt *IPTables) ListByID(table, chain string, id int) (string, error) {
+	args := []string{"-t", table, "-S", chain, strconv.Itoa(id)}
+	rule, err := ipt.executeList(args)
+	if err != nil {
+		return "", err
+	}
+	return rule[0], nil
+}
+
+// List rules in specified table/chain
+func (ipt *IPTables) List(table, chain string) ([]string, error) {
+	args := []string{"-t", table, "-S", chain}
+	return ipt.executeList(args)
+}
+
+// List rules (with counters) in specified table/chain
+func (ipt *IPTables) ListWithCounters(table, chain string) ([]string, error) {
+	args := []string{"-t", table, "-v", "-S", chain}
+	return ipt.executeList(args)
+}
+
+// ListChains returns a slice containing the name of each chain in the specified table.
+func (ipt *IPTables) ListChains(table string) ([]string, error) {
+	args := []string{"-t", table, "-S"}
+
+	result, err := ipt.executeList(args)
+	if err != nil {
+		return nil, err
+	}
+
+	// Iterate over rules to find all default (-P) and user-specified (-N) chains.
+	// Chains definition always come before rules.
+	// Format is the following:
+	// -P OUTPUT ACCEPT
+	// -N Custom
+	var chains []string
+	for _, val := range result {
+		if strings.HasPrefix(val, "-P") || strings.HasPrefix(val, "-N") {
+			chains = append(chains, strings.Fields(val)[1])
+		} else {
+			break
+		}
+	}
+	return chains, nil
+}
+
+// '-S' is fine with non existing rule index as long as the chain exists
+// therefore pass index 1 to reduce overhead for large chains
+func (ipt *IPTables) ChainExists(table, chain string) (bool, error) {
+	err := ipt.run("-t", table, "-S", chain, "1")
+	eerr, eok := err.(*Error)
+	switch {
+	case err == nil:
+		return true, nil
+	case eok && eerr.ExitStatus() == 1:
+		return false, nil
+	default:
+		return false, err
+	}
+}
+
+// Stats lists rules including the byte and packet counts
+func (ipt *IPTables) Stats(table, chain string) ([][]string, error) {
+	args := []string{"-t", table, "-L", chain, "-n", "-v", "-x"}
+	lines, err := ipt.executeList(args)
+	if err != nil {
+		return nil, err
+	}
+
+	appendSubnet := func(addr string) string {
+		if strings.IndexByte(addr, byte('/')) < 0 {
+			if strings.IndexByte(addr, '.') < 0 {
+				return addr + "/128"
+			}
+			return addr + "/32"
+		}
+		return addr
+	}
+
+	ipv6 := ipt.proto == ProtocolIPv6
+
+	rows := [][]string{}
+	for i, line := range lines {
+		// Skip over chain name and field header
+		if i < 2 {
+			continue
+		}
+
+		// Fields:
+		// 0=pkts 1=bytes 2=target 3=prot 4=opt 5=in 6=out 7=source 8=destination 9=options
+		line = strings.TrimSpace(line)
+		fields := strings.Fields(line)
+
+		// The ip6tables verbose output cannot be naively split due to the default "opt"
+		// field containing 2 single spaces.
+		if ipv6 {
+			// Check if field 6 is "opt" or "source" address
+			dest := fields[6]
+			ip, _, _ := net.ParseCIDR(dest)
+			if ip == nil {
+				ip = net.ParseIP(dest)
+			}
+
+			// If we detected a CIDR or IP, the "opt" field is empty.. insert it.
+			if ip != nil {
+				f := []string{}
+				f = append(f, fields[:4]...)
+				f = append(f, "  ") // Empty "opt" field for ip6tables
+				f = append(f, fields[4:]...)
+				fields = f
+			}
+		}
+
+		// Adjust "source" and "destination" to include netmask, to match regular
+		// List output
+		fields[7] = appendSubnet(fields[7])
+		fields[8] = appendSubnet(fields[8])
+
+		// Combine "options" fields 9... into a single space-delimited field.
+		options := fields[9:]
+		fields = fields[:9]
+		fields = append(fields, strings.Join(options, " "))
+		rows = append(rows, fields)
+	}
+	return rows, nil
+}
+
+// ParseStat parses a single statistic row into a Stat struct. The input should
+// be a string slice that is returned from calling the Stat method.
+func (ipt *IPTables) ParseStat(stat []string) (parsed Stat, err error) {
+	// For forward-compatibility, expect at least 10 fields in the stat
+	if len(stat) < 10 {
+		return parsed, fmt.Errorf("stat contained fewer fields than expected")
+	}
+
+	// Convert the fields that are not plain strings
+	parsed.Packets, err = strconv.ParseUint(stat[0], 0, 64)
+	if err != nil {
+		return parsed, fmt.Errorf(err.Error(), "could not parse packets")
+	}
+	parsed.Bytes, err = strconv.ParseUint(stat[1], 0, 64)
+	if err != nil {
+		return parsed, fmt.Errorf(err.Error(), "could not parse bytes")
+	}
+	_, parsed.Source, err = net.ParseCIDR(stat[7])
+	if err != nil {
+		return parsed, fmt.Errorf(err.Error(), "could not parse source")
+	}
+	_, parsed.Destination, err = net.ParseCIDR(stat[8])
+	if err != nil {
+		return parsed, fmt.Errorf(err.Error(), "could not parse destination")
+	}
+
+	// Put the fields that are strings
+	parsed.Target = stat[2]
+	parsed.Protocol = stat[3]
+	parsed.Opt = stat[4]
+	parsed.Input = stat[5]
+	parsed.Output = stat[6]
+	parsed.Options = stat[9]
+
+	return parsed, nil
+}
+
+// StructuredStats returns statistics as structured data which may be further
+// parsed and marshaled.
+func (ipt *IPTables) StructuredStats(table, chain string) ([]Stat, error) {
+	rawStats, err := ipt.Stats(table, chain)
+	if err != nil {
+		return nil, err
+	}
+
+	structStats := []Stat{}
+	for _, rawStat := range rawStats {
+		stat, err := ipt.ParseStat(rawStat)
+		if err != nil {
+			return nil, err
+		}
+		structStats = append(structStats, stat)
+	}
+
+	return structStats, nil
+}
+
+func (ipt *IPTables) executeList(args []string) ([]string, error) {
+	var stdout bytes.Buffer
+	if err := ipt.runWithOutput(args, &stdout); err != nil {
+		return nil, err
+	}
+
+	rules := strings.Split(stdout.String(), "\n")
+
+	// strip trailing newline
+	if len(rules) > 0 && rules[len(rules)-1] == "" {
+		rules = rules[:len(rules)-1]
+	}
+
+	for i, rule := range rules {
+		rules[i] = filterRuleOutput(rule)
+	}
+
+	return rules, nil
+}
+
+// NewChain creates a new chain in the specified table.
+// If the chain already exists, it will result in an error.
+func (ipt *IPTables) NewChain(table, chain string) error {
+	return ipt.run("-t", table, "-N", chain)
+}
+
+const existsErr = 1
+
+// ClearChain flushed (deletes all rules) in the specified table/chain.
+// If the chain does not exist, a new one will be created
+func (ipt *IPTables) ClearChain(table, chain string) error {
+	err := ipt.NewChain(table, chain)
+
+	eerr, eok := err.(*Error)
+	switch {
+	case err == nil:
+		return nil
+	case eok && eerr.ExitStatus() == existsErr:
+		// chain already exists. Flush (clear) it.
+		return ipt.run("-t", table, "-F", chain)
+	default:
+		return err
+	}
+}
+
+// RenameChain renames the old chain to the new one.
+func (ipt *IPTables) RenameChain(table, oldChain, newChain string) error {
+	return ipt.run("-t", table, "-E", oldChain, newChain)
+}
+
+// DeleteChain deletes the chain in the specified table.
+// The chain must be empty
+func (ipt *IPTables) DeleteChain(table, chain string) error {
+	return ipt.run("-t", table, "-X", chain)
+}
+
+func (ipt *IPTables) ClearAndDeleteChain(table, chain string) error {
+	exists, err := ipt.ChainExists(table, chain)
+	if err != nil || !exists {
+		return err
+	}
+	err = ipt.run("-t", table, "-F", chain)
+	if err == nil {
+		err = ipt.run("-t", table, "-X", chain)
+	}
+	return err
+}
+
+func (ipt *IPTables) ClearAll() error {
+	return ipt.run("-F")
+}
+
+func (ipt *IPTables) DeleteAll() error {
+	return ipt.run("-X")
+}
+
+// ChangePolicy changes policy on chain to target
+func (ipt *IPTables) ChangePolicy(table, chain, target string) error {
+	return ipt.run("-t", table, "-P", chain, target)
+}
+
+// Check if the underlying iptables command supports the --random-fully flag
+func (ipt *IPTables) HasRandomFully() bool {
+	return ipt.hasRandomFully
+}
+
+// Return version components of the underlying iptables command
+func (ipt *IPTables) GetIptablesVersion() (int, int, int) {
+	return ipt.v1, ipt.v2, ipt.v3
+}
+
+// run runs an iptables command with the given arguments, ignoring
+// any stdout output
+func (ipt *IPTables) run(args ...string) error {
+	return ipt.runWithOutput(args, nil)
+}
+
+// runWithOutput runs an iptables command with the given arguments,
+// writing any stdout output to the given writer
+func (ipt *IPTables) runWithOutput(args []string, stdout io.Writer) error {
+	args = append([]string{ipt.path}, args...)
+	if ipt.hasWait {
+		args = append(args, "--wait")
+		if ipt.timeout != 0 && ipt.waitSupportSecond {
+			args = append(args, strconv.Itoa(ipt.timeout))
+		}
+	} else {
+		fmu, err := newXtablesFileLock()
+		if err != nil {
+			return err
+		}
+		ul, err := fmu.tryLock()
+		if err != nil {
+			syscall.Close(fmu.fd)
+			return err
+		}
+		defer func() {
+			_ = ul.Unlock()
+		}()
+	}
+
+	var stderr bytes.Buffer
+	cmd := exec.Cmd{
+		Path:   ipt.path,
+		Args:   args,
+		Stdout: stdout,
+		Stderr: &stderr,
+	}
+
+	if err := cmd.Run(); err != nil {
+		switch e := err.(type) {
+		case *exec.ExitError:
+			return &Error{*e, cmd, stderr.String(), nil}
+		default:
+			return err
+		}
+	}
+
+	return nil
+}
+
+// getIptablesCommand returns the correct command for the given protocol, either "iptables" or "ip6tables".
+func getIptablesCommand(proto Protocol, nftables bool) string {
+	if proto == ProtocolIPv6 {
+		if nftables {
+			return "ip6tables-nft"
+		}
+		return "ip6tables-legacy"
+	}
+
+	if nftables {
+		return "iptables-nft"
+	}
+	return "iptables-legacy"
+}
+
+// Checks if iptables has the "-C" and "--wait" flag
+func getIptablesCommandSupport(v1 int, v2 int, v3 int) (bool, bool, bool, bool) {
+	return iptablesHasCheckCommand(v1, v2, v3), iptablesHasWaitCommand(v1, v2, v3), iptablesWaitSupportSecond(v1, v2), iptablesHasRandomFully(v1, v2, v3)
+}
+
+// getIptablesVersion returns the first three components of the iptables version
+// and the operating mode (e.g. nf_tables or legacy)
+// e.g. "iptables v1.3.66" would return (1, 3, 66, legacy, nil)
+func extractIptablesVersion(str string) (int, int, int, string, error) {
+	versionMatcher := regexp.MustCompile(`v([0-9]+)\.([0-9]+)\.([0-9]+)(?:\s+\((\w+))?`)
+	result := versionMatcher.FindStringSubmatch(str)
+	if result == nil {
+		return 0, 0, 0, "", fmt.Errorf("no iptables version found in string: %s", str)
+	}
+
+	v1, err := strconv.Atoi(result[1])
+	if err != nil {
+		return 0, 0, 0, "", err
+	}
+
+	v2, err := strconv.Atoi(result[2])
+	if err != nil {
+		return 0, 0, 0, "", err
+	}
+
+	v3, err := strconv.Atoi(result[3])
+	if err != nil {
+		return 0, 0, 0, "", err
+	}
+
+	mode := "legacy"
+	if result[4] != "" {
+		mode = result[4]
+	}
+	return v1, v2, v3, mode, nil
+}
+
+// Runs "iptables --version" to get the version string
+func getIptablesVersionString(path string) (string, error) {
+	cmd := exec.Command(path, "--version")
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err := cmd.Run()
+	if err != nil {
+		return "", err
+	}
+	return out.String(), nil
+}
+
+// Checks if an iptables version is after 1.4.11, when --check was added
+func iptablesHasCheckCommand(v1 int, v2 int, v3 int) bool {
+	if v1 > 1 {
+		return true
+	}
+	if v1 == 1 && v2 > 4 {
+		return true
+	}
+	if v1 == 1 && v2 == 4 && v3 >= 11 {
+		return true
+	}
+	return false
+}
+
+// Checks if an iptables version is after 1.4.20, when --wait was added
+func iptablesHasWaitCommand(v1 int, v2 int, v3 int) bool { //nolint
+	if v1 > 1 {
+		return true
+	}
+	if v1 == 1 && v2 > 4 {
+		return true
+	}
+	if v1 == 1 && v2 == 4 && v3 >= 20 {
+		return true
+	}
+	return false
+}
+
+// Checks if an iptablse version is after 1.6.0, when --wait support second
+func iptablesWaitSupportSecond(v1 int, v2 int) bool {
+	if v1 > 1 {
+		return true
+	}
+	if v1 == 1 && v2 >= 6 {
+		return true
+	}
+	return false
+}
+
+// Checks if an iptables version is after 1.6.2, when --random-fully was added
+func iptablesHasRandomFully(v1 int, v2 int, v3 int) bool {
+	if v1 > 1 {
+		return true
+	}
+	if v1 == 1 && v2 > 6 {
+		return true
+	}
+	if v1 == 1 && v2 == 6 && v3 >= 2 {
+		return true
+	}
+	return false
+}
+
+// Checks if a rule specification exists for a table
+func (ipt *IPTables) existsForOldIptables(table, chain string, rulespec []string) (bool, error) {
+	rs := strings.Join(append([]string{"-A", chain}, rulespec...), " ")
+	args := []string{"-t", table, "-S"}
+	var stdout bytes.Buffer
+	err := ipt.runWithOutput(args, &stdout)
+	if err != nil {
+		return false, err
+	}
+	return strings.Contains(stdout.String(), rs), nil
+}
+
+// counterRegex is the regex used to detect nftables counter format
+var counterRegex = regexp.MustCompile(`^\[([0-9]+):([0-9]+)\] `)
+
+// filterRuleOutput works around some inconsistencies in output.
+// For example, when iptables is in legacy vs. nftables mode, it produces
+// different results.
+func filterRuleOutput(rule string) string {
+	out := rule
+
+	// work around an output difference in nftables mode where counters
+	// are output in iptables-save format, rather than iptables -S format
+	// The string begins with "[0:0]"
+	//
+	// Fixes #49
+	if groups := counterRegex.FindStringSubmatch(out); groups != nil {
+		// drop the brackets
+		out = out[len(groups[0]):]
+		out = fmt.Sprintf("%s -c %s %s", out, groups[1], groups[2])
+	}
+
+	return out
+}
+
+func GetIPTablesRuleSpecification(rule, specification string) string {
+	parts := strings.Split(rule, " ")
+	for i, part := range parts {
+		if part == specification && i+1 < len(parts) {
+			return parts[i+1]
+		}
+	}
+
+	return ""
+}

pkg/iptables/lock.go

@@ -0,0 +1,84 @@
+// Copyright 2015 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package iptables
+
+import (
+	"os"
+	"sync"
+	"syscall"
+)
+
+const (
+	// In earlier versions of iptables, the xtables lock was implemented
+	// via a Unix socket, but now flock is used via this lockfile:
+	// http://git.netfilter.org/iptables/commit/?id=aa562a660d1555b13cffbac1e744033e91f82707
+	// Note the LSB-conforming "/run" directory does not exist on old
+	// distributions, so assume "/var" is symlinked
+	xtablesLockFilePath = "/var/run/xtables.lock"
+
+	defaultFilePerm = 0600
+)
+
+type Unlocker interface {
+	Unlock() error
+}
+
+type nopUnlocker struct{}
+
+func (n nopUnlocker) Unlock() error { return nil }
+
+type fileLock struct {
+	// mu is used to protect against concurrent invocations from within this process
+	mu sync.Mutex
+	fd int
+}
+
+// tryLock takes an exclusive lock on the xtables lock file without blocking.
+// This is best-effort only: if the exclusive lock would block (i.e. because
+// another process already holds it), no error is returned. Otherwise, any
+// error encountered during the locking operation is returned.
+// The returned Unlocker should be used to release the lock when the caller is
+// done invoking iptables commands.
+func (l *fileLock) tryLock() (Unlocker, error) {
+	l.mu.Lock()
+	err := syscall.Flock(l.fd, syscall.LOCK_EX|syscall.LOCK_NB)
+	switch err {
+	case syscall.EWOULDBLOCK:
+		l.mu.Unlock()
+		return nopUnlocker{}, nil
+	case nil:
+		return l, nil
+	default:
+		l.mu.Unlock()
+		return nil, err
+	}
+}
+
+// Unlock closes the underlying file, which implicitly unlocks it as well. It
+// also unlocks the associated mutex.
+func (l *fileLock) Unlock() error {
+	defer l.mu.Unlock()
+	return syscall.Close(l.fd)
+}
+
+// newXtablesFileLock opens a new lock on the xtables lockfile without
+// acquiring the lock
+func newXtablesFileLock() (*fileLock, error) {
+	fd, err := syscall.Open(xtablesLockFilePath, os.O_CREATE, defaultFilePerm)
+	if err != nil {
+		return nil, err
+	}
+	return &fileLock{fd: fd}, nil
+}

pkg/iptables/version.go

@@ -0,0 +1,91 @@
+package iptables
+
+import (
+	"fmt"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+type Version struct {
+	Major       int
+	Minor       int
+	Patch       int
+	BackendMode string
+}
+
+func (v Version) String() string {
+	return fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)
+}
+
+func (v Version) Compare(other Version) int {
+	if v.Major != other.Major {
+		return v.Major - other.Major
+	}
+	if v.Minor != other.Minor {
+		return v.Minor - other.Minor
+	}
+	return v.Patch - other.Patch
+}
+
+func ParseVersion(versionString string) (Version, error) {
+	re := regexp.MustCompile(`v([0-9]+)\.([0-9]+)\.([0-9]+)`)
+	match := re.FindStringSubmatch(versionString)
+	if len(match) != 4 {
+		return Version{}, fmt.Errorf("invalid version string: %s", versionString)
+	}
+	major, _ := strconv.Atoi(match[1])
+	minor, _ := strconv.Atoi(match[2])
+	patch, _ := strconv.Atoi(match[3])
+	return Version{Major: major, Minor: minor, Patch: patch}, nil
+}
+
+func GetVersion() (Version, error) {
+	ver := Version{}
+	cmd := exec.Command("iptables", "--version")
+	out, err := cmd.Output()
+	if err != nil {
+		return ver, fmt.Errorf("run cmd 'iptables --version' wtith error: %v", err)
+	}
+
+	ver, err = ParseVersion(string(out))
+	if err != nil {
+		return ver, err
+	}
+
+	nft4 := getOutput("iptables-nft-save")
+	legacy4 := getOutput("iptables-legacy-save")
+
+	nft6 := getOutput("ip6tables-nft-save")
+	legacy6 := getOutput("ip6tables-legacy-save")
+
+	if strings.Contains(nft4, "KUBE-IPTABLES") ||
+		strings.Contains(nft6, "KUBE-IPTABLES") ||
+		strings.Contains(nft4, "KUBE-KUBELET") ||
+		strings.Contains(nft6, "KUBE-KUBELET") {
+		ver.BackendMode = "nft"
+	} else if strings.Contains(legacy4, "KUBE-IPTABLES") ||
+		strings.Contains(legacy6, "KUBE-IPTABLES") ||
+		strings.Contains(legacy4, "KUBE-KUBELET") ||
+		strings.Contains(legacy6, "KUBE-KUBELET") {
+		ver.BackendMode = "legacy"
+	} else {
+		nftCount := strings.Count(nft4, "\n") + strings.Count(nft6, "\n")
+		legacyCount := strings.Count(legacy4, "\n") + strings.Count(legacy6, "\n")
+
+		if nftCount >= legacyCount {
+			ver.BackendMode = "nft"
+		} else {
+			ver.BackendMode = "legacy"
+		}
+	}
+
+	return ver, nil
+}
+
+func getOutput(name string) string {
+	cmd := exec.Command(name)
+	out, _ := cmd.Output()
+	return string(out)
+}

pkg/k8s/client.go

@@ -0,0 +1,123 @@
+package k8s
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net"
+	"time"
+
+	log "log/slog"
+
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+const (
+	defaultTimeout = 10 * time.Second
+)
+
+// NewClientset takes REST config and returns k8s clientest.
+func NewClientset(config *rest.Config) (*kubernetes.Clientset, error) {
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating kubernetes client: %s", err.Error())
+	}
+	return clientset, nil
+}
+
+// NewRestConfig takes an optional configPath and creates a new REST config for clientset.
+// If the configPath is not specified, and inCluster is true, then an
+// InClusterConfig is used.
+// Also takes a hostname which allow for overriding the config's hostname.
+func NewRestConfig(configPath string, inCluster bool, hostname string) (*rest.Config, error) {
+	config, err := restConfig(configPath, inCluster, defaultTimeout)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create rest config: %w", err)
+	}
+
+	if len(hostname) > 0 {
+		config.Host = hostname
+	}
+
+	return config, nil
+}
+
+func restConfig(kubeconfig string, inCluster bool, timeout time.Duration) (*rest.Config, error) {
+	var cfg *rest.Config
+	var err error
+	if inCluster {
+		if cfg, err = rest.InClusterConfig(); err != nil {
+			return nil, fmt.Errorf("failed to get incluster config: %w", err)
+		}
+	} else if kubeconfig != "" {
+		if cfg, err = clientcmd.BuildConfigFromFlags("", kubeconfig); err != nil {
+			return nil, fmt.Errorf("failed to build config from file '%s': %w", kubeconfig, err)
+		}
+	} else {
+		return nil, fmt.Errorf("failed to build config from file: path to KubeConfig not specified")
+	}
+
+	// Override some of the defaults allowing a little bit more flexibility speaking with the API server
+	// these should hopefully be redundant, however issues will still be logged.
+	cfg.QPS = 100
+	cfg.Burst = 250
+	cfg.Timeout = timeout
+	return cfg, nil
+}
+
+func findAddressFromRemoteCert(address string) ([]net.IP, error) {
+
+	// TODO: should we care at this point, probably not as we just want the certificates
+	conf := &tls.Config{
+		MinVersion:         tls.VersionTLS12,
+		InsecureSkipVerify: true, //nolint
+	}
+	d := &net.Dialer{
+		Timeout: time.Duration(3) * time.Second,
+	}
+
+	// Create the TCP connection
+	conn, err := tls.DialWithDialer(d, "tcp", address, conf)
+	if err != nil {
+		return nil, err
+	}
+
+	defer conn.Close()
+	// Grab the certificactes
+	certs := conn.ConnectionState().PeerCertificates
+	if len(certs) > 1 {
+		return nil, fmt.Errorf("[k8s client] not designed to recive multiple certs from API server")
+	}
+
+	return certs[0].IPAddresses, nil
+}
+
+func FindWorkingKubernetesAddress(configPath string, inCluster bool) (*rest.Config, error) {
+	// check with loopback, and retrieve its certificate
+	ips, err := findAddressFromRemoteCert("127.0.0.1:6443")
+	if err != nil {
+		return nil, err
+	}
+	for x := range ips {
+		log.Debug("[k8s client] testing", "address", ips[x].String())
+		c, err := NewRestConfig(configPath, inCluster, net.JoinHostPort(ips[x].String(), "6443"))
+		if err != nil {
+			log.Error("failed to create k8s REST config", "err", err)
+		}
+
+		c.Timeout = 2 * time.Second
+		k, err := NewClientset(c)
+		if err != nil {
+			log.Error("failed to create k8s clientset", "err", err)
+		}
+
+		_, err = k.DiscoveryClient.ServerVersion()
+		if err == nil {
+			log.Info("[k8s client] working", "address", ips[x].String())
+			c.Timeout = defaultTimeout
+			return c, nil
+		}
+	}
+	return nil, fmt.Errorf("unable to find a working address for the local API server [%v]", err)
+}

pkg/k8s/client_test.go

@@ -0,0 +1,70 @@
+package k8s
+
+//"192.168.0.174:6443"
+
+// func Test_findAddressFromRemoteCert(t *testing.T) {
+// 	type args struct {
+// 		address string
+// 	}
+// 	tests := []struct {
+// 		name    string
+// 		args    args
+// 		want    []net.IP
+// 		wantErr bool
+// 	}{
+// 		{
+// 			name:    "test server",
+// 			args:    args{address: "192.168.0.174:6443"},
+// 			want:    []net.IP{net.IPv4(10, 96, 0, 1), net.IPv4(192, 168, 0, 174)},
+// 			wantErr: false,
+// 		},
+// 	}
+// 	for _, tt := range tests {
+// 		t.Run(tt.name, func(t *testing.T) {
+// 			got, err := findAddressFromRemoteCert(tt.args.address)
+// 			if (err != nil) != tt.wantErr {
+// 				t.Errorf("findAddressFromRemoteCert() error = %v, wantErr %v", err, tt.wantErr)
+// 				return
+// 			}
+// 			if !reflect.DeepEqual(got, tt.want) {
+// 				t.Errorf("findAddressFromRemoteCert() = %v, want %v", got, tt.want)
+// 			}
+// 		})
+// 	}
+// }
+
+// //findAddressFromRemoteCert() =
+// //[10.96.0.1 192.168.0.174]
+// //[10.96.0.1 192.168.0.174]
+
+// func Test_findWorkingKubernetesAddress(t *testing.T) {
+// 	type args struct {
+// 		configPath string
+// 		inCluster  bool
+// 	}
+// 	tests := []struct {
+// 		name    string
+// 		args    args
+// 		want    *kubernetes.Clientset
+// 		wantErr bool
+// 	}{
+// 		{
+// 			name: "test",
+// 			args: args{
+// 				configPath: "/home/dan/super-admin.conf",
+// 				inCluster:  false,
+// 			},
+// 			wantErr: false,
+// 		},
+// 	}
+// 	for _, tt := range tests {
+// 		t.Run(tt.name, func(t *testing.T) {
+// 			_, err := FindWorkingKubernetesAddress(tt.args.configPath, tt.args.inCluster)
+// 			if (err != nil) != tt.wantErr {
+// 				t.Errorf("findWorkingKubernetesAddress() error = %v, wantErr %v", err, tt.wantErr)
+// 				return
+// 			}
+
+// 		})
+// 	}
+// }

pkg/kubevip/annotations.go

@@ -0,0 +1,49 @@
+package kubevip
+
+const (
+	// Hardware address of the host that has the VIP
+	HwAddrKey = "kube-vip.io/hwaddr"
+
+	// The IP address that is requested
+	RequestedIP = "kube-vip.io/requestedIP"
+
+	// The host that has the VIP
+	VipHost = "kube-vip.io/vipHost"
+
+	// Enable Egress on a service
+	Egress = "kube-vip.io/egress"
+
+	// Enable internal Egress
+	EgressInternal = "kube-vip.io/egress-internal"
+
+	// Egress should be IPv6
+	EgressIPv6 = "kube-vip.io/egress-ipv6"
+
+	// Ports that traffic is allowed to access from the egress VIP
+	EgressDestinationPorts = "kube-vip.io/egress-destination-ports"
+
+	// Allowed incoming ports to the VIP
+	EgressSourcePorts = "kube-vip.io/egress-source-ports"
+
+	// Allowed networks for the Egress to be enabled for
+	EgressAllowedNetworks = "kube-vip.io/egress-allowed-networks"
+
+	// Networks that we wont Egress for
+	EgressDeniedNetworks = "kube-vip.io/egress-denied-networks"
+
+	// The current active endpoint(pod) for the Egress VIP
+	ActiveEndpoint = "kube-vip.io/active-endpoint"
+
+	// The current active endpoint(pod) for the Egress VIP (v6)
+	ActiveEndpointIPv6 = "kube-vip.io/active-endpoint-ipv6"
+
+	// Flush the conntrack rules (remove existing sessions) once Egress is configured
+	FlushContrack = "kube-vip.io/flush-conntrack"
+
+	LoadbalancerIPAnnotation = "kube-vip.io/loadbalancerIPs"
+	LoadbalancerHostname     = "kube-vip.io/loadbalancerHostname"
+	ServiceInterface         = "kube-vip.io/serviceInterface"
+	UpnpEnabled              = "kube-vip.io/forwardUPNP"
+
+	RPFilter = "kube-vip.io/rp_filter" // Set the return path filter for a specific service interface
+)

pkg/kubevip/config_endpoints.go

@@ -1,71 +0,0 @@
-package kubevip
-
-import (
-	"fmt"
-	"net/url"
-	"strconv"
-
-	log "github.com/sirupsen/logrus"
-)
-
-func init() {
-	// Start the index negative as it will be incrememnted of first approach
-	endPointIndex = -1
-}
-
-// ValidateBackEndURLS will run through the endpoints and ensure that they're a valid URL
-func ValidateBackEndURLS(endpoints *[]BackEnd) error {
-
-	for i := range *endpoints {
-		log.Debugf("Parsing [%s]", (*endpoints)[i].RawURL)
-		u, err := url.Parse((*endpoints)[i].RawURL)
-		if err != nil {
-			return err
-		}
-
-		// No error is returned if the prefix/schema is missing
-		// If the Host is empty then we were unable to parse correctly (could be prefix is missing)
-		if u.Host == "" {
-			return fmt.Errorf("Unable to parse [%s], ensure it's prefixed with http(s)://", (*endpoints)[i].RawURL)
-		}
-		(*endpoints)[i].Address = u.Hostname()
-		// if a port is specified then update the internal endpoint stuct, if not rely on the schema
-		if u.Port() != "" {
-			portNum, err := strconv.Atoi(u.Port())
-			if err != nil {
-				return err
-			}
-			(*endpoints)[i].Port = portNum
-		}
-		(*endpoints)[i].ParsedURL = u
-	}
-	return nil
-}
-
-// ReturnEndpointAddr - returns an endpoint
-func (lb LoadBalancer) ReturnEndpointAddr() (string, error) {
-	if len(lb.Backends) == 0 {
-		return "", fmt.Errorf("No Backends configured")
-	}
-	if endPointIndex < len(lb.Backends)-1 {
-		endPointIndex++
-	} else {
-		// reset the index to the beginning
-		endPointIndex = 0
-	}
-	// TODO - weighting, decision algorythmn
-	return fmt.Sprintf("%s:%d", lb.Backends[endPointIndex].Address, lb.Backends[endPointIndex].Port), nil
-}
-
-// ReturnEndpointURL - returns an endpoint
-func (lb LoadBalancer) ReturnEndpointURL() *url.URL {
-
-	if endPointIndex != len(lb.Backends)-1 {
-		endPointIndex++
-	} else {
-		// reset the index to the beginning
-		endPointIndex = 0
-	}
-	// TODO - weighting, decision algorythmn
-	return lb.Backends[endPointIndex].ParsedURL
-}

pkg/kubevip/config_environment.go

@@ -0,0 +1,671 @@
+package kubevip
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	"math/bits"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/detector"
+)
+
+// ParseEnvironment - will popultate the configuration from environment variables
+func ParseEnvironment(c *Config) error {
+	if c == nil {
+		return nil
+	}
+	// Ensure that logging is set through the environment variables
+	env := os.Getenv(vipLogLevel)
+
+	if env != "" {
+		logLevel, err := strconv.ParseInt(env, 10, 32)
+		if err != nil {
+			panic("Unable to parse environment variable [vip_loglevel], should be int")
+		}
+		c.Logging = int32(logLevel)
+	}
+
+	// Find interface
+	env = os.Getenv(vipInterface)
+	if env != "" {
+		c.Interface = env
+	}
+
+	env = os.Getenv(vipInterfaceLoGlobal)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.LoInterfaceGlobalScope = b
+	}
+
+	// Find (services) interface
+	env = os.Getenv(vipServicesInterface)
+	if env != "" {
+		c.ServicesInterface = env
+	}
+
+	// Find Kubernetes Leader Election configuration
+	env = os.Getenv(vipLeaderElection)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.EnableLeaderElection = b
+	}
+
+	// Attempt to find the Lease name from the environment variables
+	env = os.Getenv(vipLeaseName)
+	if env != "" {
+		c.LeaseName = env
+	}
+
+	// Attempt to find the Lease configuration from the environment variables
+	env = os.Getenv(vipLeaseDuration)
+	if env != "" {
+		i, err := strconv.ParseInt(env, 10, 32)
+		if err != nil {
+			return err
+		}
+		c.LeaseDuration = int(i)
+	}
+
+	env = os.Getenv(vipRenewDeadline)
+	if env != "" {
+		i, err := strconv.ParseInt(env, 10, 32)
+		if err != nil {
+			return err
+		}
+		c.RenewDeadline = int(i)
+	}
+
+	env = os.Getenv(vipRetryPeriod)
+	if env != "" {
+		i, err := strconv.ParseInt(env, 10, 32)
+		if err != nil {
+			return err
+		}
+		c.RetryPeriod = int(i)
+	}
+
+	// Attempt to find the Lease annotations from the environment variables
+	env = os.Getenv(vipLeaseAnnotations)
+	if env != "" {
+		err := json.Unmarshal([]byte(env), &c.LeaseAnnotations)
+		if err != nil {
+			return err
+		}
+	}
+
+	env = os.Getenv(nodeName)
+	if env != "" {
+		c.NodeName = env
+	}
+
+	// Find vip address
+	env = os.Getenv(vipAddress)
+	if env != "" {
+		// TODO - parse address net.Host()
+		c.VIP = env
+		// } else {
+		// 	c.VIP = os.Getenv(address)
+	}
+
+	// Find address
+	env = os.Getenv(address)
+	if env != "" {
+		// TODO - parse address net.Host()
+		c.Address = env
+	}
+
+	// Find vip port
+	env = os.Getenv(port)
+	if env != "" {
+		i, err := strconv.ParseUint(env, 10, 16)
+		if err != nil {
+			return err
+		}
+		c.Port = uint16(i)
+	}
+
+	// Find vipDdns
+	env = os.Getenv(vipDdns)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.DDNS = b
+	}
+
+	// Find the namespace that the control plane should use (for leaderElection lock)
+	env = os.Getenv(cpNamespace)
+	if env != "" {
+		c.Namespace = env
+	}
+
+	// Find controlplane toggle
+	env = os.Getenv(cpEnable)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.EnableControlPlane = b
+	}
+
+	// Find controlplane toggle
+	env = os.Getenv(cpDetect)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.DetectControlPlane = b
+	}
+
+	env = os.Getenv(kubernetesAddr)
+	if env != "" {
+		c.KubernetesAddr = env
+	}
+
+	// Find Services toggle
+	env = os.Getenv(svcEnable)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.EnableServices = b
+
+		// Find Services leader Election
+		env = os.Getenv(svcElection)
+		if env != "" {
+			b, err := strconv.ParseBool(env)
+			if err != nil {
+				return err
+			}
+			c.EnableServicesElection = b
+		}
+
+		// Find load-balancer class only
+		env = os.Getenv(lbClassOnly)
+		if env != "" {
+			b, err := strconv.ParseBool(env)
+			if err != nil {
+				return err
+			}
+			c.LoadBalancerClassOnly = b
+		}
+
+		// Load-balancer class name
+		env, exists := os.LookupEnv(lbClassName)
+		if exists {
+			c.LoadBalancerClassName = env
+		}
+
+		// Load-balancer class legacy handling
+		env = os.Getenv(lbClassLegacyHandling)
+		if env != "" {
+			b, err := strconv.ParseBool(env)
+			if err != nil {
+				return err
+			}
+			c.LoadBalancerClassLegacyHandling = b
+		}
+
+		// Find the namespace that the control plane should use (for leaderElection lock)
+		env = os.Getenv(svcNamespace)
+		if env != "" {
+			c.ServiceNamespace = env
+		}
+
+		// Gets the leaseName for services in arp mode
+		env = os.Getenv(svcLeaseName)
+		if env != "" {
+			c.ServicesLeaseName = env
+		}
+	}
+
+	// Find vip address subnet
+	env = os.Getenv(vipSubnet)
+	if env != "" {
+		c.VIPSubnet = env
+	}
+
+	// Find Single Node
+	env = os.Getenv(vipSingleNode)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.SingleNode = b
+	}
+
+	// Find annotation configuration
+	env = os.Getenv(annotations)
+	if env != "" {
+		c.Annotations = env
+	}
+
+	// Find Start As Leader
+	// TODO - does this need deprecating?
+	// Required when the host sets itself as leader before the state change
+	env = os.Getenv(vipStartLeader)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.StartAsLeader = b
+	}
+
+	// Find if ARP is enabled
+	env = os.Getenv(vipArp)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.EnableARP = b
+	}
+
+	// Find if ARP is enabled
+	env = os.Getenv(vipArpRate)
+	if env != "" {
+		i64, err := strconv.ParseInt(env, 10, 32)
+		if err != nil {
+			return err
+		}
+		c.ArpBroadcastRate = i64
+	} else {
+		// default to three seconds
+		c.ArpBroadcastRate = 3000
+	}
+
+	// Wireguard Mode
+	env = os.Getenv(vipWireguard)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.EnableWireguard = b
+	}
+
+	// Routing Table Mode
+	env = os.Getenv(vipRoutingTable)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.EnableRoutingTable = b
+	}
+
+	// Routing Table ID
+	env = os.Getenv(vipRoutingTableID)
+	if env != "" {
+		i, err := strconv.ParseInt(env, 10, 64)
+		if err != nil {
+			return err
+		}
+		if i >= 0 && i <= math.MaxInt {
+			c.RoutingTableID = int(i)
+		} else if i < 0 {
+			return fmt.Errorf("no support of negative [%d] in env var %q", i, vipRoutingTableID)
+		} else {
+			// +1 for the signing bit as it is 0 for positive integers
+			return fmt.Errorf("no support for int64, system natively supports [int%d]", bits.OnesCount(math.MaxInt)+1)
+		}
+	}
+
+	// Routing Table Type
+	env = os.Getenv(vipRoutingTableType)
+	if env != "" {
+		i, err := strconv.ParseInt(env, 10, 32)
+		if err != nil {
+			return err
+		}
+		c.RoutingTableType = int(i)
+	}
+
+	// Routing protocol
+	env = os.Getenv(vipRoutingProtocol)
+	if env != "" {
+		i, err := strconv.ParseInt(env, 10, 32)
+		if err != nil {
+			return err
+		}
+		c.RoutingProtocol = int(i)
+	}
+
+	// Clean routing table
+	env = os.Getenv(vipCleanRoutingTable)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.CleanRoutingTable = b
+	}
+
+	// DNS mode
+	env = os.Getenv(dnsMode)
+	if env != "" {
+		c.DNSMode = env
+	}
+
+	// Disable updates for services (status.LoadBalancer.Ingress will not be updated)
+	env = os.Getenv(disableServiceUpdates)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.DisableServiceUpdates = b
+	}
+
+	// BGP Server options
+	env = os.Getenv(bgpEnable)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.EnableBGP = b
+	}
+
+	// BGP Router interface determines an interface that we can use to find an address for
+	env = os.Getenv(bgpRouterInterface)
+	if env != "" {
+		_, address, err := detector.FindIPAddress(env)
+		if err != nil {
+			return err
+		}
+		c.BGPConfig.RouterID = address
+	}
+
+	// RouterID
+	env = os.Getenv(bgpRouterID)
+	if env != "" {
+		c.BGPConfig.RouterID = env
+	}
+
+	// AS
+	env = os.Getenv(bgpRouterAS)
+	if env != "" {
+		u64, err := strconv.ParseUint(env, 10, 32)
+		if err != nil {
+			return err
+		}
+		c.BGPConfig.AS = uint32(u64)
+	}
+
+	// Peer AS
+	env = os.Getenv(bgpPeerAS)
+	if env != "" {
+		u64, err := strconv.ParseUint(env, 10, 32)
+		if err != nil {
+			return err
+		}
+		c.BGPPeerConfig.AS = uint32(u64)
+	}
+
+	// Peer AS
+	env = os.Getenv(bgpPeers)
+	if env != "" {
+		peers, err := bgp.ParseBGPPeerConfig(env)
+		if err != nil {
+			return err
+		}
+		c.BGPConfig.Peers = peers
+	}
+
+	// MPBGP mode
+	env = os.Getenv(mpbgpNexthop)
+	if env != "" {
+		c.BGPConfig.MpbgpNexthop = env
+	}
+
+	// MPBGP fixed IPv4
+	env = os.Getenv(mpbgpIPv4)
+	if env != "" {
+		c.BGPConfig.MpbgpIPv4 = env
+	}
+
+	// MPBGP fixed IPv6
+	env = os.Getenv(mpbgpIPv6)
+	if env != "" {
+		c.BGPConfig.MpbgpIPv6 = env
+	}
+
+	// BGP Peer mutlihop
+	env = os.Getenv(bgpMultiHop)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.BGPPeerConfig.MultiHop = b
+	}
+
+	// BGP Peer password
+	env = os.Getenv(bgpPeerPassword)
+	if env != "" {
+		c.BGPPeerConfig.Password = env
+	}
+
+	// BGP Source Interface
+	env = os.Getenv(bgpSourceIF)
+	if env != "" {
+		c.BGPConfig.SourceIF = env
+	}
+
+	// BGP Source Address
+	env = os.Getenv(bgpSourceIP)
+	if env != "" {
+		c.BGPConfig.SourceIP = env
+	}
+
+	// BGP Peer options, add them if relevant
+	env = os.Getenv(bgpPeerAddress)
+	if env != "" {
+		c.BGPPeerConfig.Address = env
+		// If we've added in a peer configuration, then we should add it to the BGP configuration
+		c.BGPConfig.Peers = append(c.BGPConfig.Peers, c.BGPPeerConfig)
+	}
+
+	// BGP Timers options
+	env = os.Getenv(bgpHoldTime)
+	if env != "" {
+		u64, err := strconv.ParseUint(env, 10, 32)
+		if err != nil {
+			return err
+		}
+		c.BGPConfig.HoldTime = u64
+	}
+	env = os.Getenv(bgpKeepaliveInterval)
+	if env != "" {
+		u64, err := strconv.ParseUint(env, 10, 32)
+		if err != nil {
+			return err
+		}
+		c.BGPConfig.KeepaliveInterval = u64
+	}
+
+	env = os.Getenv(zebraEnable)
+	if env != "" {
+		result, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.BGPConfig.Zebra.Enabled = result
+	}
+
+	env = os.Getenv(zebraURL)
+	if env != "" {
+		c.BGPConfig.Zebra.URL = env
+	}
+
+	env = os.Getenv(zebraVersion)
+	if env != "" {
+		u64, err := strconv.ParseUint(env, 10, 32)
+		if err != nil {
+			return err
+		}
+		c.BGPConfig.Zebra.Version = uint32(u64)
+	}
+
+	env = os.Getenv(zebraSoftwareName)
+	if env != "" {
+		c.BGPConfig.Zebra.SoftwareName = env
+	}
+
+	// Enable the load-balancer
+	env = os.Getenv(lbEnable)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.EnableLoadBalancer = b
+	}
+
+	// Find loadbalancer port
+	env = os.Getenv(lbPort)
+	if env != "" {
+		i, err := strconv.ParseUint(env, 10, 16)
+		if err != nil {
+			return err
+		}
+		c.LoadBalancerPort = uint16(i)
+	}
+
+	// Find loadbalancer forwarding method
+	env = os.Getenv(lbForwardingMethod)
+	if env != "" {
+		c.LoadBalancerForwardingMethod = env
+	}
+
+	env = os.Getenv(EnableServiceSecurity)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.EnableServiceSecurity = b
+	}
+
+	// Find if node labeling is enabled
+	env = os.Getenv(EnableNodeLabeling)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.EnableNodeLabeling = b
+	}
+
+	// Find Prometheus configuration
+	env = os.Getenv(prometheusServer)
+	if env != "" {
+		c.PrometheusHTTPServer = env
+	}
+
+	// Set Egress configuration(s)
+	env = os.Getenv(egressPodCidr)
+	if env != "" {
+		c.EgressPodCidr = env
+	}
+
+	env = os.Getenv(egressServiceCidr)
+	if env != "" {
+		c.EgressServiceCidr = env
+	}
+
+	// if this is set then we're enabling nftables
+	env = os.Getenv(egressWithNftables)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.EgressWithNftables = b
+	}
+
+	// check to see if we're using a specific path to the Kubernetes config file
+	env = os.Getenv(k8sConfigFile)
+	if env != "" {
+		c.K8sConfigFile = env
+	}
+
+	env = os.Getenv(enableEndpoints)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.EnableEndpoints = b
+	}
+
+	env = os.Getenv(mirrorDestInterface)
+	if env != "" {
+		c.MirrorDestInterface = env
+	}
+
+	env = os.Getenv(iptablesBackend)
+	if env != "" {
+		c.IptablesBackend = env
+	}
+
+	env = os.Getenv(backendHealthCheckInterval)
+	if env != "" {
+		i, err := strconv.ParseInt(env, 10, 32)
+		if err != nil {
+			return err
+		}
+		c.BackendHealthCheckInterval = int(i)
+	}
+
+	env = os.Getenv(healthCheckPort)
+	if env != "" {
+		i, err := strconv.ParseInt(env, 10, 32)
+		if err != nil {
+			return err
+		}
+		if i < 1024 {
+			return fmt.Errorf("health check port should be > 1024")
+		}
+		c.HealthCheckPort = int(i)
+	}
+
+	env = os.Getenv(enableUPNP)
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.EnableUPNP = b
+	}
+
+	if env = os.Getenv(egressClean); env == "" {
+		env = os.Getenv(strings.ToUpper(egressClean))
+	}
+	if env != "" {
+		b, err := strconv.ParseBool(env)
+		if err != nil {
+			return err
+		}
+		c.EgressClean = b
+	}
+
+	return nil
+}

pkg/kubevip/config_envvar.go

@@ -0,0 +1,237 @@
+package kubevip
+
+// Environment variables
+const (
+
+	// vipArp - defines if the arp broadcast should be enabled
+	vipArp = "vip_arp"
+
+	// vip_arpRate - defines the rate of gARP broadcasts
+	vipArpRate = "vip_arpRate"
+
+	// vipLeaderElection - defines if the kubernetes algorithm should be used
+	vipLeaderElection = "vip_leaderelection"
+
+	// vipLeaseName - defines the name of the lease lock
+	vipLeaseName = "vip_leasename"
+
+	// vipLeaderElection - defines if the kubernetes algorithm should be used
+	vipLeaseDuration = "vip_leaseduration"
+
+	// vipLeaderElection - defines if the kubernetes algorithm should be used
+	vipRenewDeadline = "vip_renewdeadline"
+
+	// vipLeaderElection - defines if the kubernetes algorithm should be used
+	vipRetryPeriod = "vip_retryperiod"
+
+	// vipLeaderElection - defines the annotations given to the lease lock
+	vipLeaseAnnotations = "vip_leaseannotations"
+
+	// vipLogLevel - defines the level of logging to produce (5 being the most verbose)
+	vipLogLevel = "vip_loglevel"
+
+	// vipInterface - defines the interface that the vip should bind too
+	vipInterface = "vip_interface"
+
+	// vipInterfaceLoGlobal - defines if the lo interface (if used) should have a global scope
+	vipInterfaceLoGlobal = "vip_interfaceloglobal"
+
+	// vipServicesInterface - defines the interface that the service vips should bind too
+	vipServicesInterface = "vip_servicesinterface"
+
+	// vipSubnet - defines the subnet that the vip will use
+	vipSubnet = "vip_subnet"
+
+	// egressPodCidr - defines the cidr that egress will ignore
+	egressPodCidr = "egress_podcidr"
+
+	// egressServiceCidr - defines the cidr that egress will ignore
+	egressServiceCidr = "egress_servicecidr"
+
+	// egressWithNftables - enables using nftables over iptables
+	egressWithNftables = "egress_withnftables"
+
+	/////////////////////////////////////
+	// TO DO:
+	// Determine how to tidy this mess up
+	/////////////////////////////////////
+
+	// vipAddress - defines the address that the vip will expose
+	// DEPRECATED: will be removed in a next release
+	vipAddress = "vip_address"
+
+	// address - defines the address that would be used as a vip
+	// it may be an IP or a DNS name, in case of a DNS name
+	// kube-vip will try to resolve it and use the IP as a VIP
+	address = "address"
+
+	// port - defines the port for the VIP
+	port = "port"
+
+	// annotations
+	annotations = "annotation"
+
+	// vipDdns - defines if use dynamic dns to allocate IP for "address"
+	vipDdns = "vip_ddns"
+
+	// vipLeaseNodeName defines the node name that is used to acquire leases
+	nodeName = "vip_nodename"
+
+	// vipSingleNode - defines the vip start as a single node cluster
+	vipSingleNode = "vip_singlenode"
+
+	// vipStartLeader - will start this instance as the leader of the cluster
+	vipStartLeader = "vip_startleader"
+
+	// bgpEnable defines if BGP should be enabled
+	bgpEnable = "bgp_enable"
+	// bgpRouterID defines the routerID for the BGP server
+	bgpRouterID = "bgp_routerid"
+	// bgpRouterInterface defines the interface that we can find the address for
+	bgpRouterInterface = "bgp_routerinterface"
+	// bgpRouterAS defines the AS for the BGP server
+	bgpRouterAS = "bgp_as"
+	// bgpPeerAddress defines the address for a BGP peer
+	bgpPeerAddress = "bgp_peeraddress"
+	// bgpPeers defines the address for a BGP peer
+	bgpPeers = "bgp_peers"
+	// bgpPeerAS defines the AS for a BGP peer
+	bgpPeerAS = "bgp_peeras"
+	// bgpPeerAS defines the AS for a BGP peer
+	bgpPeerPassword = "bgp_peerpass" // nolint
+	// bgpMultiHop enables mulithop routing
+	bgpMultiHop = "bgp_multihop"
+	// bgpSourceIF defines the source interface for BGP peering
+	bgpSourceIF = "bgp_sourceif"
+	// bgpSourceIP defines the source address for BGP peering
+	bgpSourceIP = "bgp_sourceip"
+	// bgpHoldTime defines bgp timers hold time
+	bgpHoldTime = "bgp_hold_time"
+	// bgpKeepaliveInterval defines bgp timers keepalive interval
+	bgpKeepaliveInterval = "bgp_keepalive_interval"
+
+	// zebraEnable defines if Zebra integraton should be enabled
+	zebraEnable = "zebra_enable"
+	// zebraUrl specifies path to the unix domain socket for connecting to Zebra daemon
+	zebraURL = "zebra_url"
+	// zebraVersion specifies Zebra API Version
+	zebraVersion = "zebra_version"
+	// zebraSoftwareName specifies Software Name for Zebra
+	zebraSoftwareName = "zebra_software_name"
+
+	// mpbgpNexthop defines MPBGP mode
+	mpbgpNexthop = "mpbgp_nexthop"
+	// mpbgpIPv4 defines fixed IPv4 to be used with MPBGP
+	mpbgpIPv4 = "mpbgp_ipv4"
+	// mpbgpIPv6 defines fixed IPv6 to be used with MPBGP
+	mpbgpIPv6 = "mpbgp_ipv6"
+
+	// vipWireguard - defines if wireguard will be used for vips
+	vipWireguard = "vip_wireguard" //nolint
+
+	// vipRoutingTable - defines if table mode will be used for vips
+	vipRoutingTable = "vip_routingtable" //nolint
+
+	// vipRoutingTableID - defines which table mode will be used for vips
+	vipRoutingTableID = "vip_routingtableid" //nolint
+
+	// vipRoutingTableType - defines which table type will be used for vip routes
+	// 						 valid values for this variable can be found in:
+	//						 https://pkg.go.dev/golang.org/x/sys/unix#RTN_UNSPEC
+	//						 Note that route type have the prefix `RTN_`, and you
+	//						 specify the integer value, not the name. For example:
+	//						 you should say `vip_routingtabletype=2` for RTN_LOCAL
+	vipRoutingTableType = "vip_routingtabletype" //nolint
+
+	// vipRoutingProtocol - defines what value will be used as protocol when creating routes
+	vipRoutingProtocol = "vip_routingprotocol" //nolint
+
+	// vipCleanRoutingTable - defines if routing table will be cleaned of redundant routes on kube-vip's start
+	vipCleanRoutingTable = "vip_cleanroutingtable" //nolint
+
+	// cpNamespace defines the namespace the control plane pods will run in
+	cpNamespace = "cp_namespace"
+
+	// cpEnable enables the control plane feature
+	cpEnable = "cp_enable"
+
+	// cpDetect will attempt to automatically find a working address for the control plane from loopback
+	cpDetect = "cp_detect"
+
+	// kubernetesAddr，is the address of the Kubernetes API server on this machine
+	kubernetesAddr = "kubernetes_addr"
+
+	// svcEnable enables the Kubernetes service feature
+	svcEnable = "svc_enable"
+
+	// svcNamespace defines the namespace the service pods will run in
+	svcNamespace = "svc_namespace"
+
+	// svcElection enables election per Kubernetes service
+	svcElection = "svc_election"
+
+	// svcLeaseName Name of the lease that is used for leader election for services (in arp mode)
+	svcLeaseName = "svc_leasename"
+
+	// lbClassOnly enables load-balancer for class "kube-vip.io/kube-vip-class" only
+	lbClassOnly = "lb_class_only"
+
+	// lbClassName enables load-balancer for a specific class only
+	lbClassName = "lb_class_name"
+
+	// lbClassLegacyHandling enables legacy handing of load-balancer class
+	lbClassLegacyHandling = "lb_class_legacy_handling"
+
+	// lbEnable defines if the load-balancer should be enabled
+	lbEnable = "lb_enable"
+
+	// lbPort defines the port of load-balancer
+	lbPort = "lb_port"
+
+	// lbForwardingMethod defines the forwarding method of load-balancer
+	lbForwardingMethod = "lb_fwdmethod"
+
+	// EnableServiceSecurity defines if the load-balancer should only allow traffic to service ports
+	EnableServiceSecurity = "enable_service_security"
+
+	// EnableNodeLabeling, will enable node labeling as the node becomes leader
+	EnableNodeLabeling = "enable_node_labeling"
+
+	// prometheusServer defines the address prometheus listens on
+	prometheusServer = "prometheus_server"
+
+	// vipConfigMap defines the configmap that kube-vip will watch for service definitions
+	// vipConfigMap = "vip_configmap"
+
+	// k8sConfigFile defines the path to the configfile used to speak with the API server
+	k8sConfigFile = "k8s_config_file"
+
+	// dnsMode defines mode that DNS lookup will be performed with (first, ipv4, ipv6, dual)
+	dnsMode = "dns_mode"
+
+	// disableServiceUpdates disables service updating
+	disableServiceUpdates = "disable_service_updates"
+
+	// enableEndpoints enables use of Endpoints instead of EndpointSlices
+	enableEndpoints = "enable_endpoints"
+
+	// mirrorDestInterface is the network interface where all traffics that go through service interface
+	// will be mirrored to. The source interface is ServicesInterface by default, fall back to Interface if not set.
+	// + optional
+	mirrorDestInterface = "mirror_dest_interface"
+
+	// iptablesBackend iptables backend, can be specified as `nft` or `legacy`. If not set, it defaults to automatic detection.
+	iptablesBackend = "iptables_backend"
+
+	// backendHealthCheckInterval Interval in seconds for checking backend health.
+	backendHealthCheckInterval = "backend_health_check_interval"
+
+	// healthCheckPort, if set to non-zero will be the port the health check will listen on
+	healthCheckPort = "health_check_port"
+
+	// enableUPNP enables UPNP functions
+	enableUPNP = "enable_upnp"
+
+	// egressClean enables egress cleaning on kube-vip's start
+	egressClean = "egress_clean"
+)

pkg/kubevip/config_generator_test.go

@@ -0,0 +1,23 @@
+package kubevip
+
+import "testing"
+
+func TestParseEnvironment(t *testing.T) {
+
+	tests := []struct {
+		name    string
+		c       *Config
+		wantErr bool
+	}{
+		{"", nil, false},
+		{"", &Config{Interface: "eth0", ServicesInterface: "eth1"}, false},
+	}
+	for _, tt := range tests {
+		t.Logf("%v", tt.c)
+		t.Run(tt.name, func(t *testing.T) {
+			if err := ParseEnvironment(tt.c); (err != nil) != tt.wantErr {
+				t.Errorf("ParseEnvironment() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

pkg/kubevip/config_manager.go

@@ -2,182 +2,56 @@ package kubevip
 
 import (
 	"fmt"
-	"io/ioutil"
-	"os"
-	"strconv"
-	"strings"
 
-	"github.com/ghodss/yaml"
-	log "github.com/sirupsen/logrus"
+	log "log/slog"
+
+	"github.com/vishvananda/netlink"
 )
 
-var endPointIndex int // Holds the previous endpoint (for determining decisions on next endpoint)
+const (
+	Auto = "auto"
+)
 
-//ParseBackendConfig -
-func ParseBackendConfig(ep string) (*BackEnd, error) {
-	endpoint := strings.Split(ep, ":")
-	if len(endpoint) != 2 {
-		return nil, fmt.Errorf("Ensure a backend is in in the format address:port, e.g. 10.0.0.1:8080")
-	}
-	p, err := strconv.Atoi(endpoint[1])
-	if err != nil {
-		return nil, err
-	}
-	return &BackEnd{Address: endpoint[0], Port: p}, nil
-}
-
-//ParsePeerConfig -
-func ParsePeerConfig(ep string) (*RaftPeer, error) {
-	endpoint := strings.Split(ep, ":")
-	if len(endpoint) != 3 {
-		return nil, fmt.Errorf("Ensure a peer is in in the format id:address:port, e.g. server1:10.0.0.1:8080")
-	}
-	p, err := strconv.Atoi(endpoint[2])
-	if err != nil {
-		return nil, err
-	}
-	return &RaftPeer{ID: endpoint[0], Address: endpoint[1], Port: p}, nil
-}
-
-//OpenConfig will attempt to read a file and parse it's contents into a configuration
-func OpenConfig(path string) (*Config, error) {
-	if path == "" {
-		return nil, fmt.Errorf("Path cannot be blank")
-	}
-
-	log.Infof("Reading configuration from [%s]", path)
-
-	// Check the actual path from the string
-	if _, err := os.Stat(path); !os.IsNotExist(err) {
-		// Attempt to read the data
-		configData, err := ioutil.ReadFile(path)
-		if err != nil {
-			return nil, err
+func (c *Config) CheckInterface() error {
+	if c.Interface != "" {
+		if err := isValidInterface(c.Interface); err != nil {
+			return fmt.Errorf("%s is not valid interface, reason: %w", c.Interface, err)
 		}
-
-		// If data is read succesfully parse the yaml
-		var c Config
-		err = yaml.Unmarshal(configData, &c)
-		if err != nil {
-			return nil, err
-		}
-		return &c, nil
-
-	}
-	return nil, fmt.Errorf("Error reading [%s]", path)
-}
-
-//PrintConfig - will print out an instance of the kubevip config
-func (c *Config) PrintConfig() {
-	b, _ := yaml.Marshal(c)
-
-	fmt.Printf(string(b))
-}
-
-//ParseFlags will write the current configuration to a specified [path]
-func (c *Config) ParseFlags(localPeer string, remotePeers, backends []string) error {
-	// Parse localPeer
-	p, err := ParsePeerConfig(localPeer)
-	if err != nil {
-		return err
-	}
-	c.LocalPeer = *p
-
-	// Parse remotePeers
-	//Iterate backends
-	for i := range remotePeers {
-		p, err := ParsePeerConfig(remotePeers[i])
-		if err != nil {
-			return err
-
-		}
-		c.RemotePeers = append(c.RemotePeers, *p)
 	}
 
-	//Iterate backends
-	for i := range backends {
-		b, err := ParseBackendConfig(backends[i])
-		if err != nil {
-			return err
+	if c.ServicesInterface != "" {
+		if err := isValidInterface(c.ServicesInterface); err != nil {
+			return fmt.Errorf("%s is not valid interface, reason: %w", c.ServicesInterface, err)
 		}
-		c.LoadBalancers[0].Backends = append(c.LoadBalancers[0].Backends, *b)
 	}
 
 	return nil
 }
 
-//SampleConfig will create an example configuration and write it to the specified [path]
-func SampleConfig() {
-
-	// Generate Sample configuration
-	c := &Config{
-		// Generate sample peers
-		RemotePeers: []RaftPeer{
-			{
-				ID:      "server2",
-				Address: "192.168.0.2",
-				Port:    10000,
-			},
-			{
-				ID:      "server3",
-				Address: "192.168.0.3",
-				Port:    10000,
-			},
-		},
-		LocalPeer: RaftPeer{
-			ID:      "server1",
-			Address: "192.168.0.1",
-			Port:    10000,
-		},
-		// Virtual IP address
-		VIP: "192.168.0.100",
-		// Interface to bind to
-		Interface: "eth0",
-		// Load Balancer Configuration
-		LoadBalancers: []LoadBalancer{
-			{
-				Name:      "Kubernetes Control Plane",
-				Type:      "http",
-				Port:      6443,
-				BindToVip: true,
-				Backends: []BackEnd{
-					{
-						Address: "192.168.0.100",
-						Port:    6443,
-					},
-					{
-						Address: "192.168.0.101",
-						Port:    6443,
-					},
-					{
-						Address: "192.168.0.102",
-						Port:    6443,
-					},
-				},
-			},
-		},
+func isValidInterface(iface string) error {
+	// auto interface discovery for services is enabled
+	if iface == Auto {
+		return nil
 	}
-	b, _ := yaml.Marshal(c)
-
-	fmt.Printf(string(b))
-}
-
-//WriteConfig will write the current configuration to a specified [path]
-func (c *Config) WriteConfig(path string) error {
-	f, err := os.Create(path)
+	l, err := netlink.LinkByName(iface)
 	if err != nil {
-		return err
+		return fmt.Errorf("get %s failed, error: %w", iface, err)
 	}
-	defer f.Close()
+	attrs := l.Attrs()
 
-	b, err := yaml.Marshal(c)
-	if err != nil {
-		return err
+	// Some interfaces (included but not limited to lo and point-to-point
+	//	interfaces) do not provide a operational status but are safe to use.
+	// From kernek.org: "Interface is in unknown state, neither driver nor
+	// userspace has set operational state. Interface must be considered for user
+	// data as setting operational state has not been implemented in every driver."
+	if attrs.OperState == netlink.OperUnknown {
+		log.Warn(
+			"the status of the interface is unknown. Ensure your interface is ready to accept traffic, if so you can safely ignore this message", "interface",
+			iface,
+		)
+	} else if attrs.OperState != netlink.OperUp {
+		return fmt.Errorf("%s is not up", iface)
 	}
-	bytesWritten, err := f.Write(b)
-	if err != nil {
-		return err
-	}
-	log.Debugf("wrote %d bytes\n", bytesWritten)
+
 	return nil
 }

pkg/kubevip/config_types.go

@@ -1,34 +1,98 @@
 package kubevip
 
 import (
-	"net/url"
-
-	"github.com/plunder-app/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/bgp"
 )
 
-// Config defines all of the settings for the Virtual IP / Load-balancer
+// Config defines all of the settings for the Kube-Vip Pod
 type Config struct {
+	// Logging, settings
+	Logging int32 `yaml:"logging"`
 
-	// LeaderElection defines the settings around Kubernetes LeaderElection
-	LeaderElection
+	// EnableARP, will use ARP to advertise the VIP address
+	EnableARP bool `yaml:"enableARP"`
 
-	// LocalPeer is the configuration of this host
-	LocalPeer RaftPeer `yaml:"localPeer"`
+	// EnableBGP, will use BGP to advertise the VIP address
+	EnableBGP bool `yaml:"enableBGP"`
 
-	// Peers are all of the peers within the RAFT cluster
-	RemotePeers []RaftPeer `yaml:"remotePeers"`
+	// EnableWireguard, will use wireguard to advertise the VIP address
+	EnableWireguard bool `yaml:"enableWireguard"`
+
+	// EnableRoutingTable, will use the routing table to advertise the VIP address
+	EnableRoutingTable bool `yaml:"enableRoutingTable"`
+
+	// EnableControlPlane, will enable the control plane functionality (used for hybrid behaviour)
+	EnableControlPlane bool `yaml:"enableControlPlane"`
+
+	// DetectControlPlane, will attempt to find the control plane from loopback (127.0.0.1)
+	DetectControlPlane bool `yaml:"detectControlPlane"`
+
+	// KubernetesAddr，is the address of the Kubernetes API server on this machine
+	KubernetesAddr string `yaml:"kubernetesAddr"`
+
+	// EnableServices, will enable the services functionality (used for hybrid behaviour)
+	EnableServices bool `yaml:"enableServices"`
+
+	// EnableServicesElection, will enable leaderElection per service
+	EnableServicesElection bool `yaml:"enableServicesElection"`
+
+	// EnableNodeLabeling, will enable node labeling as it becomes leader
+	EnableNodeLabeling bool `yaml:"enableNodeLabeling"`
+
+	// LoadBalancerClassOnly, will enable load balancing only for services with LoadBalancerClass set to "kube-vip.io/kube-vip-class"
+	LoadBalancerClassOnly bool `yaml:"lbClassOnly"`
+
+	// LoadBalancerClassName, will limit the load balancing services to services with LoadBalancerClass set to this value
+	LoadBalancerClassName string `yaml:"lbClassName"`
+
+	// LoadBalancerClassLegacyHandling, will enable legacy loadbalancer class handling which does not force service loadbalancer class and kube-vip's loadbalancer class to be the same.
+	LoadBalancerClassLegacyHandling bool `yaml:"lbClassNameLegacyHandling"`
+
+	// EnableServiceSecurity, will enable the use of iptables to secure services
+	EnableServiceSecurity bool `yaml:"EnableServiceSecurity"`
+
+	// ArpBroadcastRate, defines how often kube-vip will update the network about updates to the network
+	ArpBroadcastRate int64 `yaml:"arpBroadcastRate"`
+
+	// Annotations will define if we're going to wait and lookup configuration from Kubernetes node annotations
+	Annotations string
+
+	// LeaderElectionType defines the backend to run the leader election: kubernetes or etcd. Defaults to kubernetes.
+	// Etcd doesn't support load balancer mode (EnableLoadBalancer=true) or any other feature that depends on the kube-api server.
+	LeaderElectionType string `yaml:"leaderElectionType"`
+
+	// KubernetesLeaderElection defines the settings around Kubernetes KubernetesLeaderElection
+	KubernetesLeaderElection
+
+	// Etcd defines all the settings for the etcd client.
+	Etcd Etcd
 
 	// AddPeersAsBackends, this will automatically add RAFT peers as backends to a loadbalancer
 	AddPeersAsBackends bool `yaml:"addPeersAsBackends"`
 
-	// VIP is the Virtual IP address exposed for the cluster
+	// VIP is the Virtual IP address exposed for the cluster (TODO: deprecate)
 	VIP string `yaml:"vip"`
 
-	// VIPCIDR is cidr range for the VIP (primarily needed for BGP)
-	VIPCIDR string `yaml:"vipCidr"`
+	// VipSubnet is the Subnet that is applied to the VIP
+	VIPSubnet string `yaml:"vipSubnet"`
 
-	// GratuitousARP will broadcast an ARP update when the VIP changes host
-	GratuitousARP bool `yaml:"gratuitousARP"`
+	// Address is the IP or DNS Name to use as a VirtualIP
+	Address string `yaml:"address"`
+
+	// Listen port for the VirtualIP
+	Port uint16 `yaml:"port"`
+
+	// Namespace will define which namespace the control plane pods will run in
+	Namespace string `yaml:"namespace"`
+
+	// Namespace will define which namespace the control plane pods will run in
+	ServiceNamespace string `yaml:"serviceNamespace"`
+
+	// use DDNS to allocate IP when Address is set to a DNS Name
+	DDNS bool `yaml:"ddns"`
+
+	// NodeName - used for matching node name from pod spec
+	NodeName string `yaml:"leaseNodeName"`
 
 	// SingleNode will start the cluster as a single Node (Raft disabled)
 	SingleNode bool `yaml:"singleNode"`
@@ -39,55 +103,118 @@ type Config struct {
 	// Interface is the network interface to bind to (default: First Adapter)
 	Interface string `yaml:"interface,omitempty"`
 
+	// ServicesInterface is the network interface to bind to for services (optional)
+	ServicesInterface string `yaml:"servicesInterface,omitempty"`
+
 	// EnableLoadBalancer, provides the flexibility to make the load-balancer optional
 	EnableLoadBalancer bool `yaml:"enableLoadBalancer"`
 
-	// EnableBGP, will use BGP to advertise the VIP address
-	EnableBGP bool `yaml:"enableBGP"`
+	// Listen port for the IPVS Service
+	LoadBalancerPort uint16 `yaml:"lbPort"`
+
+	// Forwarding method for the IPVS Service
+	LoadBalancerForwardingMethod string `yaml:"lbForwardingMethod"`
+
+	// Routing Table ID for when using routing table mode
+	RoutingTableID int `yaml:"routingTableID"`
+
+	// Routing Table Type, what sort of route should be added to the routing table
+	RoutingTableType int `yaml:"routingTableType"`
+
+	// Routing Protocol, value that will be used as protocol when creating rutes
+	RoutingProtocol int `yaml:"routingProtocol"`
+
+	// Clean routing table of redundant routes on start
+	CleanRoutingTable bool `yaml:"cleanRoutingTable"`
 
 	// BGP Configuration
 	BGPConfig     bgp.Config
 	BGPPeerConfig bgp.Peer
-
-	// EnablePacket, will use the packet API to update the EIP <-> VIP (if BGP is enabled then BGP will be used)
-	EnablePacket bool `yaml:"enablePacket"`
-
-	// PacketAPIKey, is the API token used to authenticate to the API
-	PacketAPIKey string
-
-	// PacketProject, is the name of a particular defined project
-	PacketProject string
+	BGPPeers      []string
 
 	// LoadBalancers are the various services we can load balance over
 	LoadBalancers []LoadBalancer `yaml:"loadBalancers,omitempty"`
+
+	// The hostport used to expose Prometheus metrics over an HTTP server
+	PrometheusHTTPServer string `yaml:"prometheusHTTPServer,omitempty"`
+
+	// Egress configuration
+
+	// EgressPodCidr, this contains the pod cidr range to ignore Egress
+	EgressPodCidr string
+
+	// EgressServiceCidr, this contains the service cidr range to ignore
+	EgressServiceCidr string
+
+	// EgressWithNftables, this will use the iptables-nftables OVER iptables
+	EgressWithNftables bool
+
+	// ServicesLeaseName, this will set the lease name for services leader in arp mode
+	ServicesLeaseName string `yaml:"servicesLeaseName"`
+
+	// K8sConfigFile, this is the path to the config file used to speak with the API server
+	K8sConfigFile string `yaml:"k8sConfigFile"`
+
+	// DNSMode, this will set the mode DSN lookup will be performed (first, ipv4, ipv6, dual)
+	DNSMode string `yaml:"dnsDualStackMode"`
+
+	// DisableServiceUpdates, if true, kube-vip will only advertise service, but it will not update service's Status.LoadBalancer.Ingress slice
+	DisableServiceUpdates bool `yaml:"disableServiceUpdates"`
+
+	// EnableEndpoints, if enabled, Endpoints will be used instead of EndpointSlices
+	EnableEndpoints bool `yaml:"enableEndpoints"`
+
+	// MirrorDestInterface is the network interface where all traffics that go through service interface
+	// will be mirrored to. If ServicesInterface is not set, fall back to Interface.
+	// + optional
+	MirrorDestInterface string `yaml:"mirrorDestInterface"`
+
+	// IptablesBackend iptables backend, can be specified as `nft` or `legacy`. If not set, it defaults to automatic detection.
+	IptablesBackend string `yaml:"iptablesBackend"`
+
+	// BackendHealthCheckInterval Interval in seconds for checking backend health.
+	BackendHealthCheckInterval int `yaml:"backendHealthCheckInterval"`
+
+	// LoInterfaceGlobalScope, if true will set global scope when using the lo interface, otherwise a host scope will be used
+	LoInterfaceGlobalScope bool `yaml:"loInterfaceGlobalScope"`
+
+	// HealthCheckPort, if non-zero then will enable the healthcheck to return ok on this port
+	HealthCheckPort int `yaml:"healthCheckPort"`
+
+	// EnableUPNP, enables UPNP functions
+	EnableUPNP bool `yaml:"enableUPNP"`
+
+	// EgressClean, enables egress cleaning on Kube-vip's start
+	EgressClean bool `yaml:"egressClean"`
 }
 
-// LeaderElection defines all of the settings for Kubernetes LeaderElection
-type LeaderElection struct {
-
-	// EnableLeaderElection will use the Kubernetes leader election algorithim
+// KubernetesLeaderElection defines all of the settings for Kubernetes KubernetesLeaderElection
+type KubernetesLeaderElection struct {
+	// EnableLeaderElection will use the Kubernetes leader election algorithm
 	EnableLeaderElection bool `yaml:"enableLeaderElection"`
 
+	// LeaseName - name of the lease for leader election
+	LeaseName string `yaml:"leaseName"`
+
 	// Lease Duration - length of time a lease can be held for
 	LeaseDuration int
 
 	// RenewDeadline - length of time a host can attempt to renew its lease
 	RenewDeadline int
 
-	// RetryPerion - Number of times the host will retry to hold a lease
+	// RetryPeriod - length of time (in seconds) the LeaderElector clients should wait between tries of actions
 	RetryPeriod int
+
+	// LeaseAnnotations - annotations which will be given to the lease object
+	LeaseAnnotations map[string]string
 }
 
-// RaftPeer details the configuration of all cluster peers
-type RaftPeer struct {
-	// ID is the unique identifier a peer instance
-	ID string `yaml:"id"`
-
-	// IP Address of a peer instance
-	Address string `yaml:"address"`
-
-	// Listening port of this peer instance
-	Port int `yaml:"port"`
+// Etcd defines all the settings for the etcd client.
+type Etcd struct {
+	CAFile         string
+	ClientCertFile string
+	ClientKeyFile  string
+	Endpoints      []string
 }
 
 // LoadBalancer contains the configuration of a load balancing instance
@@ -95,33 +222,20 @@ type LoadBalancer struct {
 	// Name of a LoadBalancer
 	Name string `yaml:"name"`
 
-	// Type of LoadBalancer, either TCP of HTTP(s)
-	Type string `yaml:"type"`
-
-	// Listening frontend port of this LoadBalancer instance
-	Port int `yaml:"port"`
+	//Ports exposed by a LoadBalancer
+	Ports []Port
 
 	// BindToVip will bind the load balancer port to the VIP itself
 	BindToVip bool `yaml:"bindToVip"`
 
-	//BackendPort, is a port that all backends are listening on (To be used to simplify building a list of backends)
-	BackendPort int `yaml:"backendPort"`
-
-	//Backends, is an array of backend servers
-	Backends []BackEnd `yaml:"backends"`
+	// Forwarding method of LoadBalancer, either Local, Tunnel, DirectRoute or Bypass
+	ForwardingMethod string `yaml:"forwardingMethod"`
 }
 
-// BackEnd is a server we will load balance over
-type BackEnd struct {
-	// Backend Port to Load Balance to
+type Port struct {
+	// Type of LoadBalancer, either TCP or UDP
+	Type string `yaml:"type"`
+
+	// Listening frontend port of this LoadBalancer instance
 	Port int `yaml:"port"`
-
-	// Address of a server/service
-	Address string `yaml:"address"`
-
-	// URL is a raw URL to a backend service
-	RawURL string `yaml:"url,omitempty"`
-
-	// ParsedURL - A validated URL to a backend
-	ParsedURL *url.URL `yaml:"parsedURL,omitempty"`
 }

pkg/leaderElection/OWNERS

@@ -1,14 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-
-approvers:
-- mikedanese
-- timothysc
-reviewers:
-- wojtek-t
-- deads2k
-- mikedanese
-- gmarek
-- eparis
-- timothysc
-- ingvagabund
-- resouer

pkg/leaderElection/healthzadaptor.go

@@ -1,69 +0,0 @@
-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package leaderelection
-
-import (
-	"net/http"
-	"sync"
-	"time"
-)
-
-// HealthzAdaptor associates the /healthz endpoint with the LeaderElection object.
-// It helps deal with the /healthz endpoint being set up prior to the LeaderElection.
-// This contains the code needed to act as an adaptor between the leader
-// election code the health check code. It allows us to provide health
-// status about the leader election. Most specifically about if the leader
-// has failed to renew without exiting the process. In that case we should
-// report not healthy and rely on the kubelet to take down the process.
-type HealthzAdaptor struct {
-	pointerLock sync.Mutex
-	le          *LeaderElector
-	timeout     time.Duration
-}
-
-// Name returns the name of the health check we are implementing.
-func (l *HealthzAdaptor) Name() string {
-	return "leaderElection"
-}
-
-// Check is called by the healthz endpoint handler.
-// It fails (returns an error) if we own the lease but had not been able to renew it.
-func (l *HealthzAdaptor) Check(req *http.Request) error {
-	l.pointerLock.Lock()
-	defer l.pointerLock.Unlock()
-	if l.le == nil {
-		return nil
-	}
-	return l.le.Check(l.timeout)
-}
-
-// SetLeaderElection ties a leader election object to a HealthzAdaptor
-func (l *HealthzAdaptor) SetLeaderElection(le *LeaderElector) {
-	l.pointerLock.Lock()
-	defer l.pointerLock.Unlock()
-	l.le = le
-}
-
-// NewLeaderHealthzAdaptor creates a basic healthz adaptor to monitor a leader election.
-// timeout determines the time beyond the lease expiry to be allowed for timeout.
-// checks within the timeout period after the lease expires will still return healthy.
-func NewLeaderHealthzAdaptor(timeout time.Duration) *HealthzAdaptor {
-	result := &HealthzAdaptor{
-		timeout: timeout,
-	}
-	return result
-}

pkg/leaderElection/healthzadaptor_test.go

@@ -1,177 +0,0 @@
-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package leaderelection
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"net/http"
-
-	"k8s.io/apimachinery/pkg/util/clock"
-	rl "k8s.io/client-go/tools/leaderelection/resourcelock"
-)
-
-type fakeLock struct {
-	identity string
-}
-
-// Get is a dummy to allow us to have a fakeLock for testing.
-func (fl *fakeLock) Get(ctx context.Context) (ler *rl.LeaderElectionRecord, rawRecord []byte, err error) {
-	return nil, nil, nil
-}
-
-// Create is a dummy to allow us to have a fakeLock for testing.
-func (fl *fakeLock) Create(ctx context.Context, ler rl.LeaderElectionRecord) error {
-	return nil
-}
-
-// Update is a dummy to allow us to have a fakeLock for testing.
-func (fl *fakeLock) Update(ctx context.Context, ler rl.LeaderElectionRecord) error {
-	return nil
-}
-
-// RecordEvent is a dummy to allow us to have a fakeLock for testing.
-func (fl *fakeLock) RecordEvent(string) {}
-
-// Identity is a dummy to allow us to have a fakeLock for testing.
-func (fl *fakeLock) Identity() string {
-	return fl.identity
-}
-
-// Describe is a dummy to allow us to have a fakeLock for testing.
-func (fl *fakeLock) Describe() string {
-	return "Dummy implementation of lock for testing"
-}
-
-// TestLeaderElectionHealthChecker tests that the healthcheck for leader election handles its edge cases.
-func TestLeaderElectionHealthChecker(t *testing.T) {
-	current := time.Now()
-	req := &http.Request{}
-
-	tests := []struct {
-		description    string
-		expected       error
-		adaptorTimeout time.Duration
-		elector        *LeaderElector
-	}{
-		{
-			description:    "call check before leader elector initialized",
-			expected:       nil,
-			adaptorTimeout: time.Second * 20,
-			elector:        nil,
-		},
-		{
-			description:    "call check when the lease is far expired",
-			expected:       fmt.Errorf("failed election to renew leadership on lease %s", "foo"),
-			adaptorTimeout: time.Second * 20,
-			elector: &LeaderElector{
-				config: LeaderElectionConfig{
-					Lock:          &fakeLock{identity: "healthTest"},
-					LeaseDuration: time.Minute,
-					Name:          "foo",
-				},
-				observedRecord: rl.LeaderElectionRecord{
-					HolderIdentity: "healthTest",
-				},
-				observedTime: current,
-				clock:        clock.NewFakeClock(current.Add(time.Hour)),
-			},
-		},
-		{
-			description:    "call check when the lease is far expired but held by another server",
-			expected:       nil,
-			adaptorTimeout: time.Second * 20,
-			elector: &LeaderElector{
-				config: LeaderElectionConfig{
-					Lock:          &fakeLock{identity: "healthTest"},
-					LeaseDuration: time.Minute,
-					Name:          "foo",
-				},
-				observedRecord: rl.LeaderElectionRecord{
-					HolderIdentity: "otherServer",
-				},
-				observedTime: current,
-				clock:        clock.NewFakeClock(current.Add(time.Hour)),
-			},
-		},
-		{
-			description:    "call check when the lease is not expired",
-			expected:       nil,
-			adaptorTimeout: time.Second * 20,
-			elector: &LeaderElector{
-				config: LeaderElectionConfig{
-					Lock:          &fakeLock{identity: "healthTest"},
-					LeaseDuration: time.Minute,
-					Name:          "foo",
-				},
-				observedRecord: rl.LeaderElectionRecord{
-					HolderIdentity: "healthTest",
-				},
-				observedTime: current,
-				clock:        clock.NewFakeClock(current),
-			},
-		},
-		{
-			description:    "call check when the lease is expired but inside the timeout",
-			expected:       nil,
-			adaptorTimeout: time.Second * 20,
-			elector: &LeaderElector{
-				config: LeaderElectionConfig{
-					Lock:          &fakeLock{identity: "healthTest"},
-					LeaseDuration: time.Minute,
-					Name:          "foo",
-				},
-				observedRecord: rl.LeaderElectionRecord{
-					HolderIdentity: "healthTest",
-				},
-				observedTime: current,
-				clock:        clock.NewFakeClock(current.Add(time.Minute).Add(time.Second)),
-			},
-		},
-	}
-
-	for _, test := range tests {
-		adaptor := NewLeaderHealthzAdaptor(test.adaptorTimeout)
-		if adaptor.le != nil {
-			t.Errorf("[%s] leaderChecker started with a LeaderElector %v", test.description, adaptor.le)
-		}
-		if test.elector != nil {
-			test.elector.config.WatchDog = adaptor
-			adaptor.SetLeaderElection(test.elector)
-			if adaptor.le == nil {
-				t.Errorf("[%s] adaptor failed to set the LeaderElector", test.description)
-			}
-		}
-		err := adaptor.Check(req)
-		if test.expected == nil {
-			if err == nil {
-				continue
-			}
-			t.Errorf("[%s] called check, expected no error but received \"%v\"", test.description, err)
-		} else {
-			if err == nil {
-				t.Errorf("[%s] called check and failed to received the expected error \"%v\"", test.description, test.expected)
-			}
-			if err.Error() != test.expected.Error() {
-				t.Errorf("[%s] called check, expected %v, received %v", test.description, test.expected, err)
-			}
-		}
-	}
-}

pkg/leaderElection/leaderelection.go

@@ -1,390 +0,0 @@
-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package leaderelection implements leader election of a set of endpoints.
-// It uses an annotation in the endpoints object to store the record of the
-// election state. This implementation does not guarantee that only one
-// client is acting as a leader (a.k.a. fencing).
-//
-// A client only acts on timestamps captured locally to infer the state of the
-// leader election. The client does not consider timestamps in the leader
-// election record to be accurate because these timestamps may not have been
-// produced by a local clock. The implemention does not depend on their
-// accuracy and only uses their change to indicate that another client has
-// renewed the leader lease. Thus the implementation is tolerant to arbitrary
-// clock skew, but is not tolerant to arbitrary clock skew rate.
-//
-// However the level of tolerance to skew rate can be configured by setting
-// RenewDeadline and LeaseDuration appropriately. The tolerance expressed as a
-// maximum tolerated ratio of time passed on the fastest node to time passed on
-// the slowest node can be approximately achieved with a configuration that sets
-// the same ratio of LeaseDuration to RenewDeadline. For example if a user wanted
-// to tolerate some nodes progressing forward in time twice as fast as other nodes,
-// the user could set LeaseDuration to 60 seconds and RenewDeadline to 30 seconds.
-//
-// While not required, some method of clock synchronization between nodes in the
-// cluster is highly recommended. It's important to keep in mind when configuring
-// this client that the tolerance to skew rate varies inversely to master
-// availability.
-//
-// Larger clusters often have a more lenient SLA for API latency. This should be
-// taken into account when configuring the client. The rate of leader transitions
-// should be monitored and RetryPeriod and LeaseDuration should be increased
-// until the rate is stable and acceptably low. It's important to keep in mind
-// when configuring this client that the tolerance to API latency varies inversely
-// to master availability.
-//
-// DISCLAIMER: this is an alpha API. This library will likely change significantly
-// or even be removed entirely in subsequent releases. Depend on this API at
-// your own risk.
-package leaderelection
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"time"
-
-	"k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/clock"
-	"k8s.io/apimachinery/pkg/util/runtime"
-	"k8s.io/apimachinery/pkg/util/wait"
-	rl "k8s.io/client-go/tools/leaderelection/resourcelock"
-
-	"k8s.io/klog"
-)
-
-const (
-	JitterFactor = 1.2
-)
-
-// NewLeaderElector creates a LeaderElector from a LeaderElectionConfig
-func NewLeaderElector(lec LeaderElectionConfig) (*LeaderElector, error) {
-	if lec.LeaseDuration <= lec.RenewDeadline {
-		return nil, fmt.Errorf("leaseDuration must be greater than renewDeadline")
-	}
-	if lec.RenewDeadline <= time.Duration(JitterFactor*float64(lec.RetryPeriod)) {
-		return nil, fmt.Errorf("renewDeadline must be greater than retryPeriod*JitterFactor")
-	}
-	if lec.LeaseDuration < 1 {
-		return nil, fmt.Errorf("leaseDuration must be greater than zero")
-	}
-	if lec.RenewDeadline < 1 {
-		return nil, fmt.Errorf("renewDeadline must be greater than zero")
-	}
-	if lec.RetryPeriod < 1 {
-		return nil, fmt.Errorf("retryPeriod must be greater than zero")
-	}
-	if lec.Callbacks.OnStartedLeading == nil {
-		return nil, fmt.Errorf("OnStartedLeading callback must not be nil")
-	}
-	if lec.Callbacks.OnStoppedLeading == nil {
-		return nil, fmt.Errorf("OnStoppedLeading callback must not be nil")
-	}
-
-	if lec.Lock == nil {
-		return nil, fmt.Errorf("Lock must not be nil.")
-	}
-	le := LeaderElector{
-		config:  lec,
-		clock:   clock.RealClock{},
-		metrics: globalMetricsFactory.newLeaderMetrics(),
-	}
-	le.metrics.leaderOff(le.config.Name)
-	return &le, nil
-}
-
-type LeaderElectionConfig struct {
-	// Lock is the resource that will be used for locking
-	Lock rl.Interface
-
-	// LeaseDuration is the duration that non-leader candidates will
-	// wait to force acquire leadership. This is measured against time of
-	// last observed ack.
-	//
-	// A client needs to wait a full LeaseDuration without observing a change to
-	// the record before it can attempt to take over. When all clients are
-	// shutdown and a new set of clients are started with different names against
-	// the same leader record, they must wait the full LeaseDuration before
-	// attempting to acquire the lease. Thus LeaseDuration should be as short as
-	// possible (within your tolerance for clock skew rate) to avoid a possible
-	// long waits in the scenario.
-	//
-	// Core clients default this value to 15 seconds.
-	LeaseDuration time.Duration
-	// RenewDeadline is the duration that the acting master will retry
-	// refreshing leadership before giving up.
-	//
-	// Core clients default this value to 10 seconds.
-	RenewDeadline time.Duration
-	// RetryPeriod is the duration the LeaderElector clients should wait
-	// between tries of actions.
-	//
-	// Core clients default this value to 2 seconds.
-	RetryPeriod time.Duration
-
-	// Callbacks are callbacks that are triggered during certain lifecycle
-	// events of the LeaderElector
-	Callbacks LeaderCallbacks
-
-	// WatchDog is the associated health checker
-	// WatchDog may be null if its not needed/configured.
-	WatchDog *HealthzAdaptor
-
-	// ReleaseOnCancel should be set true if the lock should be released
-	// when the run context is cancelled. If you set this to true, you must
-	// ensure all code guarded by this lease has successfully completed
-	// prior to cancelling the context, or you may have two processes
-	// simultaneously acting on the critical path.
-	ReleaseOnCancel bool
-
-	// Name is the name of the resource lock for debugging
-	Name string
-}
-
-// LeaderCallbacks are callbacks that are triggered during certain
-// lifecycle events of the LeaderElector. These are invoked asynchronously.
-//
-// possible future callbacks:
-//  * OnChallenge()
-type LeaderCallbacks struct {
-	// OnStartedLeading is called when a LeaderElector client starts leading
-	OnStartedLeading func(context.Context)
-	// OnStoppedLeading is called when a LeaderElector client stops leading
-	OnStoppedLeading func()
-	// OnNewLeader is called when the client observes a leader that is
-	// not the previously observed leader. This includes the first observed
-	// leader when the client starts.
-	OnNewLeader func(identity string)
-}
-
-// LeaderElector is a leader election client.
-type LeaderElector struct {
-	config LeaderElectionConfig
-	// internal bookkeeping
-	observedRecord    rl.LeaderElectionRecord
-	observedRawRecord []byte
-	observedTime      time.Time
-	// used to implement OnNewLeader(), may lag slightly from the
-	// value observedRecord.HolderIdentity if the transition has
-	// not yet been reported.
-	reportedLeader string
-
-	// clock is wrapper around time to allow for less flaky testing
-	clock clock.Clock
-
-	metrics leaderMetricsAdapter
-
-	// name is the name of the resource lock for debugging
-	name string
-}
-
-// Run starts the leader election loop
-func (le *LeaderElector) Run(ctx context.Context) {
-	defer func() {
-		runtime.HandleCrash()
-		le.config.Callbacks.OnStoppedLeading()
-	}()
-	if !le.acquire(ctx) {
-		return // ctx signalled done
-	}
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-	go le.config.Callbacks.OnStartedLeading(ctx)
-	le.renew(ctx)
-}
-
-// RunOrDie starts a client with the provided config or panics if the config
-// fails to validate.
-func RunOrDie(ctx context.Context, lec LeaderElectionConfig) {
-	le, err := NewLeaderElector(lec)
-	if err != nil {
-		panic(err)
-	}
-	if lec.WatchDog != nil {
-		lec.WatchDog.SetLeaderElection(le)
-	}
-	le.Run(ctx)
-}
-
-// GetLeader returns the identity of the last observed leader or returns the empty string if
-// no leader has yet been observed.
-func (le *LeaderElector) GetLeader() string {
-	return le.observedRecord.HolderIdentity
-}
-
-// IsLeader returns true if the last observed leader was this client else returns false.
-func (le *LeaderElector) IsLeader() bool {
-	return le.observedRecord.HolderIdentity == le.config.Lock.Identity()
-}
-
-// acquire loops calling tryAcquireOrRenew and returns true immediately when tryAcquireOrRenew succeeds.
-// Returns false if ctx signals done.
-func (le *LeaderElector) acquire(ctx context.Context) bool {
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-	succeeded := false
-	desc := le.config.Lock.Describe()
-	klog.Infof("attempting to acquire leader lease  %v...", desc)
-	wait.JitterUntil(func() {
-		succeeded = le.tryAcquireOrRenew(ctx)
-		le.maybeReportTransition()
-		if !succeeded {
-			klog.V(4).Infof("failed to acquire lease %v", desc)
-			return
-		}
-		le.config.Lock.RecordEvent("became leader")
-		le.metrics.leaderOn(le.config.Name)
-		klog.Infof("successfully acquired lease %v", desc)
-		cancel()
-	}, le.config.RetryPeriod, JitterFactor, true, ctx.Done())
-	return succeeded
-}
-
-// renew loops calling tryAcquireOrRenew and returns immediately when tryAcquireOrRenew fails or ctx signals done.
-func (le *LeaderElector) renew(ctx context.Context) {
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-	wait.Until(func() {
-		timeoutCtx, timeoutCancel := context.WithTimeout(ctx, le.config.RenewDeadline)
-		defer timeoutCancel()
-		err := wait.PollImmediateUntil(le.config.RetryPeriod, func() (bool, error) {
-			return le.tryAcquireOrRenew(timeoutCtx), nil
-		}, timeoutCtx.Done())
-
-		le.maybeReportTransition()
-		desc := le.config.Lock.Describe()
-		if err == nil {
-			klog.V(5).Infof("successfully renewed lease %v", desc)
-			return
-		}
-		le.config.Lock.RecordEvent("stopped leading")
-		le.metrics.leaderOff(le.config.Name)
-		klog.Infof("failed to renew lease %v: %v", desc, err)
-		cancel()
-	}, le.config.RetryPeriod, ctx.Done())
-
-	// if we hold the lease, give it up
-	if le.config.ReleaseOnCancel {
-		le.release()
-	}
-}
-
-// release attempts to release the leader lease if we have acquired it.
-func (le *LeaderElector) release() bool {
-	if !le.IsLeader() {
-		return true
-	}
-	leaderElectionRecord := rl.LeaderElectionRecord{
-		LeaseDurationSeconds: le.observedRecord.LeaseDurationSeconds,
-		LeaderTransitions:    le.observedRecord.LeaderTransitions,
-	}
-	if err := le.config.Lock.Update(context.TODO(), leaderElectionRecord); err != nil {
-		klog.Errorf("Failed to release lock: %v", err)
-		return false
-	}
-	le.observedRecord = leaderElectionRecord
-	le.observedTime = le.clock.Now()
-	return true
-}
-
-// tryAcquireOrRenew tries to acquire a leader lease if it is not already acquired,
-// else it tries to renew the lease if it has already been acquired. Returns true
-// on success else returns false.
-func (le *LeaderElector) tryAcquireOrRenew(ctx context.Context) bool {
-	now := metav1.Now()
-	leaderElectionRecord := rl.LeaderElectionRecord{
-		HolderIdentity:       le.config.Lock.Identity(),
-		LeaseDurationSeconds: int(le.config.LeaseDuration / time.Second),
-		RenewTime:            now,
-		AcquireTime:          now,
-	}
-
-	// 1. obtain or create the ElectionRecord
-	oldLeaderElectionRecord, oldLeaderElectionRawRecord, err := le.config.Lock.Get(ctx)
-	if err != nil {
-		if !errors.IsNotFound(err) {
-			klog.Errorf("error retrieving resource lock %v: %v", le.config.Lock.Describe(), err)
-			return false
-		}
-		if err = le.config.Lock.Create(ctx, leaderElectionRecord); err != nil {
-			klog.Errorf("error initially creating leader election record: %v", err)
-			return false
-		}
-		le.observedRecord = leaderElectionRecord
-		le.observedTime = le.clock.Now()
-		return true
-	}
-
-	// 2. Record obtained, check the Identity & Time
-	if !bytes.Equal(le.observedRawRecord, oldLeaderElectionRawRecord) {
-		le.observedRecord = *oldLeaderElectionRecord
-		le.observedRawRecord = oldLeaderElectionRawRecord
-		le.observedTime = le.clock.Now()
-	}
-	if len(oldLeaderElectionRecord.HolderIdentity) > 0 &&
-		le.observedTime.Add(le.config.LeaseDuration).After(now.Time) &&
-		!le.IsLeader() {
-		klog.V(4).Infof("lock is held by %v and has not yet expired", oldLeaderElectionRecord.HolderIdentity)
-		return false
-	}
-
-	// 3. We're going to try to update. The leaderElectionRecord is set to it's default
-	// here. Let's correct it before updating.
-	if le.IsLeader() {
-		leaderElectionRecord.AcquireTime = oldLeaderElectionRecord.AcquireTime
-		leaderElectionRecord.LeaderTransitions = oldLeaderElectionRecord.LeaderTransitions
-	} else {
-		leaderElectionRecord.LeaderTransitions = oldLeaderElectionRecord.LeaderTransitions + 1
-	}
-
-	// update the lock itself
-	if err = le.config.Lock.Update(ctx, leaderElectionRecord); err != nil {
-		klog.Errorf("Failed to update lock: %v", err)
-		return false
-	}
-
-	le.observedRecord = leaderElectionRecord
-	le.observedTime = le.clock.Now()
-	return true
-}
-
-func (le *LeaderElector) maybeReportTransition() {
-	if le.observedRecord.HolderIdentity == le.reportedLeader {
-		return
-	}
-	le.reportedLeader = le.observedRecord.HolderIdentity
-	if le.config.Callbacks.OnNewLeader != nil {
-		go le.config.Callbacks.OnNewLeader(le.reportedLeader)
-	}
-}
-
-// Check will determine if the current lease is expired by more than timeout.
-func (le *LeaderElector) Check(maxTolerableExpiredLease time.Duration) error {
-	if !le.IsLeader() {
-		// Currently not concerned with the case that we are hot standby
-		return nil
-	}
-	// If we are more than timeout seconds after the lease duration that is past the timeout
-	// on the lease renew. Time to start reporting ourselves as unhealthy. We should have
-	// died but conditions like deadlock can prevent this. (See #70819)
-	if le.clock.Since(le.observedTime) > le.config.LeaseDuration+maxTolerableExpiredLease {
-		return fmt.Errorf("failed election to renew leadership on lease %s", le.config.Name)
-	}
-
-	return nil
-}

pkg/leaderElection/leaderelection_test.go

@@ -1,919 +0,0 @@
-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package leaderelection
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"sync"
-	"testing"
-	"time"
-
-	coordinationv1 "k8s.io/api/coordination/v1"
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/equality"
-	"k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/util/clock"
-	"k8s.io/apimachinery/pkg/util/diff"
-	"k8s.io/client-go/kubernetes/fake"
-	fakeclient "k8s.io/client-go/testing"
-	rl "k8s.io/client-go/tools/leaderelection/resourcelock"
-	"k8s.io/client-go/tools/record"
-)
-
-func createLockObject(t *testing.T, objectType, namespace, name string, record *rl.LeaderElectionRecord) (obj runtime.Object) {
-	objectMeta := metav1.ObjectMeta{
-		Namespace: namespace,
-		Name:      name,
-	}
-	if record != nil {
-		recordBytes, _ := json.Marshal(record)
-		objectMeta.Annotations = map[string]string{
-			rl.LeaderElectionRecordAnnotationKey: string(recordBytes),
-		}
-	}
-	switch objectType {
-	case "endpoints":
-		obj = &corev1.Endpoints{ObjectMeta: objectMeta}
-	case "configmaps":
-		obj = &corev1.ConfigMap{ObjectMeta: objectMeta}
-	case "leases":
-		var spec coordinationv1.LeaseSpec
-		if record != nil {
-			spec = rl.LeaderElectionRecordToLeaseSpec(record)
-		}
-		obj = &coordinationv1.Lease{ObjectMeta: objectMeta, Spec: spec}
-	default:
-		t.Fatal("unexpected objType:" + objectType)
-	}
-	return
-}
-
-// Will test leader election using endpoints as the resource
-func TestTryAcquireOrRenewEndpoints(t *testing.T) {
-	testTryAcquireOrRenew(t, "endpoints")
-}
-
-type Reactor struct {
-	verb       string
-	objectType string
-	reaction   fakeclient.ReactionFunc
-}
-
-func testTryAcquireOrRenew(t *testing.T, objectType string) {
-	future := time.Now().Add(1000 * time.Hour)
-	past := time.Now().Add(-1000 * time.Hour)
-
-	tests := []struct {
-		name           string
-		observedRecord rl.LeaderElectionRecord
-		observedTime   time.Time
-		reactors       []Reactor
-
-		expectSuccess    bool
-		transitionLeader bool
-		outHolder        string
-	}{
-		{
-			name: "acquire from no object",
-			reactors: []Reactor{
-				{
-					verb: "get",
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName())
-					},
-				},
-				{
-					verb: "create",
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.CreateAction).GetObject(), nil
-					},
-				},
-			},
-			expectSuccess: true,
-			outHolder:     "baz",
-		},
-		{
-			name: "acquire from object without annotations",
-			reactors: []Reactor{
-				{
-					verb: "get",
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), nil), nil
-					},
-				},
-				{
-					verb: "update",
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.CreateAction).GetObject(), nil
-					},
-				},
-			},
-			expectSuccess:    true,
-			transitionLeader: true,
-			outHolder:        "baz",
-		},
-		{
-			name: "acquire from unled object",
-			reactors: []Reactor{
-				{
-					verb: "get",
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{}), nil
-					},
-				},
-				{
-					verb: "update",
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.CreateAction).GetObject(), nil
-					},
-				},
-			},
-
-			expectSuccess:    true,
-			transitionLeader: true,
-			outHolder:        "baz",
-		},
-		{
-			name: "acquire from led, unacked object",
-			reactors: []Reactor{
-				{
-					verb: "get",
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil
-					},
-				},
-				{
-					verb: "update",
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.CreateAction).GetObject(), nil
-					},
-				},
-			},
-			observedRecord: rl.LeaderElectionRecord{HolderIdentity: "bing"},
-			observedTime:   past,
-
-			expectSuccess:    true,
-			transitionLeader: true,
-			outHolder:        "baz",
-		},
-		{
-			name: "acquire from empty led, acked object",
-			reactors: []Reactor{
-				{
-					verb: "get",
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: ""}), nil
-					},
-				},
-				{
-					verb: "update",
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.CreateAction).GetObject(), nil
-					},
-				},
-			},
-			observedTime: future,
-
-			expectSuccess:    true,
-			transitionLeader: true,
-			outHolder:        "baz",
-		},
-		{
-			name: "don't acquire from led, acked object",
-			reactors: []Reactor{
-				{
-					verb: "get",
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil
-					},
-				},
-			},
-			observedTime: future,
-
-			expectSuccess: false,
-			outHolder:     "bing",
-		},
-		{
-			name: "renew already acquired object",
-			reactors: []Reactor{
-				{
-					verb: "get",
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "baz"}), nil
-					},
-				},
-				{
-					verb: "update",
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.CreateAction).GetObject(), nil
-					},
-				},
-			},
-			observedTime:   future,
-			observedRecord: rl.LeaderElectionRecord{HolderIdentity: "baz"},
-
-			expectSuccess: true,
-			outHolder:     "baz",
-		},
-	}
-
-	for i := range tests {
-		test := &tests[i]
-		t.Run(test.name, func(t *testing.T) {
-			// OnNewLeader is called async so we have to wait for it.
-			var wg sync.WaitGroup
-			wg.Add(1)
-			var reportedLeader string
-			var lock rl.Interface
-
-			objectMeta := metav1.ObjectMeta{Namespace: "foo", Name: "bar"}
-			resourceLockConfig := rl.ResourceLockConfig{
-				Identity:      "baz",
-				EventRecorder: &record.FakeRecorder{},
-			}
-			c := &fake.Clientset{}
-			for _, reactor := range test.reactors {
-				c.AddReactor(reactor.verb, objectType, reactor.reaction)
-			}
-			c.AddReactor("*", "*", func(action fakeclient.Action) (bool, runtime.Object, error) {
-				t.Errorf("unreachable action. testclient called too many times: %+v", action)
-				return true, nil, fmt.Errorf("unreachable action")
-			})
-
-			switch objectType {
-			case "endpoints":
-				lock = &rl.EndpointsLock{
-					EndpointsMeta: objectMeta,
-					LockConfig:    resourceLockConfig,
-					Client:        c.CoreV1(),
-				}
-			case "configmaps":
-				lock = &rl.ConfigMapLock{
-					ConfigMapMeta: objectMeta,
-					LockConfig:    resourceLockConfig,
-					Client:        c.CoreV1(),
-				}
-			case "leases":
-				lock = &rl.LeaseLock{
-					LeaseMeta:  objectMeta,
-					LockConfig: resourceLockConfig,
-					Client:     c.CoordinationV1(),
-				}
-			}
-
-			lec := LeaderElectionConfig{
-				Lock:          lock,
-				LeaseDuration: 10 * time.Second,
-				Callbacks: LeaderCallbacks{
-					OnNewLeader: func(l string) {
-						defer wg.Done()
-						reportedLeader = l
-					},
-				},
-			}
-			observedRawRecord := GetRawRecordOrDie(t, objectType, test.observedRecord)
-			le := &LeaderElector{
-				config:            lec,
-				observedRecord:    test.observedRecord,
-				observedRawRecord: observedRawRecord,
-				observedTime:      test.observedTime,
-				clock:             clock.RealClock{},
-			}
-			if test.expectSuccess != le.tryAcquireOrRenew(context.Background()) {
-				t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", !test.expectSuccess)
-			}
-
-			le.observedRecord.AcquireTime = metav1.Time{}
-			le.observedRecord.RenewTime = metav1.Time{}
-			if le.observedRecord.HolderIdentity != test.outHolder {
-				t.Errorf("expected holder:\n\t%+v\ngot:\n\t%+v", test.outHolder, le.observedRecord.HolderIdentity)
-			}
-			if len(test.reactors) != len(c.Actions()) {
-				t.Errorf("wrong number of api interactions")
-			}
-			if test.transitionLeader && le.observedRecord.LeaderTransitions != 1 {
-				t.Errorf("leader should have transitioned but did not")
-			}
-			if !test.transitionLeader && le.observedRecord.LeaderTransitions != 0 {
-				t.Errorf("leader should not have transitioned but did")
-			}
-
-			le.maybeReportTransition()
-			wg.Wait()
-			if reportedLeader != test.outHolder {
-				t.Errorf("reported leader was not the new leader. expected %q, got %q", test.outHolder, reportedLeader)
-			}
-		})
-	}
-}
-
-// Will test leader election using configmap as the resource
-func TestTryAcquireOrRenewConfigMaps(t *testing.T) {
-	testTryAcquireOrRenew(t, "configmaps")
-}
-
-// Will test leader election using lease as the resource
-func TestTryAcquireOrRenewLeases(t *testing.T) {
-	testTryAcquireOrRenew(t, "leases")
-}
-
-func TestLeaseSpecToLeaderElectionRecordRoundTrip(t *testing.T) {
-	holderIdentity := "foo"
-	leaseDurationSeconds := int32(10)
-	leaseTransitions := int32(1)
-	oldSpec := coordinationv1.LeaseSpec{
-		HolderIdentity:       &holderIdentity,
-		LeaseDurationSeconds: &leaseDurationSeconds,
-		AcquireTime:          &metav1.MicroTime{time.Now()},
-		RenewTime:            &metav1.MicroTime{time.Now()},
-		LeaseTransitions:     &leaseTransitions,
-	}
-
-	oldRecord := rl.LeaseSpecToLeaderElectionRecord(&oldSpec)
-	newSpec := rl.LeaderElectionRecordToLeaseSpec(oldRecord)
-
-	if !equality.Semantic.DeepEqual(oldSpec, newSpec) {
-		t.Errorf("diff: %v", diff.ObjectReflectDiff(oldSpec, newSpec))
-	}
-
-	newRecord := rl.LeaseSpecToLeaderElectionRecord(&newSpec)
-
-	if !equality.Semantic.DeepEqual(oldRecord, newRecord) {
-		t.Errorf("diff: %v", diff.ObjectReflectDiff(oldRecord, newRecord))
-	}
-}
-
-func multiLockType(t *testing.T, objectType string) (primaryType, secondaryType string) {
-	switch objectType {
-	case rl.EndpointsLeasesResourceLock:
-		return rl.EndpointsResourceLock, rl.LeasesResourceLock
-	case rl.ConfigMapsLeasesResourceLock:
-		return rl.ConfigMapsResourceLock, rl.LeasesResourceLock
-	default:
-		t.Fatal("unexpected objType:" + objectType)
-	}
-	return
-}
-
-func GetRawRecordOrDie(t *testing.T, objectType string, ler rl.LeaderElectionRecord) (ret []byte) {
-	var err error
-	switch objectType {
-	case "endpoints", "configmaps", "leases":
-		ret, err = json.Marshal(ler)
-		if err != nil {
-			t.Fatalf("lock %s get raw record %v failed: %v", objectType, ler, err)
-		}
-	case "endpointsleases", "configmapsleases":
-		recordBytes, err := json.Marshal(ler)
-		if err != nil {
-			t.Fatalf("lock %s get raw record %v failed: %v", objectType, ler, err)
-		}
-		ret = rl.ConcatRawRecord(recordBytes, recordBytes)
-	default:
-		t.Fatal("unexpected objType:" + objectType)
-	}
-	return
-}
-
-func testTryAcquireOrRenewMultiLock(t *testing.T, objectType string) {
-	future := time.Now().Add(1000 * time.Hour)
-	past := time.Now().Add(-1000 * time.Hour)
-	primaryType, secondaryType := multiLockType(t, objectType)
-	tests := []struct {
-		name              string
-		observedRecord    rl.LeaderElectionRecord
-		observedRawRecord []byte
-		observedTime      time.Time
-		reactors          []Reactor
-
-		expectSuccess    bool
-		transitionLeader bool
-		outHolder        string
-	}{
-		{
-			name: "acquire from no object",
-			reactors: []Reactor{
-				{
-					verb:       "get",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName())
-					},
-				},
-				{
-					verb:       "create",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.CreateAction).GetObject(), nil
-					},
-				},
-				{
-					verb:       "create",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.CreateAction).GetObject(), nil
-					},
-				},
-			},
-			expectSuccess: true,
-			outHolder:     "baz",
-		},
-		{
-			name: "acquire from unled old object",
-			reactors: []Reactor{
-				{
-					verb:       "get",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, primaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{}), nil
-					},
-				},
-				{
-					verb:       "get",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName())
-					},
-				},
-				{
-					verb:       "update",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.UpdateAction).GetObject(), nil
-					},
-				},
-				{
-					verb:       "get",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName())
-					},
-				},
-				{
-					verb:       "create",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.CreateAction).GetObject(), nil
-					},
-				},
-			},
-			expectSuccess:    true,
-			transitionLeader: true,
-			outHolder:        "baz",
-		},
-		{
-			name: "acquire from unled transition object",
-			reactors: []Reactor{
-				{
-					verb:       "get",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, primaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{}), nil
-					},
-				},
-				{
-					verb:       "get",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, secondaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{}), nil
-					},
-				},
-				{
-					verb:       "update",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.UpdateAction).GetObject(), nil
-					},
-				},
-				{
-					verb:       "get",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, secondaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{}), nil
-					},
-				},
-				{
-					verb:       "update",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.UpdateAction).GetObject(), nil
-					},
-				},
-			},
-			expectSuccess:    true,
-			transitionLeader: true,
-			outHolder:        "baz",
-		},
-		{
-			name: "acquire from led, unack old object",
-			reactors: []Reactor{
-				{
-					verb:       "get",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, primaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil
-					},
-				},
-				{
-					verb:       "get",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName())
-					},
-				},
-				{
-					verb:       "update",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.UpdateAction).GetObject(), nil
-					},
-				},
-				{
-					verb:       "get",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, secondaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil
-					},
-				},
-				{
-					verb:       "create",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.CreateAction).GetObject(), nil
-					},
-				},
-			},
-			observedRecord:    rl.LeaderElectionRecord{HolderIdentity: "bing"},
-			observedRawRecord: GetRawRecordOrDie(t, primaryType, rl.LeaderElectionRecord{HolderIdentity: "bing"}),
-			observedTime:      past,
-
-			expectSuccess:    true,
-			transitionLeader: true,
-			outHolder:        "baz",
-		},
-		{
-			name: "acquire from led, unack transition object",
-			reactors: []Reactor{
-				{
-					verb:       "get",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, primaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil
-					},
-				},
-				{
-					verb:       "get",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, secondaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil
-					},
-				},
-				{
-					verb:       "update",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.UpdateAction).GetObject(), nil
-					},
-				},
-				{
-					verb:       "get",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, secondaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil
-					},
-				},
-				{
-					verb:       "update",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.UpdateAction).GetObject(), nil
-					},
-				},
-			},
-			observedRecord:    rl.LeaderElectionRecord{HolderIdentity: "bing"},
-			observedRawRecord: GetRawRecordOrDie(t, objectType, rl.LeaderElectionRecord{HolderIdentity: "bing"}),
-			observedTime:      past,
-
-			expectSuccess:    true,
-			transitionLeader: true,
-			outHolder:        "baz",
-		},
-		{
-			name: "acquire from conflict led, ack transition object",
-			reactors: []Reactor{
-				{
-					verb:       "get",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, primaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil
-					},
-				},
-				{
-					verb:       "get",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, secondaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "baz"}), nil
-					},
-				},
-			},
-			observedRecord:    rl.LeaderElectionRecord{HolderIdentity: "bing"},
-			observedRawRecord: GetRawRecordOrDie(t, objectType, rl.LeaderElectionRecord{HolderIdentity: "bing"}),
-			observedTime:      future,
-
-			expectSuccess: false,
-			outHolder:     rl.UnknownLeader,
-		},
-		{
-			name: "acquire from led, unack unknown object",
-			reactors: []Reactor{
-				{
-					verb:       "get",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, primaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: rl.UnknownLeader}), nil
-					},
-				},
-				{
-					verb:       "get",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, secondaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: rl.UnknownLeader}), nil
-					},
-				},
-				{
-					verb:       "update",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.UpdateAction).GetObject(), nil
-					},
-				},
-				{
-					verb:       "get",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, secondaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: rl.UnknownLeader}), nil
-					},
-				},
-				{
-					verb:       "update",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.UpdateAction).GetObject(), nil
-					},
-				},
-			},
-			observedRecord:    rl.LeaderElectionRecord{HolderIdentity: rl.UnknownLeader},
-			observedRawRecord: GetRawRecordOrDie(t, objectType, rl.LeaderElectionRecord{HolderIdentity: rl.UnknownLeader}),
-			observedTime:      past,
-
-			expectSuccess:    true,
-			transitionLeader: true,
-			outHolder:        "baz",
-		},
-		{
-			name: "don't acquire from led, ack old object",
-			reactors: []Reactor{
-				{
-					verb:       "get",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, primaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil
-					},
-				},
-				{
-					verb:       "get",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName())
-					},
-				},
-			},
-			observedRecord:    rl.LeaderElectionRecord{HolderIdentity: "bing"},
-			observedRawRecord: GetRawRecordOrDie(t, primaryType, rl.LeaderElectionRecord{HolderIdentity: "bing"}),
-			observedTime:      future,
-
-			expectSuccess: false,
-			outHolder:     "bing",
-		},
-		{
-			name: "don't acquire from led, acked new object, observe new record",
-			reactors: []Reactor{
-				{
-					verb:       "get",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, primaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "baz"}), nil
-					},
-				},
-				{
-					verb:       "get",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, secondaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil
-					},
-				},
-			},
-			observedRecord:    rl.LeaderElectionRecord{HolderIdentity: "bing"},
-			observedRawRecord: GetRawRecordOrDie(t, secondaryType, rl.LeaderElectionRecord{HolderIdentity: "bing"}),
-			observedTime:      future,
-
-			expectSuccess: false,
-			outHolder:     rl.UnknownLeader,
-		},
-		{
-			name: "don't acquire from led, acked new object, observe transition record",
-			reactors: []Reactor{
-				{
-					verb:       "get",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, primaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil
-					},
-				},
-				{
-					verb:       "get",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, secondaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil
-					},
-				},
-			},
-			observedRecord:    rl.LeaderElectionRecord{HolderIdentity: "bing"},
-			observedRawRecord: GetRawRecordOrDie(t, objectType, rl.LeaderElectionRecord{HolderIdentity: "bing"}),
-			observedTime:      future,
-
-			expectSuccess: false,
-			outHolder:     "bing",
-		},
-		{
-			name: "renew already required object",
-			reactors: []Reactor{
-				{
-					verb:       "get",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, primaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "baz"}), nil
-					},
-				},
-				{
-					verb:       "get",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, secondaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "baz"}), nil
-					},
-				},
-				{
-					verb:       "update",
-					objectType: primaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.UpdateAction).GetObject(), nil
-					},
-				},
-				{
-					verb:       "get",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, createLockObject(t, secondaryType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "baz"}), nil
-					},
-				},
-				{
-					verb:       "update",
-					objectType: secondaryType,
-					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
-						return true, action.(fakeclient.UpdateAction).GetObject(), nil
-					},
-				},
-			},
-			observedRecord:    rl.LeaderElectionRecord{HolderIdentity: "baz"},
-			observedRawRecord: GetRawRecordOrDie(t, objectType, rl.LeaderElectionRecord{HolderIdentity: "baz"}),
-			observedTime:      future,
-
-			expectSuccess: true,
-			outHolder:     "baz",
-		},
-	}
-
-	for i := range tests {
-		test := &tests[i]
-		t.Run(test.name, func(t *testing.T) {
-			// OnNewLeader is called async so we have to wait for it.
-			var wg sync.WaitGroup
-			wg.Add(1)
-			var reportedLeader string
-			var lock rl.Interface
-
-			objectMeta := metav1.ObjectMeta{Namespace: "foo", Name: "bar"}
-			resourceLockConfig := rl.ResourceLockConfig{
-				Identity:      "baz",
-				EventRecorder: &record.FakeRecorder{},
-			}
-			c := &fake.Clientset{}
-			for _, reactor := range test.reactors {
-				c.AddReactor(reactor.verb, reactor.objectType, reactor.reaction)
-			}
-			c.AddReactor("*", "*", func(action fakeclient.Action) (bool, runtime.Object, error) {
-				t.Errorf("unreachable action. testclient called too many times: %+v", action)
-				return true, nil, fmt.Errorf("unreachable action")
-			})
-
-			switch objectType {
-			case rl.EndpointsLeasesResourceLock:
-				lock = &rl.MultiLock{
-					Primary: &rl.EndpointsLock{
-						EndpointsMeta: objectMeta,
-						LockConfig:    resourceLockConfig,
-						Client:        c.CoreV1(),
-					},
-					Secondary: &rl.LeaseLock{
-						LeaseMeta:  objectMeta,
-						LockConfig: resourceLockConfig,
-						Client:     c.CoordinationV1(),
-					},
-				}
-			case rl.ConfigMapsLeasesResourceLock:
-				lock = &rl.MultiLock{
-					Primary: &rl.ConfigMapLock{
-						ConfigMapMeta: objectMeta,
-						LockConfig:    resourceLockConfig,
-						Client:        c.CoreV1(),
-					},
-					Secondary: &rl.LeaseLock{
-						LeaseMeta:  objectMeta,
-						LockConfig: resourceLockConfig,
-						Client:     c.CoordinationV1(),
-					},
-				}
-			}
-
-			lec := LeaderElectionConfig{
-				Lock:          lock,
-				LeaseDuration: 10 * time.Second,
-				Callbacks: LeaderCallbacks{
-					OnNewLeader: func(l string) {
-						defer wg.Done()
-						reportedLeader = l
-					},
-				},
-			}
-			le := &LeaderElector{
-				config:            lec,
-				observedRecord:    test.observedRecord,
-				observedRawRecord: test.observedRawRecord,
-				observedTime:      test.observedTime,
-				clock:             clock.RealClock{},
-			}
-			if test.expectSuccess != le.tryAcquireOrRenew(context.Background()) {
-				t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", !test.expectSuccess)
-			}
-
-			le.observedRecord.AcquireTime = metav1.Time{}
-			le.observedRecord.RenewTime = metav1.Time{}
-			if le.observedRecord.HolderIdentity != test.outHolder {
-				t.Errorf("expected holder:\n\t%+v\ngot:\n\t%+v", test.outHolder, le.observedRecord.HolderIdentity)
-			}
-			if len(test.reactors) != len(c.Actions()) {
-				t.Errorf("wrong number of api interactions")
-			}
-			if test.transitionLeader && le.observedRecord.LeaderTransitions != 1 {
-				t.Errorf("leader should have transitioned but did not")
-			}
-			if !test.transitionLeader && le.observedRecord.LeaderTransitions != 0 {
-				t.Errorf("leader should not have transitioned but did")
-			}
-
-			le.maybeReportTransition()
-			wg.Wait()
-			if reportedLeader != test.outHolder {
-				t.Errorf("reported leader was not the new leader. expected %q, got %q", test.outHolder, reportedLeader)
-			}
-		})
-	}
-}
-
-// Will test leader election using endpointsleases as the resource
-func TestTryAcquireOrRenewEndpointsLeases(t *testing.T) {
-	testTryAcquireOrRenewMultiLock(t, "endpointsleases")
-}
-
-// Will test leader election using configmapsleases as the resource
-func TestTryAcquireOrRenewConfigMapsLeases(t *testing.T) {
-	testTryAcquireOrRenewMultiLock(t, "configmapsleases")
-}

pkg/leaderElection/metrics.go

@@ -1,109 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package leaderelection
-
-import (
-	"sync"
-)
-
-// This file provides abstractions for setting the provider (e.g., prometheus)
-// of metrics.
-
-type leaderMetricsAdapter interface {
-	leaderOn(name string)
-	leaderOff(name string)
-}
-
-// GaugeMetric represents a single numerical value that can arbitrarily go up
-// and down.
-type SwitchMetric interface {
-	On(name string)
-	Off(name string)
-}
-
-type noopMetric struct{}
-
-func (noopMetric) On(name string)  {}
-func (noopMetric) Off(name string) {}
-
-// defaultLeaderMetrics expects the caller to lock before setting any metrics.
-type defaultLeaderMetrics struct {
-	// leader's value indicates if the current process is the owner of name lease
-	leader SwitchMetric
-}
-
-func (m *defaultLeaderMetrics) leaderOn(name string) {
-	if m == nil {
-		return
-	}
-	m.leader.On(name)
-}
-
-func (m *defaultLeaderMetrics) leaderOff(name string) {
-	if m == nil {
-		return
-	}
-	m.leader.Off(name)
-}
-
-type noMetrics struct{}
-
-func (noMetrics) leaderOn(name string)  {}
-func (noMetrics) leaderOff(name string) {}
-
-// MetricsProvider generates various metrics used by the leader election.
-type MetricsProvider interface {
-	NewLeaderMetric() SwitchMetric
-}
-
-type noopMetricsProvider struct{}
-
-func (_ noopMetricsProvider) NewLeaderMetric() SwitchMetric {
-	return noopMetric{}
-}
-
-var globalMetricsFactory = leaderMetricsFactory{
-	metricsProvider: noopMetricsProvider{},
-}
-
-type leaderMetricsFactory struct {
-	metricsProvider MetricsProvider
-
-	onlyOnce sync.Once
-}
-
-func (f *leaderMetricsFactory) setProvider(mp MetricsProvider) {
-	f.onlyOnce.Do(func() {
-		f.metricsProvider = mp
-	})
-}
-
-func (f *leaderMetricsFactory) newLeaderMetrics() leaderMetricsAdapter {
-	mp := f.metricsProvider
-	if mp == (noopMetricsProvider{}) {
-		return noMetrics{}
-	}
-	return &defaultLeaderMetrics{
-		leader: mp.NewLeaderMetric(),
-	}
-}
-
-// SetProvider sets the metrics provider for all subsequently created work
-// queues. Only the first call has an effect.
-func SetProvider(metricsProvider MetricsProvider) {
-	globalMetricsFactory.setProvider(metricsProvider)
-}

pkg/loadbalancer/ipvs.go

@@ -0,0 +1,391 @@
+package loadbalancer
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/netip"
+	"os"
+	"strings"
+	"sync"
+
+	log "log/slog"
+
+	"github.com/cloudflare/ipvs"
+	"github.com/cloudflare/ipvs/netmask"
+	"github.com/kube-vip/kube-vip/pkg/backend"
+	"github.com/kube-vip/kube-vip/pkg/sysctl"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	"github.com/vishvananda/netlink"
+)
+
+/*
+IPVS Architecture - for those that are interested
+
+There are going to be a large number of end users that are using a VIP that exists within the same subnet
+as the back end servers. This unfortunately will result in "packet" confusion with the destingation and
+source becoming messed up by the IPVS NAT.
+
+The solution is to perform two things !
+
+First:
+Set up kube-vip TCP port forwarder from the VIP:PORT to the IPVS:PORT
+
+Second:
+Start up a node watcher and a IPVS load balancer, the node balancer is responsible for adding/removing
+the nodes from the IPVS load-balancer.
+
+*/
+
+const (
+	ROUNDROBIN = "rr"
+)
+
+type IPVSLoadBalancer struct {
+	client              ipvs.Client
+	loadBalancerService ipvs.Service
+	Port                uint16
+	forwardingMethod    ipvs.ForwardType
+	backendMap          backend.Map
+	interval            int
+	lock                sync.Mutex
+	stop                chan struct{}
+	networkInterface    string
+	leaderCancel        context.CancelFunc
+	signal              chan os.Signal
+	address             string
+	family              ipvs.AddressFamily
+}
+
+func NewIPVSLB(address string, port uint16, forwardingMethod string, backendHealthCheckInterval int, networkInterface string, leaderCancel context.CancelFunc, signal chan os.Signal) (*IPVSLoadBalancer, error) {
+	log.Info("Starting IPVS LoadBalancer", "address", address)
+
+	// Create IPVS client
+	c, err := ipvs.New()
+	if err != nil {
+		log.Error("ensure IPVS kernel modules are loaded")
+		log.Error("Error starting IPVS", "err", err)
+		panic("")
+	}
+	i, err := c.Info()
+	if err != nil {
+		log.Error("ensure IPVS kernel modules are loaded")
+		log.Error("Error retrieving IPVS info", "err", err)
+		if errors.Is(err, os.ErrPermission) {
+			log.Error("no permission to get IPVS info - please ensure that kube-vip is running with proper capabilities/privileged mode")
+		}
+		panic("")
+	}
+	log.Info("IPVS Loadbalancer enabled", "version", fmt.Sprintf("%d.%d.%d", i.Version[0], i.Version[1], i.Version[2]))
+
+	ip, family := ipAndFamily(address)
+
+	if strings.ToLower(forwardingMethod) == "masquerade" {
+		enableProcSys("/proc/sys/net/ipv4/vs/conntrack", "net.ipv4.vs.conntrack")
+		if family == ipvs.INET6 {
+			enableProcSys("/proc/sys/net/ipv6/conf/all/forwarding", "net.ipv6.conf.all.forwarding")
+		} else {
+			enableProcSys("/proc/sys/net/ipv4/ip_forward", "net.ipv4.ip_forward")
+		}
+	}
+
+	netMask := netmask.MaskFrom(31, 32) // For ipv4
+	if family == ipvs.INET6 {
+		netMask = netmask.MaskFrom(128, 128) // For ipv6
+	}
+
+	// Generate out API Server LoadBalancer instance
+	svc := ipvs.Service{
+		Netmask:   netMask,
+		Family:    family,
+		Protocol:  ipvs.TCP,
+		Port:      port,
+		Address:   ip,
+		Scheduler: ROUNDROBIN,
+	}
+
+	var m ipvs.ForwardType
+	switch strings.ToLower(forwardingMethod) {
+	case "masquerade":
+		m = ipvs.Masquerade
+	case "local":
+		m = ipvs.Local
+	case "tunnel":
+		m = ipvs.Tunnel
+	case "directroute":
+		m = ipvs.DirectRoute
+	case "bypass":
+		m = ipvs.Bypass
+	default:
+		m = ipvs.Local
+		log.Warn("unknown forwarding method. Defaulting to Local")
+	}
+
+	if backendHealthCheckInterval <= 0 {
+		backendHealthCheckInterval = 5
+	}
+
+	lb := &IPVSLoadBalancer{
+		Port:                port,
+		client:              c,
+		loadBalancerService: svc,
+		forwardingMethod:    m,
+		interval:            backendHealthCheckInterval,
+		backendMap:          make(backend.Map),
+		stop:                make(chan struct{}),
+		networkInterface:    networkInterface,
+		leaderCancel:        leaderCancel,
+		signal:              signal,
+		address:             address,
+		family:              family,
+	}
+
+	go lb.healthCheck()
+
+	// Return our created load-balancer
+	return lb, nil
+}
+
+func enableProcSys(path, name string) {
+	isSet, err := sysctl.EnableProcSys(path)
+	if err != nil {
+		log.Error(fmt.Sprintf("ensuring %s enabled", name), "err", err)
+		panic("")
+	}
+	if isSet {
+		log.Info(fmt.Sprintf("sysctl set %s to 1", name))
+	}
+}
+
+func (lb *IPVSLoadBalancer) RemoveIPVSLB() error {
+	log.Info("Stopping IPVS LoadBalancer", "address", lb.address)
+	close(lb.stop)
+	err := lb.client.RemoveService(lb.loadBalancerService)
+	if err != nil {
+		return fmt.Errorf("error removing existing IPVS service: %v", err)
+	}
+	return nil
+}
+
+func (lb *IPVSLoadBalancer) AddBackend(address string, port uint16) error {
+	isLocal := false
+	var err error
+
+	// Discard backend if it is of different IP family than LB address.
+	if _, family := ipAndFamily(address); family != lb.family {
+		return nil
+	}
+
+	if lb.forwardingMethod == ipvs.Local {
+		log.Info("checking if backend is local", "addr", address)
+		isLocal, err = lb.isLocal(address)
+		if err != nil {
+			log.Error("checking if backend is local", "err", err)
+		}
+	}
+
+	backend := backend.Entry{Addr: address, Port: port, IsLocal: isLocal}
+
+	lb.lock.Lock()
+	defer lb.lock.Unlock()
+	if _, ok := lb.backendMap[backend]; !ok {
+		isHealth := backend.Check()
+		if isHealth {
+			err := lb.addBackend(address, port)
+			if err != nil {
+				return err
+			}
+		}
+		lb.backendMap[backend] = isHealth
+	}
+	return nil
+}
+
+func (lb *IPVSLoadBalancer) addBackend(address string, port uint16) error {
+	backend := backend.Entry{Addr: address, Port: port}
+	// Check if this is the first backend
+	backends, err := lb.client.Destinations(lb.loadBalancerService)
+	if err != nil && strings.Contains(err.Error(), "file does not exist") {
+		log.Error("querying backends", "err", err)
+	}
+	// If this is our first backend, then we can create the load-balancer service and add a backend
+	if len(backends) == 0 {
+		err = lb.client.CreateService(lb.loadBalancerService)
+		// If we've an error it could be that the IPVS lb instance has been left from a previous leadership
+		if err != nil && strings.Contains(err.Error(), "file exists") {
+			log.Warn("load balancer for API server already exists, attempting to remove and re-create")
+			err = lb.client.RemoveService(lb.loadBalancerService)
+			if err != nil {
+				return fmt.Errorf("error re-creating IPVS service: %v", err)
+			}
+			err = lb.client.CreateService(lb.loadBalancerService)
+			if err != nil {
+				return fmt.Errorf("error re-creating IPVS service: %v", err)
+			}
+		} else if err != nil {
+			// Fatal error at this point as IPVS is probably not working
+			log.Error("Unable to create an IPVS service, ensure IPVS kernel modules are loaded")
+			log.Error("IPVS service", "err", err)
+			panic("")
+
+		}
+		log.Info("load-Balancer services created", "address", lb.addrString(), "port", lb.Port)
+	}
+
+	ip, family := ipAndFamily(backend.Addr)
+
+	// Ignore backends that use a different address family.
+	// Looks like different families could be supported in tunnel mode...
+	if family != lb.loadBalancerService.Family {
+		return nil
+	}
+
+	dst := ipvs.Destination{
+		Address:   ip,
+		Port:      backend.Port,
+		Family:    family,
+		Weight:    1,
+		FwdMethod: lb.forwardingMethod,
+	}
+
+	err = lb.client.CreateDestination(lb.loadBalancerService, dst)
+	// Swallow error of existing back end, the node watcher may attempt to apply
+	// the same back end multiple times
+	if err != nil {
+		if !strings.Contains(err.Error(), "file exists") {
+			return fmt.Errorf("error creating backend: %v", err)
+		}
+		// file exists is fine, we will just return at this point
+		return nil
+	}
+	log.Info("backend added", "src addr", lb.addrString(), "src port", lb.Port, "dst addr", backend.Addr, "dst port", backend.Port)
+
+	return nil
+}
+
+func (lb *IPVSLoadBalancer) RemoveBackend(address string, port uint16) error {
+	backend := backend.Entry{Addr: address, Port: port}
+
+	lb.lock.Lock()
+	defer lb.lock.Unlock()
+
+	if _, ok := lb.backendMap[backend]; ok {
+		err := lb.removeBackend(address, port)
+		if err != nil {
+			return err
+		}
+		delete(lb.backendMap, backend)
+	}
+
+	return nil
+}
+
+func (lb *IPVSLoadBalancer) removeBackend(address string, port uint16) error {
+	ip, family := ipAndFamily(address)
+	if family != lb.loadBalancerService.Family {
+		return nil
+	}
+
+	dst := ipvs.Destination{
+		Address: ip,
+		Port:    port,
+		Family:  family,
+		Weight:  1,
+	}
+	err := lb.client.RemoveDestination(lb.loadBalancerService, dst)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("error removing backend: %v", err)
+	}
+	return nil
+}
+
+func (lb *IPVSLoadBalancer) addrString() string {
+	return lb.loadBalancerService.Address.String()
+}
+
+func ipAndFamily(address string) (netip.Addr, ipvs.AddressFamily) {
+	ipAddr := net.ParseIP(address)
+	if ipAddr.To4() == nil {
+		return netip.AddrFrom16([16]byte(ipAddr.To16())), ipvs.INET6
+	}
+	return netip.AddrFrom4([4]byte(ipAddr.To4())), ipvs.INET
+}
+
+func (lb *IPVSLoadBalancer) healthCheck() {
+	backend.Watch(func() {
+		lb.lock.Lock()
+		defer lb.lock.Unlock()
+		for backend, oldStatus := range lb.backendMap {
+			newStatus := backend.Check()
+			if newStatus {
+				// old status -> health
+				if !oldStatus {
+					err := lb.addBackend(backend.Addr, backend.Port)
+					if err != nil {
+						log.Error("add backend", "err", err)
+					}
+					lb.backendMap[backend] = newStatus
+				}
+			} else {
+				// old status -> not health
+				if oldStatus {
+					log.Info("healthCheck failed - removing backend", "address", backend.Addr, "port", backend.Port)
+					err := lb.removeBackend(backend.Addr, backend.Port)
+					if err != nil {
+						log.Error("failed to remove backend", "address", backend.Addr, "port", backend.Port, "err", err)
+					}
+					lb.backendMap[backend] = newStatus
+				}
+				if lb.forwardingMethod == ipvs.Local && !lb.localBackendExists() {
+					if lb.signal != nil {
+						close(lb.signal)
+					}
+
+					if lb.leaderCancel != nil {
+						lb.leaderCancel()
+					}
+				}
+			}
+		}
+	}, lb.interval, lb.stop)
+}
+
+func (lb *IPVSLoadBalancer) isLocal(address string) (bool, error) {
+	link, err := netlink.LinkByName(lb.networkInterface)
+	if err != nil {
+		return false, fmt.Errorf("getting link '%s': %w", lb.networkInterface, err)
+	}
+
+	family := netlink.FAMILY_V6
+	if vip.IsIPv4(address) {
+		family = netlink.FAMILY_V4
+	}
+
+	target := net.ParseIP(address)
+	if target == nil {
+		return false, fmt.Errorf("address '%s' is not a valid IP address", address)
+	}
+
+	addrs, err := netlink.AddrList(link, family)
+	if err != nil {
+		return false, fmt.Errorf("listing addresses for link '%s': %w", lb.networkInterface, err)
+	}
+
+	for _, addr := range addrs {
+		if addr.IP.Equal(target) {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+func (lb *IPVSLoadBalancer) localBackendExists() bool {
+	for backend, isHealthy := range lb.backendMap {
+		if backend.IsLocal && isHealthy {
+			return true
+		}
+	}
+	return false
+}

pkg/loadbalancer/ipvs_test.go

@@ -0,0 +1,49 @@
+package loadbalancer
+
+import (
+	"net/netip"
+	"reflect"
+	"testing"
+
+	"github.com/cloudflare/ipvs"
+)
+
+func Test_ipAndFamily(t *testing.T) {
+	type args struct {
+		address string
+	}
+	tests := []struct {
+		name  string
+		args  args
+		want  netip.Addr
+		want1 ipvs.AddressFamily
+	}{
+		{
+			name: "IPv4",
+			args: args{
+				address: "192.168.0.20",
+			},
+			want:  netip.AddrFrom4([4]byte{192, 168, 0, 20}),
+			want1: ipvs.INET,
+		},
+		{
+			name: "IPv6",
+			args: args{
+				address: "ff02::3",
+			},
+			want:  netip.AddrFrom16([16]byte{0: 0xff, 1: 0x02, 15: 0x03}),
+			want1: ipvs.INET6,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, got1 := ipAndFamily(tt.args.address)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ipAndFamily() got = %v, want %v", got, tt.want)
+			}
+			if !reflect.DeepEqual(got1, tt.want1) {
+				t.Errorf("ipAndFamily() got1 = %v, want %v", got1, tt.want1)
+			}
+		})
+	}
+}

pkg/loadbalancer/lb_connections.go

@@ -1,129 +0,0 @@
-package loadbalancer
-
-import (
-	"io"
-	"net"
-	"sync"
-	"time"
-
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
-)
-
-// 1. Load balancer port is exposed
-// 2. We listen
-// 3. On connection we connect to an endpoint
-// [loop]
-// 4. We read from the load balancer port
-// 5. We write traffic to the endpoint
-// 6. We read response from endpoint
-// 7. We write response to load balancer
-// [goto loop]
-
-func persistentConnection(frontendConnection net.Conn, lb *kubevip.LoadBalancer) {
-
-	var endpoint net.Conn
-	defer frontendConnection.Close()
-	// Makes sure we close the connections to the endpoint when we've completed
-
-	// Set a timeout for connecting to an endpoint
-	dialer := net.Dialer{Timeout: time.Millisecond * 500}
-	for {
-
-		// Connect to Endpoint
-		ep, err := lb.ReturnEndpointAddr()
-		if err != nil {
-			return
-		}
-
-		// We now dial to an endpoint with a timeout of half a second
-		// TODO - make this adjustable
-		endpoint, err = dialer.Dial("tcp", ep)
-		if err != nil {
-			log.Debugf("%v", err)
-			log.Warnf("[%s]---X [FAILED] X-->[%s]", frontendConnection.RemoteAddr(), ep)
-		} else {
-			log.Debugf("[%s]---->[ACCEPT]---->[%s]", frontendConnection.RemoteAddr(), ep)
-			defer endpoint.Close()
-			break
-		}
-	}
-
-	wg := &sync.WaitGroup{}
-	wg.Add(1)
-
-	// Begin copying incoming (frontend -> to an endpoint)
-	go func() {
-		bytes, err := io.Copy(endpoint, frontendConnection)
-		log.Debugf("[%d] bytes of data sent to endpoint", bytes)
-		if err != nil {
-			log.Warnf("Error sending data to endpoint [%s] [%v]", endpoint.RemoteAddr(), err)
-		}
-		wg.Done()
-	}()
-	// go func() {
-	// Begin copying recieving (endpoint -> back to frontend)
-	bytes, err := io.Copy(frontendConnection, endpoint)
-	log.Debugf("[%d] bytes of data sent to client", bytes)
-	if err != nil {
-		log.Warnf("Error sending data to frontend [%s] [%s]", frontendConnection.RemoteAddr(), err)
-	}
-	// 	wg.Done()
-	// 	endpoint.Close()
-	// }()
-	wg.Wait()
-}
-
-func persistentUDPConnection(frontendConnection net.Conn, lb *kubevip.LoadBalancer) {
-
-	var endpoint net.Conn
-	defer frontendConnection.Close()
-	// Makes sure we close the connections to the endpoint when we've completed
-
-	// Set a timeout for connecting to an endpoint
-	dialer := net.Dialer{Timeout: time.Millisecond * 500}
-	for {
-
-		// Connect to Endpoint
-		ep, err := lb.ReturnEndpointAddr()
-		if err != nil {
-			return
-		}
-
-		// We now dial to an endpoint with a timeout of half a second
-		// TODO - make this adjustable
-		endpoint, err = dialer.Dial("udp", ep)
-		if err != nil {
-			log.Debugf("%v", err)
-			log.Warnf("[%s]---X [FAILED] X-->[%s]", frontendConnection.RemoteAddr(), ep)
-		} else {
-			log.Debugf("[%s]---->[ACCEPT]---->[%s]", frontendConnection.RemoteAddr(), ep)
-			defer endpoint.Close()
-			break
-		}
-	}
-
-	wg := &sync.WaitGroup{}
-	wg.Add(1)
-
-	// Begin copying incoming (frontend -> to an endpoint)
-	go func() {
-		bytes, err := io.Copy(endpoint, frontendConnection)
-		log.Debugf("[%d] bytes of data sent to endpoint", bytes)
-		if err != nil {
-			log.Warnf("Error sending data to endpoint [%s] [%v]", endpoint.RemoteAddr(), err)
-		}
-		wg.Done()
-	}()
-	// go func() {
-	// Begin copying recieving (endpoint -> back to frontend)
-	bytes, err := io.Copy(frontendConnection, endpoint)
-	log.Debugf("[%d] bytes of data sent to client", bytes)
-	if err != nil {
-		log.Warnf("Error sending data to frontend [%s] [%s]", frontendConnection.RemoteAddr(), err)
-	}
-	// 	wg.Done()
-	// 	endpoint.Close()
-	// }()
-	wg.Wait()
-}

pkg/loadbalancer/lb_http.go

@@ -1,126 +0,0 @@
-package loadbalancer
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"net/http/httputil"
-	"net/url"
-	"time"
-
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
-)
-
-func (lb *LBInstance) startHTTP(bindAddress string) error {
-	log.Infof("Starting TCP Load Balancer for service [%s]", lb.instance.Name)
-
-	// Validate the back end URLS
-	err := kubevip.ValidateBackEndURLS(&lb.instance.Backends)
-	if err != nil {
-		return err
-	}
-
-	frontEnd := fmt.Sprintf("%s:%d", bindAddress, lb.instance.Port)
-
-	handler := func(w http.ResponseWriter, req *http.Request) {
-		// parse the url
-		url, _ := url.Parse(lb.instance.ReturnEndpointURL().String())
-
-		// create the reverse proxy
-		proxy := httputil.NewSingleHostReverseProxy(url)
-
-		// Update the headers to allow for SSL redirection
-		req.URL.Host = url.Host
-		req.URL.Scheme = url.Scheme
-		req.Header.Set("X-Forwarded-Host", req.Host)
-		req.Host = url.Host
-
-		//Print out the response (if debug logging)
-		if log.GetLevel() == log.DebugLevel {
-			fmt.Printf("Host:\t%s\n", req.Host)
-			fmt.Printf("Request:\t%s\n", req.Method)
-			fmt.Printf("URI:\t%s\n", req.RequestURI)
-
-			for key, value := range req.Header {
-				fmt.Println("Header:", key, "Value:", value)
-			}
-		}
-
-		// Note that ServeHttp is non blocking and uses a go routine under the hood
-		proxy.ServeHTTP(w, req)
-	}
-
-	mux := http.NewServeMux()
-	mux.HandleFunc("/", handler)
-	log.Infof("Starting server listening [%s]", frontEnd)
-
-	server := &http.Server{Addr: frontEnd, Handler: mux}
-
-	go func() error {
-		if err := server.ListenAndServe(); err != nil {
-			return err
-		}
-		return nil
-	}()
-
-	// If the stop channel is closed then the server will be gracefully shut down
-	<-lb.stop
-	log.Infof("Stopping the load balancer [%s] bound to [%s] with 5sec timeout", lb.instance.Name, frontEnd)
-
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	if err := server.Shutdown(ctx); err != nil {
-		return err
-	}
-	close(lb.stopped)
-	return nil
-}
-
-//StartHTTP - begins the HTTP load balancer
-func StartHTTP(lb *kubevip.LoadBalancer, address string) error {
-	log.Infof("Starting TCP Load Balancer for service [%s]", lb.Name)
-
-	// Validate the back end URLS
-	err := kubevip.ValidateBackEndURLS(&lb.Backends)
-	if err != nil {
-		return err
-	}
-
-	frontEnd := fmt.Sprintf("%s:%d", address, lb.Port)
-
-	handler := func(w http.ResponseWriter, req *http.Request) {
-		// parse the url
-		url, _ := url.Parse(lb.ReturnEndpointURL().String())
-
-		// create the reverse proxy
-		proxy := httputil.NewSingleHostReverseProxy(url)
-
-		// Update the headers to allow for SSL redirection
-		req.URL.Host = url.Host
-		req.URL.Scheme = url.Scheme
-		req.Header.Set("X-Forwarded-Host", req.Host)
-		req.Host = url.Host
-
-		//Print out the response (if debug logging)
-		if log.GetLevel() == log.DebugLevel {
-			fmt.Printf("Host:\t%s\n", req.Host)
-			fmt.Printf("Request:\t%s\n", req.Method)
-			fmt.Printf("URI:\t%s\n", req.RequestURI)
-
-			for key, value := range req.Header {
-				fmt.Println("Header:", key, "Value:", value)
-			}
-		}
-
-		// Note that ServeHttp is non blocking and uses a go routine under the hood
-		proxy.ServeHTTP(w, req)
-	}
-
-	mux := http.NewServeMux()
-	mux.HandleFunc("/", handler)
-	log.Infof("Starting server listening [%s]", frontEnd)
-	http.ListenAndServe(frontEnd, mux)
-	// Should never get here
-	return nil
-}

pkg/loadbalancer/lb_tcp.go

@@ -1,178 +0,0 @@
-package loadbalancer
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"net"
-	"time"
-
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
-)
-
-// StartTCP a TCP load balancer server instane
-func (lb *LBInstance) startTCP(bindAddress string) error {
-	fullAddress := fmt.Sprintf("%s:%d", bindAddress, lb.instance.Port)
-	log.Infof("Starting TCP Load Balancer for service [%s]", fullAddress)
-
-	laddr, err := net.ResolveTCPAddr("tcp", fullAddress)
-	if nil != err {
-		log.Errorln(err)
-	}
-	l, err := net.ListenTCP("tcp", laddr)
-	if nil != err {
-		return fmt.Errorf("Unable to bind [%s]", err.Error())
-	}
-	go func() {
-		for {
-			select {
-
-			case <-lb.stop:
-				log.Debugln("Closing listener")
-
-				// We've closed the stop channel
-				err = l.Close()
-				if err != nil {
-					return
-				}
-				// Close the stopped channel as the listener has been stopped
-				close(lb.stopped)
-			default:
-
-				err = l.SetDeadline(time.Now().Add(200 * time.Millisecond))
-				if err != nil {
-					log.Errorf("Error setting TCP deadline", err)
-				}
-				fd, err := l.Accept()
-				if err != nil {
-					// Check it it's an accept timeout
-					if opErr, ok := err.(*net.OpError); ok && opErr.Timeout() {
-						continue
-					} else if err != io.EOF {
-						log.Errorf("TCP Accept error [%s]", err)
-					}
-				}
-				go persistentConnection(fd, lb.instance)
-			}
-		}
-	}()
-	log.Infof("Load Balancer [%s] started", lb.instance.Name)
-
-	return nil
-}
-
-// startTCPDNU - Start TCP service Do not use
-// This stops the service by closing the listener and then ignorning the error from Accept()
-func (lb *LBInstance) startTCPDNU(bindAddress string) error {
-	fullAddress := fmt.Sprintf("%s:%d", bindAddress, lb.instance.Port)
-	log.Infof("Starting TCP Load Balancer for service [%s]", fullAddress)
-
-	//l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", bindAddress, lb.instance.Port))
-	laddr, err := net.ResolveTCPAddr("tcp", fullAddress)
-	if nil != err {
-		log.Errorln(err)
-	}
-	l, err := net.ListenTCP("tcp", laddr)
-	if nil != err {
-		return fmt.Errorf("Unable to bind [%s]", err.Error())
-	}
-	go func() {
-		<-lb.stop
-		log.Debugln("Closing listener")
-
-		// We've closed the stop channel
-		err = l.Close()
-		if err != nil {
-			return
-		}
-		// Close the stopped channel as the listener has been stopped
-		close(lb.stopped)
-
-	}()
-
-	go func() {
-		for {
-
-			fd, err := l.Accept()
-			if err != nil {
-				// Check it it's an accept timeout
-				if opErr, ok := err.(*net.OpError); ok && opErr.Timeout() {
-					continue
-				} else if err != io.EOF {
-					log.Errorf("TCP Accept error [%s]", err)
-					return
-				}
-			}
-			go persistentConnection(fd, lb.instance)
-			//go processRequests(lb.instance, fd)
-		}
-		//	}
-	}()
-	log.Infof("Load Balancer [%s] started", lb.instance.Name)
-
-	return nil
-}
-
-// user -> [LB]
-// [LB] (select end pot) -> [endpoint]
-//
-//
-//
-//
-//
-//
-
-func processRequests(lb *kubevip.LoadBalancer, frontendConnection net.Conn) {
-	for {
-		// READ FROM client
-		buf := make([]byte, 1024*1024)
-		datalen, err := frontendConnection.Read(buf)
-		if err != nil {
-			log.Fatalf("%v", err)
-		}
-		log.Debugf("Sent [%d] bytes to the LB", datalen)
-		data := buf[0:datalen]
-
-		// Connect to Endpoint
-		ep, err := lb.ReturnEndpointAddr()
-		if err != nil {
-			log.Errorf("No Backends available")
-		}
-		log.Debugf("Attempting endpoint [%s]", ep)
-
-		endpoint, err := net.Dial("tcp", ep)
-		if err != nil {
-			fmt.Println("dial error:", err)
-			// return nil, err
-		}
-		log.Debugf("succesfully connected to [%s]", ep)
-
-		// Set a timeout
-		endpoint.SetReadDeadline(time.Now().Add(time.Second * 1))
-
-		b, err := endpointRequest(endpoint, ep, string(data))
-
-		_, err = frontendConnection.Write(b)
-		if err != nil {
-			log.Fatal("Write: ", err)
-		}
-	}
-}
-
-// endpointRequest will take an endpoint address and send the data and wait for the response
-func endpointRequest(endpoint net.Conn, endpointAddr, request string) ([]byte, error) {
-
-	// defer conn.Close()
-	datalen, err := fmt.Fprintf(endpoint, request)
-	if err != nil {
-		fmt.Println("dial error:", err)
-		return nil, err
-	}
-	log.Debugf("Sent [%d] bytes to the endpoint", datalen)
-
-	var b bytes.Buffer
-	io.Copy(&b, endpoint)
-	log.Debugf("Recieved [%d] from the endpoint", b.Len())
-	return b.Bytes(), nil
-}

pkg/loadbalancer/lb_udp.go

@@ -1,103 +0,0 @@
-package loadbalancer
-
-import (
-	"fmt"
-	"net"
-
-	log "github.com/sirupsen/logrus"
-)
-
-// StartTCP a TCP load balancer server instane
-func (lb *LBInstance) startUDP(bindAddress string) error {
-	fullAddress := fmt.Sprintf("%s:%d", bindAddress, lb.instance.Port)
-	log.Infof("Starting UDP Load Balancer for service [%s]", fullAddress)
-
-	laddr, err := net.ResolveUDPAddr("udp", fullAddress)
-	if nil != err {
-		log.Errorln(err)
-	}
-	l, err := net.ListenUDP("udp", laddr)
-	if nil != err {
-		return fmt.Errorf("Unable to bind [%s]", err.Error())
-	}
-	//stopListener := make(chan bool)
-
-	go func() {
-		for {
-			select {
-
-			case <-lb.stop:
-				log.Debugln("Closing listener")
-
-				// We've closed the stop channel
-				err = l.Close()
-				if err != nil {
-					return
-				}
-				// Close the stopped channel as the listener has been stopped
-				close(lb.stopped)
-			default:
-
-				// err = l.SetDeadline(time.Now().Add(200 * time.Millisecond))
-				// if err != nil {
-				// 	log.Errorf("Error setting TCP deadline", err)
-				// }
-				// fd, err := l.Accept()
-				// if err != nil {
-				// 	// Check it it's an accept timeout
-				// 	if opErr, ok := err.(*net.OpError); ok && opErr.Timeout() {
-				// 		continue
-				// 	} else if err != io.EOF {
-				// 		log.Errorf("TCP Accept error [%s]", err)
-				// 	}
-				// }
-				go persistentUDPConnection(l, lb.instance)
-			}
-		}
-	}()
-	log.Infof("Load Balancer [%s] started", lb.instance.Name)
-
-	// go func() {
-	// 	<-lb.stop
-	// 	log.Debugln("Closing listener")
-
-	// 	// We've closed the stop channel
-	// 	err = l.Close()
-	// 	if err != nil {
-	// 		return
-	// 	}
-	// 	// Close the stopped channel as the listener has been stopped
-	// 	close(stopListener)
-	// }()
-
-	// // Create a listener per CPU
-	// // TODO - make this customisable?
-
-	// //for i := 0; i < runtime.NumCPU(); i++ {
-	// go persistentUDPConnection(l, lb.instance)
-	// //}
-
-	//<-stopListener // hang until an error
-
-	log.Infof("Load Balancer [%s] started", lb.instance.Name)
-
-	return nil
-}
-
-// func listen(connection *net.UDPConn, quit chan bool) {
-// 	buffer := make([]byte, 1024)
-// 	n, remoteAddr, err := 0, new(net.UDPAddr), error(nil)
-// 	for err == nil {
-
-// 		//TODO -
-
-// 		n, remoteAddr, err = connection.ReadFromUDP(buffer)
-// 		// you might copy out the contents of the packet here, to
-// 		// `var r myapp.Request`, say, and `go handleRequest(r)` (or
-// 		// send it down a channel) to free up the listening
-// 		// goroutine. you do *need* to copy then, though,
-// 		// because you've only made one buffer per listen().
-// 		fmt.Println("from", remoteAddr, "-", buffer[:n])
-// 	}
-// 	fmt.Println("listener failed - ", err)
-// }

pkg/loadbalancer/manager.go

@@ -1,78 +0,0 @@
-package loadbalancer
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/plunder-app/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
-)
-
-//LBInstance - manages the state of load balancer instances
-type LBInstance struct {
-	stop     chan bool             // Asks LB to stop
-	stopped  chan bool             // LB is stopped
-	instance *kubevip.LoadBalancer // pointer to a LB instance
-	//	mux      sync.Mutex
-}
-
-//LBManager - will manage a number of load blancer instances
-type LBManager struct {
-	loadBalancer []LBInstance
-}
-
-//Add - handles the building of the load balancers
-func (lm *LBManager) Add(bindAddress string, lb *kubevip.LoadBalancer) error {
-	newLB := LBInstance{
-		stop:     make(chan bool, 1),
-		stopped:  make(chan bool, 1),
-		instance: lb,
-	}
-
-	switch strings.ToLower(lb.Type) {
-	case "tcp":
-		err := newLB.startTCP(bindAddress)
-		if err != nil {
-			return err
-		}
-	case "udp":
-		err := newLB.startUDP(bindAddress)
-		if err != nil {
-			return err
-		}
-	case "http":
-		err := newLB.startHTTP(bindAddress)
-		if err != nil {
-			return err
-		}
-	default:
-		return fmt.Errorf("Unknown Load Balancer type [%s]", lb.Type)
-	}
-
-	lm.loadBalancer = append(lm.loadBalancer, newLB)
-	return nil
-}
-
-//StopAll - handles the building of the load balancers
-func (lm *LBManager) StopAll() error {
-	log.Debugf("Stopping [%d] loadbalancer instances", len(lm.loadBalancer))
-	for x := range lm.loadBalancer {
-		err := lm.loadBalancer[x].Stop()
-		if err != nil {
-			return err
-		}
-	}
-	// Reset the loadbalancer entries
-	lm.loadBalancer = nil
-	return nil
-}
-
-//Stop - handles the building of the load balancers
-func (l *LBInstance) Stop() error {
-
-	close(l.stop)
-
-	<-l.stopped
-	log.Infof("Load Balancer instance [%s] has stopped", l.instance.Name)
-	return nil
-}

pkg/manager/cluster.go

@@ -0,0 +1,30 @@
+package manager
+
+import (
+	"github.com/pkg/errors"
+
+	"github.com/kube-vip/kube-vip/pkg/cluster"
+	"github.com/kube-vip/kube-vip/pkg/etcd"
+)
+
+func initClusterManager(sm *Manager) (*cluster.Manager, error) {
+	m := &cluster.Manager{
+		SignalChan: sm.signalChan,
+	}
+
+	switch sm.config.LeaderElectionType {
+	case "kubernetes", "":
+		m.KubernetesClient = sm.clientSet
+		m.RetryWatcherClient = sm.rwClientSet
+	case "etcd":
+		client, err := etcd.NewClient(sm.config)
+		if err != nil {
+			return nil, err
+		}
+		m.EtcdClient = client
+	default:
+		return nil, errors.Errorf("invalid LeaderElectionMode %s not supported", sm.config.LeaderElectionType)
+	}
+
+	return m, nil
+}