Unintentionally created duplicate functions that can't be used

Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# Enable GitHub funding
+
+github: [kube-vip]
+

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: gomod
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: /
+    schedule:
+      interval: weekly

.github/workflows/anchore-syft.yml

@@ -0,0 +1,31 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# This workflow checks out code, builds an image, performs a container image
+# scan with Anchore's Syft tool, and uploads the results to the GitHub Dependency
+# submission API.
+
+# For more information on the Anchore sbom-action usage
+# and parameters, see https://github.com/anchore/sbom-action. For more
+# information about the Anchore SBOM tool, Syft, see
+# https://github.com/anchore/syft
+name: Anchore Syft SBOM scan
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  sbom:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref_name }}
+      - name: Anchore SBOM Action
+        uses: anchore/sbom-action@v0.20.4
+        with:
+          format: cyclonedx-json

.github/workflows/ci-pull-request.yaml

@@ -0,0 +1,51 @@
+name: For each PR
+on:
+  pull_request:
+jobs:
+  e2e-tests:
+    runs-on: ubuntu-latest
+    name: E2E tests
+    strategy:
+      matrix:
+        mode: ["arp", "rt", "bgp"]
+      fail-fast: true
+      max-parallel: 1
+    steps:
+      - name: Ensure fs wont cause issues
+        run: sudo sysctl fs.inotify.max_user_instances=8192 &&  sudo sysctl fs.inotify.max_user_watches=524288
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Build image locally
+        run: make dockerx86Local
+      - name: Run Manifest generation tests
+        run: make manifest-test
+      - name: Run ARP mode tests v1.29.0 onwards
+        run: make e2e-tests129-arp
+        if: matrix.mode== 'arp'
+      - name: Run RT mode tests v1.29.0 onwards
+        run: make e2e-tests129-rt
+        if: matrix.mode== 'rt'
+      - name: Get GoBGP binaries
+        run: make get-gobgp
+        if: matrix.mode== 'bgp'
+      - name: Run BGP mode tests v1.29.0 onwards
+        run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 make e2e-tests129-bgp
+        if: matrix.mode== 'bgp'
+  service-e2e-tests:
+    runs-on: ubuntu-latest
+    name: E2E service tests
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Build image with iptables
+        run: make dockerx86ActionIPTables
+      - name: Run tests
+        run: DOCKERTAG=action make service-tests

.github/workflows/ci.yaml

@@ -1,27 +1,62 @@
-name: For each commit and PR
+name: For each commit
 on:
   push:
   pull_request:
-
 jobs:
   validation:
     runs-on: ubuntu-latest
-    env:
-      CGO_ENABLED: 0
+    name: Checks and linters
     steps:
-    - name: Init
-      run: sudo apt-get update && sudo apt-get install -y build-essential golint
-    - name: Checkout code
-      uses: actions/checkout@v2
-    - name: Install Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: '1.15'
-    - name: checks
-      run: make check
-    - name: test docker build
-      run: make dockerx86Action
-    - name: Manifest generate
-      run: ./testing/testing.sh
-    - name: e2e tests
-      run: DOCKERTAG=action make e2e-tests
+      - name: Init
+        run: sudo apt-get update && sudo apt-get install -y build-essential &&  sudo sysctl fs.inotify.max_user_instances=8192 &&  sudo sysctl fs.inotify.max_user_watches=524288
+      - name: Install golangci-lint
+        run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.64.8
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: All checks
+        run: make check
+  unit-tests:
+    runs-on: ubuntu-latest
+    name: Unit tests
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Run tests
+        run: make unit-tests
+  integration-tests:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Run tests
+        run: make integration-tests
+  image-vul-check:
+    runs-on: ubuntu-latest
+    name: Image vulnerability scan
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Build image with iptables
+        run: make dockerx86ActionIPTables
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: "plndr/kube-vip:action"
+          format: "table"
+          exit-code: "1"
+          ignore-unfixed: true
+          vuln-type: "os,library"
+          severity: "CRITICAL,HIGH"

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,75 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '17 10 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      
+    - name: Install Go
+      uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+        
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v3
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3

.github/workflows/main.yaml

@@ -1,43 +1,52 @@
-name: Publish the latest dev image
+name: Build and publish main image regularly
 
 on:
-  push:
-    branches:
-      - 'master'
+  schedule:
+    - cron: '25 0 * * *'
+  workflow_dispatch:
+
 
 jobs:
-  docker:
+  nightly_build:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Prepare Names
-        id: prep
-        run: |
-          DOCKER_IMAGE=plndr/kube-vip
-          VERSION=$(echo ${GITHUB_SHA} | cut -c1-8)
-          TAGS="${DOCKER_IMAGE}:${VERSION}"
-          TAGS="$TAGS,${DOCKER_IMAGE}:nightly"
-          echo ::set-output name=tags::${TAGS}
-
+        uses: actions/checkout@v4
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Build and push main branch 
-        uses: docker/build-push-action@v2
+      - name: Login to Github Packages
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}   
+      - name: Build standard version
+        id: docker_build
+        uses: docker/build-push-action@v6
         with:
           context: .
-          platforms: linux/amd64,linux/arm/v7,linux/arm64
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
           push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.prep.outputs.tags }}
-
+          tags: >-
+            plndr/kube-vip:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip:${{ github.ref_name }}
+      - name: Build iptables version
+        id: docker_build_iptables
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile_iptables
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: >-
+            plndr/kube-vip-iptables:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip-iptables:${{ github.ref_name }}
       - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
+        run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/release.yaml

@@ -1,48 +1,64 @@
-name: Publish Releases to Docker Hub
+name: Publish Releases to Docker Hub and GitHub Container Registry
 
 on:
   push:
     tags:
       - '*'
+  workflow_dispatch:
+
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Prepare Names
-        id: prep
-        run: |
-          DOCKER_IMAGE=plndr/kube-vip
-          VERSION=${GITHUB_REF#refs/tags/}
-          TAGS="${DOCKER_IMAGE}:${VERSION},ghcr.io/kube-vip/kube-vip:${VERSION}"
-          TAGS="$TAGS,${DOCKER_IMAGE}:latest,ghcr.io/kube-vip/kube-vip:latest"
-          echo ::set-output name=tags::${TAGS}
-
+        uses: actions/checkout@v4
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Login to Github Packages
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and push main branch 
-        uses: docker/build-push-action@v2
+      - name: Generate Metadata
+        uses: docker/metadata-action@v5.7.0
+        id: metadata
+        with:
+          labels: |
+            org.opencontainers.image.documentation=https://kube-vip.io/docs/
+      - name: Build and push main branch
+        id: docker_build
+        uses: docker/build-push-action@v6
         with:
           context: .
-          platforms: linux/amd64,linux/arm/v7,linux/arm64
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
           push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.prep.outputs.tags }}
-
+          labels: ${{ steps.metadata.outputs.labels }}
+          tags: >-
+            plndr/kube-vip:${{ github.ref_name }},
+            plndr/kube-vip:latest,
+            ghcr.io/kube-vip/kube-vip:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip:latest
+      - name: Build iptables version and push main branch
+        id: docker_build_iptables
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile_iptables
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          labels: ${{ steps.metadata.outputs.labels }}
+          tags: >-
+            plndr/kube-vip-iptables:${{ github.ref_name }},
+            plndr/kube-vip-iptables:latest,
+            ghcr.io/kube-vip/kube-vip-iptables:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip-iptables:latest
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}

.gitignore

@@ -0,0 +1,7 @@
+.idea
+kube-vip
+.vscode
+bin
+testing/e2e/etcd/certs
+pkg/etcd/etcd.pid
+pkg/etcd/etcd-data

.golangci.yml

@@ -0,0 +1,13 @@
+run:
+  timeout: 10m
+  
+linters:
+  enable:
+  - bodyclose
+  - gofmt
+  - goimports
+  - revive
+  - gosec
+  - misspell
+  - unconvert
+  - unparam

CONTRIBUTING.md

@@ -154,7 +154,7 @@ the `$GOPATH`.
 
 To develop locally, you can follow these steps:
 
- 1. [Install Go 1.15](https://golang.org/doc/install)
+ 1. [Install Go 1.19](https://golang.org/doc/install)
  2. Checkout your feature branch and `cd` into it.
  3. To build all Go files and install them under `bin`, run `make bin`
  4. To run all Go unit tests, run `make test-unit`

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:experimental
 
-FROM golang:1.16.5-alpine3.13 as dev
+FROM golang:1.24.5-alpine3.22 as dev
 RUN apk add --no-cache git ca-certificates make
 RUN adduser -D appuser
 COPY . /src/
@@ -16,4 +16,4 @@ FROM scratch
 COPY --from=dev /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 # Add kube-vip binary
 COPY --from=dev /src/kube-vip /
-ENTRYPOINT ["/kube-vip"]
+ENTRYPOINT ["/kube-vip"]

Dockerfile_iptables

@@ -0,0 +1,21 @@
+# syntax=docker/dockerfile:experimental
+
+FROM golang:1.24.5-alpine3.22 as dev
+RUN apk add --no-cache git make
+RUN adduser -D appuser
+COPY . /src/
+WORKDIR /src
+
+ENV GO111MODULE=on
+RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
+    --mount=type=cache,sharing=locked,id=goroot,target=/root/.cache/go-build \
+    CGO_ENABLED=0 GOOS=linux make build
+
+FROM alpine:3.22.1
+# Update pkgs and add iptables
+RUN apk upgrade && \
+    apk add --no-cache iptables iptables-legacy
+
+# Add kube-vip binary
+COPY --from=dev /src/kube-vip /
+ENTRYPOINT ["/kube-vip"]

Makefile

@@ -2,10 +2,11 @@ SHELL := /bin/sh
 
 # The name of the executable (default is current directory name)
 TARGET := kube-vip
-.DEFAULT_GOAL: $(TARGET)
+.DEFAULT_GOAL := $(TARGET)
 
 # These will be provided to the target
-VERSION := v0.3.5
+VERSION := v1.0.0
+
 BUILD := `git rev-parse HEAD`
 
 # Operating System Default (LINUX)
@@ -14,9 +15,9 @@ TARGETOS=linux
 # Use linker flags to provide version/build settings to the target
 LDFLAGS=-ldflags "-s -w -X=main.Version=$(VERSION) -X=main.Build=$(BUILD) -extldflags -static"
 DOCKERTAG ?= $(VERSION)
-REPOSITORY = plndr
+REPOSITORY ?= plndr
 
-.PHONY: all build clean install uninstall fmt simplify check run e2e-tests
+.PHONY: all build clean install uninstall simplify check run e2e-tests
 
 all: check install
 
@@ -36,17 +37,30 @@ install:
 uninstall: clean
 	@rm -f $$(which ${TARGET})
 
-fmt:
-	@gofmt -l -w ./...
-
 demo:
 	@cd demo
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 	@cd ..
 
 ## Remote (push of images)
 # This build a local docker image (x86 only) for quick testing
+
+dockerx86Dev:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 --push -t $(REPOSITORY)/$(TARGET):dev .
+	@echo New single x86 Architecture Docker image created
+
+dockerx86Iptables:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 -f ./Dockerfile_iptables --push -t $(REPOSITORY)/$(TARGET):dev .
+	@echo New single x86 Architecture Docker image created
+
+dockerx86IptablesLocal:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 -f ./Dockerfile_iptables -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@echo New single x86 Architecture Docker image created
+
 dockerx86:
 	@-rm ./kube-vip
 	@docker buildx build  --platform linux/amd64 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
@@ -54,7 +68,7 @@ dockerx86:
 
 docker:
 	@-rm ./kube-vip
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 ## Local (docker load of images)
@@ -69,21 +83,26 @@ dockerx86Action:
 	@docker buildx build  --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):action .
 	@echo New Multi Architecture Docker image created
 
+dockerx86ActionIPTables:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 -f ./Dockerfile_iptables --load -t $(REPOSITORY)/$(TARGET):action .
+	@echo New Multi Architecture Docker image created
+
 dockerLocal:
 	@-rm ./kube-vip
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7 --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 simplify:
-	@gofmt -s -l -w ./...
+	@gofmt -s -l -w *.go pkg cmd
 
 check:
 	go mod tidy
 	test -z "$(git status --porcelain)"
-	test -z $(shell gofmt -l main.go | tee /dev/stderr) || echo "[WARN] Fix formatting issues with 'make fmt'"
-	golint ./...
+	test -z $(shell gofmt -l *.go pkg cmd) || echo "[WARN] Fix formatting issues with 'make simplify'"
+	golangci-lint run
 	go vet ./...
-	
+
 run: install
 	@$(TARGET)
 
@@ -91,11 +110,70 @@ manifests:
 	@make build
 	@mkdir -p ./docs/manifests/$(VERSION)/
 	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-arp.yaml
+	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-lb.yaml
 	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --bgp --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-bgp.yaml
 	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
 	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
-	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster --provider-config /etc/cloud-sa/cloud-sa.json > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
 	@-rm ./kube-vip
 
+manifest-test: 
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --enableLoadBalancer
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --bgp --controlplane --services 
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster
+
+unit-tests:
+	go test ./...
+
+integration-tests:
+	go test -tags=integration,e2e -v ./pkg/etcd
+
 e2e-tests:
-	E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/ginkgo -v -p testing/e2e
+	GOMAXPROCS=4 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd
+
+e2e-tests129-arp:
+	GOMAXPROCS=4 TEST_MODE=arp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
+
+e2e-tests129-rt:
+	GOMAXPROCS=4 TEST_MODE=rt V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
+
+e2e-tests129-bgp:
+	GOMAXPROCS=4 TEST_MODE=bgp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
+
+e2e-tests129: e2e-tests129-arp e2e-tests129-rt e2e-tests129-bgp
+
+service-tests:
+	E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/services -Services -simple -deployments -leaderActive -leaderFailover -localDeploy -egress -egressIPv6 -dualStack
+
+trivy: dockerx86ActionIPTables
+	docker run -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy:0.47.0 \
+		image  \
+		--format table \
+		--exit-code  1 \
+		--ignore-unfixed \
+		--vuln-type  'os,library' \
+		--severity  'CRITICAL,HIGH'  \
+		$(REPOSITORY)/$(TARGET):action
+
+kind-quick:
+	echo "Standing up your cluster"
+	kind create cluster --config ./testing/kind/kind.yaml --name kube-vip
+	kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
+	kubectl create configmap --namespace kube-system kubevip --from-literal range-global=172.18.100.10-172.18.100.30
+	kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
+	kind load docker-image --name kube-vip $(REPOSITORY)/$(TARGET):$(DOCKERTAG)
+	docker run --network host --rm $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --services --inCluster --arp --servicesElection --interface  eth0 | kubectl apply -f -
+
+kind-reload:
+	kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name kube-vip
+	kubectl rollout restart -n kube-system daemonset/kube-vip-ds
+
+get-gobgp:
+	mkdir -p bin
+	wget -nc --directory-prefix=bin https://github.com/osrg/gobgp/releases/download/v3.37.0/gobgp_3.37.0_linux_amd64.tar.gz
+	tar -xvzf bin/gobgp_3.37.0_linux_amd64.tar.gz -C bin
+

README.md

@@ -1,8 +1,10 @@
 # kube-vip
 
-High Availability and Load-Balancing 
+High Availability and Load-Balancing
 
-![](https://kube-vip.io/kube-vip.png)
+![](https://github.com/kube-vip/kube-vip/raw/main/kube-vip.png)
+
+[![Build and publish main image regularly](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml/badge.svg)](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml) [![LFX Health Score](https://img.shields.io/static/v1?label=Health%20Score&message=Healthy&color=A7F3D0&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip) [![LFX Active Contributors](https://img.shields.io/static/v1?label=Active%20contributors%20(1Y)&message=212&color=0094FF&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip)
 
 ## Overview
 Kubernetes Virtual IP and Load-Balancer for both control plane and Kubernetes services
@@ -30,7 +32,8 @@ Kube-Vip was originally created to provide a HA solution for the Kubernetes cont
 - Service LoadBalancer address pools per namespace or global
 - Service LoadBalancer address via (existing network DHCP)
 - Service LoadBalancer address exposure to gateway via UPNP
-- ... manifest generation, vendor API integrations and many nore... 
+- Egress! Kube-vip will utilise a service loadbalancer as both the ingress and **egress** for a pod. 
+- ... manifest generation, vendor API integrations and many more...
 
 ## Why?
 
@@ -56,3 +59,13 @@ All of these would require a separate level of configuration and in some infrast
 ## Troubleshooting and Feedback
 
 Please raise issues on the GitHub repository and as mentioned check the documentation at [https://kube-vip.io](https://kube-vip.io/).
+
+## Contributing
+
+Thanks for taking the time to join our community and start contributing! We welcome pull requests. Feel free to dig through the [issues](https://github.com/kube-vip/kube-vip/issues) and jump in.
+
+:warning: This project has issue compiling on MacOS, please compile it on linux distribution
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=kube-vip/kube-vip&type=Date)](https://star-history.com/#kube-vip/kube-vip&Date)

ROADMAP.md

@@ -1,10 +1,10 @@
 # Kube-Vip Roadmap
 
-This document outlines the roadmap for the **kube-vip** project and only covers the technologies within this particular project, other projects that augment or provide additional functionality (such as cloud-providers) may have their own roadmaps in future. The functionality for **kube-vip** has grown either been developed organically or through real-world needs, and this is the first attempt to put into words a plan for the future of **kube-vip** and will additional evolve over time. This means that items listed or detailed here are not neccessarily set in stone and the roadmap can grow/shrink as the project matures. We definitely welcome suggestions and ideas from everyone about the roadmap and **kube-vip** features. Reach us through Issues, Slack or email <catch-all>@kube-vip.io.
-  
+This document outlines the roadmap for the **kube-vip** project and only covers the technologies within this particular project, other projects that augment or provide additional functionality (such as cloud-providers) may have their own roadmaps in future. The functionality for **kube-vip** has grown either been developed organically or through real-world needs, and this is the first attempt to put into words a plan for the future of **kube-vip** and will additional evolve over time. This means that items listed or detailed here are not necessarily set in stone and the roadmap can grow/shrink as the project matures. We definitely welcome suggestions and ideas from everyone about the roadmap and **kube-vip** features. Reach us through Issues, Slack or email <catch-all>@kube-vip.io.
+
 ## Release methodology
 
-The **kube-vip** project attempts to follow a tick-tock release cycle, this typically means that one release will come **packed** with new features where the following release will come with fixes, code sanitation and performane enhancements.
+The **kube-vip** project attempts to follow a tick-tock release cycle, this typically means that one release will come **packed** with new features where the following release will come with fixes, code sanitation and performance enhancements.
 
 ## Roadmap
 
@@ -13,12 +13,12 @@ The **kube-vip** project offers two main areas of functionality:
 - HA Kubernetes clusters through a control-plane VIP
 - Kubernetes `service type:LoadBalancer`
 
-Whilst both of these functions share underlying technologies and code they will have slightly differening roadmaps. 
+Whilst both of these functions share underlying technologies and code they will have slightly differing roadmaps.
 
 ### HA Kubernetes Control Plane
 
-- **Re-implememt LoadBalancing** - due to a previous request the HTTP loadbalancing was removed leaving just HA for the control plane. This functionality will be re-implemented either through the original round-robin HTTP requests or utilising IPVS. 
-- **Utilise the Kubernetes API to determine additional Control Plane members** - Once a single node cluster is running **kube-vip** could use the API to determine the additional members, at this time a Cluster-API provider needs to drop a static manifest per CP node. 
+- **Re-implement LoadBalancing** - due to a previous request the HTTP loadbalancing was removed leaving just HA for the control plane. This functionality will be re-implemented either through the original round-robin HTTP requests or utilising IPVS.
+- **Utilise the Kubernetes API to determine additional Control Plane members** - Once a single node cluster is running **kube-vip** could use the API to determine the additional members, at this time a Cluster-API provider needs to drop a static manifest per CP node.
 - **Re-evaluate raft** - **kube-vip** is mainly designed to run within a Kubernetes cluster, however it's original design was a raft cluster external to Kubernetes. Unfortunately given some of the upgrade paths identified in things like CAPV moving to leaderElection within Kubernetes became a better idea.
 
 ## Kubernetes `service type:LoadBalancer`
@@ -29,7 +29,7 @@ Whilst both of these functions share underlying technologies and code they will
 ## Global **Kube-Vip** items
 
 - **Improved metrics** - At this time the scaffolding for monitoring exists, however this needs drastically extending to provide greater observability to what is happening within **kube-vip**
-- **Windows support** - The Go SDK didn't support the capability for low-levels sockets for ARP originally, this should be revisted. 
+- **Windows support** - The Go SDK didn't support the capability for low-levels sockets for ARP originally, this should be revisited.
 - **Additional BGP features** :
   - Communities
   - BFD

cmd/kube-vip-config.go

@@ -1,146 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-
-	"github.com/ghodss/yaml"
-	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	appv1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-// [sample configuration] - flags
-var cliConfig kubevip.Config
-var cliConfigLB kubevip.LoadBalancer
-var cliLocalPeer string
-var cliRemotePeers, cliBackends []string
-
-func init() {
-	kubeVipSampleConfig.Flags().StringVar(&cliConfig.Interface, "interface", "eth0", "Name of the interface to bind to")
-	kubeVipSampleConfig.Flags().StringVar(&cliConfig.VIP, "vip", "192.168.0.1", "The Virtual IP address")
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfig.SingleNode, "singleNode", false, "Start this instance as a single node")
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfig.StartAsLeader, "startAsLeader", false, "Start this instance as the cluster leader")
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfig.EnableARP, "arp", true, "Use ARP broadcasts to improve VIP re-allocations")
-	kubeVipSampleConfig.Flags().StringVar(&cliLocalPeer, "localPeer", "server1:192.168.0.1:10000", "Settings for this peer, format: id:address:port")
-	kubeVipSampleConfig.Flags().StringSliceVar(&cliRemotePeers, "remotePeers", []string{"server2:192.168.0.2:10000", "server3:192.168.0.3:10000"}, "Comma seperated remotePeers, format: id:address:port")
-	// Load Balancer flags
-	kubeVipSampleConfig.Flags().BoolVar(&cliConfigLB.BindToVip, "lbBindToVip", false, "Bind example load balancer to VIP")
-	kubeVipSampleConfig.Flags().StringVar(&cliConfigLB.Type, "lbType", "tcp", "Type of load balancer instance (TCP/HTTP)")
-	kubeVipSampleConfig.Flags().StringVar(&cliConfigLB.Name, "lbName", "Example Load Balancer", "The name of a load balancer instance")
-	kubeVipSampleConfig.Flags().IntVar(&cliConfigLB.Port, "lbPort", 8080, "Port that load balancer will expose on")
-	kubeVipSampleConfig.Flags().StringSliceVar(&cliBackends, "lbBackends", []string{"192.168.0.1:8080", "192.168.0.2:8080"}, "Comma seperated backends, format: address:port")
-}
-
-var kubeVipSampleConfig = &cobra.Command{
-	Use:   "config",
-	Short: "Generate a Sample configuration",
-	Run: func(cmd *cobra.Command, args []string) {
-
-		// // Parse localPeer
-		// p, err := kubevip.ParsePeerConfig(cliLocalPeer)
-		// if err != nil {
-		// 	cmd.Help()
-		// 	log.Fatalln(err)
-		// }
-		// cliConfig.LocalPeer = *p
-
-		// // Parse remotePeers
-		// //Iterate backends
-		// for i := range cliRemotePeers {
-		// 	p, err := kubevip.ParsePeerConfig(cliRemotePeers[i])
-		// 	if err != nil {
-		// 		cmd.Help()
-		// 		log.Fatalln(err)
-		// 	}
-		// 	cliConfig.RemotePeers = append(cliConfig.RemotePeers, *p)
-		// }
-
-		// //Iterate backends
-		// for i := range cliBackends {
-		// 	b, err := kubevip.ParseBackendConfig(cliBackends[i])
-		// 	if err != nil {
-		// 		cmd.Help()
-		// 		log.Fatalln(err)
-		// 	}
-		// 	cliConfigLB.Backends = append(cliConfigLB.Backends, *b)
-		// }
-
-		// Add the basic Load-Balancer to the configuration
-		cliConfig.LoadBalancers = append(cliConfig.LoadBalancers, cliConfigLB)
-
-		err := cliConfig.ParseFlags(cliLocalPeer, cliRemotePeers, cliBackends)
-		if err != nil {
-			cmd.Help()
-			log.Fatalln(err)
-		}
-
-		err = kubevip.ParseEnvironment(&cliConfig)
-		if err != nil {
-			cmd.Help()
-			log.Fatalln(err)
-		}
-
-		cliConfig.PrintConfig()
-	},
-}
-
-var kubeVipSampleManifest = &cobra.Command{
-	Use:   "manifest",
-	Short: "Generate a Sample kubernetes manifest",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Generate the sample manifest specification
-		p := &appv1.Pod{
-			TypeMeta: metav1.TypeMeta{
-				Kind:       "Pod",
-				APIVersion: "v1",
-			},
-			ObjectMeta: metav1.ObjectMeta{
-				Name:      "kube-vip",
-				Namespace: "kube-system",
-			},
-			Spec: appv1.PodSpec{
-				Containers: []appv1.Container{
-					{
-						Name:  "kube-vip",
-						Image: fmt.Sprintf("docker.io/plndr/kube-vip:%s", Release.Version),
-						SecurityContext: &appv1.SecurityContext{
-							Capabilities: &appv1.Capabilities{
-								Add: []appv1.Capability{
-									"NET_ADMIN",
-									"SYS_TIME",
-								},
-							},
-						},
-						Args: []string{
-							"start",
-							"-c",
-							"/etc/kube-vip/config.yaml",
-						},
-						VolumeMounts: []appv1.VolumeMount{
-							{
-								Name:      "config",
-								MountPath: "/etc/kube-vip/",
-							},
-						},
-					},
-				},
-				Volumes: []appv1.Volume{
-					{
-						Name: "config",
-						VolumeSource: appv1.VolumeSource{
-							HostPath: &appv1.HostPathVolumeSource{
-								Path: "/etc/kube-vip/",
-							},
-						},
-					},
-				},
-				HostNetwork: true,
-			},
-		}
-
-		b, _ := yaml.Marshal(p)
-		fmt.Printf(string(b))
-	},
-}

cmd/kube-vip-kubeadm.go

@@ -1,18 +1,13 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
-	"net"
 	"os"
 
+	log "log/slog"
+
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
 )
 
 // kubeadm adds two subcommands for managing a vip during a kubeadm init/join
@@ -28,8 +23,8 @@ func init() {
 var kubeKubeadm = &cobra.Command{
 	Use:   "kubeadm",
 	Short: "Kubeadm functions",
-	Run: func(cmd *cobra.Command, args []string) {
-		cmd.Help()
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		_ = cmd.Help()
 		// TODO - A load of text detailing what's actually happening
 	},
 }
@@ -38,124 +33,84 @@ var kubeKubeadmInit = &cobra.Command{
 	Use:   "init",
 	Short: "kube-vip init",
 	Long:  "The \"init\" subcommand will generate the Kubernetes manifest that will be started by kubeadm through the kubeadm init process",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
 		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
+		err := kubevip.ParseEnvironment(&initConfig)
+		if err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+
 		// TODO - check for certain things VIP/interfaces
 		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+			_ = cmd.Help()
+			log.Error("No interface is specified for kube-vip to bind to")
+			return
 		}
 
 		if initConfig.VIP == "" && initConfig.Address == "" {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
-		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
 
-		fmt.Println(cfg)
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
+			if err != nil {
+				log.Error("generating VIPSubnet", "err", err)
+				return
+			}
+		}
+
+		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
+		fmt.Println(cfg) // output manifest to stdout
 	},
 }
 
 var kubeKubeadmJoin = &cobra.Command{
 	Use:   "join",
 	Short: "kube-vip join",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
 		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
+		err := kubevip.ParseEnvironment(&initConfig)
+		if err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+
 		// TODO - check for certain things VIP/interfaces
 		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+			_ = cmd.Help()
+			log.Error("No interface is specified for kube-vip to bind to")
+			return
 		}
 
 		if initConfig.VIP == "" && initConfig.Address == "" {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
 
 		if _, err := os.Stat(kubeConfigPath); os.IsNotExist(err) {
-			log.Fatalf("Unable to find file [%s]", kubeConfigPath)
+			log.Error("kubeConfig not found", "Path", kubeConfigPath)
+			return
 		}
 
-		// We will use kubeconfig in order to find all the master nodes
-		// use the current context in kubeconfig
-		config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
-		if err != nil {
-			log.Fatal(err.Error())
-		}
-
-		// create the clientset
-		clientset, err := kubernetes.NewForConfig(config)
-		if err != nil {
-			log.Fatal(err.Error())
-		}
-
-		opts := metav1.ListOptions{}
-		opts.LabelSelector = "node-role.kubernetes.io/master"
-		nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), opts)
-
-		// Iterate over all nodes that are masters and find the details to build a peer list
-		for x := range nodes.Items {
-			// Get hostname and address
-			var nodeAddress, nodeHostname string
-			for y := range nodes.Items[x].Status.Addresses {
-				switch nodes.Items[x].Status.Addresses[y].Type {
-				case corev1.NodeHostName:
-					nodeHostname = nodes.Items[x].Status.Addresses[y].Address
-				case corev1.NodeInternalIP:
-					nodeAddress = nodes.Items[x].Status.Addresses[y].Address
-				}
-			}
-
-			newPeer, err := kubevip.ParsePeerConfig(fmt.Sprintf("%s:%s:%d", nodeHostname, nodeAddress, 10000))
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
 			if err != nil {
-				panic(err.Error())
+				log.Error("generating VIPSubnet", "err", err)
+				return
 			}
-			initConfig.RemotePeers = append(initConfig.RemotePeers, *newPeer)
-
 		}
-		// Generate manifest and print
+
 		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
-		fmt.Println(cfg)
+		fmt.Println(cfg) // output manifest to stdout
 	},
 }
-
-func autoGenLocalPeer() (*kubevip.RaftPeer, error) {
-	// hostname // address // defaultport
-	h, err := os.Hostname()
-	if err != nil {
-		return nil, err
-	}
-
-	var a string
-	addrs, err := net.InterfaceAddrs()
-	if err != nil {
-		return nil, err
-	}
-	for _, address := range addrs {
-		// check the address type and if it is not a loopback the display it
-		if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
-			if ipnet.IP.To16() != nil {
-				a = ipnet.IP.String()
-				break
-			}
-		}
-	}
-	if a == "" {
-		return nil, fmt.Errorf("Unable to find local address")
-	}
-	return &kubevip.RaftPeer{
-		ID:      h,
-		Address: a,
-		Port:    10000,
-	}, nil
-
-}

cmd/kube-vip-manifests.go

@@ -3,8 +3,9 @@ package cmd
 import (
 	"fmt"
 
+	log "log/slog"
+
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
@@ -12,23 +13,26 @@ import (
 // manifests will be used to generate:
 // - Pod spec manifest, mainly used for a static pod (kubeadm)
 // - Daemonset manifest, mainly used to run kube-vip as a deamonset within Kubernetes (k3s/rke)
+// - RBAC manifest, used to generate the RBAC permissions for kube-vip
 
-//var inCluster bool
-var taint bool
+var taint, role, rolebinding bool
 
 func init() {
 	kubeManifest.PersistentFlags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
 	kubeManifestDaemon.PersistentFlags().BoolVar(&taint, "taint", false, "Taint the manifest for only running on control planes")
+	kubeManifestRbac.PersistentFlags().BoolVar(&role, "role", false, "Generate only a Role inside the serviceNamespace access")
+	kubeManifestRbac.PersistentFlags().BoolVar(&rolebinding, "rolebinding", false, "Generate only a RoleBinding for namespaced access")
 
 	kubeManifest.AddCommand(kubeManifestPod)
 	kubeManifest.AddCommand(kubeManifestDaemon)
+	kubeManifest.AddCommand(kubeManifestRbac)
 }
 
 var kubeManifest = &cobra.Command{
 	Use:   "manifest",
 	Short: "Manifest functions",
-	Run: func(cmd *cobra.Command, args []string) {
-		cmd.Help()
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		_ = cmd.Help()
 		// TODO - A load of text detailing what's actually happening
 	},
 }
@@ -36,51 +40,111 @@ var kubeManifest = &cobra.Command{
 var kubeManifestPod = &cobra.Command{
 	Use:   "pod",
 	Short: "Generate a Pod Manifest",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		var err error
+
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
 		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
-		// TODO - check for certain things VIP/interfaces
-		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
+			log.Error("parsing environment", "err", err)
+			return
 		}
 
 		// The control plane has a requirement for a VIP being specified
-		if initConfig.EnableControlPane && (initConfig.VIP == "" && initConfig.Address == "" && initConfig.DDNS == false) {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
-		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
 
-		fmt.Println(cfg)
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
+			if err != nil {
+				log.Error("config parse", "err", err)
+				return
+			}
+		}
+
+		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
+		fmt.Println(cfg) // output manifest to stdout
 	},
 }
 
 var kubeManifestDaemon = &cobra.Command{
 	Use:   "daemonset",
 	Short: "Generate a Daemonset Manifest",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		var err error
+
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
 		// TODO - A load of text detailing what's actually happening
-		kubevip.ParseEnvironment(&initConfig)
-		// TODO - check for certain things VIP/interfaces
-		if initConfig.Interface == "" {
-			cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+		// The control plane has a requirement for a VIP being specified
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
+		}
+
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
+			if err != nil {
+				log.Error("config parse", "err", err)
+				return
+			}
+		}
+
+		cfg := kubevip.GenerateDaemonsetManifestFromConfig(&initConfig, Release.Version, inCluster, taint)
+		fmt.Println(cfg) // output manifest to stdout
+	},
+}
+
+var kubeManifestRbac = &cobra.Command{
+	Use:   "rbac",
+	Short: "Generate an RBAC Manifest",
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		var err error
+
+		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
+		// TODO - A load of text detailing what's actually happening
+		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
+			log.Error("parsing environment", "err", err)
+			return
 		}
 
 		// The control plane has a requirement for a VIP being specified
-		if initConfig.EnableControlPane && (initConfig.VIP == "" && initConfig.Address == "" && initConfig.DDNS == false) {
-			cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
-		cfg := kubevip.GenerateDeamonsetManifestFromConfig(&initConfig, Release.Version, inCluster, taint)
 
-		fmt.Println(cfg)
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
+			if err != nil {
+				log.Error("generating VIPSubnet", "err", err)
+				return
+			}
+		}
+		saCfg := kubevip.GenerateSA(&initConfig)
+		roleCfg := kubevip.GenerateRole(&initConfig, role)
+		if role {
+			rolebinding = true
+		}
+		roleBindingCfg := kubevip.GenerateRoleBinding(rolebinding, saCfg, roleCfg)
+
+		// Output the YAML manifests to stdout
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(saCfg))
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(roleCfg))
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(roleBindingCfg))
 	},
 }

cmd/kube-vip-start.go

@@ -1,136 +0,0 @@
-package cmd
-
-import (
-	"os"
-	"os/signal"
-
-	"github.com/kube-vip/kube-vip/pkg/bgp"
-	"github.com/kube-vip/kube-vip/pkg/cluster"
-	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-)
-
-// Start as a single node (no cluster), start as a leader in the cluster
-var startConfig kubevip.Config
-var startConfigLB kubevip.LoadBalancer
-var startLocalPeer, startKubeConfigPath string
-var startRemotePeers, startBackends []string
-var inCluster bool
-
-func init() {
-	// Get the configuration file
-	kubeVipStart.Flags().StringVarP(&configPath, "config", "c", "", "Path to a kube-vip configuration")
-	kubeVipStart.Flags().BoolVarP(&disableVIP, "disableVIP", "d", false, "Disable the VIP functionality")
-
-	// Pointers so we can see if they're nil (and not called)
-	kubeVipStart.Flags().StringVar(&startConfig.Interface, "interface", "eth0", "Name of the interface to bind to")
-	kubeVipStart.Flags().StringVar(&startConfig.VIP, "vip", "192.168.0.1", "The Virtual IP address")
-	kubeVipStart.Flags().StringVar(&startConfig.Address, "address", "", "an address (IP or DNS name) to use as a VIP")
-	kubeVipStart.Flags().IntVar(&startConfig.Port, "port", 6443, "listen port for the VIP")
-	kubeVipStart.Flags().BoolVar(&startConfig.DDNS, "ddns", false, "use Dynamic DNS + DHCP to allocate VIP for address")
-	kubeVipStart.Flags().BoolVar(&startConfig.SingleNode, "singleNode", false, "Start this instance as a single node")
-	kubeVipStart.Flags().BoolVar(&startConfig.StartAsLeader, "startAsLeader", false, "Start this instance as the cluster leader")
-	kubeVipStart.Flags().BoolVar(&startConfig.EnableARP, "arp", false, "Use ARP broadcasts to improve VIP re-allocations")
-	kubeVipStart.Flags().StringVar(&startLocalPeer, "localPeer", "server1:192.168.0.1:10000", "Settings for this peer, format: id:address:port")
-	kubeVipStart.Flags().StringSliceVar(&startRemotePeers, "remotePeers", []string{"server2:192.168.0.2:10000", "server3:192.168.0.3:10000"}, "Comma seperated remotePeers, format: id:address:port")
-	// Load Balancer flags
-	kubeVipStart.Flags().BoolVar(&startConfigLB.BindToVip, "lbBindToVip", false, "Bind example load balancer to VIP")
-	kubeVipStart.Flags().StringVar(&startConfigLB.Type, "lbType", "tcp", "Type of load balancer instance (TCP/HTTP)")
-	kubeVipStart.Flags().StringVar(&startConfigLB.Name, "lbName", "Example Load Balancer", "The name of a load balancer instance")
-	kubeVipStart.Flags().IntVar(&startConfigLB.Port, "lbPort", 8080, "Port that load balancer will expose on")
-	kubeVipStart.Flags().IntVar(&startConfigLB.BackendPort, "lbBackEndPort", 6443, "A port that all backends may be using (optional)")
-	kubeVipStart.Flags().StringSliceVar(&startBackends, "lbBackends", []string{"192.168.0.1:8080", "192.168.0.2:8080"}, "Comma seperated backends, format: address:port")
-
-	// Cluster configuration
-	kubeVipStart.Flags().StringVar(&startKubeConfigPath, "kubeConfig", "/etc/kubernetes/admin.conf", "The path of a kubernetes configuration file")
-	kubeVipStart.Flags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
-	kubeVipStart.Flags().BoolVar(&startConfig.EnableLeaderElection, "leaderElection", false, "Use the Kubernetes leader election mechanism for clustering")
-
-	// This sets the namespace that the lock should exist in
-	kubeVipStart.Flags().StringVarP(&startConfig.Namespace, "namespace", "n", "kube-system", "The configuration map defined within the cluster")
-}
-
-var kubeVipStart = &cobra.Command{
-	Use:   "start",
-	Short: "Start the Virtual IP / Load balancer",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
-		var err error
-
-		// If a configuration file is loaded, then it will overwrite flags
-
-		if configPath != "" {
-			c, err := kubevip.OpenConfig(configPath)
-			if err != nil {
-				log.Fatalf("%v", err)
-			}
-			startConfig = *c
-		}
-
-		// parse environment variables, these will overwrite anything loaded or flags
-		err = kubevip.ParseEnvironment(&startConfig)
-		if err != nil {
-			log.Fatalln(err)
-		}
-
-		newCluster, err := cluster.InitCluster(&startConfig, disableVIP)
-		if err != nil {
-			log.Fatalf("%v", err)
-		}
-		var bgpServer *bgp.Server
-		if startConfig.SingleNode {
-			// If the Virtual IP isn't disabled then create the netlink configuration
-			// Start a single node cluster
-			newCluster.StartSingleNode(&startConfig, disableVIP)
-		} else {
-			if disableVIP {
-				log.Fatalln("Cluster mode requires the Virtual IP to be enabled, use single node with no VIP")
-			}
-
-			if startConfig.EnableLeaderElection {
-				cm, err := cluster.NewManager(startKubeConfigPath, inCluster, startConfig.Port)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-
-				if startConfig.EnableBGP {
-					log.Info("Starting the BGP server to adverise VIP routes to VGP peers")
-					bgpServer, err = bgp.NewBGPServer(&startConfig.BGPConfig)
-					if err != nil {
-						log.Fatalf("%v", err)
-					}
-
-					// Defer a function to check if the bgpServer has been created and if so attempt to close it
-					defer func() {
-						if bgpServer != nil {
-							bgpServer.Close()
-						}
-					}()
-				}
-
-				// Leader Cluster will block
-				err = newCluster.StartLeaderCluster(&startConfig, cm, bgpServer)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-			} else {
-
-				// // Start a multi-node (raft) cluster, this doesn't block so will wait on signal
-				err = newCluster.StartRaftCluster(&startConfig)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-				signalChan := make(chan os.Signal, 1)
-				signal.Notify(signalChan, os.Interrupt)
-
-				<-signalChan
-
-				newCluster.Stop()
-			}
-
-		}
-
-	},
-}

cmd/kube-vip.go

@@ -3,42 +3,33 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"net"
 	"net/http"
 	"os"
+	"strings"
 	"time"
 
+	log "log/slog"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/spf13/cobra"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
 	"github.com/kube-vip/kube-vip/pkg/manager"
-	"github.com/kube-vip/kube-vip/pkg/packet"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
+	"github.com/kube-vip/kube-vip/pkg/vip"
 )
 
-// Path to the configuration file
-var configPath string
-
-// Path to the configuration file
-var namespace string
-
-// Disable the Virtual IP (bind to the existing network stack)
-var disableVIP bool
-
-// Disable the Virtual IP (bind to the existing network stack)
-var controlPlane bool
-
-// Run as a load balancer service (within a pod / kubernetes)
-var serviceArp bool
+// Is kube-vip running within cluster
+var inCluster bool
 
 // ConfigMap name within a Kubernetes cluster
 var configMap string
 
-// Configure the level of loggin
-var logLevel uint32
-
-// Provider Config
-var providerConfig string
+// Points to a kubernetes configuration file
+var kubeConfigPath string
 
 // Release - this struct contains the release information populated when building kube-vip
 var Release struct {
@@ -47,11 +38,10 @@ var Release struct {
 }
 
 // Structs used via the various subcommands
-var initConfig kubevip.Config
-var initLoadBalancer kubevip.LoadBalancer
-
-// Points to a kubernetes configuration file
-var kubeConfigPath string
+var (
+	initConfig       kubevip.Config
+	initLoadBalancer kubevip.LoadBalancer
+)
 
 var kubeVipCmd = &cobra.Command{
 	Use:   "kube-vip",
@@ -59,84 +49,110 @@ var kubeVipCmd = &cobra.Command{
 }
 
 func init() {
-
-	localpeer, err := autoGenLocalPeer()
-	if err != nil {
-		log.Fatalln(err)
-	}
-	initConfig.LocalPeer = *localpeer
-	//initConfig.Peers = append(initConfig.Peers, *localpeer)
+	// Basic flags
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Interface, "interface", "", "Name of the interface to bind to")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesInterface, "serviceInterface", "", "Name of the interface to bind to (for services)")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIP, "vip", "", "The Virtual IP address")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPSubnet, "vipSubnet", "", "The Virtual IP address subnet e.g. /32 /24 /8 etc.. (Default to 32 for IPv4 and 128 for IPv6)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.NodeName, "nodeName", "", "Name to be used for lease holder. Must be unique for each node/instance")
+
+	// VIP flags
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Address, "address", "", "an address (IP or DNS name) to use as a VIP")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.Port, "port", 6443, "listen port for the VIP")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPCIDR, "cidr", "32", "The CIDR range for the virtual IP address")
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for Vip changes")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Annotations, "annotations", "", "Set Node annotations prefix for parsing")
+	kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.Port, "port", 6443, "Port for the VIP")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for VIP changes")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableWireguard, "wireguard", false, "Enable Wireguard for services VIPs")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableRoutingTable, "table", false, "Enable Routing Table for services VIPs")
+
+	// LoadBalancer flags
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "enableLoadBalancer", false, "enable loadbalancing on the VIP with IPVS")
+	kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.LoadBalancerPort, "lbPort", 6443, "loadbalancer port for the VIP")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerForwardingMethod, "lbForwardingMethod", "local", "loadbalancer forwarding method")
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DDNS, "ddns", false, "use Dynamic DNS + DHCP to allocate VIP for address")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MirrorDestInterface, "mirrorDestInterface", "", "network interface where all traffic that traverses the service interface will be mirrored to. Source interface will use default interface is servicesInterface is not set.")
 
 	// Clustering type (leaderElection)
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLeaderElection, "leaderElection", false, "Use the Kubernetes leader election mechanism for clustering")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 5, "Length of time a Kubernetes leader lease can be held for")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time a Kubernetes leader can attempt to renew its lease")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Number of times the host will retry to hold a lease")
-
-	// Clustering type (raft)
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.StartAsLeader, "startAsLeader", false, "Start this instance as the cluster leader")
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.AddPeersAsBackends, "addPeersToLB", true, "Add raft peers to the load-balancer")
-
-	// Packet flags
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableMetal, "metal", false, "This will use the Equinix Metal API (requires the token ENV) to update the EIP <-> VIP")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalAPIKey, "metalKey", "", "The API token for authenticating with the Equinix Metal API")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalProject, "metalProject", "", "The name of project already created within Equinix Metal")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalProjectID, "metalProjectID", "", "The ID of project already created within Equinix Metal")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ProviderConfig, "provider-config", "", "The path to a provider configuration")
-
-	// Load Balancer flags
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "lbEnable", false, "Enable a load-balancer on the VIP")
-	kubeVipCmd.PersistentFlags().BoolVar(&initLoadBalancer.BindToVip, "lbBindToVip", true, "Bind example load balancer to VIP")
-	kubeVipCmd.PersistentFlags().StringVar(&initLoadBalancer.Type, "lbType", "tcp", "Type of load balancer instance (TCP/HTTP)")
-	kubeVipCmd.PersistentFlags().StringVar(&initLoadBalancer.Name, "lbName", "Kubeadm Load Balancer", "The name of a load balancer instance")
-	kubeVipCmd.PersistentFlags().IntVar(&initLoadBalancer.Port, "lbPort", 6443, "Port that load balancer will expose on")
-	kubeVipCmd.PersistentFlags().IntVar(&initLoadBalancer.BackendPort, "lbBackEndPort", 6444, "A port that all backends may be using (optional)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaderElectionType, "leaderElectionType", "kubernetes", "Defines the backend to run the leader election: kubernetes or etcd. Defaults to kubernetes.")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaseName, "leaseName", "plndr-cp-lock", "Name of the lease that is used for leader election")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 5, "Length of time (in seconds) a Kubernetes leader lease can be held for")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time (in seconds) a Kubernetes leader can attempt to renew its lease")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Length of time (in seconds) the LeaderElector clients should wait between tries of actions")
 
 	// BGP flags
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableBGP, "bgp", false, "This will enable BGP support within kube-vip")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.RouterID, "bgpRouterID", "", "The routerID for the bgp server")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.SourceIF, "sourceIF", "", "The source interface for bgp peering (not to be used with sourceIP)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.SourceIP, "sourceIP", "", "The source address for bgp peering (not to be used with sourceIF)")
 	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.AS, "localAS", 65000, "The local AS number for the bgp server")
+	kubeVipCmd.PersistentFlags().Uint64Var(&initConfig.BGPConfig.HoldTime, "bgpHoldTimer", 30, "The hold timer for all bgp peers (it defines the time a session is held)")
+	kubeVipCmd.PersistentFlags().Uint64Var(&initConfig.BGPConfig.KeepaliveInterval, "bgpKeepAliveInterval", 10, "The keepalive interval for all bgp peers (it defines the heartbeat of keepalive messages)")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPPeerConfig.Address, "peerAddress", "", "The address of a BGP peer")
 	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPPeerConfig.AS, "peerAS", 65000, "The AS number for a BGP peer")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPPeerConfig.Password, "peerPass", "", "The md5 password for a BGP peer")
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPPeerConfig.MultiHop, "multihop", false, "This will enable BGP multihop support")
-	kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.BGPPeers, "bgppeers", []string{}, "Comma seperated BGP Peer, format: address:as:password:multihop")
+	kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.BGPPeers, "bgppeers", []string{}, "Comma separated BGP Peer, format: address:as:password:multihop")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Annotations, "annotations", "", "Set Node annotations prefix for parsing")
 
-	// Control plane specific flags
-	kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The configuration map defined within the cluster")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPConfig.Zebra.Enabled, "zebra", false, "This will enable Zebra support within kube-vip")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.URL, "zebraUrl", "unix:/var/run/frr/zserv.api", "Path to the unix domain socket for connecting to Zebra daemon")
+	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.Zebra.Version, "zebraVersion", 6, "Zebra API Version")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.SoftwareName, "zebraSoftwareName", "frr8.3", "Software Name for Zebra")
+
+	// Namespace for kube-vip
+	kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The namespace for the configmap defined within the cluster")
 
 	// Manage logging
-	kubeVipCmd.PersistentFlags().Uint32Var(&logLevel, "log", 4, "Set the level of logging")
+	kubeVipCmd.PersistentFlags().Int32Var(&initConfig.Logging, "log", 0, "Set the level of logging")
 
 	// Service flags
 	kubeVipService.Flags().StringVarP(&configMap, "configMap", "c", "plndr", "The configuration map defined within the cluster")
 
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableControlPane, "controlplane", false, "Enable HA for control plane, hybrid mode")
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServices, "services", false, "Enable Kubernetes services, hybrid mode")
+	// Routing Table flags
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableID, "tableID", 198, "The routing table used for all table entries")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableType, "tableType", unix.RTN_UNICAST, "The type of route that will be added to the routing table")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingProtocol, "routingProtocol", 248, "The routing protocol value used to create routes")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.CleanRoutingTable, "cleanRoutingTable", false, "Clean routing table of redundant routes on start")
+
+	// Behaviour flags
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableControlPlane, "controlplane", false, "Enable HA for control plane")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DetectControlPlane, "autodetectcp", false, "Determine working address for control plane (from loopback)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServices, "services", false, "Enable Kubernetes services")
+
+	// Extended behaviour flags
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServicesElection, "servicesElection", false, "Enable leader election per kubernetes service")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassOnly, "lbClassOnly", false, "Enable load balancing only for services with LoadBalancerClass \"kube-vip.io/kube-vip-class\"")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerClassName, "lbClassName", "kube-vip.io/kube-vip-class", "Name of load balancer class for kube-VIP, defaults to \"kube-vip.io/kube-vip-class\"")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassLegacyHandling, "lbClassNameLegacyHandling", true, "Use legacy LoadBalancer class name handling (e.g. accepting services both with empty and non-empty class)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServiceSecurity, "onlyAllowTrafficServicePorts", false, "Only allow traffic to service ports, others will be dropped, defaults to false")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableNodeLabeling, "enableNodeLabeling", false, "Enable leader node labeling with \"kube-vip.io/has-ip=<VIP address>\", defaults to false")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesLeaseName, "servicesLeaseName", "plndr-svcs-lock", "Name of the lease that is used for leader election for services (in arp mode)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.DNSMode, "dnsMode", "first", "Name of the mode that DNS lookup will be performed (first, ipv4, ipv6, dual)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DisableServiceUpdates, "disableServiceUpdates", false, "If true, kube-vip will process services as usual, but will not update service's Status.LoadBalancer.Ingress slice")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpoints, "enableEndpoints", false, "If enabled, kube-vip will only advertise services, but will use the (deprecated since v1.33) endpoints for IP addresses")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoInterfaceGlobalScope, "loInterfaceGlobalScope", false, "If true, kube-vip will set global scope when using the lo interface, otherwise a host scope will be used by default")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.HealthCheckPort, "healthCheckPort", 0, "If set to non-zero (> 1024), then this is the port that the healthcheck will listen on")
 
 	// Prometheus HTTP Server
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.PrometheusHTTPServer, "promethuesHTTPServer", ":2112", "Host and port used to expose Promethues metrics via an HTTP server")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.PrometheusHTTPServer, "prometheusHTTPServer", ":2112", "Host and port used to expose Prometheus metrics via an HTTP server")
+
+	// Etcd
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.CAFile, "etcdCACert", "", "Verify certificates of TLS-enabled secure servers using this CA bundle file")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.ClientCertFile, "etcdCert", "", "Identify secure client using this TLS certificate file")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.ClientKeyFile, "etcdKey", "", "Identify secure client using this TLS key file")
+	kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.Etcd.Endpoints, "etcdEndpoints", nil, "Etcd member endpoints")
+
+	// Kubernetes client specific flags
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.K8sConfigFile, "k8sConfigPath", "/etc/kubernetes/admin.conf", "Path to the configuration file used with the Kubernetes client")
 
 	kubeVipCmd.AddCommand(kubeKubeadm)
 	kubeVipCmd.AddCommand(kubeManifest)
 	kubeVipCmd.AddCommand(kubeVipManager)
 	kubeVipCmd.AddCommand(kubeVipSample)
 	kubeVipCmd.AddCommand(kubeVipService)
-	kubeVipCmd.AddCommand(kubeVipStart)
 	kubeVipCmd.AddCommand(kubeVipVersion)
 
-	// Sample commands
-	kubeVipSample.AddCommand(kubeVipSampleConfig)
-	kubeVipSample.AddCommand(kubeVipSampleManifest)
-
+	// Set the logging level for all subsequent functions
+	log.SetLogLoggerLevel(log.Level(initConfig.Logging))
 }
 
 // Execute - starts the command parsing process
@@ -150,7 +166,7 @@ func Execute() {
 var kubeVipVersion = &cobra.Command{
 	Use:   "version",
 	Short: "Version and Release information about the Kubernetes Virtual IP Server",
-	Run: func(cmd *cobra.Command, args []string) {
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 		fmt.Printf("Kube-VIP Release Information\n")
 		fmt.Printf("Version:  %s\n", Release.Version)
 		fmt.Printf("Build:    %s\n", Release.Build)
@@ -160,22 +176,34 @@ var kubeVipVersion = &cobra.Command{
 var kubeVipSample = &cobra.Command{
 	Use:   "sample",
 	Short: "Generate a Sample configuration",
-	Run: func(cmd *cobra.Command, args []string) {
-		cmd.Help()
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		_ = cmd.Help()
 	},
 }
 
 var kubeVipService = &cobra.Command{
 	Use:   "service",
 	Short: "Start the Virtual IP / Load balancer as a service within a Kubernetes cluster",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 
 		// parse environment variables, these will overwrite anything loaded or flags
 		err := kubevip.ParseEnvironment(&initConfig)
 		if err != nil {
-			log.Fatalln(err)
+			log.Error("parsing env", "err", err)
+			return
+		}
+
+		// Change RTN_UNSPEC to default type
+		if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
+			initConfig.RoutingProtocol = unix.RTN_UNICAST
+		}
+
+		// Set the logging level for all subsequent functions
+		log.SetLogLoggerLevel(log.Level(initConfig.Logging))
+
+		if err := initConfig.CheckInterface(); err != nil {
+			log.Error("checking interface", "err", err)
+			return
 		}
 
 		// User Environment variables as an option to make manifest clearer
@@ -184,16 +212,27 @@ var kubeVipService = &cobra.Command{
 			configMap = envConfigMap
 		}
 
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
+			if err != nil {
+				log.Error("generating CIDR", "err", err)
+				return
+			}
+		}
+
 		// Define the new service manager
 		mgr, err := manager.New(configMap, &initConfig)
 		if err != nil {
-			log.Fatalf("%v", err)
+			log.Error("new manager", "err", err)
+			return
 		}
 
 		// Start the service manager, this will watch the config Map and construct kube-vip services for it
 		err = mgr.Start()
 		if err != nil {
-			log.Fatalf("%v", err)
+			log.Error("manager start", "err", err)
+			return
 		}
 	},
 }
@@ -201,18 +240,134 @@ var kubeVipService = &cobra.Command{
 var kubeVipManager = &cobra.Command{
 	Use:   "manager",
 	Short: "Start the kube-vip manager",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
-
-		go servePrometheusHTTPServer(cmd.Context(), PrometheusHTTPServerConfig{
-			Addr: initConfig.PrometheusHTTPServer,
-		})
-
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 		// parse environment variables, these will overwrite anything loaded or flags
 		err := kubevip.ParseEnvironment(&initConfig)
 		if err != nil {
-			log.Fatalln(err)
+			log.Error("parsing environment", "err", err)
+			return
+		}
+
+		// Change RTN_UNSPEC to default type
+		if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
+			initConfig.RoutingProtocol = unix.RTN_UNICAST
+		}
+
+		// Set the logging level for all subsequent functions
+		log.SetLogLoggerLevel(log.Level(initConfig.Logging))
+
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
+			if err != nil {
+				log.Error("No interface is specified for kube-vip to bind to")
+				return
+			}
+		}
+
+		// Welome messages
+		log.Info("kube-vip.io", "version", Release.Version, "build", Release.Build)
+
+		// start prometheus server
+		if initConfig.PrometheusHTTPServer != "" {
+			go servePrometheusHTTPServer(cmd.Context(), PrometheusHTTPServerConfig{
+				Addr: initConfig.PrometheusHTTPServer,
+			})
+		}
+
+		// Determine the kube-vip mode
+		var mode string
+		if initConfig.EnableARP {
+			mode = "ARP"
+		}
+
+		if initConfig.EnableBGP {
+			mode = "BGP"
+		}
+
+		if initConfig.EnableWireguard {
+			mode = "Wireguard"
+		}
+
+		if initConfig.EnableRoutingTable {
+			mode = "Routing Table"
+		}
+
+		// Provide configuration to output/logging
+		log.Info("starting", "namespace", initConfig.Namespace, "Mode", mode, "Control Plane", initConfig.EnableControlPlane, "Services", initConfig.EnableServices)
+
+		// End if nothing is enabled
+		if !initConfig.EnableServices && !initConfig.EnableControlPlane {
+			log.Error("no features are enabled")
+			return
+		}
+
+		if !initConfig.EnableARP && strings.Contains(initConfig.VIPSubnet, kubevip.Auto) {
+			log.Error("auto subnet discovery cannot be used outside ARP mode")
+			return
+		}
+
+		if strings.Contains(initConfig.VIPSubnet, kubevip.Auto) && initConfig.Address != "" {
+			log.Error("auto subnet discovery cannot be used if VIP address was provided")
+			return
+		}
+
+		// If we're using wireguard then all traffic goes through the wg0 interface
+		if initConfig.EnableWireguard {
+			if initConfig.Interface == "" {
+				// Set the vip interface to the wireguard interface
+				initConfig.Interface = "wg0"
+			}
+
+			log.Info("configuring Wireguard networking")
+			l, err := netlink.LinkByName(initConfig.Interface)
+			if err != nil {
+				if strings.Contains(err.Error(), "Link not found") {
+					log.Warn("attempting to create wireguard interface", "interface not found", initConfig.Interface)
+					err = netlink.LinkAdd(&netlink.Wireguard{LinkAttrs: netlink.LinkAttrs{Name: initConfig.Interface}})
+					if err != nil {
+						log.Error("adding link", "err", err)
+						return
+					}
+					l, err = netlink.LinkByName(initConfig.Interface)
+					if err != nil {
+						log.Error("finding link", "err", err)
+						return
+					}
+				}
+			}
+			err = netlink.LinkSetUp(l)
+			if err != nil {
+				log.Error("setting link UP", "err", err)
+				return
+			}
+
+		} else { // if we're not using Wireguard then we'll need to use an actual interface
+			// Check if the interface needs auto-detecting
+			if initConfig.Interface == "" {
+				log.Info("No interface is specified for VIP in config, auto-detecting default Interface")
+				defaultIF, err := vip.GetDefaultGatewayInterface()
+				if err != nil {
+					_ = cmd.Help()
+					log.Error("detecting interface", "err", err)
+					return
+				}
+				initConfig.Interface = defaultIF.Name
+				log.Info("kube-vip bind", "interface", initConfig.Interface)
+
+				go func() {
+					if err := vip.MonitorDefaultInterface(context.TODO(), defaultIF); err != nil {
+
+						log.Error("interface monitor", "err", err)
+						return
+					}
+				}()
+			}
+		}
+		// Perform a check on the state of the interface
+		if err := initConfig.CheckInterface(); err != nil {
+			log.Error("checking interface", "err", err)
+			return
 		}
 
 		// User Environment variables as an option to make manifest clearer
@@ -221,22 +376,11 @@ var kubeVipManager = &cobra.Command{
 			configMap = envConfigMap
 		}
 
-		// If Packet is enabled and there is a provider configuration passed
-		if initConfig.EnableMetal {
-			if providerConfig != "" {
-				providerAPI, providerProject, err := packet.GetPacketConfig(providerConfig)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-				initConfig.MetalAPIKey = providerAPI
-				initConfig.MetalProject = providerProject
-			}
-		}
-
 		// Define the new service manager
 		mgr, err := manager.New(configMap, &initConfig)
 		if err != nil {
-			log.Fatalf("%v", err)
+			log.Error("new manager", "err", err)
+			return
 		}
 
 		prometheus.MustRegister(mgr.PrometheusCollector()...)
@@ -244,11 +388,13 @@ var kubeVipManager = &cobra.Command{
 		// Start the service manager, this will watch the config Map and construct kube-vip services for it
 		err = mgr.Start()
 		if err != nil {
-			log.Fatalf("%v", err)
+			log.Error("start manager", "err", err)
+			return
 		}
 	},
 }
 
+// PrometheusHTTPServerConfig defines the Prometheus server configuration.
 type PrometheusHTTPServerConfig struct {
 	// Addr sets the http server address used to expose the metric endpoint
 	Addr string
@@ -258,23 +404,34 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
 	var err error
 	mux := http.NewServeMux()
 	mux.Handle("/metrics", promhttp.Handler())
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { //nolint TODO
+		_, _ = w.Write([]byte(`<html>
+			<head><title>kube-vip</title></head>
+			<body>
+			<h1>kube-vip Metrics</h1>
+			<p><a href="` + "/metrics" + `">Metrics</a></p>
+			</body>
+			</html>`))
+	})
 
 	srv := &http.Server{
-		Addr:    config.Addr,
-		Handler: mux,
+		Addr:              config.Addr,
+		Handler:           mux,
+		ReadHeaderTimeout: 2 * time.Second,
 	}
 
 	go func() {
 		if err = srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
-			log.Fatalf("listen:%+s\n", err)
+			log.Error("prometheus HTTP server", "err", err)
+			return
 		}
 	}()
 
-	log.Printf("server started")
+	log.Info("prometheus HTTP server started")
 
 	<-ctx.Done()
 
-	log.Printf("server stopped")
+	log.Info("prometheus HTTP server stopped")
 
 	ctxShutDown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer func() {
@@ -282,11 +439,35 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
 	}()
 
 	if err = srv.Shutdown(ctxShutDown); err != nil {
-		log.Fatalf("server Shutdown Failed:%+s", err)
+		log.Error("shutting down prometheus HTTP server", "err", err)
+		return
 	}
 
 	if err == http.ErrServerClosed {
 		err = nil
 	}
-
+}
+
+func GenerateCidrRange(address string) (string, error) {
+	var cidrs []string
+
+	addresses := strings.Split(address, ",")
+	for _, a := range addresses {
+		ip := net.ParseIP(a)
+		if ip == nil {
+			ips, err := net.LookupIP(a)
+			if len(ips) == 0 || err != nil {
+				return "", fmt.Errorf("invalid IP address: %s from [%s], %v", a, address, err)
+			}
+			ip = ips[0]
+		}
+
+		if ip.To4() != nil {
+			cidrs = append(cidrs, "32")
+		} else {
+			cidrs = append(cidrs, "128")
+		}
+	}
+
+	return strings.Join(cidrs, ","), nil
 }

demo/README.md

@@ -0,0 +1,46 @@
+# Demo client-server
+
+This contains some example code to determine how long "failovers" are taking within kube-vip, the server component should live within the cluster and the client should be externally.
+
+## Deploy the server
+
+Simply apply the manifest to a working cluster that has kube-vip deployed:
+
+```
+kubectl apply -f ./demo/server/deploy.yaml
+```
+
+Retrieve the loadBalancer IP that is fronting the service:
+
+```
+kubectl get svc demo-service
+NAME           TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)           AGE
+demo-service   LoadBalancer   10.104.18.147   192.168.0.217   10002:32529/UDP   117m
+```
+
+## Connect the client
+
+From elsewhere, clone the kube-vip repository and connect the client to the server endpoint (loadBalancer IP) with the following command:
+
+```
+go run ./demo/client/main.go -address=<vip>
+```
+
+You will only see output when the client has reconcilled the connection to a pod beneath the service, where it will print the timestamp to reconnection along with the time in milliseconds it took:
+
+```
+15:58:35.916952 3008
+15:58:45.947506 2005
+15:58:57.983151 3007
+15:59:08.013450 2005
+15:59:20.046491 3008
+15:59:30.076341 2507
+15:59:42.110747 3008
+```
+
+## Kill some pods to test
+
+On a machine or control plane that has `kubectl` and has the credentials to speak to the cluster we will run a command to find the demo pod and kill it every 10 seconds:
+
+`while true ; do kubectl delete pod $(kubectl get pods | grep -v NAME | grep vip| awk '{ print $1 }'); sleep 10; done`
+

demo/client/main.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"net"
+	"time"
+)
+
+const udpdata = "a3ViZS12aXAK=kube-vip"
+
+func main() {
+	address := flag.String("address", "127.0.0.1", "The address of the server")
+	port := flag.Int("port", 10002, "the port of the server")
+	interval := flag.Float64("interval", 1000, "Interval in milliseconds")
+	flag.Parse()
+	var errorTime time.Time
+	var errorOccurred bool
+	for {
+		p := make([]byte, 2048)
+		conn, err := net.Dial("udp", fmt.Sprintf("%s:%d", *address, *port))
+		if err != nil {
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			continue
+		}
+
+		err = conn.SetDeadline(time.Now().Add(time.Duration(*interval) * time.Millisecond))
+		if err != nil {
+			//fmt.Printf("Connectivity error [%v]", err)
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			if err = conn.Close(); err != nil {
+				fmt.Printf("Error closing connection [%v]", err)
+			}
+			continue
+		}
+
+		_, err = fmt.Fprint(conn, udpdata)
+		if err != nil {
+			fmt.Printf("Error writing data [%v]", err)
+		}
+
+		_, err = bufio.NewReader(conn).Read(p)
+		if err != nil {
+			//fmt.Printf("read error %v\n", err)
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			if err = conn.Close(); err != nil {
+				fmt.Printf("Error closing connection [%v]", err)
+			}
+			continue
+		}
+		time.Sleep(time.Duration(*interval) * time.Millisecond)
+		if errorOccurred {
+			finishTime := time.Since(errorTime)
+			//fmt.Printf("connectivity reconciled in %dms\n", finishTime.Milliseconds())
+			//t :=time.Now().Format("15:04:05.000000")
+			fmt.Printf("%s %d\n", time.Now().Format("15:04:05.000000"), finishTime.Milliseconds())
+
+			errorOccurred = false
+		}
+	}
+}

demo/server/Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:experimental
 
-FROM golang:1.13-alpine as dev
+FROM golang:1.19-alpine as dev
 RUN apk add --no-cache git ca-certificates
 RUN adduser -D appuser
 COPY main.go /src/
@@ -13,4 +13,4 @@ RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
 
 FROM scratch
 COPY --from=dev /src/demo /
-CMD ["/demo"]
+CMD ["/demo"]

demo/server/Makefile

@@ -45,7 +45,7 @@ fmt:
 	@gofmt -l -w $(SRC)
 
 docker:
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 simplify:

demo/server/deploy.yaml

@@ -6,7 +6,7 @@ metadata:
     app: kube-vip-demo
   name: kube-vip-demo
 spec:
-  replicas: 3
+  replicas: 1
   selector:
     matchLabels:
       app: kube-vip-demo
@@ -30,4 +30,25 @@ spec:
         ports:
             - containerPort: 10001
             - containerPort: 10002
-status: {}
+status: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: demo-service
+  namespace: default
+  labels:
+    app: demo-service
+  annotations:
+    kube-vip.io/egress: "true"
+spec:
+  type: LoadBalancer
+  # "Local" preserves the client source IP and avoids a second hop for
+  # LoadBalancer and NodePort
+  externalTrafficPolicy: Local
+  ports:
+  - name: demo-udp
+    port: 10002
+    protocol: UDP
+  selector:
+    app: kube-vip-demo

demo/server/main.go

@@ -32,7 +32,7 @@ func main() {
 		}
 	}
 
-	if serverType == strings.ToLower("udp") {
+	if serverType == "udp" {
 		// Start the UDP echo server
 
 		ServerAddr, err := net.ResolveUDPAddr("udp", ":10002")

docs/Dockerfile

@@ -1,5 +0,0 @@
-FROM nginx:latest
-RUN apt-get update; apt-get install -y nodejs npm; npm install -g markdown-styles;
-COPY . /docs
-WORKDIR /docs
-RUN generate-md --layout github --input ./ --output /usr/share/nginx/html/

docs/architecture/index.md

@@ -1,135 +0,0 @@
-# **kube-vip** architecture
-
-This section covers two parts of the architecture: 
-
-1. The technical capabilities of `kube-vip`
-2. The components to build a load-balancing service within [Kubernetes](https://kubernetes.io)
-
-The `kube-vip` project is designed to provide both a highly available networking endpoint and load-balancing functionality for underlying networking services. The project was originally designed for the purpose of providing a resilient control-plane for Kubernetes, it has since expanded to provide the same functionality for applications within a Kubernetes cluster.
-
-Additionally `kube-vip` is designed to be lightweight and **multi-architecture**, all of the components are built for Linux but are also built for both `x86` and `armv7`,`armhvf`. This means that `kube-vip` will run fine in **bare-metal**, **virtual** and **edge** (raspberry pi or small arm SoC devices). 
-
-## Technologies
-
-There are a number of technologies or functional design choices that provide high-availability or networking functions as part of a VIP/Load-balancing solution.
-
-### Cluster
-
-The `kube-vip` service builds a multi-node or multi-pod cluster to provide High-Availability. In ARP mode a leader is elected, this node will inherit the Virtual IP and become the leader of the load-balancing within the cluster, whereas with BGP all nodes will advertise the VIP address.
-
-When using ARP or layer2 it will use [leader election](https://godoc.org/k8s.io/client-go/tools/leaderelection)
-
-It is also possible to use [raft](https://en.wikipedia.org/wiki/Raft_(computer_science) clustering technology, but this approach has largely been superseded by leader election especially when running in cluster.
-
-### Virtual IP
-
-The leader within the cluster will assume the **vip** and will have it bound to the selected interface that is declared within the configuration. When the leader changes it will evacuate the **vip** first or in failure scenarios the **vip** will be directly assumed by the next elected leader.
-
-When the **vip** moves from one host to another any host that has been using the **vip** will retain the previous `vip <-> MAC address` mapping until the ARP (Address resolution protocol) expires the old entry (typically 30 seconds) and retrieves a new `vip <-> MAC` mapping. This can be improved using Gratuitous ARP broadcasts (when enabled), this is detailed below.
-
-### ARP
-
-(Optional) The `kube-vip` can be configured to broadcast a [gratuitous arp](https://wiki.wireshark.org/Gratuitous_ARP) that will typically immediately notify all local hosts that the `vip <-> MAC` has changed.
-
-**Below** we can see that the failover is typically done within a few seconds as the ARP broadcast is recieved.
-
-```
-64 bytes from 192.168.0.75: icmp_seq=146 ttl=64 time=0.258 ms
-64 bytes from 192.168.0.75: icmp_seq=147 ttl=64 time=0.240 ms
-92 bytes from 192.168.0.70: Redirect Host(New addr: 192.168.0.75)
-Vr HL TOS  Len   ID Flg  off TTL Pro  cks      Src      Dst
- 4  5  00 0054 bc98   0 0000  3f  01 3d16 192.168.0.95  192.168.0.75 
-
-Request timeout for icmp_seq 148
-92 bytes from 192.168.0.70: Redirect Host(New addr: 192.168.0.75)
-Vr HL TOS  Len   ID Flg  off TTL Pro  cks      Src      Dst
- 4  5  00 0054 75ff   0 0000  3f  01 83af 192.168.0.95  192.168.0.75 
-
-Request timeout for icmp_seq 149
-92 bytes from 192.168.0.70: Redirect Host(New addr: 192.168.0.75)
-Vr HL TOS  Len   ID Flg  off TTL Pro  cks      Src      Dst
- 4  5  00 0054 2890   0 0000  3f  01 d11e 192.168.0.95  192.168.0.75 
-
-Request timeout for icmp_seq 150
-64 bytes from 192.168.0.75: icmp_seq=151 ttl=64 time=0.245 ms
-```
-
-### Load Balancing
-
-Within a Kubernetes cluster, the load-balancing is managed by the `plndr-cloud-provider` which watches all service that are created, and for those of `type=LoadBalancer` will create the configuration for `kube-vip` to consume.
-
-#### Load Balancing (Inside a cluster)
-
-When using `type=LoadBalancer` within a Kubernetes cluster `kube-vip` will assign the VIP to the leader (when using ARP) or to all running Pods (when using BGP). When traffic is directed to a node with the VIP then the rules configured by `kube-proxy` will redirect the traffic to one of the pods running in the service.
-
-#### Load Balancing (Outside a cluster)
-
-Within the configuration of `kube-vip` multiple load-balancers can be created, below is the example load-balancer for a Kubernetes Control-plane:
-
-```
-loadBalancers:
-- name: Kubernetes Control Plane
-  type: tcp
-  port: 6443
-  bindToVip: true
-  backends:
-  - port: 6444
-    address: 192.168.0.70
-  - port: 6444
-    address: 192.168.0.71
-  - port: 6444
-    address: 192.168.0.72
-```
-
-The above load balancer will create an instance that listens on port `6443` and will forward traffic to the array of backend addresses. If the load-balancer type is `tcp` then the backends will be IP addresses, however if the backend is set to `http` then the backends should be URLs:
-
-```
-  type: http
-  port: 6443
-  bindToVip: true
-  backends:
-  - port: 6444
-    address: https://192.168.0.70
-```
-
-Additionally the load-balancing within `kibe-vip` has two modes of operation:
-
-`bindToVip: false` - will result in every node in the cluster binding all load-balancer port(s) to all interfaces on the host itself
-
-`bindToVip: true` - The load-balancer will only **bind** to the VIP address.
-
-
-## Components within a Kubernetes Cluster
-
-The `kube-vip` kubernetes load-balancer requires a number of components in order to function:
-
-- The Plunder Cloud Provider -> [https://github.com/kube-vip/plndr-cloud-provider](https://github.com/kube-vip/plndr-cloud-provider)
-- The Kube-Vip Deployment -> [https://github.com/kube-vip/kube-vip](https://github.com/kube-vip/kube-vip)
-
-### Architecture overview
-
-![kubernetes-vip-architecture.png](kubernetes-vip-architecture.png)
-
-### Plunder Cloud Provider
-
-The cloud provider works like all Kubernetes cloud providers and is built using the Kubernetes cloud-provider SDK. It's role is to provide the same cloud "like" services one would expect from services such as AWS / Azure / GCP etc.. in that when a user requests functionality then the cloud provider can speak natively to the underlying vendor and provision the required service 
-
-e.g. _In AWS when requesting a Kubernetes LoadBalancer, the cloud provider will provision an **ELB**_ 
-
-The `Plunder cloud Provider` is *currently* only designed to intercept the creation of LoadBalancers and translate that into a `kube-vip` load balancer. 
-
-It is configured by a `configMap` within the `kube-system` namespace that contains the ranges of addresses that the other `kube-vip` load-balancers can use, it will also manage the allocation of addresses and then build the configMap configurations in these namespaces for consumption by `kube-vip`. The IP addresses for each namespace should be in the structure `cidr-<namespace>` followed by the cidr range for the address pool.
-
-**Example `ConfigMap`**
-
-```
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: plndr
-  namespace: kube-system
-data:
-  cidr-default: 192.168.0.200/29
-  cidr-plunder: 192.168.0.210/29
-  cidr-testing: 192.168.0.220/29
-``` 

docs/control-plane/0.1.4/index.md

@@ -1,147 +0,0 @@
-# Load Balancing a Kubernetes Cluster (Control-Plane) (pre 0.1.5)
-
-This document covers all of the details for using `kube-vip` to build a HA Kubernetes cluster
-
-`tl;dr version`
-- Generate/modify first node `kube-vip` config/manifest
-- `init` first node
-- `join` remaining nodes
-- Add remaining config/manifests
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.75 |
-| controlPlane01 | 10.0.0.70 |
-| controlPlane02 | 10.0.0.71 |
-| controlPlane03 | 10.0.0.72 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.17.0.
-
-### Generate the `kube-vip` configuration
-
-Make sure that the config directory exists: `sudo mkdir -p /etc/kube-vip/`, this directory can be any directory however the `hostPath` in the manifest will need modifying to point to the correct path.
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1 /kube-vip sample config | sudo tee /etc/kube-vip/config.yaml
-```
-
-### Modify the configuration
-
-**Cluster Configuration**
-Modify the `remotePeers` to point to the correct addresses of the other two nodes, ensure that their `id` is unique otherwise this will confuse the raft algorithm. The `localPeer` should be the configuration of the current node (`controlPlane01`), which is where this instance of the cluster will run. 
-
-As this node will be the first node, it will need to elect itself leader as until this occurs the VIP won’t be activated!
-
-`startAsLeader: true`
-
-**VIP Config**
-We will need to set our VIP address to `192.168.0.75` and to ensure all hosts are updated when the VIP moves we will enable ARP broadcasts `gratuitousARP: true`
-
-**Load Balancer**
-We will configure the load balancer to sit on the standard API-Server port `6443` and we will configure the backends to point to the API-servers that will be configured to run on port `6444`. Also for the Kubernetes Control Plane we will configure the load balancer to be of `type: tcp`.
-
-We can also use `6443` for both the VIP and the API-Servers, in order to do this we need to specify that the api-server is bound to it's local IP. To do this we use the `--apiserver-advertise-address` flag as part of the `init`, this means that we can then bind the same port to the VIP and we wont have a port conflict.
-
-**config.yaml**
-
-`user@controlPlane01:/etc/kube-vip$ cat config.yaml`
-
-...
-
-``` 
-remotePeers:
-- id: server2
-  address: 192.168.0.71
-  port: 10000
-- id: server3
-  address: 192.168.0.72
-  port: 10000
-localPeer:
-  id: server1
-  address: 192.168.0.70
-  port: 10000
-vip: 192.168.0.75
-gratuitousARP: true
-singleNode: false
-startAsLeader: true
-interface: ens192
-loadBalancers:
-- name: Kubernetes Control Plane
-  type: tcp
-  port: 6443
-  bindToVip: true
-  backends:
-  - port: 6444
-    address: 192.168.0.70
-  - port: 6444
-    address: 192.168.0.71
-  - port: 6444
-    address: 192.168.0.72
-```
-
-### First Node
-
-To generate the basic Kubernetes static pod `yaml` configuration:
-
-Make sure that the manifest directory exists: `sudo mkdir -p /etc/kubernetes/manifests/`
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1 /kube-vip sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Ensure that `image: plndr/kube-vip:<x>` is modified to point to a specific version (`0.1` at the time of writing), refer to [docker hub](https://hub.docker.com/r/plndr/kube-vip/tags) for details. Also ensure that the `hostPath` points to the correct `kube-vip` configuration, if it isn’t the above path. 
-
-The **vip** is set to `192.168.0.75` and this first node will elect itself as leader, and as part of the `kubeadm init` it will use the VIP in order to speak back to the initialising api-server.
-
-`sudo kubeadm init --control-plane-endpoint “192.168.0.75:6443” --apiserver-bind-port 6444 --upload-certs --kubernetes-version “v1.17.0”`
-
-Once this node is up and running we will be able to see the control-plane pods, including the `kube-vip` pod:
-
-```
-$ kubectl get pods -A
-NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
-<...>
-kube-system   kube-vip-controlplane01                  1/1     Running   0          10m
-```
-
-### Remaining Nodes
-
-We first will need to create the `kube-vip` configuration that resides in `/etc/kube-vip/config.yaml` or we can regenerate it from scratch using the above example. Ensure that the configuration is almost identical with the `localPeer` and `remotePeers` sections are updated for each node. Finally, ensure that the remaining nodes will behave as standard cluster nodes by setting `startAsLeader: false`.
-
-At this point **DON’T** generate the manifests, this is due to some bizarre `kubeadm/kubelet` behaviour.
-
-```
-  kubeadm join 192.168.0.75:6443 --token <tkn> \
-    --discovery-token-ca-cert-hash sha256:<hash> \
-    --control-plane --certificate-key <key> 
-
-```
-
-**After** this node has been added to the cluster, we can add the manifest to also add this node as a `kube-vip` member. (Adding the manifest afterwards doesn’t interfere with `kubeadm`). 
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1 /kube-vip sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Once this node is added we will be able to see that the `kube-vip` pod is up and running as expected:
-
-```
-user@controlPlane01:~$ kubectl get pods -A | grep vip
-kube-system   kube-vip-controlplane01                  1/1     Running             1          16m
-kube-system   kube-vip-controlplane02                  1/1     Running             0          18m
-kube-system   kube-vip-controlplane03                  1/1     Running             0          20m
-
-```
-
-If we look at the logs, we can see that the VIP is running on the second node and we’re waiting for our third node to join the cluster:
-
-```
-$ kubectl logs kube-vip-controlplane02  -n kube-system
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-
-```

docs/control-plane/0.1.5/index.md

@@ -1,195 +0,0 @@
-# Load Balancing a Kubernetes Cluster (Control-Plane)
-
-This document covers the newer (post `0.1.5`) method for using `kube-vip` to provide HA for a Kubernetes Cluster. The documentation for older releases can be found [here](./0.1.4/)
-
-This document covers all of the details for using `kube-vip` to build a HA Kubernetes cluster
-
-`tl;dr version`
-- Generate/modify first node `kube-vip` config/manifest
-- `init` first node
-- `join` remaining nodes
-- Add remaining config/manifests
-
-Below are examples of the steps required:
-
-```
-# First Node
-sudo docker run --network host --rm plndr/kube-vip:0.1.5 kubeadm init --interface ens192 --vip 192.168.0.81 --startAsLeader=true | sudo tee /etc/kubernetes/manifests/vip.yaml
-
-sudo kubeadm init --kubernetes-version 1.17.0 --control-plane-endpoint 192.168.0.81 --upload-certs
-
-# Additional Node(s)
-
-sudo kubeadm join 192.168.0.81:6443 --token w5atsr.blahblahblah --control-plane --certificate-key abc123
-
-sudo docker run -v /etc/kubernetes/admin.conf:/etc/kubernetes/admin.conf --network host --rm plndr/kube-vip:0.1.5 kubeadm join --interface ens192 --vip 192.168.0.81 --startAsLeader=false | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.75 |
-| controlPlane01 | 10.0.0.70 |
-| controlPlane02 | 10.0.0.71 |
-| controlPlane03 | 10.0.0.72 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.17.0.
-
-### Generate the `kube-vip` configuration
-
-Kube-Vip no longer requires storing it's configuration in a seperate directory and will now store its configuration in the actual manifest that defines the static pods. 
-
-```
-sudo docker run --network host \
-	--rm plndr/kube-vip:0.1.5 \
-	kubeadm init \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--startAsLeader=true | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-The above command will "initialise" the manifest within the `/etc/kubernetes/manifests` directory, that will be started when we actually initialise our Kubernetes cluster with `kubeadm init`
-
-### Modify the configuration
-
-**Cluster Configuration**
-As this node will be the first node, it will need to elect itself leader as until this occurs the VIP won’t be activated!
-
-`--startAsLeader=true`
-
-**VIP Config**
-We will need to set our VIP address to `192.168.0.75` with `--vip 192.168.0.75` and to ensure all hosts are updated when the VIP moves we will enable ARP broadcasts `--arp` (defaults to `true`)
-
-**Load Balancer**
-We will configure the load balancer to sit on the standard API-Server port `6443` and we will configure the backends to point to the API-servers that will be configured to run on port `6444`. Also for the Kubernetes Control Plane we will configure the load balancer to be of `type: tcp`.
-
-We can also use `6443` for both the VIP and the API-Servers, in order to do this we need to specify that the api-server is bound to it's local IP. To do this we use the `--apiserver-advertise-address` flag as part of the `init`, this means that we can then bind the same port to the VIP and we wont have a port conflict.
-
-**vip.yaml** Static-pod Manifest
-
-`$ sudo cat /etc/kubernetes/manifests/vip.yaml`
-
-
-``` 
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - start
-    env:
-    - name: vip_arp
-      value: "true"
-    - name: vip_interface
-      value: ens192
-    - name: vip_address
-      value: 192.168.0.81
-    - name: vip_startleader
-      value: "true"
-    - name: vip_addpeerstolb
-      value: "true"
-    - name: vip_localpeer
-      value: controlPlane01:192.168.0.70:10000
-    - name: lb_backendport
-      value: "6443"
-    - name: lb_name
-      value: Kubeadm Load Balancer
-    - name: lb_type
-      value: tcp
-    - name: lb_bindtovip
-      value: "true"
-    image: plndr/kube-vip:0.1.5
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - SYS_TIME
-  hostNetwork: true
-status: {}
-```
-
-### First Node
-
-To generate the basic Kubernetes static pod `yaml` configuration:
-
-Make sure that the manifest directory exists: `sudo mkdir -p /etc/kubernetes/manifests/`
-
-```
-sudo docker run --network host \
-	--rm plndr/kube-vip:0.1.5 \
-	kubeadm init \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--startAsLeader=true | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-Ensure that `image: plndr/kube-vip:<x>` is modified to point to a specific version (`0.1.5` at the time of writing), refer to [docker hub](https://hub.docker.com/r/plndr/kube-vip/tags) for details. 
-
-The **vip** is set to `192.168.0.75` and this first node will elect itself as leader, and as part of the `kubeadm init` it will use the VIP in order to speak back to the initialising api-server.
-
-`sudo kubeadm init --control-plane-endpoint “192.168.0.75:6443” --apiserver-bind-port 6444 --upload-certs --kubernetes-version “v1.17.0”`
-
-Once this node is up and running we will be able to see the control-plane pods, including the `kube-vip` pod:
-
-```
-$ kubectl get pods -A
-NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
-<...>
-kube-system   kube-vip-controlplane01                  1/1     Running   0          10m
-```
-
-### Remaining Nodes
-
-
-At this point **DON’T** generate the manifests, this is due to some bizarre `kubeadm/kubelet` behaviour.
-
-```
-  kubeadm join 192.168.0.75:6443 --token <tkn> \
-    --discovery-token-ca-cert-hash sha256:<hash> \
-    --control-plane --certificate-key <key> 
-
-```
-
-**After** this node has been added to the cluster, we can add the manifest to also add this node as a `kube-vip` member. (Adding the manifest afterwards doesn’t interfere with `kubeadm`). 
-
-```
-sudo docker run \
-	-v /etc/kubernetes/admin.conf:/etc/kubernetes/admin.conf \
-	--network host \
-	--rm plndr/kube-vip:0.1.5 \
-	kubeadm join \
-	--interface ens192 \
-	--vip 192.168.0.81 \
-	--startAsLeader=false | sudo tee /etc/kubernetes/manifests/vip.yaml
-
-```
-
-Once this node is added we will be able to see that the `kube-vip` pod is up and running as expected:
-
-```
-user@controlPlane01:~$ kubectl get pods -A | grep vip
-kube-system   kube-vip-controlplane01                  1/1     Running             1          16m
-kube-system   kube-vip-controlplane02                  1/1     Running             0          18m
-kube-system   kube-vip-controlplane03                  1/1     Running             0          20m
-
-```
-
-If we look at the logs, we can see that the VIP is running on the second node and we’re waiting for our third node to join the cluster:
-
-```
-$ kubectl logs kube-vip-controlplane02  -n kube-system
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-
-```

docs/control-plane/index.md

@@ -1,421 +0,0 @@
-# Load Balancing a Kubernetes Cluster (Control-Plane)
-
-**Note**: The most common deployment currently for HA Kubernetes clusters w/`kube-vip` involved `kubeadm`, however recently we've worked to bring a method of bringing `kube-vip` to other types of Kubernetes cluster. Typically this deployment method makes use of a daemonset that is usually brought up during the cluster instantiation.. So for those wanting to deploy [k3s](https://k3s.io), we now have installation steps available [here](https://kube-vip.io/control-plane/#k3s),
-
-This document covers the newer (post `0.1.6`) method for using `kube-vip` to provide HA for a Kubernetes Cluster. The documentation for older releases can be found [here](./0.1.5/)
-
-From version `0.1.6` we've moved `kube-vip` from raft to leaderElection within the Kubernetes cluster. After a lot of testing it became clear that the leaderElection gave quicker reconciliation when removing nodes etc.. during upgrades and failures.
-
-For **more** configuration around LeaderElection click [here](https://kube-vip.io/control-plane/#leaderelection-configuration).
-
-This document covers all of the details for using `kube-vip` to build a HA Kubernetes cluster
-
-`tl;dr version`
-- Generate/modify first node `kube-vip` config/manifest
-- `init` first node
-- `join` remaining nodes
-- Add remaining config/manifests
-
-Below are examples of the steps required:
-
-```
-# First Node
-sudo docker run --network host --rm plndr/kube-vip:0.2.1 manifest pod \
---interface ens192 \
---vip 192.168.0.75 \
---arp \
---leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-
-sudo kubeadm init --kubernetes-version 1.17.0 --control-plane-endpoint 192.168.0.75 --upload-certs
-
-# Additional Node(s)
-
-sudo kubeadm join 192.168.0.75:6443 --token w5atsr.blahblahblah --control-plane --certificate-key abc123
-
-sudo docker run --network host --rm plndr/kube-vip:0.2.1 manifest pod \
---interface ens192 \
---vip 192.168.0.75 \
---arp \
---leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.75 |
-| controlPlane01 | 10.0.0.70 |
-| controlPlane02 | 10.0.0.71 |
-| controlPlane03 | 10.0.0.72 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.17.0.
-
-### Generate the `kube-vip` configuration
-
-`kube-vip` no longer requires storing its configuration in a separate directory and will now store its configuration in the actual manifest that defines the static pods. 
-
-```
-sudo docker run --network host \
-	--rm plndr/kube-vip:0.2.1 \
-	manifest pod \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--arp \
-	--leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-The above command will "initialise" the manifest within the `/etc/kubernetes/manifests` directory, that will be started when we actually initialise our Kubernetes cluster with `kubeadm init`
-
-### Modify the configuration
-
-**Cluster Configuration**
-To enable Kubernetes leader Election passing the `--leaderElection` flag will enable `kube-vip` to use the Kubernetes leaderElection functionality to work out which member is the leader.
-
-**VIP Config**
-We will need to set our VIP address to `192.168.0.75` with `--vip 192.168.0.75` and to ensure all hosts are updated when the VIP moves we will enable ARP broadcasts `--arp` (defaults to `true`)
-
-**vip.yaml** Static-pod Manifest
-
-`$ sudo cat /etc/kubernetes/manifests/vip.yaml`
-
-
-``` 
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - start
-    env:
-    - name: vip_arp
-      value: "true"
-    - name: vip_interface
-      value: ens160
-    - name: vip_leaderelection
-      value: "true"
-    - name: vip_leaseduration
-      value: "5"
-    - name: vip_renewdeadline
-      value: "3"
-    - name: vip_retryperiod
-      value: "1"
-    - name: vip_address
-      value: 192.168.0.75
-    image: plndr/kube-vip:0.2.1
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - SYS_TIME
-  hostNetwork: true
-status: {}
-```
-
-### First Node
-
-To generate the basic Kubernetes static pod `yaml` configuration:
-
-Make sure that the manifest directory exists: `sudo mkdir -p /etc/kubernetes/manifests/`
-
-```
-sudo docker run --network host \
-	--rm plndr/kube-vip:0.2.1 \
-	manifest pod \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--arp \
-	--leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-Ensure that `image: plndr/kube-vip:<x>` is modified to point to a specific version (`0.1.8` at the time of writing), refer to [docker hub](https://hub.docker.com/r/plndr/kube-vip/tags) for details. 
-
-The **vip** is set to `192.168.0.75` and this first node will elect itself as leader, and as part of the `kubeadm init` it will use the VIP in order to speak back to the initialising api-server.
-
-`sudo kubeadm init --control-plane-endpoint “192.168.0.75:6443” --upload-certs --kubernetes-version “v1.17.0”`
-
-Once this node is up and running we will be able to see the control-plane pods, including the `kube-vip` pod:
-
-```
-$ kubectl get pods -A
-NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
-<...>
-kube-system   kube-vip-controlplane01                  1/1     Running   0          10m
-```
-
-### Remaining Nodes
-
-
-At this point **DON’T** generate the manifests, this is due to some bizarre `kubeadm/kubelet` behaviour.
-
-```
-  kubeadm join 192.168.0.75:6443 --token <tkn> \
-    --discovery-token-ca-cert-hash sha256:<hash> \
-    --control-plane --certificate-key <key> 
-
-```
-
-**After** this node has been added to the cluster, we can add the manifest to also add this node as a `kube-vip` member. (Adding the manifest afterwards doesn’t interfere with `kubeadm`). 
-
-```
-sudo docker run --network host \
-	--rm plndr/kube-vip:0.2.1 \
-	manifest pod \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--arp \
-	--leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-
-```
-
-Once this node is added we will be able to see that the `kube-vip` pod is up and running as expected:
-
-```
-user@controlPlane01:~$ kubectl get pods -A | grep vip
-kube-system   kube-vip-controlplane01                  1/1     Running             1          16m
-kube-system   kube-vip-controlplane02                  1/1     Running             0          18m
-kube-system   kube-vip-controlplane03                  1/1     Running             0          20m
-
-```
-
-## DNS Support
-
-### Static DNS Support (added in 0.2.0)
-
-A new flag `--address` is introduced to support using a DNS record as the control plane endpoint. `kube-vip` will do a dns lookup to retrieve the IP for the DNS record, and use that IP as the VIP. An `dnsUpdater` periodically checks and updates the system if IP changes for the DNS record.
-
-### Dynamic DNS Support (added in 0.2.1)
-
-`kube-vip` was also updated to support DHCP + [Dynamic DNS](https://en.wikipedia.org/wiki/Dynamic_DNS), for the use case where it's not able to reserve a static IP for the control plane endpoint.
-
-A new flag `--ddns` is introduced. Once enabled, `kube-vip` expects the input `--address` will be a FQDN without binding to an IP. Then `kube-vip` will start a dhcp client to allocate an IP for the hostname of FQDN, and maintain the lease for it.
-
-Once DHCP returns an IP for the FQDN, the same `dnsUpdater` runs to periodically checks and updates if IP got changed.
-
-## BGP Support (added in 0.1.8)
-
-In version `0.1.8`+ `kube-vip` was updated to support [BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol) as a VIP failover mechanism. When a node is elected as a leader then it will update it's peers so that they are aware to route traffic to that node in order to access the VIP. 
-
-The following new flags are used:
-
-- `--bgp` This will enable BGP support within kube-vip
-- `--localAS` The local AS number
-- `--bgpRouterID` The local router address
-- `--peerAS` The AS number for a BGP peer
-- `--peerAddress` The address of a BGP peer
-
-### BGP Packet support
-
-If the `--bgp` flag is passed alone with the Packet flags `packet, packetKey and packetProject`, then the Packet API will be used in order to determine the BGP configuration for the nodes being used in the cluster. This automates a lot of the process and makes using BGP within Packet much simpler.
-
-## Packet Support (added in 0.1.7)
-
-Recently in version `0.1.7` of `kube-vip` we added the functionality to use a Packet Elastic IP as the virtual IP fronting the Kubernetes Control plane cluster. In order to first get out virtual IP we will need to use our Packet account and create a EIP (either public (eek) or private). We will only need a single address so a `/32` will suffice, once this is created as part of a Packet project we can now apply this address to the servers that live in the same project. 
-
-In this example we've logged into the UI can created a new EIP of `147.75.1.2`, and we've deployed three small server instances with Ubuntu.
-
-The following new flags are used:
-
-- `--packet` which enables the use of the Packet API
-- `--packetKey` which is our API key
-- `--packetProject`which is the name of our Packet project where our servers and EIP are located.
-
-*Also* the `--arp` flag should NOT be used as it wont work within the Packet network.
-
-### Variables
-
-```
-export EIP=1.1.1.1
-export PACKET_AUTH_TOKEN=XYZ
-```
-
-### First node
-
-```
-# Generate the manifest
-
-sudo docker run --network host --rm plndr/kube-vip:0.2.1 manifest pod \
---arp=false \
---interface lo \
---vip $EIP \
---leaderElection \
---packet \
---packetKey $PACKET_AUTH_TOKEN \
---packetProject vipTest | sudo tee /etc/kubernetes/manifests/vip.yaml\
-
-# Init Kubernetes
-
-sudo kubeadm init --kubernetes-version 1.18.5 --control-plane-endpoint $EIP --upload-certs
-```
-
-### Other nodes
-
-```
-# Join
-kubeadm join $EIP:6443 --token BLAH --control-plane --certificate-key BLAH --discovery-token-ca-cert-hash sha:blah
-
-# Generate Manifest
-
-sudo docker run --network host --rm plndr/kube-vip:0.2.1 manifest pod \
---arp=false \
---interface lo \
---vip $EIP \
---leaderElection \
---packet \
---packetKey $PACKET_AUTH_TOKEN \
---packetProject vipTest | sudo tee /etc/kubernetes/manifests/vip.yaml\
-```
-
-The Elastic IP failover takes some time (30+ seconds) to move from a failed host to a new leader, so in this release it is mainly for testing. 
-
-
-## Upgrades
-
-From above we have a 3 node cluster and the controlPlane01 is leader:
-
-```
-$ kubectl logs -n kube-system kube-vip-controlplane01 -f
-time="2020-07-04T15:12:52Z" level=info msg="Beginning cluster membership, namespace [kube-system], lock name [plunder-lock], id [controlPlane01]"
-I0704 15:12:52.290420       1 leaderelection.go:242] attempting to acquire leader lease  kube-system/plunder-lock...
-I0704 15:12:56.373113       1 leaderelection.go:252] successfully acquired lease kube-system/plunder-lock
-time="2020-07-04T15:12:56Z" level=info msg="This node is assuming leadership of the cluster"
-time="2020-07-04T15:12:56Z" level=error msg="This node is leader and is adopting the virtual IP"
-time="2020-07-04T15:12:56Z" level=info msg="Starting TCP Load Balancer for service [192.168.0.81:0]"
-time="2020-07-04T15:12:56Z" level=info msg="Load Balancer [Kubeadm Load Balancer] started"
-time="2020-07-04T15:12:56Z" level=info msg="Broadcasting ARP update for 192.168.0.81 (00:50:56:a5:69:a1) via ens192"
-time="2020-07-04T15:12:56Z" level=info msg="Starting TCP Load Balancer for service [192.168.0.81:0]"
-time="2020-07-04T15:12:56Z" level=info msg="Load Balancer [Kubeadm Load Balancer] started"
-time="2020-07-04T15:12:56Z" level=info msg="Broadcasting ARP update for 192.168.0.81 (00:50:56:a5:69:a1) via ens192"
-time="2020-07-04T15:12:56Z" level=info msg="new leader elected: controlPlane01"
-```
-
-We will kill this node and watch `kube-vip` logs from another node:
-
-#### Pinging VIP
-
-```
-64 bytes from 192.168.0.81: icmp_seq=667 ttl=64 time=0.387 ms
-Request timeout for icmp_seq 668
-Request timeout for icmp_seq 669
-Request timeout for icmp_seq 670
-Request timeout for icmp_seq 671
-Request timeout for icmp_seq 672
-64 bytes from 192.168.0.81: icmp_seq=673 ttl=64 time=0.453 ms
-```
-
-#### Logs 
-```
-$ kubectl logs -n kube-system kube-vip-controlplane03 -f
-time="2020-07-04T15:17:53Z" level=info msg="Beginning cluster membership, namespace [kube-system], lock name [plunder-lock], id [controlPlane03]"
-I0704 15:17:53.484698       1 leaderelection.go:242] attempting to acquire leader lease  kube-system/plunder-lock...
-time="2020-07-04T15:17:53Z" level=info msg="new leader elected: controlPlane01"
-E0704 15:20:18.864141       1 leaderelection.go:331] error retrieving resource lock kube-system/plunder-lock: etcdserver: request timed out
-time="2020-07-04T15:20:20Z" level=info msg="new leader elected: controlPlane02"
-```
-
-#### Adding `controlPlane04`
-
-A kubeadm join will fail as the `controlPlane01` still exists as an endpoint, so we have two options (manual steps and configmap edit to remove all mention of this node, or we can bring this node up and `kubeadm reset` the node (which we will do)). 
-
-```
-$ kubectl get nodes
-NAME             STATUS     ROLES    AGE   VERSION
-controlplane01   NotReady   master   14m   v1.17.0
-controlplane02   Ready      master   13m   v1.17.2
-controlplane03   Ready      master   13m   v1.17.0
-controlplane04   NotReady   master   9s    v1.17.0
-```
- After this we can add this node into `kube-vip` with the same manifest created by `docker run`.
-
-## LeaderElection configuration
-
-The Kubernetes LeaderElection that is used to manage the election of a new leader now supports having it's settings managed through flags. 
-
- - `--leaseDuration` Length of time a Kubernetes leader lease can be held for
- - `--leaseRenewDuration` Length of time a Kubernetes leader can attempt to renew its lease
- - `--leaseRetry` Number of times the host will retry to hold a lease
-
-For larger clusters the `--leaseDuration` and `--leaseRenewDuration` may need extending due to slower `etcd` performance. (Tested with 2000 nodes)
-
-## k3s
-
-This section details the steps required to deploye `k3s` in a Highly available manner, using kube-vip deployed within k3s as a daemonset on the control plane nodes. As of `k3s` v1 the persistent datastore is back to etcd, however this guide will also include the steps for using `mysql`.
-
-### Example MySQL deployment (optional)
-
-To quickly validate this we can use docker on a host to quickly spin up a mysql database to store the persistent Kubernetes data.
-
-#### Create local directory for BD storage
-`mkdir mysql`
-
-#### Start Docker MySQL container
-`sudo docker run --cap-add SYS_NICE -p 3306:3306 --name k3s-mysql -v /home/dan/mysql:/var/lib/mysql -e MYSQL_ROOT_PASSWORD=k3s-password -d mysql:8`
-
-### Create `kube-vip` manifest
-
-The `kube-vip` manifest contains all the configuration for starting up `kube-vip` within the `k3s` cluster, it runs as a daemonset with affinity/taints for the control-plane nodes. As `k3s` starts it will parse all manifests in the manifests folder and start the highly available VIP across all control plane nodes in the cluster.
-
-#### Create the `k3` manifests directory
-
-Create the manifests directory, this directory is used by `k3s` for all of it's other deployments once it's up and running.
-`sudo mkdir -p /var/lib/rancher/k3s/server/manifests/`
-
-#### Generate the manifest
-
-Modify the `vipAddress` and `vipInterface` to match the floating IP address you'd like to use and the interface it should bind to.
-
-`curl -sL kube-vip.io/k3s | vipAddress=192.168.0.10 vipInterface=ens192 sh | sudo tee /var/lib/rancher/k3s/server/manifests/vip.yaml`
-
-### Start `k3s`
-
-Set the VIP **first**
-
-`export VIP=192.168.0.10`
-
-
-From online `-->`
-
-```
-curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--write-kubeconfig-mode 644 \
--t agent-secret --tls-san $VIP" sh -
-```
-
-From local `-->`
-
-```
-sudo ./k3s server --tls-san $VIP
-```
-
-#### With MySQL
-
-From online `-->`
-
-```
-curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--write-kubeconfig-mode 644 \
---datastore-endpoint mysql://root:k3s-password@tcp(192.168.0.43:3306)/kubernetes \
--t agent-secret --tls-san $VIP" sh -
-```
-
-From local `-->`
-
-```
-sudo ./k3s server --tls-san $VIP \
---datastore-endpoint="mysql://root:k3s-password@tcp(192.168.0.43:3306)/kubernetes"
-```
-
-### Get a `kubeconfig` that uses the vip
-
-````
-mkdir -p $HOME/.kube
-sudo cat /etc/rancher/k3s/k3s.yaml | sed 's/127.0.0.1/'$VIP'/g' > $HOME/.kube/config
-sudo chown $(id -u):$(id -g) $HOME/.kube/config
-```

docs/flags/index.md

@@ -1,84 +0,0 @@
-# Kube-Vip Flag / Environment Variable reference
-
-## Flags
-
-These flags are typically used in manifest generation.
-
-| Category     | Flag | Usage | Notes |
-|--------------|------|-------|-------|
-|**Mode**          ||||
-|              |`--controlPlane`|Enables `kube-vip` control-plane functionality||
-|              |`--services`|Enables `kube-vip` to watch services of type:LoadBalancer||
-|**Vip Config**   ||||
-|              |`--arp`|Enables ARP brodcasts from Leader||
-|              |`--bgp`|Enables BGP peering from `kube-vip`||
-|              |`--vip`|`<IP Address>`|(deprecated)|
-|              |`--address`|`<IP Address>` or `<DNS name>`||
-|              |`--interface`|`<linux interface>`||
-|              |`--leaderElection`|Enables Kubernetes LeaderElection|Used by ARP, as only the leader can broadcast|
-|**Services**||||
-|              |`--cidr`|Defaults "32"|Used when advertising BGP addresses (typically as `x.x.x.x/32`)|
-|**Kubernetes**||||
-|              |`--inCluster`|Defaults to looking inside the Pod for the token||
-|              |`--taint`|Enables a taint, stopping control plane daemonset being on workers||
-|**LeaderElection**||||
-|              |`--leaseDuration`|default 5|Seconds a lease is held for|
-|              |`--leaseRenewDuration`|default 3|Seconds a leader can attempt to renew the lease|
-|              |`--leaseRetry`|default 1|Number of times the leader will hold the lease for|
-|              |`--namespace`|"kube-vip"|The namespace where the lease will reside|
-|**BGP**||||
-|              |`--bgpRouterID`|`<IP Address>`|Typically the address of the local node|
-|              |`--localAS`|default 65000|The AS we peer from|
-|              |`--bgppeers`|`<address:AS:password:mutlihop>`|Comma seperate list of BGP peers|
-|              |`--peerAddress`|`<IP Address>`|Address of a single BGP Peer|
-|              |`--peerAS`|default 65000|AS of a single BGP Peer|
-|              |`--peerPass`|""| Password to work with a single BGP Peer|
-|              |`--multiHop`|Enables eBGP MultiHop| Enable multiHop with a single BGP Peer|
-|              |`--annotaions`|`<provider string>`|Startup will be paused until the node annotaions contain the BGP configuration|
-|**Equinix Metal**|||(May be deprecated)|
-|              |`--metal`|Enables Equinix Metal API calls||
-|              |`--metalKey`|Equinix Metal API token||
-|              |`--metalProject`|Equinix Metal Project (Name)||
-|              |`--metalProjectID`|Equinix Metal Project (UUID)||
-|              |`--provider-config`|Path to the Equinix Metal provider configuration|Requires the Equinix Metal CCM|
-
-## Environment Variables
-
-These environment variables are usually part of a kube-vip manifest.
-
-More environment variables can be read through the `pkg/kubevip/config_envvar.go` file.
-
-| Category     | Environment Variable | Usage | Notes |
-|--------------|------|-------|-------|
-|**Mode**          ||||
-|              |`cp_enable`|Enables `kube-vip` control-plane functionality||
-|              |`svc_enable`|Enables `kube-vip` to watch services of `type:LoadBalancer`||
-|**Vip Config**   ||||
-|              |`vip_arp`|Enables ARP brodcasts from Leader||
-|              |`bgp_enable`|Enables BGP peering from `kube-vip`||
-|              |`vip_address`|`<IP Address>`|(deprecated)|
-|              |`address`|`<IP Address>` or `<DNS name>`||
-|              |`vip_interface`|`<linux interface>`||
-|              |`vip_leaderelection`|Enables Kubernetes LeaderElection|Used by ARP, as only the leader can broadcast|
-|**Services**||||
-|              |`vip_cidr`|Defaults "32"|Used when advertising BGP addresses (typically as `x.x.x.x/32`)|
-|**LeaderElection**||||
-|              |`vip_leaseduration`|default 5|Seconds a lease is held for|
-|              |`vip_renewdeadline`|default 3|Seconds a leader can attempt to renew the lease|
-|              |`vip_retryperiod`|default 1|Number of times the leader will hold the lease for|
-|              |`cp_namespace`|"kube-vip"|The namespace where the lease will reside|
-|**BGP**||||
-|              |`bgp_routerid`|`<IP Address>`|Typically the address of the local node|
-|              |`bgp_as`|default 65000|The AS we peer from|
-|              |`bgp_peers`|`<address:AS:password:mutlihop>`|Comma seperate list of BGP peers|
-|              |`bgp_peeraddress`|`<IP Address>`|Address of a single BGP Peer|
-|              |`bgp_peeras`|default 65000|AS of a single BGP Peer|
-|              |`bgp_peerpass`|""| Password to work with a single BGP Peer|
-|              |`bgp_multihop`|Enables eBGP MultiHop| Enable multiHop with a single BGP Peer|
-|              |`annotaions`|`<provider string>`|Startup will be paused until the node annotaions contain the BGP configuration|
-|**Equinix Metal**|||(May be deprecated)|
-|              |`vip_packet`|Enables Equinix Metal API calls||
-|              |`PACKET_AUTH_TOKEN`|Equinix Metal API token||
-|              |`vip_packetproject`|Equinix Metal Project (Name)||
-|              |`vip_packetprojectid`|Equinix Metal Project (UUID)||
-|              |`provider_config`|Path to the Equinix Metal provider configuration|Requires the Equinix Metal CCM|

docs/hybrid/daemonset/index.md

@@ -1,226 +0,0 @@
-# Kube-Vip as a daemonset
-
-In Hybrid mode `kube-vip` will manage a virtual IP address that is passed through it's configuration for a Highly Available Kubernetes cluster, it will also "watch" services of `type:LoadBalancer` and once their `spec.LoadBalancerIP` is updated (typically by a cloud controller) it will advertise this address using BGP/ARP.
-
-
-**Note about Daemonsets**
-
-The "hybrid" mode is now the default mode in `kube-vip` from `0.2.3` onwards, and allows both modes to be enabled at the same time. 
-
-If the Kubernetes installer allows for adding a Virtual IP as an additional [SAN](https://en.wikipedia.org/wiki/Subject_Alternative_Name) to the API server certificate then we can apply `kube-vip` to the cluster once the first node has been brought up. 
-
-Unlike generating the static manifest there are a few more things that may need configuring, this page will cover most scenarios.
-
-## Create the RBAC settings
-
-As a daemonSet runs within the Kubernetes cluster it needs the correct access to be able to watch Kubernetes services and other objects. In order to do this we create a User, Role, and a binding.. we can apply this with the command:
-
-```
-kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
-```
-
-## Generating a Manifest
-
-This section only covers generating a simple *BGP* configuration, as the main focus is will be on additional changes to the manifest. For more examples we can look at [here](/hybrid/static/).
-
-**Note:** Pay attention if using the "static" examples, as the `manifest` subcommand should use `daemonset` and NOT `pod`.
-
-### Set configuration details
-
-`export VIP=192.168.0.40`
-
-`export INTERFACE=<interface>`
-
-### Configure to use a container runtime
-
-The easiest method to generate a manifest is using the container itself, below will create an alias for different container runtimes.
-
-#### containerd
-`alias kube-vip="ctr run --rm --net-host docker.io/plndr/kube-vip:0.3.1 vip"`
-
-#### Docker
-`alias kube-vip="docker run --network host --rm plndr/kube-vip:0.3.1"`
-
-### BGP Example
-
-This configuration will create a manifest that will start `kube-vip` providing **controlplane** and **services** management. **Unlike** ARP, all nodes in the BGP configuration will advertise virtual IP addresses. 
-
-**Note** we bind the address to `lo` as we don't want multiple devices that have the same address on public interfaces. We can specify all the peers in a comma seperate list in the format of `address:AS:password:multihop`.
-
-**Note 2** we pass the `--inCluster` flag as this is running as a daemonSet within the Kubernetes cluster and therefore will have access to the token inside the running pod.
-
-**Note 2** we pass the `--taint` flag as we're deploying `kube-vip` as both a daemonset and as advertising controlplane, we want to taint this daemonset to only run on the worker nodes.
-
-`export INTERFACE=lo`
-
-```
-kube-vip manifest daemonset \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --inCluster \
-    --taint \
-    --bgp \
-    --bgppeers 192.168.0.10:65000::false,192.168.0.11:65000::false
-```
-
-### Generated Manifest
-
-```
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: lo
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: "true"
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_peers
-          value: "192.168.0.10:65000::false,192.168.0.11:65000::false"
-        - name: vip_address
-          value: 192.168.0.40
-        image: plndr/kube-vip:0.2.3
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      nodeSelector:
-        node-role.kubernetes.io/master: "true"
-      tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
-  updateStrategy: {}
-```
-
-### Manifest Overview
-
-- `nodeSelector` - Ensures that this particular daemonset only runs on control plane nodes
-- `serviceAccountName: kube-vip` - this specifies the user in the `rbac` that will give us the permissions to get/update services.
-- `hostNetwork: true` - This pod will need to modify interfaces (for VIPs)
-- `env {...}` - We pass the configuration into the kube-vip pod through environment variables.
-
-## Equinix Metal Overview (using the [Equinix Metal CCM](https://github.com/packethost/packet-ccm))
-
-The below example is for running `type:LoadBalancer` services on worker nodes only and will create a daemonset that will run `kube-vip`. 
-
-**NOTE** This use-case requires the [Equinix Metal CCM](https://github.com/packethost/packet-ccm) to be installed and that the cluster/kubelet is configured to use an "external" cloud provider.
-
-This is important as the CCM will apply the BGP configuration to the [node annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) making it easy for `kube-vip` to find the networking configuration it needs to expose load balancer addresses. The `--annotations metal.equinix.com` will cause kube-vip to "watch" the annotations of the worker node that it is running on, once all of the configuarion has been applied by the CCM then the `kube-vip` pod is ready to advertise BGP addresses for the service.
-
-```
-kube-vip manifest daemonset \
-  --interface $INTERFACE \
-  --services \
-  --bgp \
-  --annotations metal.equinix.com \
-  --inCluster | k apply -f -
-```
-
-### Troubleshooting
-
-If `kube-vip` has been sat waiting for a long time then you may need to investigate that the annotations have been applied correctly by doing running the `describe` on the node:
-
-```
-kubectl describe node k8s.bgp02
-...
-Annotations:        kubeadm.alpha.kubernetes.io/cri-socket: /var/run/dockershim.sock
-                    node.alpha.kubernetes.io/ttl: 0
-                    metal.equinix.com/node-asn: 65000
-                    metal.equinix.com/peer-asn: 65530
-                    metal.equinix.com/peer-ip: x.x.x.x
-                    metal.equinix.com/src-ip: x.x.x.x
-```
-
-If there are errors regarding `169.254.255.1` or `169.254.255.2` in the `kube-vip` logs then the routes to the ToR switches that provide BGP peering may by missing from the nodes. They can be replaced with the below command:
-
-```
-GATEWAY_IP=$(curl https://metadata.platformequinix.com/metadata | jq -r ".network.addresses[] | select(.public == false) | .gateway")
-ip route add 169.254.255.1 via $GATEWAY_IP
-ip route add 169.254.255.2 via $GATEWAY_IP
-```
-
-Additionally examining the logs of the Packet CCM may reveal why the node is not yet ready.
-
-## K3s overview (on Equinix Metal)
-
-### Step 1: TIDY (best if something was running before)
-`rm -rf /var/lib/rancher /etc/rancher ~/.kube/*; ip addr flush dev lo; ip addr add 127.0.0.1/8 dev lo; mkdir -p /var/lib/rancher/k3s/server/manifests/`
-
-### Step 2: Get rbac
-`curl https://kube-vip.io/manifests/rbac.yaml > /var/lib/rancher/k3s/server/manifests/rbac.yaml`
-
-### Step 3: Generate kube-vip (get EIP from CLI or UI)
-
-```
-export EIP=x.x.x.x
-export INTERFACE=lo
-```
-
-```
-kube-vip manifest daemonset \
-  --interface $INTERFACE \
-  --vip $EIP \
-  --controlplane \
-  --services \
-  --inCluster \
-  --taint \
-  --bgp \
-  --metal \
-  --provider-config /etc/cloud-sa/cloud-sa.json | tee /var/lib/rancher/k3s/server/manifests/vip.yaml
-```
-
-NOTE: the `—provider-config` actually comes from the secret we apply in step 5 (this will leave kube-vip waiting to start)
-
-### Step 4: Up Cluster
-`K3S_TOKEN=SECRET k3s server --cluster-init --tls-san $EIP --no-deploy servicelb --disable-cloud-controller`
-
-### Step 5: Add CCM
-
-`alias k="k3s kubectl"`
-`k apply -f ./secret.yaml`
-
-(^ https://github.com/packethost/packet-ccm/blob/master/deploy/template/secret.yaml)
-
-`k apply -f https://gist.githubusercontent.com/thebsdbox/c86dd970549638105af8d96439175a59/raw/4abf90fb7929ded3f7a201818efbb6164b7081f0/ccm.yaml`
-
-### Step 6: Demo !
-`k apply -f https://k8s.io/examples/application/deployment.yaml`
-`k expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx`
-
-### Step 7 watch and test:
-`k get svc --watch`

docs/hybrid/index.md

@@ -1,141 +0,0 @@
-# Using `kube-vip` in Hybrid Mode
-
-We can deploy kube-vip in two different methods, which completely depends on your use-case and method for installing Kubernetes:
-
-- Static Pods (hybrid)
-- Daemonset (hybrid, requires taint)
-
-## **Prerequisites**
-
-In order for `kube-vip` to be able to speak with the Kubernetes API server, we need to be able to resolve the hostname within the pod. In order to ensure this will work as expected the `/etc/hosts` file should have the `hostname` of the server within it. The `/etc/hosts` file is passed into the running container and will ensure that the pod isn't "confused" by any Kubernetes networking.
-
-## Kubernetes Services (`type:LoadBalancer`)
-
-To learn more about how `kube-vip` in hybrid works with the LoadBalancer services within a kubernetes cluster the documentation is [here](./services/). To get `kube-vip` deployed read on !
-
-## Static Pods
-
-Static pods are a Kubernetes pod that is ran by the `kubelet` on a single node, and is **not** managed by the Kubernetes cluster itself. This means that whilst the pod can appear within Kubernetes it can't make use of a variety of kubernetes functionality (such as the kubernetes token or `configMaps`). The static pod approach is primarily required for [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/), this is due to the sequence of actions performed by `kubeadm`. Ideally we want `kube-vip` to be part of the kubernetes cluster, for various bits of functionality we also need `kube-vip` to provide a HA virtual IP as part of the installation. 
-
-The sequence of events for this to work follows:
-1. Generate a `kube-vip` manifest in the static pods manifest folder
-2. Run `kubeadm init`, this generates the manifests for the control plane and wait to connect to the VIP
-3. The `kubelet` will parse and execute all manifest, including the `kube-vip` manifest
-4. `kube-vip` starts and advertises our VIP
-5. The `kubeadm init` finishes succesfully.
-
-## Daemonset
-
-Other Kubernetes distributions can bring up a Kubernetes cluster, without depending on a VIP (BUT they are configured to support one). A prime example of this would be k3s, that can be configured to start and also sign the certificates to allow incoming traffic to a virtual IP. Given we don't need the VIP to exist **before** the cluster, we can bring up the k3s node(s) and then add `kube-vip` as a daemonset for all control plane nodes.
-
-# Deploying `kube-vip`
-
-The simplest method for generating the Kubernetes manifests is with `kube-vip` itself.. The subcommand `manifest pod|daemonset` can be used to generate specific types of Kubernetes manifests for use in a cluster. These subcommands can be configured with additional flags to enable/disable BGP/ARP/LeaderElection and a host of other options.
-
-Both Examples will use the same Architecture:
-
-#### Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.40 |
-| controlPlane01 | 10.0.0.41 |
-| controlPlane02 | 10.0.0.42 |
-| controlPlane03 | 10.0.0.43 |
-| worker01       | 10.0.0.44 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.19.0, we only have one worker as we're going to use our controlPlanes in "hybrid" mode.
-
-## As a static Pod (for kubeadm)
-
-The details for creating a static pod are available [here](./static/)
-
-## As a daemonset
-
-When using `kube-vip` as a daemonset the details are available [here](./daemonset/)
-
-# Kube-Vip flag reference
-
-| Category     | Flag | Usage | Notes |
-|--------------|------|-------|-------|
-|**Mode**          ||||
-|              |`--controlPlane`|Enables `kube-vip` control-plane functionality||
-|              |`--services`|Enables `kube-vip` to watch services of type:LoadBalancer||
-|**Vip Config**   ||||
-|              |`--arp`|Enables ARP brodcasts from Leader||
-|              |`--bgp`|Enables BGP peering from `kube-vip`||
-|              |`--vip`|`<IP Address>`|(deprecated)|
-|              |`--address`|`<IP Address>` or `<DNS name>`||
-|              |`--interface`|`<linux interface>`||
-|              |`--leaderElection`|Enables Kubernetes LeaderElection|Used by ARP, as only the leader can broadcast|
-|**Services**||||
-|              |`--cidr`|Defaults "32"|Used when advertising BGP addresses (typically as `x.x.x.x/32`)|
-|**Kubernetes**||||
-|              |`--inCluster`|Defaults to looking inside the Pod for the token||
-|              |`--taint`|Enables a taint, stopping control plane daemonset being on workers||
-|**LeaderElection**||||
-|              |`--leaseDuration`|default 5|Seconds a lease is held for|
-|              |`--leaseRenewDuration`|default 3|Seconds a leader can attempt to renew the lease|
-|              |`--leaseRetry`|default 1|Number of times the leader will hold the lease for|
-|              |`--namespace`|"kube-vip"|The namespace where the lease will reside|
-|**BGP**||||
-|              |`--bgpRouterID`|`<IP Address>`|Typically the address of the local node|
-|              |`--localAS`|default 65000|The AS we peer from|
-|              |`--bgppeers`|`<address:AS:password:mutlihop>`|Comma seperate list of BGP peers|
-|              |`--peerAddress`|`<IP Address>`|Address of a single BGP Peer|
-|              |`--peerAS`|default 65000|AS of a single BGP Peer|
-|              |`--peerPass`|""| Password to work with a single BGP Peer|
-|              |`--multiHop`|Enables eBGP MultiHop| Enable multiHop with a single BGP Peer|
-|              |`--annotaions`|`<provider string>`|Startup will be paused until the node annotaions contain the BGP configuration|
-|**Equinix Metal**|||(May be deprecated)|
-|              |`--metal`|Enables Equinix Metal API calls||
-|              |`--metalKey`|Equinix Metal API token||
-|              |`--metalProject`|Equinix Metal Project (Name)||
-|              |`--metalProjectID`|Equinix Metal Project (UUID)||
-|              |`--provider-config`|Path to the Equinix Metal provider configuration|Requires the Equinix Metal CCM|
-
-## Changelog
-
-### Static DNS Support (added in 0.2.0)
-
-A new flag `--address` is introduced to support using a DNS record as the control plane endpoint. `kube-vip` will do a dns lookup to retrieve the IP for the DNS record, and use that IP as the VIP. An `dnsUpdater` periodically checks and updates the system if IP changes for the DNS record.
-
-### Dynamic DNS Support (added in 0.2.1)
-
-`kube-vip` was also updated to support DHCP + [Dynamic DNS](https://en.wikipedia.org/wiki/Dynamic_DNS), for the use case where it's not able to reserve a static IP for the control plane endpoint.
-
-A new flag `--ddns` is introduced. Once enabled, `kube-vip` expects the input `--address` will be a FQDN without binding to an IP. Then `kube-vip` will start a dhcp client to allocate an IP for the hostname of FQDN, and maintain the lease for it.
-
-Once DHCP returns an IP for the FQDN, the same `dnsUpdater` runs to periodically checks and updates if IP got changed.
-
-## BGP Support (added in 0.1.8)
-
-In version `0.1.8` `kube-vip` was updated to support [BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol) as a VIP failover mechanism. When a node is elected as a leader then it will update it's peers so that they are aware to route traffic to that node in order to access the VIP. 
-
-The following new flags are used:
-
-- `--bgp` This will enable BGP support within kube-vip
-- `--localAS` The local AS number
-- `--bgpRouterID` The local router address
-- `--peerAS` The AS number for a BGP peer
-- `--peerAddress` The address of a BGP peer
-
-### Equinix Metal BGP support
-
-If the `--bgp` flag is passed along with the Equinix Metal flags `metal, metalKey and metalProject`, then Equinix Metal  API will be used in order to determine the BGP configuration for the nodes being used in the cluster. This automates a lot of the process and makes using BGP within Equinix Metal much simpler.
-
-## Equinix Metal Control Plane Support (added in 0.1.8)
-
-Recently in version `0.1.7` of `kube-vip` we added the functionality to use a Equinix Metal Elastic IP as the virtual IP fronting the Kubernetes Control plane cluster. In order to first get out virtual IP we will need to use our Equinix Metal account and create a EIP (either public or private). We will only need a single address so a `/32` will suffice, once this is created as part of a Equinix Metal project we can now apply this address to the servers that live in the same project. 
-
-In this example we've logged into the UI can created a new EIP of `147.75.1.2`, and we've deployed three small server instances with Ubuntu.
-
-The following new flags are used:
-
-- `--metal` which enables the use of the Equinix Metal API
-- `--metalKey` which is our API key
-- `--metalProject`which is the name of our Equinix Metal project where our servers and EIP are located.
-
-*Also* the `--arp` flag should NOT be used as it wont work within the Equinix Metal network.

docs/hybrid/services/index.md

@@ -1,269 +0,0 @@
-# Kube-vip services 
-
-We've designed `kube-vip` to be as de-coupled or agnostic from other components that may exist within a Kubernetes cluster as possible. This has lead to `kube-vip` having a very simplistic but robust approach to advertising Kubernetes services to the outside world and marking these services as ready to use.
-
-## Flow
-
-This section details the flow of events in order for `kube-vip` to advertise a Kubernetes service:
-
-1. An end user exposes a application through Kubernetes as a LoadBalancer => `kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx`
-2. Within the Kubernetes cluster a service object is created with the `svc.Spec.Type = ServiceTypeLoadBalancer`
-3. A controller (typically a Cloud Controller) has a loop that "watches" for services of the type `LoadBalancer`.
-4. The controller now has the responsibility of providing an IP address for this service along with doing anything that is network specific for the environment where the cluster is running.
-5. Once the controller has an IP address it will update the service `svc.Spec.LoadBalancerIP` with it's new IP address.
-6. The `kube-vip` pods also implement a "watcher" for services that have a `svc.Spec.LoadBalancerIP` address attached.
-7. When a new service appears `kube-vip` will start advertising this address to the wider network (through BGP/ARP) which will allow traffic to come into the cluster and hit the service network.
-8. Finally `kube-vip` will update the service status so that the API reflects that this LoadBalancer is ready. This is done by updating the `svc.Status.LoadBalancer.Ingress` with the VIP address.
-
-## CCM
-
-We can see from the [flow](#Flow) above that `kube-vip` isn't coupled to anything other than the Kubernetes API, and will only act upon an existing Kubernetes primative (in this case the object of type `Service`). This makes it easy for existing CCMs to simply apply their logic to services of type LoadBalancer and leave `kube-vip` to take the next steps to advertise these load-balancers to the outside world. 
-
-
-## Using the Kube-vip Cloud Provider
-
-The below instructions *should just work* on Kubernetes regardless of architecture (Linux Operating System is the only requirement) - you can quickly install the "latest" components:
-
-**Install the `kube-vip-cloud-provider`**
-
-```
-$ kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
-```
-
-It uses a `statefulSet` and can always be viewed with the following command:
-
-```
-kubectl describe pods -n kube-system kube-vip-cloud-provider-0
-```
-
-**Create a global CIDR or IP Range**
-
-Any `service` in any `namespace` can use an address from the global CIDR `cidr-global` or range `range-global`
-
-```
-kubectl create configmap --namespace kube-system kubevip --from-literal cidr-global=192.168.0.220/29
-```
-or
-```
-kubectl create configmap --namespace kube-system kubevip --from-literal range-global=192.168.1.220-192.168.1.230
-```
-
-Creating services of `type: LoadBalancer` in *any namespace* will now take addresses from the **global** cidr defined in the `configmap` unless a specific 
-
-
-## The Detailed guide
-
-### Deploy the Kube-vip Cloud Provider
-
-**Install the `kube-vip-cloud-provider`**
-
-```
-$ kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
-```
-
-The following output should appear when the manifest is applied: 
-
-```
-serviceaccount/kube-vip-cloud-controller created
-clusterrole.rbac.authorization.k8s.io/system:kube-vip-cloud-controller-role created
-clusterrolebinding.rbac.authorization.k8s.io/system:kube-vip-cloud-controller-binding created
-statefulset.apps/kube-vip-cloud-provider created
-```
-
-We can validate the cloud provider by examining the pods and following the logs:
-
-```
-kubectl describe pods -n kube-system kube-vip-cloud-provider-0
-kubectl logs -n kube-system kube-vip-cloud-provider-0 -f
-```
-
-### The Kube-vip Cloud Provider `configmap`
-
-To manage the IP address ranges for the load balancer instances the `kube-vip-cloud-provider` uses a `configmap` held in the `kube-system` namespace. IP address ranges can be configured using:
-- IP address pools by CIDR
-- IP ranges [start address - end address]
-- Multiple pools by CIDR per namespace
-- Multiple IP ranges per namespace (handles overlapping ranges)
-- Setting of static addresses through --load-balancer-ip=x.x.x.x
-
-To control which IP address range is used for which service the following rules are applied:
-- Global address pools (`cidr-global` or `range-global`) are available for use by *any* `service` in *any* `namespace`
-- Namespace specific address pools (`cidr-<namespace>` or `range-<namespace>`) are *only* available for use by `service` in the *specific* `namespace`
-- Static IP addresses can be applied to a load balancer `service` using the `loadbalancerIP` setting, even outside of the assigned ranges
-
-Example Configmap:
-
-```
-$ kubectl get configmap -n kube-system kubevip -o yaml
-
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kubevip
-  namespace: kube-system
-data:
-  cidr-default: 192.168.0.200/29                      # CIDR-based IP range for use in the default namespace
-  range-development: 192.168.0.210-192.168.0.219      # Range-based IP range for use in the development namespace
-  cidr-finance: 192.168.0.220/29,192.168.0.230/29     # Multiple CIDR-based ranges for use in the finance namespace
-  cidr-global: 192.168.0.240/29                       # CIDR-based range which can be used in any namespace
-```
-
-### Expose a service
-
-We can now expose a service and once the cloud provider has provided an address `kube-vip` will start to advertise that address to the outside world as shown below!
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx
-```
-
-or via a `service` YAML definition
-
-```
-apiVersion: v1
-kind: Service
-metadata:
-  name: nginx
-spec:
-  ports:
-  - name: http
-    port: 80
-    protocol: TCP
-  selector:
-    app: nginx
-  type: LoadBalancer
-  ```
-
-
-We can also expose a specific address by specifying it on the command line:
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```
-
-or including it in the `service` definition:
-
-```
-apiVersion: v1
-kind: Service
-metadata:
-  name: nginx
-spec:
-  ports:
-  - name: http
-    port: 80
-    protocol: TCP
-  selector:
-    app: nginx
-  type: LoadBalancer
-  loadBalancerIP: "1.1.1.1"
-```
-
-### Using DHCP for Load Balancers (experimental)
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to use the local network DHCP server to provide `kube-vip` with a load-balancer address that can be used to access a
- Kubernetes service on the network. 
-
-In order to do this we need to signify to `kube-vip` and the cloud-provider that we don't need one of their managed addresses. We do this by explicitly exposing a service on the 
-address `0.0.0.0`. When `kube-vip` sees a service on this address it will create a `macvlan` interface on the host and request a DHCP address, once this address is provided it will assign it as the VIP and update the Kubernetes service!
-
-```
-$ k expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx-dhcp --load-balancer-ip=0.0.0.0; k get svc
-service/nginx-dhcp exposed
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   0.0.0.0         80:31184/TCP   0s
-
-{ ... a second or so later ... }
-
-$ k get svc
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   192.168.0.155   80:31184/TCP   3s
-```
-
-### Using UPNP to expose a service to the outside world
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to expose a load-balancer on a specific port and using UPNP (on a supported gateway) expose this service to the inte
-rnet.
-
-Most simple networks look something like the following:
-
-`<----- <internal network 192.168.0.0/24> <Gateway / router> <external network address> ----> Internet`
-
-Using UPNP we can create a matching port on the `<external network address>` allowing your service to be exposed to the internet.
-
-#### Enable UPNP
-
-Add the following to the `kube-vip` `env:` section, and the rest should be completely automated. 
-
-**Note** some environments may require (Unifi) will require `Secure mode` being `disabled` (this allows a host with a different address to register a port)
-
-```
-- name: enableUPNP
-  value: "true"
-```
-
-#### Exposing a service
-
-To expose a port successfully we'll need to change the command slightly:
-
-`--target-port=80` the port of the application in the pods (HTT/NGINX)
-`--port=32380` the port the service will be exposed on (and what you should connect to in order to receive traffic from the service)
-
-`kubectl expose deployment plunder-nginx --port=32380 --target-port=80 --type=LoadBalancer --namespace plunder`
-
-The above example should expose a port on your external (internet facing address), that can be tested externally with:
-
-```
-$ curl externalIP:32380
-<!DOCTYPE html>
-<html>
-...
-```
-
-### Expose with Equinix Metal (using the `kube-vip-cloud-provider`)
-
-Either through the CLI or through the UI, create a public IPv4 EIP address.. and this is the address you can expose through BGP!
-
-```
-# packet ip request -p xxx-bbb-ccc -f ams1 -q 1 -t public_ipv4                                                                   
-+-------+---------------+--------+----------------------+
-|   ID  |    ADDRESS    | PUBLIC |       CREATED        |
-+-------+---------------+--------+----------------------+
-| xxxxx |     1.1.1.1   | true   | 2020-11-10T15:57:39Z |
-+-------+---------------+--------+----------------------+
-
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```
-
-## Equinix Metal Overview (using the [Equinix Metal CCM](https://github.com/packethost/packet-ccm))
-
-Below are two examples for running `type:LoadBalancer` services on worker nodes only and will create a daemonset that will run `kube-vip`. 
-
-**NOTE** This use-case requires the [Equinix Metal CCM](https://github.com/packethost/packet-ccm) to be installed and that the cluster/kubelet is configured to use an "external" cloud provider.
-
-### Using Annotations
-
-This is important as the CCM will apply the BGP configuration to the [node annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) making it easy for `kube-vip` to find the networking configuration it needs to expose load balancer addresses. The `--annotations metal.equinix.com` will cause kube-vip to "watch" the annotations of the worker node that it is running on, once all of the configuarion has been applied by the CCM then the `kube-vip` pod is ready to advertise BGP addresses for the service.
-
-```
-kube-vip manifest daemonset \
-  --interface $INTERFACE \
-  --services \
-  --bgp \
-  --annotations metal.equinix.com \
-  --inCluster | k apply -f -
-```
-
-### Using the existing CCM secret 
-
-Alternatively it is possible to create a daemonset that will use the existing CCM secret to do an API lookup, this will allow for discovering the networking configuration needed to advertise loadbalancer addresses through BGP.
-
-```
-kube-vip manifest daemonset --interface $INTERFACE \
---services \
---inCluster \
---bgp \
---metal \
---provider-config /etc/cloud-sa/cloud-sa.json | kubectl apply -f -
-```

docs/hybrid/static/index.md

@@ -1,129 +0,0 @@
-# Kube-vip as a Static Pod
-
-In Hybrid mode `kube-vip` will manage a virtual IP address that is passed through it's configuration for a Highly Available Kubernetes cluster, it will also "watch" services of `type:LoadBalancer` and once their `spec.LoadBalancerIP` is updated (typically by a cloud controller) it will advertise this address using BGP/ARP.
-
-The "hybrid" mode is now the default mode in `kube-vip` from `0.2.3` onwards, and allows both modes to be enabled at the same time. 
-
-## Generating a Manifest
-
-This section details creating a number of manifests for various use cases
-
-### Set configuration details
-
-`export VIP=192.168.0.40`
-
-`export INTERFACE=<interface>`
-
-### Configure to use a container runtime
-
-The easiest method to generate a manifest is using the container itself, below will create an alias for different container runtimes.
-
-#### containerd
-`alias kube-vip="ctr run --rm --net-host docker.io/plndr/kube-vip:0.3.1 vip /kube-vip"`
-
-#### Docker
-`alias kube-vip="docker run --network host --rm plndr/kube-vip:0.3.1"`
-
-
-### ARP
-
-This configuration will create a manifest that starts `kube-vip` providing **controlplane** and **services** management, using **leaderElection**. When this instance is elected as the leader it will bind the `vip` to the specified `interface`, this is also the same for services of `type:LoadBalancer`.
-
-`export INTERFACE=eth0`
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --arp \
-    --leaderElection | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-### BGP
-
-This configuration will create a manifest that will start `kube-vip` providing **controlplane** and **services** management. **Unlike** ARP, all nodes in the BGP configuration will advertise virtual IP addresses. 
-
-**Note** we bind the address to `lo` as we don't want multiple devices that have the same address on public interfaces. We can specify all the peers in a comma seperate list in the format of `address:AS:password:multihop`.
-
-`export INTERFACE=lo`
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --localAS 65000 \
-    --bgpRouterID 192.168.0.2 \
-    --bgppeers 192.168.0.10:65000::false,192.168.0.11:65000::false | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-### BGP with Equinix Metal
-
-When deploying Kubernetes with Equinix Metal with the `--controlplane` functionality we need to pre-populate the BGP configuration in order for the control plane to be advertised and work in a HA scenario. Luckily Equinix Metal provides the capability to "look up" the configuration details (for BGP) that we need in order to advertise our virtual IP for HA functionality. We can either make use of the [Equinix Metal API](https://metal.equinix.com/developers/api/) or we can parse the [Equinix Metal Metadata service](https://metal.equinix.com/developers/docs/servers/metadata/).
-
-**Note** If this cluster will be making use of Equinix Metal for `type:LoadBalancer` (by using the [Equinix Metal CCM](https://github.com/packethost/packet-ccm)) then we will need to ensure that nodes are set to use an external cloud-provider. Before doing a `kubeadm init|join` ensure the kubelet has the correct flags by using the following command `echo KUBELET_EXTRA_ARGS=\"--cloud-provider=external\" > /etc/default/kubelet`.
-
-#### Creating a manifest using the API
-
-We can enable `kube-vip` with the capability to discover the required configuration for BGP by passing the `--metal` flag and the API Key and our project ID.
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE\
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --metal \
-    --metalKey xxxxxxx \
-    --metalProjectID xxxxx | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-#### Creating a manifest using the metadata
-
-We can parse the metadata, *however* it requires that the tools `curl` and `jq` are installed. 
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE\
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --peerAS $(curl https://metadata.platformequinix.com/metadata | jq '.bgp_neighbors[0].peer_as') \
-    --peerAddress $(curl https://metadata.platformequinix.com/metadata | jq -r '.bgp_neighbors[0].peer_ips[0]') \
-    --localAS $(curl https://metadata.platformequinix.com/metadata | jq '.bgp_neighbors[0].customer_as') \
-    --bgpRouterID $(curl https://metadata.platformequinix.com/metadata | jq -r '.bgp_neighbors[0].customer_ip') | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-## Deploy your Kubernetes Cluster
-
-### First node
-
-```
-sudo kubeadm init \
-    --kubernetes-version 1.19.0 \
-    --control-plane-endpoint $VIP \
-    --upload-certs
-```
-
-### Additional Node(s)
-
-Due to an oddity with `kubeadm` we can't have our `kube-vip` manifest present **before** joining our additional nodes. So on these control plane nodes we will add them first to the cluster.
-
-```
-sudo kubeadm join $VIP:6443 \
-    --token w5atsr.blahblahblah 
-    --control-plane \
-    --certificate-key abc123
-```
-
-**Once**, joined these nodes can have the same command that we ran on the first node to populate the `/etc/kubernetes/manifests/` folder with the `kube-vip` manifest.
-
-## Services
-
-At this point your `kube-vip` static pods will be up and running and where used with the `--services` flag will also be watching for Kubernetes services that they can advertise. In order for `kube-vip` to advertise a service it needs a CCM or other controller to apply an IP address to the `spec.LoadBalancerIP`, which marks the loadbalancer as defined. 

docs/index.md

@@ -1,74 +0,0 @@
-# Kube-vip
-
-![kube-vip.png](kube-vip.png)
-
-## Overview
-Kubernetes Virtual IP and Load-Balancer for both control plane and Kubernetes services
-
-The idea behind `kube-vip` is a small self-contained Highly-Available option for all environments, especially:
-
-- Bare-Metal
-- On-Prem
-- Edge (ARM / Raspberry PI)
-- Virtualisation
-- Pretty much anywhere else :)
-
-## Features
-
-Kube-Vip was originally created to provide a HA solution for the Kubernetes control plane, over time it has evolved to incorporate that same functionality into Kubernetes service type [load-balancers](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer).
-
-- VIP addresses can be both IPv4 or IPv6
-- Control Plane with ARP (Layer 2) or BGP (Layer 3)
-- Control Plane using either [leader election](https://godoc.org/k8s.io/client-go/tools/leaderelection) or [raft](https://en.wikipedia.org/wiki/Raft_(computer_science))
-- Control Plane HA with kubeadm (static Pods)
-- Control Plane HA with K3s/and others (daemonsets)
-- Service LoadBalancer using [leader election](https://godoc.org/k8s.io/client-go/tools/leaderelection) for ARP (Layer 2)
-- Service LoadBalancer using multiple nodes with BGP
-- Service LoadBalancer address pools per namespace or global
-- Service LoadBalancer address via (existing network DHCP)
-- Service LoadBalancer address exposure to gateway via UPNP
-- ... manifest generation, vendor API integrations and many nore... 
-
-## Why?
-
-The "original" purpose of `kube-vip` was to simplify the building of HA Kubernetes clusters, which at this time can involve a few components and configurations that all need to be managed. This was blogged about in detail by [thebsdbox](https://twitter.com/thebsdbox/) here -> [https://thebsdbox.co.uk/2020/01/02/Designing-Building-HA-bare-metal-Kubernetes-cluster/#Networking-load-balancing](https://thebsdbox.co.uk/2020/01/02/Designing-Building-HA-bare-metal-Kubernetes-cluster/#Networking-load-balancing). As the project evolved it now can use those same technologies to provide load-balancing capabilities within a Kubernetes Cluster.
-
-
-## Architecture
-
-The architecture for `kube-vip` (and associated kubernetes components) is covered in detail [here](/architecture/)
-
-## Installation
-
-There are two main routes for deploying `kube-vip`, either through a [static pod](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/) when bringing up a Kubernetes cluster with [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/) or as a [daemon set](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) (typically with distributions like [k3s](https://k3s.io)). 
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.40 |
-| controlPlane01 | 10.0.0.41 |
-| controlPlane02 | 10.0.0.42 |
-| controlPlane03 | 10.0.0.43 |
-| worker01       | 10.0.0.44 |
-
-All nodes are running Ubuntu 20.04, Docker CE and will use Kubernetes 1.21.0, we only have one worker as we're going to use our controlPlanes in "hybrid" mode.
-
-- [Static Pod](/install_static)
-- [Daemon Set](/install_daemonset)
-
-## Usage
-
-- [On-Prem with the kube-vip cloud controller](/usage/on-prem)
-- [KIND](/usage/kind)
-- [Equinix Metal](/usage/EquinixMetal)
-- [k3s](/usage/k3s)
-
-## Flags/Environment Variables
-
-- [Flags and Environment variables](/flags/)
-
-## Links
-
-- The Kube-Vip Cloud Provider Repository -> [https://github.com/kube-vip/kube-vip-cloud-provider](https://github.com/kube-vip/kube-vip-cloud-provider)
-- The Kube-Vip Repository -> [https://github.com/kube-vip/kube-vip](https://github.com/kube-vip/kube-vip)

docs/install_daemonset/index.md

@@ -1,167 +0,0 @@
-# Kube-Vip as a daemonset
-
-## Daemonset
-
-Other Kubernetes distributions can bring up a Kubernetes cluster, without depending on a VIP (BUT they are configured to support one). A prime example of this would be k3s, that can be configured to start and also sign the certificates to allow incoming traffic to a virtual IP. Given we don't need the VIP to exist **before** the cluster, we can bring up the k3s node(s) and then add `kube-vip` as a daemonset for all control plane nodes.
-
-If the Kubernetes installer allows for adding a Virtual IP as an additional [SAN](https://en.wikipedia.org/wiki/Subject_Alternative_Name) to the API server certificate then we can apply `kube-vip` to the cluster once the first node has been brought up. 
-
-## Kube-Vip as **HA**, **Load-Balancer** or both ` ¯\_(ツ)_/¯`
-
-When generating a manifest for `kube-vip` we will pass in the flags `--controlplane` / `--services` these will enable the various types of functionality within `kube-vip`. 
-
-With both enabled `kube-vip` will manage a virtual IP address that is passed through it's configuration for a Highly Available Kubernetes cluster, it will also "watch" services of `type:LoadBalancer` and once their `spec.LoadBalancerIP` is updated (typically by a cloud controller) it will advertise this address using BGP/ARP.
-
-**Note about Daemonsets**
-
-Unlike generating the static manifest there are a few more things that may need configuring, this page will cover most scenarios.
-
-## Create the RBAC settings
-
-As a daemonSet runs within the Kubernetes cluster it needs the correct access to be able to watch Kubernetes services and other objects. In order to do this we create a User, Role, and a binding.. we can apply this with the command:
-
-```
-kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
-```
-
-## Generating a Manifest
-
-This section only covers generating a simple *BGP* configuration, as the main focus is will be on additional changes to the manifest. For more examples we can look at [here](/hybrid/static/).
-
-**Note:** Pay attention if using the "static" examples, as the `manifest` subcommand should use `daemonset` and NOT `pod`.
-
-### Set configuration details
-
-`export VIP=192.168.0.40`
-
-`export INTERFACE=<interface>`
-
-### Configure to use a container runtime
-
-The easiest method to generate a manifest is using the container itself, below will create an alias for different container runtimes.
-
-#### containerd
-`alias kube-vip="ctr run --rm --net-host docker.io/plndr/kube-vip:0.3.1 vip"`
-
-#### Docker
-`alias kube-vip="docker run --network host --rm plndr/kube-vip:0.3.1"`
-
-### BGP Example
-
-This configuration will create a manifest that will start `kube-vip` providing **controlplane** and **services** management. **Unlike** ARP, all nodes in the BGP configuration will advertise virtual IP addresses. 
-
-**Note** we bind the address to `lo` as we don't want multiple devices that have the same address on public interfaces. We can specify all the peers in a comma seperate list in the format of `address:AS:password:multihop`.
-
-**Note 2** we pass the `--inCluster` flag as this is running as a daemonSet within the Kubernetes cluster and therefore will have access to the token inside the running pod.
-
-**Note 2** we pass the `--taint` flag as we're deploying `kube-vip` as both a daemonset and as advertising controlplane, we want to taint this daemonset to only run on the worker nodes.
-
-`export INTERFACE=lo`
-
-```
-kube-vip manifest daemonset \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --inCluster \
-    --taint \
-    --bgp \
-    --bgppeers 192.168.0.10:65000::false,192.168.0.11:65000::false
-```
-
-### Generated Manifest
-
-```
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: lo
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: bgp_peers
-          value: 192.168.0.10:65000::false,192.168.0.11:65000::false
-        - name: vip_address
-          value: 192.168.0.40
-        image: 'plndr/kube-vip:'
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      nodeSelector:
-        node-role.kubernetes.io/master: "true"
-      serviceAccountName: kube-vip
-      tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
-        operator: Exists
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-```
-
-### Managing a `routerID` as a daemonset
-
-The routerID needs to be unique on each node that participates in BGP advertisements. In order to do this we can modify the manifest so that when `kube-vip` starts it will look up its local address and use that as the routerID.
-
-```
-          - name: bgp_routerinterface
-            value: "ens160"
-```
-
-This will instruct each instance of `kube-vip` as part of the daemonset to look up the IP address on that interface and use it as the routerID.
-
-### Manifest Overview
-
-- `nodeSelector` - Ensures that this particular daemonset only runs on control plane nodes
-- `serviceAccountName: kube-vip` - this specifies the user in the `rbac` that will give us the permissions to get/update services.
-- `hostNetwork: true` - This pod will need to modify interfaces (for VIPs)
-- `env {...}` - We pass the configuration into the kube-vip pod through environment variables.

docs/install_static/index.md

@@ -1,75 +0,0 @@
-# Kube-vip as a Static Pod
-
-## Static Pods
-
-Static pods are a Kubernetes pod that is ran by the `kubelet` on a single node, and is **not** managed by the Kubernetes cluster itself. This means that whilst the pod can appear within Kubernetes it can't make use of a variety of kubernetes functionality (such as the kubernetes token or `configMaps`). The static pod approach is primarily required for [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/), this is due to the sequence of actions performed by `kubeadm`. Ideally we want `kube-vip` to be part of the kubernetes cluster, for various bits of functionality we also need `kube-vip` to provide a HA virtual IP as part of the installation. 
-
-The sequence of events for this to work follows:
-1. Generate a `kube-vip` manifest in the static pods manifest folder
-2. Run `kubeadm init`, this generates the manifests for the control plane and wait to connect to the VIP
-3. The `kubelet` will parse and execute all manifest, including the `kube-vip` manifest
-4. `kube-vip` starts and advertises our VIP
-5. The `kubeadm init` finishes succesfully.
-
-## Kube-Vip as **HA**, **Load-Balancer** or both ` ¯\_(ツ)_/¯`
-
-When generating a manifest for `kube-vip` we will pass in the flags `--controlplane` / `--services` these will enable the various types of functionality within `kube-vip`. 
-
-With both enabled `kube-vip` will manage a virtual IP address that is passed through it's configuration for a Highly Available Kubernetes cluster, it will also "watch" services of `type:LoadBalancer` and once their `spec.LoadBalancerIP` is updated (typically by a cloud controller) it will advertise this address using BGP/ARP.
-
-## Generating a Manifest
-
-This section details creating a number of manifests for various use cases
-
-### Set configuration details
-
-`export VIP=192.168.0.40`
-
-`export INTERFACE=<interface>`
-
-## Configure to use a container runtime
-
-The easiest method to generate a manifest is using the container itself, below will create an alias for different container runtimes.
-
-### containerd
-`alias kube-vip="ctr run --rm --net-host docker.io/plndr/kube-vip:0.3.1 vip /kube-vip"`
-
-### Docker
-`alias kube-vip="docker run --network host --rm plndr/kube-vip:0.3.1"`
-
-
-## ARP
-
-This configuration will create a manifest that starts `kube-vip` providing **controlplane** and **services** management, using **leaderElection**. When this instance is elected as the leader it will bind the `vip` to the specified `interface`, this is also the same for services of `type:LoadBalancer`.
-
-`export INTERFACE=eth0`
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --arp \
-    --leaderElection | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-## BGP
-
-This configuration will create a manifest that will start `kube-vip` providing **controlplane** and **services** management. **Unlike** ARP, all nodes in the BGP configuration will advertise virtual IP addresses. 
-
-**Note** we bind the address to `lo` as we don't want multiple devices that have the same address on public interfaces. We can specify all the peers in a comma seperate list in the format of `address:AS:password:multihop`.
-
-`export INTERFACE=lo`
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --localAS 65000 \
-    --bgpRouterID 192.168.0.2 \
-    --bgppeers 192.168.0.10:65000::false,192.168.0.11:65000::false | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```

docs/k3s

@@ -1,61 +0,0 @@
-#!/bin/bash
-
-echo "apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: \"true\"
-        - name: vip_interface
-          value: $vipInterface
-        - name: port
-          value: \"6443\"
-        - name: vip_cidr
-          value: \"32\"
-        - name: cp_enable
-          value: \"true\"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: \"false\"
-        - name: vip_address
-          value: $vipAddress
-        image: plndr/kube-vip:v0.3.5
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      nodeSelector:
-        node-role.kubernetes.io/master: \"true\"
-      serviceAccountName: kube-vip
-      tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0"

docs/kubernetes/arp/index.md

@@ -1,181 +0,0 @@
-# Kube-vip (Layer 2 / ARP)
-
-**BEFORE** we begin we should ensure that ipvs has `strict` ARP enabled:
-
-```
-$ kubectl describe configmap -n kube-system kube-proxy | grep ARP
-  strictARP: false
-```
-
-If this is false we can enable it with the command:
-
-```
-$ kubectl get configmap kube-proxy -n kube-system -o yaml | \
- sed -e "s/strictARP: false/strictARP: true/" | \
- kubectl apply -f - -n kube-system
-```
-
-and confirm with:
-
-```
-$ kubectl describe configmap -n kube-system kube-proxy | grep ARP
-  strictARP: true
-```
-
-## Deploy `kube-vip`
-
-To deploy the [latest] then `kubectl apply -f https://kube-vip.io/manifests/kube-vip.yaml`, specific versions should be found in the repository as detailed below:
-
-From the GitHub repository [https://github.com/kube-vip/kube-vip/tree/master/example/deploy](https://github.com/kube-vip/kube-vip/tree/master/example/deploy) find the version of the `kube-vip` to deploy (although typically the highest version number will provider more functionality/stability). The [raw] option in Github will provide the url that can be applied directly with a `kubectl apply -f <url>`.
-
-The following output should appear when the manifest is applied: 
-```
-serviceaccount/vip created
-role.rbac.authorization.k8s.io/vip-role created
-rolebinding.rbac.authorization.k8s.io/vip-role-bind created
-deployment.apps/kube-vip-cluster created
-```
-
-*NOTE* The manifest for the `kube-vip` deployment has rules to ensure affinity (pods are always distributed to different nodes for HA). By default the replicas are set to `3` in the event you have less than `3` worker nodes then those replicas will sit as `pending`. This in itself isn't an issue, it means when new workers are added then they will be scheduled. *However*, tooling such as `kapps` will inspect the manifest before it's applied an error because of issues such as this.
-
-### Editing `kube-vip` configuration
-
-Either download and edit the manifest locally or apply as above and edit the deployment with `kubectl edit deploy/kube-vip-cluster` (change namespace where appropriate `-n`)
-
-```
-		- name: vip_interface
-		  value: ens192
-		- name: vip_configmap
-		  value: plndr
-		- name: vip_arp
-		  value: "true"
-		- name: vip_loglevel
-		  value: "5"
-```
-
-- `vip_interface` - defines the interface that the VIP will bind to
-- `vip_configmap` - defines the `configmap` that `kube-vip` will watch for service configuration
-- `vip_arp` - determines if ARP broadcasts are enabled
-- `vip_loglevel` - determines the verbosity of logging
-
-## Using other namespaces
-
-In this example we'll deploy and load-balance within the namespace `plunder`
-
-### Create the namespace
-
-`kubectl create namespace plunder`
-
-### Add a network range/cidr for this namespace
-
-`kubectl edit -n kube-system configmap/plndr`
-
-We will add the range 192.168.0.210/29 for the namespace plunder underneath the existing range for the namespace default:
-
-```
-apiVersion: v1
-data:
-  cidr-default: 192.168.0.200/29
-  cidr-global: 192.168.0.210/29
-  cidr-plunder: 192.168.0.220/29
-<...>
-```
-
-### Deploy `kube-vip` in the namespace **plunder**
-
-In the same way we deployed `kube-vip` into the default namespace we can deploy the same manifest into a different namespace using `-n namespace` e.g.
-
-**Note** change the version of manifest when actually deploying!
-
-```
-kubectl apply -f https://kube-vip.io/manifests/kube-vip.yaml -n plunder
-```
-
-## Usage
-
-This example will deploy into the namespace `plunder` as mention in the [Using other namespaces](Using other namespaces) example. Remove the `-n plunder` to deploy within the `default` namespace.
-
-### Deploy nginx
-
-```
-kubectl create deployment --image nginx plunder-nginx --namespace plunder
-```
-
-### Create a load balancer
-
-```
-kubectl expose deployment plunder-nginx --port=80 --type=LoadBalancer --namespace plunder
-```
-
-## Using DHCP for Load Balancers (experimental)
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to use the local network DHCP server to provide `kube-vip` with a load-balancer address that can be used to access a Kubernetes service on the network. 
-
-In order to do this we need to signify to `kube-vip` and the cloud-provider that we don't need one of their managed addresses. We do this by explicitly exposing a service on the address `0.0.0.0`. When `kube-vip` sees a service on this address it will create a `macvlan` interface on the host and request a DHCP address, once this address is provided it will assign it as the VIP and update the Kubernetes service!
-
-```
-$ k expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx-dhcp --load-balancer-ip=0.0.0.0; k get svc
-service/nginx-dhcp exposed
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   0.0.0.0         80:31184/TCP   0s
-
-{ ... a second or so later ... }
-
-$ k get svc
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   192.168.0.155   80:31184/TCP   3s
-```
-
-## Using UPNP to expose a service to the outside world
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to expose a load-balancer on a specific port and using UPNP (on a supported gateway) expose this service to the internet.
-
-Most simple networks look something like the following:
-
-`<----- <internal network 192.168.0.0/24> <Gateway / router> <external network address> ----> Internet`
-
-Using UPNP we can create a matching port on the `<external network address>` allowing your service to be exposed to the internet.
-
-### Enable UPNP
-
-Add the following to the `kube-vip` `env:` section, and the rest should be completely automated. 
-
-**Note** some environments may require (Unifi) will require `Secure mode` being `disabled` (this allows a host with a different address to register a port)
-
-```
-- name: enableUPNP
-  value: "true"
-```
-
-### Exposing a service
-
-To expose a port successfully we'll need to change the command slightly:
-
-`--target-port=80` the port of the application in the pods (HTT/NGINX)
-`--port=32380` the port the service will be exposed on (and what you should connect to in order to receive traffic from the service)
-
-`kubectl expose deployment plunder-nginx --port=32380 --target-port=80 --type=LoadBalancer --namespace plunder`
-
-The above example should expose a port on your external (internet facing address), that can be tested externally with:
-
-```
-$ curl externalIP:32380
-<!DOCTYPE html>
-<html>
-...
-```
-
-## Troubleshooting
-
-Typically the logs from the `kube-vip` controller will reveal the most clues as to where a problem may lie.
-
-The `ClusterRoleBinding` is missing will result in the following:
-
-```
-E0229 17:36:38.014351       1 retrywatcher.go:129] Watch failed: unknown (get endpoints)
-E0229 17:36:38.014352       1 retrywatcher.go:129] Watch failed: unknown (get endpoints)
-```
-
-Additionally ensure that the vip_interface matches the correct interface from `ip addr`

docs/kubernetes/bgp/index.md

@@ -1,87 +0,0 @@
-# Kube-vip (Layer 3 / BGP)
-
-## Deploy `kube-vip`
-
-To deploy the [latest] then `kubectl apply -f https://kube-vip.io/manifests/kube-vip.yaml`, specific versions should be found in the repository as detailed below:
-
-From the GitHub repository [https://github.com/kube-vip/kube-vip/tree/master/example/deploy](https://github.com/kube-vip/kube-vip/tree/master/example/deploy) find the version of the `kube-vip` to deploy (although typically the highest version number will provider more functionality/stability). The [raw] option in Github will provide the url that can be applied directly with a `kubectl apply -f <url>`.
-
-The following output should appear when the manifest is applied: 
-```
-serviceaccount/vip created
-role.rbac.authorization.k8s.io/vip-role created
-rolebinding.rbac.authorization.k8s.io/vip-role-bind created
-deployment.apps/kube-vip-cluster created
-```
-
-*NOTE* The manifest for the `kube-vip` deployment has rules to ensure affinity (pods are always distributed to different nodes for HA). By default the replicas are set to `3` in the event you have less than `3` worker nodes then those replicas will sit as `pending`. This in itself isn't an issue, it means when new workers are added then they will be scheduled. *However*, tooling such as `kapps` will inspect the manifest before it's applied an error because of issues such as this.
-
-### Editing `kube-vip` configuration
-
-Either download and edit the manifest locally or apply as above and edit the deployment with `kubectl edit deploy/kube-vip-cluster` (change namespace where appropriate `-n`)
-
-Ensure the `vip_arp` isn't enabled as ARP and BGP can't be used at the same time (today), also that the `vip_interface` is set to localhost (`lo`).
-
-```
-        - name: vip_interface
-		  value: "lo"
-		- name: vip_configmap
-		  value: "plndr" 
-		- name: bgp_enable
-		  value: "true"
-		- name: vip_loglevel
-		  value: "5"
-```
-
-### BGP Specific configuration
-
-Additionally for BGP we'll need some configuration details, your local friendly network admin should be able to help here:
-
-```
-		- name: bgp_routerid
-		  value: "192.168.0.45"
-		- name: bgp_as
-          value: "65000" 
-		- name: bgp_peeraddress
-    	  value: "10.0.0.1"
-		- name: bgp_peeras
-    	  value: "65522"
-```
-
-### BGP on Packet
-
-If you're lucky enough to be running services on Packet then The above BGP information can be found from the API, instead of specifying the above we need to use the following:
-
-```
-		- name: vip_packet
-		  value: "true"
-		- name: vip_packetproject
-		  value: "My Project" 
-		- name: PACKET_AUTH_TOKEN
-		  value: "XXYZZYVVY"
-```
-
-With the above configuration in place, all `kube-vip` pods will start in active mode and when a service is exposed then all nodes will advertise the VIP to the routers.
-
-## Expose a service
-
-Given that `kube-vip` doesn't know your network (at this point) ask your local friendly network OPs for an address you can advertise. That is the address you can expose to the outside world as shown below!
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```
-
-## Expose with packet 
-
-Either through the CLI or through the UI, create a public IPv4 EIP address.. and this is the address you can expose through BGP!
-
-```
-# packet ip request -p xxx-bbb-ccc -f ams1 -q 1 -t public_ipv4                                                                   
-+-------+---------------+--------+----------------------+
-|   ID  |    ADDRESS    | PUBLIC |       CREATED        |
-+-------+---------------+--------+----------------------+
-| xxxxx |     1.1.1.1   | true   | 2020-11-10T15:57:39Z |
-+-------+---------------+--------+----------------------+
-
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```

docs/kubernetes/index.md

@@ -1,75 +0,0 @@
-# Usage
-
-The below instructions *should just work* on Kubernetes regardless of architecture, Linux as the Operating System is the only requirement.
-
-## The `tl;dr` guide
-
-If you just want things to "work", then you can quickly install the "latest" components:
-
-**NOTE** the `kube-vip.yaml` may need customising to set ARP/BGP OR to configure which interface to bind VIPs too.
-
-**Install the `plndr-cloud-provider`, and `kube-vip`**
-
-```
-kubectl apply -f https://kube-vip.io/manifests/controller.yaml
-kubectl apply -f https://kube-vip.io/manifests/kube-vip.yaml
-```
-
-**Create the `cidr` for the `global` namespace**
-
-```
-kubectl create configmap --namespace kube-system plndr --from-literal cidr-global=192.168.0.200/29
-```
-
-Creating services of `type: LoadBalancer` in the default namespace will now take addresses from the **global** cidr defined in the `configmap`.
-
-**Additional namespaces**
-
-Edit the `configmap` and add in the cidr ranges for those namespaces, the key in the cidr should be `cidr-<namespace>`, then ensure that `kube-vip` is deployed into that namespace with the above `apply` command with the `-n namespace` flag.
-
-## The Detailed guide
-
-### Deploy the `plndr-cloud-provider`
-
-To deploy the [latest] then `kubectl apply -f https://kube-vip.io/manifests/controller.yaml`, specific versions should be found in the repository as detailed below:
-
-From the GitHub repository [https://github.com/kube-vip/plndr-cloud-provider/tree/master/example/pod](https://github.com/kube-vip/plndr-cloud-provider/tree/master/example/pod), find the version of the plunder cloud provider manifest (although typically the highest version number will provider more functionality/stability). The [raw] option in Github will provide the url that can be applied directly with a `kubectl apply -f <url>`.
-
-The following output should appear when the manifest is applied: 
-
-```
-serviceaccount/plunder-cloud-controller created
-clusterrole.rbac.authorization.k8s.io/system:plunder-cloud-controller-role created
-clusterrolebinding.rbac.authorization.k8s.io/system:plunder-cloud-controller-binding created
-pod/plndr-cloud-provider created
-```
-
-We can validate the cloud-provider by examining the pods:
-
-`kubectl logs -n kube-system plndr-cloud-provider-0 -f`
-
-#### The `plndr-cloud-provider` `configmap`
-
-The `configmap` details a CIDR range *per* namespace, however as of (`kube-vip 0.2.1` and `plnder-cloud-provider 0.1.4`), there is now the option of having a **global** CIDR range (`cidr-global)`.  
-
-To manage the ranges for the load-balancer instances, the `plndr-cloud-provider` has a `configmap` held in the `kube-system` namespace. The structure for the key/values within the `configmap` should be that the key is in the format `cidr-<namespace>` and the value should be the cidr range.
-
-Example Configmap:
-
-```
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: plndr
-  namespace: kube-system
-data:
-  cidr-default: 192.168.0.200/29
-  cidr-global: 192.168.0.210/29
-```
-
-### Deploying `kube-vip`
-
-To use `kube-vip` in Layer2/ARP the follow this [guide](/kubernetes/arp/)
-
-To use `kube-vip` in Layer3/BGP the follow this [guide](/kubernetes/bgp/)
-

docs/manifests/0.3.1/kube-vip-arp-ds.yaml

@@ -1,63 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "true"
-        - name: vip_interface
-          value: eth0
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: vip_address
-          value: 192.168.0.1
-        image: plndr/kube-vip:0.3.1
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/0.3.1/kube-vip-bgp-ds.yaml

@@ -1,72 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: eth0
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: vip_address
-          value: 192.168.0.1
-        image: plndr/kube-vip:0.3.1
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/0.3.1/kube-vip-bgp-em-ds.yaml

@@ -1,82 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: lo
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "false"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: provider_config
-          value: /etc/cloud-sa/cloud-sa.json
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: vip_address
-          value: 192.168.0.1
-        image: plndr/kube-vip:0.3.1
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-        volumeMounts:
-        - mountPath: /etc/cloud-sa
-          name: cloud-sa-volume
-          readOnly: true
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      volumes:
-      - name: cloud-sa-volume
-        secret:
-          secretName: packet-cloud-config
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/0.3.1/kube-vip-bgp.yaml

@@ -1,62 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - manager
-    env:
-    - name: vip_arp
-      value: "false"
-    - name: vip_interface
-      value: eth0
-    - name: port
-      value: "6443"
-    - name: vip_cidr
-      value: "32"
-    - name: cp_enable
-      value: "true"
-    - name: cp_namespace
-      value: kube-system
-    - name: svc_enable
-      value: "true"
-    - name: vip_startleader
-      value: "false"
-    - name: vip_addpeerstolb
-      value: "true"
-    - name: vip_localpeer
-      value: code:192.168.0.22:10000
-    - name: bgp_enable
-      value: "true"
-    - name: bgp_routerid
-    - name: bgp_as
-      value: "65000"
-    - name: bgp_peeraddress
-    - name: bgp_peerpass
-    - name: bgp_peeras
-      value: "65000"
-    - name: vip_address
-      value: 192.168.0.1
-    image: plndr/kube-vip:0.3.1
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_RAW
-        - SYS_TIME
-    volumeMounts:
-    - mountPath: /etc/kubernetes/admin.conf
-      name: kubeconfig
-  hostNetwork: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/admin.conf
-    name: kubeconfig
-status: {}
-

docs/manifests/controller.yaml

@@ -1,70 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip-cloud-controller
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-cloud-controller-role
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "create", "update", "list", "put"]
-  - apiGroups: [""]
-    resources: ["configmaps", "endpoints","events","services/status", "leases"]
-    verbs: ["*"]
-  - apiGroups: [""]
-    resources: ["nodes", "services"]
-    verbs: ["list","get","watch","update"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-cloud-controller-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-cloud-controller-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip-cloud-controller
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: kube-vip-cloud-provider
-  namespace: kube-system
-spec:
-  serviceName: kube-vip-cloud-provider
-  podManagementPolicy: OrderedReady
-  replicas: 1
-  revisionHistoryLimit: 10
-  selector:
-    matchLabels:
-      app: kube-vip
-      component: kube-vip-cloud-provider
-  template:
-    metadata:
-      labels:
-        app: kube-vip
-        component: kube-vip-cloud-provider
-    spec:
-      containers:
-      - command:
-        - /kube-vip-cloud-provider
-        - --leader-elect-resource-name=kube-vip-cloud-controller
-        image: kubevip/kube-vip-cloud-provider:latest
-        name: kube-vip-cloud-provider
-        imagePullPolicy: Always
-        resources: {}
-      dnsPolicy: ClusterFirst
-      restartPolicy: Always
-      schedulerName: default-scheduler
-      securityContext: {}
-      terminationGracePeriodSeconds: 30
-      serviceAccountName: kube-vip-cloud-controller

docs/manifests/kube-vip-arp.yaml

@@ -1,85 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "create", "update", "list", "put"]
-  - apiGroups: [""]
-    resources: ["services"]
-    verbs: ["list","get","watch", "update"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-workers
-  name: kube-vip-workers
-  namespace: kube-system
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-workers
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-workers
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-workers
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: plndr/kube-vip:0.2.2
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        env:
-          - name: vip_interface
-            value: "ens160"
-          - name: vip_arp
-            value: "true"
-          - name: vip_loglevel
-            value: "5"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-      serviceAccountName: kube-vip
-status: {}

docs/manifests/kube-vip-bgp.yaml

@@ -1,90 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: [""]
-    resources: ["services"]
-    verbs: ["list","get","watch", "update"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-workers
-  name: kube-vip-workers
-  namespace: kube-system
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-workers
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-workers
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-workers
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: plndr/kube-vip:0.2.2
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        env:
-          - name: vip_interface
-            value: "lo"
-          - name: bgp_enable
-            value: "true"
-          - name: vip_loglevel
-            value: "5"
-          - name: bgp_routerinterface
-            value: "ens160"
-          - name: bgp_as
-            value: "64512"
-          - name: bgp_peeraddress
-            value: "192.168.0.1"
-          - name: bgp_peeras
-            value: "64512"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-      serviceAccountName: kube-vip
-status: {}

docs/manifests/kube-vip-em.yaml

@@ -1,91 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: [""]
-    resources: ["services", "services/status"]
-    verbs: ["list","get","watch", "update"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: lo
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: svc_enable
-          value: "true"
-        - name: provider_config
-          value: /etc/cloud-sa/cloud-sa.json
-        - name: vip_packet
-          value: "true"
-        - name: bgp_enable
-          value: "true"
-        image: plndr/kube-vip:0.2.3
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - SYS_TIME
-        volumeMounts:
-        - mountPath: /etc/cloud-sa
-          name: cloud-sa-volume
-          readOnly: true
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      volumes:
-      - name: cloud-sa-volume
-        secret:
-          secretName: packet-cloud-config
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0

docs/manifests/kube-vip.yaml

@@ -1,83 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: vip
----
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: vip-role
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "create", "update", "list", "put"]
-  - apiGroups: [""]
-    resources: ["configmaps", "endpoints"]
-    verbs: ["watch", "get"]
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: vip-role-bind
-subjects:
-  - kind: ServiceAccount
-    name: vip 
-    apiGroup: ""
-roleRef:
-  kind: Role
-  name: vip-role
-  apiGroup: "" 
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-cluster
-  name: kube-vip-cluster
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-cluster
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-cluster
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-cluster
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: plndr/kube-vip:0.1.3
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        env:
-          - name: vip_interface
-            value: "ens192"
-          - name: vip_configmap
-            value: "plndr" 
-          - name: vip_arp
-            value: "true"
-          - name: vip_loglevel
-            value: "5"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-      serviceAccountName: vip
-status: {}

docs/manifests/rbac.yaml

@@ -1,32 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: [""]
-    resources: ["services", "services/status", "nodes"]
-    verbs: ["list","get","watch", "update"]
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["list", "get", "watch", "update", "create"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system

docs/publish.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-echo Deploying updated documentation
-
-# Sleep for dramatic purposes!
-
-sleep 5
-
-generate-md --layout github --input ./ --output /var/www/kube-vip/

docs/usage/EquinixMetal/index.md

@@ -1,114 +0,0 @@
-# Equinix Metal Overview (using the [Equinix Metal CCM](https://github.com/packethost/packet-ccm))
-
-## BGP with Equinix Metal
-
-When deploying Kubernetes with Equinix Metal with the `--controlplane` functionality we need to pre-populate the BGP configuration in order for the control plane to be advertised and work in a HA scenario. Luckily Equinix Metal provides the capability to "look up" the configuration details (for BGP) that we need in order to advertise our virtual IP for HA functionality. We can either make use of the [Equinix Metal API](https://metal.equinix.com/developers/api/) or we can parse the [Equinix Metal Metadata service](https://metal.equinix.com/developers/docs/servers/metadata/).
-
-**Note** If this cluster will be making use of Equinix Metal for `type:LoadBalancer` (by using the [Equinix Metal CCM](https://github.com/packethost/packet-ccm)) then we will need to ensure that nodes are set to use an external cloud-provider. Before doing a `kubeadm init|join` ensure the kubelet has the correct flags by using the following command `echo KUBELET_EXTRA_ARGS=\"--cloud-provider=external\" > /etc/default/kubelet`.
-
-## Creating HA clusters in Equinix Metal
-
-### Creating a manifest using the API
-
-We can enable `kube-vip` with the capability to discover the required configuration for BGP by passing the `--metal` flag and the API Key and our project ID.
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE\
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --metal \
-    --metalKey xxxxxxx \
-    --metalProjectID xxxxx | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-### Creating a manifest using the metadata
-
-We can parse the metadata, *however* it requires that the tools `curl` and `jq` are installed. 
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE\
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --peerAS $(curl https://metadata.platformequinix.com/metadata | jq '.bgp_neighbors[0].peer_as') \
-    --peerAddress $(curl https://metadata.platformequinix.com/metadata | jq -r '.bgp_neighbors[0].peer_ips[0]') \
-    --localAS $(curl https://metadata.platformequinix.com/metadata | jq '.bgp_neighbors[0].customer_as') \
-    --bgpRouterID $(curl https://metadata.platformequinix.com/metadata | jq -r '.bgp_neighbors[0].customer_ip') | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-## Load Balancing servies on Equinix Metal
-
-Below are two examples for running `type:LoadBalancer` services on worker nodes only and will create a daemonset that will run `kube-vip`. 
-
-**NOTE** This use-case requires the [Equinix Metal CCM](https://github.com/packethost/packet-ccm) to be installed and that the cluster/kubelet is configured to use an "external" cloud provider.
-
-### Using Annotations
-
-This is important as the CCM will apply the BGP configuration to the [node annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) making it easy for `kube-vip` to find the networking configuration it needs to expose load balancer addresses. The `--annotations metal.equinix.com` will cause kube-vip to "watch" the annotations of the worker node that it is running on, once all of the configuarion has been applied by the CCM then the `kube-vip` pod is ready to advertise BGP addresses for the service.
-
-```
-kube-vip manifest daemonset \
-  --interface $INTERFACE \
-  --services \
-  --bgp \
-  --annotations metal.equinix.com \
-  --inCluster | k apply -f -
-```
-
-### Using the existing CCM secret 
-
-Alternatively it is possible to create a daemonset that will use the existing CCM secret to do an API lookup, this will allow for discovering the networking configuration needed to advertise loadbalancer addresses through BGP.
-
-```
-kube-vip manifest daemonset --interface $INTERFACE \
---services \
---inCluster \
---bgp \
---metal \
---provider-config /etc/cloud-sa/cloud-sa.json | kubectl apply -f -
-```
-
-### Expose with Equinix Metal (using the `kube-vip-cloud-provider`)
-
-Either through the CLI or through the UI, create a public IPv4 EIP address.. and this is the address you can expose through BGP!
-
-```
-# packet ip request -p xxx-bbb-ccc -f ams1 -q 1 -t public_ipv4                                                                   
-+-------+---------------+--------+----------------------+
-|   ID  |    ADDRESS    | PUBLIC |       CREATED        |
-+-------+---------------+--------+----------------------+
-| xxxxx |     1.1.1.1   | true   | 2020-11-10T15:57:39Z |
-+-------+---------------+--------+----------------------+
-
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```
-
-## Troubleshooting
-
-If `kube-vip` has been sat waiting for a long time then you may need to investigate that the annotations have been applied correctly by doing running the `describe` on the node:
-
-```
-kubectl describe node k8s.bgp02
-...
-Annotations:        kubeadm.alpha.kubernetes.io/cri-socket: /var/run/dockershim.sock
-                    node.alpha.kubernetes.io/ttl: 0
-                    metal.equinix.com/node-asn: 65000
-                    metal.equinix.com/peer-asn: 65530
-                    metal.equinix.com/peer-ip: x.x.x.x
-                    metal.equinix.com/src-ip: x.x.x.x
-```
-
-If there are errors regarding `169.254.255.1` or `169.254.255.2` in the `kube-vip` logs then the routes to the ToR switches that provide BGP peering may by missing from the nodes. They can be replaced with the below command:
-
-```
-GATEWAY_IP=$(curl https://metadata.platformequinix.com/metadata | jq -r ".network.addresses[] | select(.public == false) | .gateway")
-ip route add 169.254.255.1 via $GATEWAY_IP
-ip route add 169.254.255.2 via $GATEWAY_IP
-```
-
-Additionally examining the logs of the Packet CCM may reveal why the node is not yet ready.

docs/usage/k3s/index.md

@@ -1,82 +0,0 @@
-# K3s overview (on Equinix Metal)
-
-## Optional Tidy environment (best if something was running before)
-```
-rm -rf /var/lib/rancher /etc/rancher ~/.kube/*; \ 
-ip addr flush dev lo; \
-ip addr add 127.0.0.1/8 dev lo; 
-```
-
-## Step 1: Create Manifests folder
-
-This is required, this folder will contain all of the generated manifests that `k3s` will execute as it starts. We will create it before `k3s` and place our `kube-vip` manifests within it.
-
-```
-mkdir -p /var/lib/rancher/k3s/server/manifests/
-```
-
-## Step 2: Get rbac for `Kube-Vip`
-
-As `kube-vip` runs inside of the Kubernetes cluster, we will need to ensure that the required permissions exist.
-
-```
-curl https://kube-vip.io/manifests/rbac.yaml > /var/lib/rancher/k3s/server/manifests/rbac.yaml
-```
-
-## Step 3: Generate kube-vip (A VIP address for the network will be required)
-
-Configure your virtual IP (for the control plane) and interface that will expose this VIP first.
-
-```
-export VIP=x.x.x.x
-export INTERFACE=ethx
-```
-
-Modify the `VIP` and `INTERFACE` to match the floating IP address you'd like to use and the interface it should bind to.
-
-To generate the manifest we have two options! We can generate the manifest from [kube-vip.io](kube-vip.io) or use a kube-vip image to generate the manifest!
-
-## Step 3.1: Generate from kube-vip.io
- 
-```
-curl -sL kube-vip.io/k3s | vipAddress=$VIP vipInterface=$INTERFACE sh | sudo tee /var/lib/rancher/k3s/server/manifests/vip.yaml
-```
-
-## Step 3.2 Genereate from container image
-
-### containerd
-`alias kube-vip="ctr run --rm --net-host docker.io/plndr/kube-vip:0.3.1 vip /kube-vip"`
-
-### Docker
-`alias kube-vip="docker run --network host --rm plndr/kube-vip:0.3.1"`
-
-
-```
-kube-vip manifest daemonset \
-  --interface $INTERFACE \
-  --vip $VIP \
-  --controlplane \
-  --services \
-  --inCluster \
-  --taint \
-  --arp 
-```
-
-## Step 4: Up Cluster
-
-From online `-->`
-
-```
-curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--write-kubeconfig-mode 644 \
--t agent-secret --tls-san $VIP" sh -
-```
-
-From local `-->`
-
-```
-sudo ./k3s server --tls-san $VIP
-```
-
-## Step 5: Service Load-Balancing
-
-For this refer to the [on-prem](../on-prem) documentation 

docs/usage/kind/index.md

@@ -1,57 +0,0 @@
-# Kube-vip on KIND
-
-## Deploying KIND
-
-The documentation for KIND is fantastic and it's quickstart guide will have you up and running in no time -> [https://kind.sigs.k8s.io/docs/user/quick-start/](https://kind.sigs.k8s.io/docs/user/quick-start/)
-
-## Find Address Pool for Kube-Vip
-
-We will need to find addresses that can be used by Kube-Vip:
-
-```
-docker network inspect kind -f '{{ range $i, $a := .IPAM.Config }}{{ println .Subnet }}{{ end }}'
-```
-
-This will return a cidr range such as `172.18.0.0/16` and from here we can select a range.
-
-## Deploy the Kube-Vip Cloud Controller
-
-```
-$ kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
-```
-
-## Add our Address range
-
-```
-kubectl create configmap --namespace kube-system kubevip --from-literal range-global=172.18.100.10-172.18.100.30
-```
-
-## Install kube-vip
-
-Set the correct `alias` for `kube-vip`
-```
-alias kube-vip="docker run --network host --rm plndr/kube-vip:v0.3.5"
-```
-
-Install Kube-vip deamonset inside of KIND
-
-```
-kube-vip manifest daemonset --services --inCluster --arp --interface eth0 | ./kubectl apply -f -
-```
-
-## Test
-
-```
-kubectl apply -f https://k8s.io/examples/application/deployment.yaml
-```
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx
-```
-
-```
-kubectl get svc
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        74m
-nginx        LoadBalancer   10.96.196.235   172.18.100.11   80:31236/TCP   6s
-```

docs/usage/on-prem/index.md

@@ -1,222 +0,0 @@
-# Kube-vip on-prem 
-
-We've designed `kube-vip` to be as de-coupled or agnostic from other components that may exist within a Kubernetes cluster as possible. This has lead to `kube-vip` having a very simplistic but robust approach to advertising Kubernetes services to the outside world and marking these services as ready to use.
-
-## Flow
-
-This section details the flow of events in order for `kube-vip` to advertise a Kubernetes service:
-
-1. An end user exposes a application through Kubernetes as a LoadBalancer => `kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx`
-2. Within the Kubernetes cluster a service object is created with the `svc.Spec.Type = ServiceTypeLoadBalancer`
-3. A controller (typically a Cloud Controller) has a loop that "watches" for services of the type `LoadBalancer`.
-4. The controller now has the responsibility of providing an IP address for this service along with doing anything that is network specific for the environment where the cluster is running.
-5. Once the controller has an IP address it will update the service `svc.Spec.LoadBalancerIP` with it's new IP address.
-6. The `kube-vip` pods also implement a "watcher" for services that have a `svc.Spec.LoadBalancerIP` address attached.
-7. When a new service appears `kube-vip` will start advertising this address to the wider network (through BGP/ARP) which will allow traffic to come into the cluster and hit the service network.
-8. Finally `kube-vip` will update the service status so that the API reflects that this LoadBalancer is ready. This is done by updating the `svc.Status.LoadBalancer.Ingress` with the VIP address.
-
-## CCM
-
-We can see from the [flow](#Flow) above that `kube-vip` isn't coupled to anything other than the Kubernetes API, and will only act upon an existing Kubernetes primative (in this case the object of type `Service`). This makes it easy for existing CCMs to simply apply their logic to services of type LoadBalancer and leave `kube-vip` to take the next steps to advertise these load-balancers to the outside world. 
-
-
-## Using the Kube-vip Cloud Provider
-
-The below instructions *should just work* on Kubernetes regardless of architecture (Linux Operating System is the only requirement) - you can quickly install the "latest" components:
-
-**Install the `kube-vip-cloud-provider`**
-
-```
-$ kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
-```
-
-It uses a `statefulSet` and can always be viewed with the following command:
-
-```
-kubectl describe pods -n kube-system kube-vip-cloud-provider-0
-```
-
-**Create a global CIDR or IP Range**
-
-Any `service` in any `namespace` can use an address from the global CIDR `cidr-global` or range `range-global`
-
-```
-kubectl create configmap --namespace kube-system kubevip --from-literal cidr-global=192.168.0.220/29
-```
-or
-```
-kubectl create configmap --namespace kube-system kubevip --from-literal range-global=192.168.1.220-192.168.1.230
-```
-
-Creating services of `type: LoadBalancer` in *any namespace* will now take addresses from the **global** cidr defined in the `configmap` unless a specific 
-
-
-## The Detailed guide
-
-### Deploy the Kube-vip Cloud Provider
-
-**Install the `kube-vip-cloud-provider`**
-
-```
-$ kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
-```
-
-The following output should appear when the manifest is applied: 
-
-```
-serviceaccount/kube-vip-cloud-controller created
-clusterrole.rbac.authorization.k8s.io/system:kube-vip-cloud-controller-role created
-clusterrolebinding.rbac.authorization.k8s.io/system:kube-vip-cloud-controller-binding created
-statefulset.apps/kube-vip-cloud-provider created
-```
-
-We can validate the cloud provider by examining the pods and following the logs:
-
-```
-kubectl describe pods -n kube-system kube-vip-cloud-provider-0
-kubectl logs -n kube-system kube-vip-cloud-provider-0 -f
-```
-
-### The Kube-vip Cloud Provider `configmap`
-
-To manage the IP address ranges for the load balancer instances the `kube-vip-cloud-provider` uses a `configmap` held in the `kube-system` namespace. IP address ranges can be configured using:
-- IP address pools by CIDR
-- IP ranges [start address - end address]
-- Multiple pools by CIDR per namespace
-- Multiple IP ranges per namespace (handles overlapping ranges)
-- Setting of static addresses through --load-balancer-ip=x.x.x.x
-
-To control which IP address range is used for which service the following rules are applied:
-- Global address pools (`cidr-global` or `range-global`) are available for use by *any* `service` in *any* `namespace`
-- Namespace specific address pools (`cidr-<namespace>` or `range-<namespace>`) are *only* available for use by `service` in the *specific* `namespace`
-- Static IP addresses can be applied to a load balancer `service` using the `loadbalancerIP` setting, even outside of the assigned ranges
-
-Example Configmap:
-
-```
-$ kubectl get configmap -n kube-system kubevip -o yaml
-
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kubevip
-  namespace: kube-system
-data:
-  cidr-default: 192.168.0.200/29                      # CIDR-based IP range for use in the default namespace
-  range-development: 192.168.0.210-192.168.0.219      # Range-based IP range for use in the development namespace
-  cidr-finance: 192.168.0.220/29,192.168.0.230/29     # Multiple CIDR-based ranges for use in the finance namespace
-  cidr-global: 192.168.0.240/29                       # CIDR-based range which can be used in any namespace
-```
-
-### Expose a service
-
-We can now expose a service and once the cloud provider has provided an address `kube-vip` will start to advertise that address to the outside world as shown below!
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx
-```
-
-or via a `service` YAML definition
-
-```
-apiVersion: v1
-kind: Service
-metadata:
-  name: nginx
-spec:
-  ports:
-  - name: http
-    port: 80
-    protocol: TCP
-  selector:
-    app: nginx
-  type: LoadBalancer
-  ```
-
-
-We can also expose a specific address by specifying it on the command line:
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```
-
-or including it in the `service` definition:
-
-```
-apiVersion: v1
-kind: Service
-metadata:
-  name: nginx
-spec:
-  ports:
-  - name: http
-    port: 80
-    protocol: TCP
-  selector:
-    app: nginx
-  type: LoadBalancer
-  loadBalancerIP: "1.1.1.1"
-```
-
-### Using DHCP for Load Balancers (experimental)
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to use the local network DHCP server to provide `kube-vip` with a load-balancer address that can be used to access a
- Kubernetes service on the network. 
-
-In order to do this we need to signify to `kube-vip` and the cloud-provider that we don't need one of their managed addresses. We do this by explicitly exposing a service on the 
-address `0.0.0.0`. When `kube-vip` sees a service on this address it will create a `macvlan` interface on the host and request a DHCP address, once this address is provided it will assign it as the VIP and update the Kubernetes service!
-
-```
-$ k expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx-dhcp --load-balancer-ip=0.0.0.0; k get svc
-service/nginx-dhcp exposed
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   0.0.0.0         80:31184/TCP   0s
-
-{ ... a second or so later ... }
-
-$ k get svc
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   192.168.0.155   80:31184/TCP   3s
-```
-
-### Using UPNP to expose a service to the outside world
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to expose a load-balancer on a specific port and using UPNP (on a supported gateway) expose this service to the inte
-rnet.
-
-Most simple networks look something like the following:
-
-`<----- <internal network 192.168.0.0/24> <Gateway / router> <external network address> ----> Internet`
-
-Using UPNP we can create a matching port on the `<external network address>` allowing your service to be exposed to the internet.
-
-#### Enable UPNP
-
-Add the following to the `kube-vip` `env:` section, and the rest should be completely automated. 
-
-**Note** some environments may require (Unifi) will require `Secure mode` being `disabled` (this allows a host with a different address to register a port)
-
-```
-- name: enableUPNP
-  value: "true"
-```
-
-#### Exposing a service
-
-To expose a port successfully we'll need to change the command slightly:
-
-`--target-port=80` the port of the application in the pods (HTT/NGINX)
-`--port=32380` the port the service will be exposed on (and what you should connect to in order to receive traffic from the service)
-
-`kubectl expose deployment plunder-nginx --port=32380 --target-port=80 --type=LoadBalancer --namespace plunder`
-
-The above example should expose a port on your external (internet facing address), that can be tested externally with:
-
-```
-$ curl externalIP:32380
-<!DOCTYPE html>
-<html>
-...
-```

example/deploy/0.1.2.yaml

@@ -28,7 +28,7 @@ spec:
                   - kube-vip-cluster
             topologyKey: "kubernetes.io/hostname"
       containers:
-      - image: plndr/kube-vip:0.1.2
+      - image: ghcr.io/kube-vip/kube-vip:0.3.7
         imagePullPolicy: Always
         name: kube-vip
         command:

example/deploy/0.1.3.yaml

@@ -58,7 +58,7 @@ spec:
                   - kube-vip-cluster
             topologyKey: "kubernetes.io/hostname"
       containers:
-      - image: plndr/kube-vip:0.1.3
+      - image: ghcr.io/kube-vip/kube-vip:0.3.7
         imagePullPolicy: Always
         name: kube-vip
         command:

example/deployment.yaml

@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:latest
+        ports:
+        - containerPort: 80

example/service-interface-annotation.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-interface-ens192-service
+  annotations:
+    kube-vip.io/serviceInterface: ens192
+spec:
+  selector:
+    app: nginx
+  type: LoadBalancer
+  ports:
+  - name: http
+    port: 80
+    targetPort: 80
+    protocol: TCP

example/service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-service
+spec:
+  selector:
+    app: nginx
+  type: LoadBalancer
+  ports:
+  - name: http
+    port: 80
+    targetPort: 80
+    protocol: TCP

go.mod

@@ -1,66 +1,165 @@
 module github.com/kube-vip/kube-vip
 
-go 1.14
+go 1.24.4
 
 require (
-	github.com/armon/go-metrics v0.3.8 // indirect
-	github.com/davecgh/go-spew v1.1.1
-	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
-	github.com/fatih/color v1.10.0 // indirect
-	github.com/ghodss/yaml v1.0.0
-	github.com/golang/protobuf v1.5.2
-	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/uuid v1.2.0 // indirect
-	github.com/googleapis/gnostic v0.5.5 // indirect
-	github.com/hashicorp/go-hclog v0.16.0 // indirect
-	github.com/hashicorp/go-immutable-radix v1.3.0 // indirect
-	github.com/hashicorp/go-msgpack v1.1.5 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
-	github.com/hashicorp/raft v1.3.1
-	github.com/imdario/mergo v0.3.12 // indirect
-	github.com/insomniacslk/dhcp v0.0.0-20210621130208-1cac67f12b1e
+	github.com/cloudflare/ipvs v0.11.0
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
+	github.com/docker/docker v28.3.3+incompatible
+	github.com/florianl/go-conntrack v0.4.0
+	github.com/google/go-cmp v0.7.0
+	github.com/google/nftables v0.3.0
+	github.com/gookit/slog v0.5.8
+	github.com/huin/goupnp v1.3.0
+	github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d
 	github.com/jpillora/backoff v1.0.0
-	github.com/json-iterator/go v1.1.11 // indirect
-	github.com/k-sone/critbitgo v1.4.0 // indirect
-	github.com/kamhlos/upnp v0.0.0-20210324072331-5661950dff08
-	github.com/magiconair/properties v1.8.5 // indirect
-	github.com/mdlayher/ndp v0.0.0-20200602162440-17ab9e3e5567
-	github.com/mdlayher/raw v0.0.0-20210412142147-51b895745faf // indirect
-	github.com/mitchellh/mapstructure v1.4.1 // indirect
-	github.com/onsi/ginkgo v1.11.0
-	github.com/onsi/gomega v1.7.0
-	github.com/osrg/gobgp v2.0.0+incompatible
-	github.com/packethost/packngo v0.13.0
-	github.com/pelletier/go-toml v1.9.0 // indirect
+	github.com/mdlayher/ndp v1.1.0
+	github.com/onsi/ginkgo/v2 v2.23.4
+	github.com/onsi/gomega v1.37.0
+	github.com/osrg/gobgp/v3 v3.37.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.10.0
-	github.com/prometheus/common v0.23.0 // indirect
-	github.com/sirupsen/logrus v1.8.1
-	github.com/spf13/afero v1.6.0 // indirect
-	github.com/spf13/cast v1.3.1 // indirect
-	github.com/spf13/cobra v1.1.3
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/viper v1.7.1 // indirect
-	github.com/vishvananda/netlink v1.1.1-0.20200221165523-c79a4b7b4066
-	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
-	golang.org/x/crypto v0.0.0-20210503195802-e9a32991a82e // indirect
-	golang.org/x/net v0.0.0-20210505024714-0287a6fb4125
-	golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c // indirect
-	golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea
-	golang.org/x/term v0.0.0-20210503060354-a79de5458b56 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20210505142820-a42aa055cf76 // indirect
-	google.golang.org/grpc v1.37.0 // indirect
-	gopkg.in/ini.v1 v1.62.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/api v0.21.0
-	k8s.io/apimachinery v0.21.0
-	k8s.io/client-go v0.21.0
-	k8s.io/klog v1.0.0
-	k8s.io/klog/v2 v2.8.0
-	k8s.io/utils v0.0.0-20210305010621-2afb4311ab10 // indirect
-	sigs.k8s.io/kind v0.10.0
-	sigs.k8s.io/structured-merge-diff/v4 v4.1.1 // indirect
+	github.com/prometheus/client_golang v1.22.0
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cobra v1.9.1
+	github.com/stretchr/testify v1.10.0
+	github.com/vishvananda/netlink v1.3.1
+	go.etcd.io/etcd/api/v3 v3.6.1
+	go.etcd.io/etcd/client/pkg/v3 v3.6.1
+	go.etcd.io/etcd/client/v3 v3.6.1
+	go.uber.org/zap v1.27.0
+	golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
+	golang.org/x/sync v0.15.0
+	golang.org/x/sys v0.33.0
+	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
+	google.golang.org/grpc v1.73.0
+	google.golang.org/protobuf v1.36.6
+	k8s.io/api v0.33.3
+	k8s.io/apimachinery v0.33.3
+	k8s.io/client-go v0.33.3
+	k8s.io/klog/v2 v2.130.1
+	sigs.k8s.io/kind v0.29.0
+	sigs.k8s.io/yaml v1.4.0
 )
 
-replace github.com/osrg/gobgp v2.0.0+incompatible => github.com/osrg/gobgp v0.0.0-20191101114856-a42a1a5f6bf0
+require (
+	al.essio.dev/pkg/shellescape v1.5.1 // indirect
+	github.com/BurntSushi/toml v1.4.0 // indirect
+	github.com/Microsoft/go-winio v0.4.14 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/coreos/go-semver v0.3.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/dlclark/regexp2 v1.11.4 // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/eapache/channels v1.1.0 // indirect
+	github.com/eapache/queue v1.1.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.12.1 // indirect
+	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/gnostic-models v0.6.9 // indirect
+	github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gookit/color v1.5.4 // indirect
+	github.com/gookit/goutil v0.6.18 // indirect
+	github.com/gookit/gsr v0.1.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/josharian/native v1.1.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/k-sone/critbitgo v1.4.0 // indirect
+	github.com/magiconair/properties v1.8.9 // indirect
+	github.com/mailru/easyjson v0.9.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mdlayher/genetlink v1.3.2 // indirect
+	github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect
+	github.com/mdlayher/packet v1.1.2 // indirect
+	github.com/mdlayher/socket v0.5.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/sys/atomicwriter v0.1.0 // indirect
+	github.com/moby/term v0.5.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/pelletier/go-toml v1.9.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.62.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/sagikazarmark/locafero v0.6.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/spf13/viper v1.19.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tj/go-spin v1.1.0 // indirect
+	github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/vishvananda/netns v0.0.5 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xlab/c-for-go v1.3.0 // indirect
+	github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.37.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 // indirect
+	go.opentelemetry.io/otel/metric v1.37.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.37.0 // indirect
+	go.opentelemetry.io/otel/trace v1.37.0 // indirect
+	go.uber.org/automaxprocs v1.6.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/crypto v0.39.0 // indirect
+	golang.org/x/mod v0.25.0 // indirect
+	golang.org/x/net v0.41.0 // indirect
+	golang.org/x/oauth2 v0.30.0 // indirect
+	golang.org/x/term v0.32.0 // indirect
+	golang.org/x/text v0.26.0 // indirect
+	golang.org/x/time v0.9.0 // indirect
+	golang.org/x/tools v0.33.0 // indirect
+	golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
+	k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect
+	modernc.org/cc/v4 v4.24.4 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/opt v0.1.4 // indirect
+	modernc.org/sortutil v1.2.1 // indirect
+	modernc.org/strutil v1.2.1 // indirect
+	modernc.org/token v1.1.0 // indirect
+	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
+)

kubernetes-control-plane.md

@@ -1,147 +0,0 @@
-# Load Balancing a Kubernetes Cluster (Control-Plane)
-
-This document covers all of the details for using `kube-vip` to build a HA Kubernetes cluster
-
-`tl;dr version`
-- Generate/modify first node `kube-vip` config/manifest
-- `init` first node
-- `join` remaining nodes
-- Add remaining config/manifests
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.75 |
-| controlPlane01 | 10.0.0.70 |
-| controlPlane02 | 10.0.0.71 |
-| controlPlane03 | 10.0.0.72 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.17.0.
-
-### Generate the `kube-vip` configuration
-
-Make sure that the config directory exists: `sudo mkdir -p /etc/kube-vip/`, this directory can be any directory however the `hostPath` in the manifest will need modifying to point to the correct path.
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1.5 sample config | sudo tee /etc/kube-vip/config.yaml
-```
-
-### Modify the configuration
-
-**Cluster Configuration**
-Modify the `remotePeers` to point to the correct addresses of the other two nodes, ensure that their `id` is unique otherwise this will confuse the raft algorithm. The `localPeer` should be the configuration of the current node (`controlPlane01`), which is where this instance of the cluster will run. 
-
-As this node will be the first node, it will need to elect itself leader as until this occurs the VIP won’t be activated!
-
-`startAsLeader: true`
-
-**VIP Config**
-We will need to set our VIP address to `192.168.0.75` and to ensure all hosts are updated when the VIP moves we will enable ARP broadcasts `gratuitousARP: true`
-
-**Load Balancer**
-We will configure the load balancer to sit on the standard API-Server port `6443` and we will configure the backends to point to the API-servers that will be configured to run on port `6444`. Also for the Kubernetes Control Plane we will configure the load balancer to be of `type: tcp`.
-
-We can also use `6443` for both the VIP and the API-Servers, in order to do this we need to specify that the api-server is bound to it's local IP. To do this we use the `--apiserver-advertise-address` flag as part of the `init`, this means that we can then bind the same port to the VIP and we wont have a port conflict.
-
-**config.yaml**
-
-`user@controlPlane01:/etc/kube-vip$ cat config.yaml`
-
-...
-
-``` 
-remotePeers:
-- id: server2
-  address: 192.168.0.71
-  port: 10000
-- id: server3
-  address: 192.168.0.72
-  port: 10000
-localPeer:
-  id: server1
-  address: 192.168.0.70
-  port: 10000
-vip: 192.168.0.75
-gratuitousARP: true
-singleNode: false
-startAsLeader: true
-interface: ens192
-loadBalancers:
-- name: Kubernetes Control Plane
-  type: tcp
-  port: 6443
-  bindToVip: true
-  backends:
-  - port: 6444
-    address: 192.168.0.70
-  - port: 6444
-    address: 192.168.0.71
-  - port: 6444
-    address: 192.168.0.72
-```
-
-### First Node
-
-To generate the basic Kubernetes static pod `yaml` configuration:
-
-Make sure that the manifest directory exists: `sudo mkdir -p /etc/kubernetes/manifests/`
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1.5 sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Ensure that `image: plndr/kube-vip:<x>` is modified to point to a specific version (`0.1.5` at the time of writing), refer to [docker hub](https://hub.docker.com/r/plndr/kube-vip/tags) for details. Also ensure that the `hostPath` points to the correct `kube-vip` configuration, if it isn’t the above path.
-
-The **vip** is set to `192.168.0.75` and this first node will elect itself as leader, and as part of the `kubeadm init` it will use the VIP in order to speak back to the initialising api-server.
-
-`sudo kubeadm init --control-plane-endpoint “192.168.0.75:6443” --apiserver-bind-port 6444 --upload-certs --kubernetes-version “v1.17.0”`
-
-Once this node is up and running we will be able to see the control-plane pods, including the `kube-vip` pod:
-
-```
-$ kubectl get pods -A
-NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
-<...>
-kube-system   kube-vip-controlplane01                  1/1     Running   0          10m
-```
-
-### Remaining Nodes
-
-We first will need to create the `kube-vip` configuration that resides in `/etc/kube-vip/config.yaml` or we can regenerate it from scratch using the above example. Ensure that the configuration is almost identical with the `localPeer` and `remotePeers` sections are updated for each node. Finally, ensure that the remaining nodes will behave as standard cluster nodes by setting `startAsLeader: false`.
-
-At this point **DON’T** generate the manifests, this is due to some bizarre `kubeadm/kubelet` behaviour.
-
-```
-  kubeadm join 192.168.0.75:6443 --token <tkn> \
-    --discovery-token-ca-cert-hash sha256:<hash> \
-    --control-plane --certificate-key <key> 
-
-```
-
-**After** this node has been added to the cluster, we can add the manifest to also add this node as a `kube-vip` member. (Adding the manifest afterwards doesn’t interfere with `kubeadm`). 
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1.5 sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Once this node is added we will be able to see that the `kube-vip` pod is up and running as expected:
-
-```
-user@controlPlane01:~$ kubectl get pods -A | grep vip
-kube-system   kube-vip-controlplane01                  1/1     Running             1          16m
-kube-system   kube-vip-controlplane02                  1/1     Running             0          18m
-kube-system   kube-vip-controlplane03                  1/1     Running             0          20m
-
-```
-
-If we look at the logs, we can see that the VIP is running on the second node and we’re waiting for our third node to join the cluster:
-
-```
-$ kubectl logs kube-vip-controlplane02  -n kube-system
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-
-```

main.go

@@ -1,6 +1,8 @@
 package main
 
-import "github.com/kube-vip/kube-vip/cmd"
+import (
+	"github.com/kube-vip/kube-vip/cmd"
+)
 
 // Version is populated from the Makefile and is tied to the release TAG
 var Version string
@@ -9,6 +11,7 @@ var Version string
 var Build string
 
 func main() {
+
 	cmd.Release.Version = Version
 	cmd.Release.Build = Build
 	cmd.Execute()

pkg/arp/arp.go

@@ -0,0 +1,149 @@
+package arp
+
+import (
+	"context"
+	log "log/slog"
+	"sync"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
+
+type Manager struct {
+	instances map[string]*Instance
+	config    *kubevip.Config
+}
+
+type Instance struct {
+	network vip.Network
+	ndp     *vip.NdpResponder
+	mu      sync.Mutex
+	counter int
+}
+
+func NewManager(config *kubevip.Config) *Manager {
+	return &Manager{
+		instances: make(map[string]*Instance),
+		config:    config,
+	}
+}
+
+func NewInstance(network vip.Network, ndp *vip.NdpResponder) *Instance {
+	return &Instance{
+		ndp:     ndp,
+		network: network,
+		counter: 1,
+	}
+}
+
+func (i *Instance) Name() string {
+	return i.network.ARPName()
+}
+
+func (m *Manager) Insert(instance *Instance) {
+	i, ok := m.instances[instance.Name()]
+	if !ok {
+		log.Info("inserting ARP/NDP instance", "name", instance.Name())
+		m.instances[instance.Name()] = instance
+	} else {
+		i.mu.Lock()
+		defer i.mu.Unlock()
+		i.counter++
+	}
+}
+
+func (m *Manager) Remove(instance *Instance) {
+	if i, ok := m.instances[instance.Name()]; ok {
+		i.mu.Lock()
+		defer i.mu.Unlock()
+		if i.counter > 1 {
+			i.counter--
+		} else {
+			log.Info("removing ARP/NDP instance", "name", instance.Name())
+			delete(m.instances, instance.Name())
+		}
+	}
+}
+
+func (m *Manager) Count(name string) int {
+	if i, ok := m.instances[name]; ok {
+		i.mu.Lock()
+		defer i.mu.Unlock()
+		return i.counter
+	}
+	return 0
+}
+
+func (m *Manager) StartAdvertisement(ctx context.Context) {
+	log.Info("Starting ARP/NDP advertisement")
+	for {
+		select {
+		case <-ctx.Done(): // if cancel() execute
+			return
+		default:
+			for _, instance := range m.instances {
+				if instance.counter > 0 {
+					ensureIPAndSendGratuitous(instance)
+				}
+			}
+		}
+		if m.config.ArpBroadcastRate < 500 {
+			log.Error("arp broadcast rate is too low", "rate (ms)", m.config.ArpBroadcastRate, "setting to (ms)", "3000")
+			m.config.ArpBroadcastRate = 3000
+		}
+		time.Sleep(time.Duration(m.config.ArpBroadcastRate) * time.Millisecond)
+	}
+}
+
+// ensureIPAndSendGratuitous - adds IP to the interface if missing, and send
+// either a gratuitous ARP or gratuitous NDP. Re-adds the interface if it is IPv6
+// and in a dadfailed state.
+func ensureIPAndSendGratuitous(instance *Instance) {
+	iface := instance.network.Interface()
+	ipString := instance.network.IP()
+
+	// Check if IP is dadfailed
+	if instance.network.IsDADFAIL() {
+		log.Warn("IP address is in dadfailed state, removing config", "ip", ipString, "interface", iface)
+		deleted, err := instance.network.DeleteIP()
+		if err != nil {
+			log.Warn(err.Error())
+		}
+		if deleted {
+			log.Info("deleted and recreating address", "IP", ipString, "interface", iface)
+			// if _, err := instance.network.AddIP(false); err != nil {
+			// 	log.Error("failed to recreate address", "IP", ipString, "interface", iface)
+			// }
+		}
+	}
+
+	// Ensure the address exists on the interface before attempting to ARP
+	// if instance.network.HasEndpoints() {
+	if added, err := instance.network.AddIP(true); err != nil {
+		log.Warn(err.Error())
+	} else if added {
+		log.Warn("Re-applied the VIP configuration", "ip", ipString, "interface", iface)
+	}
+	// }
+
+	if vip.IsIPv6(ipString) {
+		// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
+		if instance.ndp == nil {
+			log.Error("NDP responder was not created")
+		} else {
+			err := instance.ndp.SendGratuitous(ipString)
+			if err != nil {
+				log.Warn(err.Error())
+			}
+		}
+
+	} else {
+		// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
+		err := vip.ARPSendGratuitous(ipString, iface)
+		if err != nil {
+			log.Warn(err.Error())
+		}
+	}
+
+}

pkg/backend/backend.go

@@ -0,0 +1,88 @@
+package backend
+
+import (
+	"fmt"
+	"time"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/k8s"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+type Entry struct {
+	Addr    string
+	Port    uint16
+	IsLocal bool
+}
+
+type Map map[Entry]bool
+
+func (e *Entry) Check() bool {
+	var client *kubernetes.Clientset
+	var err error
+	var config *rest.Config
+
+	adminConfigPath := "/etc/kubernetes/admin.conf"
+	// TODO: add one more switch case of homeConfigPath if there is such scenario in future
+	// homeConfigPath := filepath.Join(os.Getenv("HOME"), ".kube", "config")
+
+	var k8sAddr string
+	if vip.IsIPv4(e.Addr) {
+		k8sAddr = fmt.Sprintf("%s:%v", e.Addr, e.Port)
+	} else {
+		k8sAddr = fmt.Sprintf("[%s]:%v", e.Addr, e.Port)
+	}
+
+	switch {
+	case utils.FileExists(adminConfigPath):
+		config, err = k8s.NewRestConfig(adminConfigPath, false, k8sAddr)
+		if err != nil {
+			log.Error("create k8s REST config", "path", adminConfigPath, "err", err)
+			return false
+		}
+	default:
+		config, err = k8s.NewRestConfig("", true, k8sAddr)
+		if err != nil {
+			log.Error("create k8s REST config", "err", err)
+			return false
+		}
+	}
+
+	client, err = k8s.NewClientset(config)
+	if err != nil {
+		log.Error("create k8s client", "err", err)
+		return false
+	}
+
+	_, err = client.DiscoveryClient.ServerVersion()
+	if err != nil {
+		log.Error("discover k8s version", "err", err)
+		return false
+	}
+	return true
+}
+
+func Watch(tickAction func(), interval int, stop chan struct{}) {
+	if interval <= 0 {
+		interval = 5
+	}
+
+	ticker := time.NewTicker(time.Second * time.Duration(interval))
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-stop:
+			ticker.Stop()
+			return
+		case <-ticker.C:
+			ticker.Stop()
+			tickAction()
+			ticker.Reset(time.Second * time.Duration(interval))
+		}
+	}
+}

pkg/bgp/hosts.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"net"
 
-	api "github.com/osrg/gobgp/api"
+	api "github.com/osrg/gobgp/v3/api"
 )
 
 // AddHost will update peers of a host

pkg/bgp/peers.go

@@ -7,33 +7,27 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/golang/protobuf/ptypes"
-	"github.com/golang/protobuf/ptypes/any"
-	api "github.com/osrg/gobgp/api"
+	//nolint
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	api "github.com/osrg/gobgp/v3/api"
+	"github.com/osrg/gobgp/v3/pkg/server"
+	"google.golang.org/protobuf/types/known/anypb"
 )
 
-//AddPeer will add peers to the BGP configuration
+// AddPeer will add peers to the BGP configuration
 func (b *Server) AddPeer(peer Peer) (err error) {
-	port := 179
-
-	if t := strings.SplitN(peer.Address, ":", 2); len(t) == 2 {
-		peer.Address = t[0]
-
-		if port, err = strconv.Atoi(t[1]); err != nil {
-			return fmt.Errorf("Unable to parse port '%s' as int: %s", t[1], err)
-		}
-	}
-
 	p := &api.Peer{
 		Conf: &api.PeerConf{
 			NeighborAddress: peer.Address,
-			PeerAs:          peer.AS,
+			PeerAsn:         peer.AS,
 			AuthPassword:    peer.Password,
 		},
 
 		Timers: &api.Timers{
 			Config: &api.TimersConfig{
-				ConnectRetry: 10,
+				ConnectRetry:      10,
+				HoldTime:          b.c.HoldTime,
+				KeepaliveInterval: b.c.KeepaliveInterval,
 			},
 		},
 
@@ -46,95 +40,302 @@ func (b *Server) AddPeer(peer Peer) (err error) {
 		Transport: &api.Transport{
 			MtuDiscovery:  true,
 			RemoteAddress: peer.Address,
-			RemotePort:    uint32(port),
+			RemotePort:    uint32(179),
 		},
 	}
 
-	// if b.c.SourceIP != "" {
-	// 	p.Transport.LocalAddress = b.c.SourceIP
-	// }
-
-	// if b.c.SourceIF != "" {
-	// 	p.Transport.BindInterface = b.c.SourceIF
-	// }
-
-	return b.s.AddPeer(context.Background(), &api.AddPeerRequest{
-		Peer: p,
-	})
-}
-
-func (b *Server) getPath(ip net.IP) *api.Path {
-	var pfxLen uint32 = 32
-	if ip.To4() == nil {
-		if !b.c.IPv6 {
-			return nil
+	if b.c.MpbgpNexthop != "" {
+		p.AfiSafis = []*api.AfiSafi{
+			{
+				Config: &api.AfiSafiConfig{
+					Family: &api.Family{
+						Afi:  api.Family_AFI_IP,
+						Safi: api.Family_SAFI_UNICAST,
+					},
+					Enabled: true,
+				},
+			},
+			{
+				Config: &api.AfiSafiConfig{
+					Family: &api.Family{
+						Afi:  api.Family_AFI_IP6,
+						Safi: api.Family_SAFI_UNICAST,
+					},
+					Enabled: true,
+				},
+			},
 		}
 
-		pfxLen = 128
+		peer.setMpbgpOptions(b.c)
+
+		ipv4Address, ipv6Address, err := peer.findMpbgpAddresses(p, b.c)
+		if err != nil {
+			return fmt.Errorf("failed to get MP-BGP addresses: %w", err)
+		}
+
+		mask := "128"
+		address := ipv4Address
+		family := api.Family_AFI_IP
+		if vip.IsIPv4(p.Conf.NeighborAddress) {
+			mask = "32"
+			address = ipv6Address
+			family = api.Family_AFI_IP6
+		}
+
+		err = b.s.AddDefinedSet(context.Background(), &api.AddDefinedSetRequest{
+			DefinedSet: &api.DefinedSet{
+				DefinedType: api.DefinedType_NEIGHBOR,
+				Name:        fmt.Sprintf("peer-%s", p.Conf.NeighborAddress),
+				List:        []string{fmt.Sprintf("%s/%s", p.Conf.NeighborAddress, mask)},
+			},
+		})
+		if err != nil {
+			return fmt.Errorf("failed to add defined set: %v", err)
+		}
+
+		if address != "" {
+			if err := insertPolicy(b.s, address, p, family); err != nil {
+				return fmt.Errorf("failed to add policy: %w", err)
+			}
+		}
+	} else {
+		if b.c.SourceIP != "" {
+			p.Transport.LocalAddress = b.c.SourceIP
+		}
+
+		if b.c.SourceIF != "" {
+			p.Transport.BindInterface = b.c.SourceIF
+		}
 	}
 
-	nlri, _ := ptypes.MarshalAny(&api.IPAddressPrefix{
-		Prefix:    ip.String(),
-		PrefixLen: pfxLen,
-	})
+	if err := b.s.AddPeer(context.Background(), &api.AddPeerRequest{Peer: p}); err != nil {
+		return fmt.Errorf("failed to add peer: %v", err)
+	}
 
-	a1, _ := ptypes.MarshalAny(&api.OriginAttribute{
+	return nil
+}
+
+func (b *Server) getPath(ip net.IP) (path *api.Path) {
+	isV6 := ip.To4() == nil
+
+	//nolint
+	originAttr, _ := anypb.New(&api.OriginAttribute{
 		Origin: 0,
 	})
 
-	var nh string
-	if b.c.NextHop != "" {
-		nh = b.c.NextHop
-	} else if b.c.SourceIP != "" {
-		nh = b.c.SourceIP
+	if !isV6 {
+		//nolint
+		nlri, _ := anypb.New(&api.IPAddressPrefix{
+			Prefix:    ip.String(),
+			PrefixLen: 32,
+		})
+
+		//nolint
+		nhAttr, _ := anypb.New(&api.NextHopAttribute{
+			NextHop: "0.0.0.0", // gobgp will fill this
+		})
+
+		path = &api.Path{
+			Family: &api.Family{
+				Afi:  api.Family_AFI_IP,
+				Safi: api.Family_SAFI_UNICAST,
+			},
+			Nlri:   nlri,
+			Pattrs: []*anypb.Any{originAttr, nhAttr},
+		}
 	} else {
-		nh = b.c.RouterID
-	}
+		//nolint
+		nlri, _ := anypb.New(&api.IPAddressPrefix{
+			Prefix:    ip.String(),
+			PrefixLen: 128,
+		})
 
-	a2, _ := ptypes.MarshalAny(&api.NextHopAttribute{
-		NextHop: nh,
-	})
-
-	return &api.Path{
-		Family: &api.Family{
-			Afi:  api.Family_AFI_IP,
+		v6Family := &api.Family{
+			Afi:  api.Family_AFI_IP6,
 			Safi: api.Family_SAFI_UNICAST,
-		},
-		Nlri:   nlri,
-		Pattrs: []*any.Any{a1, a2},
+		}
+
+		//nolint
+		mpAttr, _ := anypb.New(&api.MpReachNLRIAttribute{
+			Family:   v6Family,
+			NextHops: []string{"::"}, // gobgp will fill this
+			Nlris:    []*anypb.Any{nlri},
+		})
+
+		path = &api.Path{
+			Family: v6Family,
+			Nlri:   nlri,
+			Pattrs: []*anypb.Any{originAttr, mpAttr},
+		}
 	}
+	return
 }
 
 // ParseBGPPeerConfig - take a string and parses it into an array of peers
 func ParseBGPPeerConfig(config string) (bgpPeers []Peer, err error) {
 	peers := strings.Split(config, ",")
 	if len(peers) == 0 {
-		return nil, fmt.Errorf("No BGP Peer configurations found")
+		return nil, fmt.Errorf("no BGP Peer configurations found")
 	}
 
 	for x := range peers {
-		peer := strings.Split(peers[x], ":")
-		if len(peer) != 4 {
-			return nil, fmt.Errorf("BGP Peer configuration format error <host>:<AS>:<password>:<multihop>")
+		peerStr := peers[x]
+		config := strings.Split(peerStr, "/")
+		peerStr = config[0]
+		if peerStr == "" {
+			continue
 		}
-		ASNumber, err := strconv.Atoi(peer[1])
+		isV6Peer := peerStr[0] == '['
+
+		address := ""
+		if isV6Peer {
+			addressEndPos := strings.IndexByte(peerStr, ']')
+			if addressEndPos == -1 {
+				return nil, fmt.Errorf("no matching ] found for IPv6 BGP Peer")
+			}
+			address = peerStr[1:addressEndPos]
+			peerStr = peerStr[addressEndPos+1:]
+		}
+
+		peer := strings.Split(peerStr, ":")
+		if len(peer) < 2 {
+			return nil, fmt.Errorf("mandatory peering params <host>:<AS> incomplete")
+		}
+
+		if !isV6Peer {
+			address = peer[0]
+		}
+
+		ASNumber, err := strconv.ParseUint(peer[1], 10, 32)
 		if err != nil {
 			return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
-
 		}
-		multiHop, err := strconv.ParseBool(peer[3])
-		if err != nil {
-			return nil, fmt.Errorf("BGP MultiHop format error (true/false) [%s]", peer[1])
+
+		password := ""
+		if len(peer) >= 3 {
+			password = peer[2]
+		}
+
+		multiHop := false
+		if len(peer) >= 4 {
+			multiHop, err = strconv.ParseBool(peer[3])
+			if err != nil {
+				return nil, fmt.Errorf("BGP MultiHop format error (true/false) [%s]", peer[1])
+			}
+		}
+
+		var port uint64
+		if len(peer) >= 5 {
+			port, err = strconv.ParseUint(peer[4], 10, 16)
+			if err != nil {
+				return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
+			}
+		} else {
+			port = 179
+		}
+
+		var mpbgpNexthop, mpbgpIPv4, mpbgpIPv6 string
+
+		if len(config) > 1 {
+			configData := strings.Split(config[1], ";")
+			for _, cfg := range configData {
+				c := strings.Split(cfg, "=")
+				switch c[0] {
+				case "mpbgp_nexthop":
+					mpbgpNexthop = c[1]
+				case "mpbgp_ipv4":
+					mpbgpIPv4 = c[1]
+				case "mpbgp_ipv6":
+					mpbgpIPv6 = c[1]
+				default:
+					return nil, fmt.Errorf("peer configuration parameter '%s' is not supported", c[0])
+				}
+			}
 		}
 
 		peerConfig := Peer{
-			Address:  peer[0],
-			AS:       uint32(ASNumber),
-			Password: peer[2],
-			MultiHop: multiHop,
+			Address:      address,
+			AS:           uint32(ASNumber),
+			Port:         uint16(port),
+			Password:     password,
+			MultiHop:     multiHop,
+			MpbgpNexthop: mpbgpNexthop,
+			MpbgpIPv4:    mpbgpIPv4,
+			MpbgpIPv6:    mpbgpIPv6,
 		}
 
 		bgpPeers = append(bgpPeers, peerConfig)
 	}
 	return
 }
+
+func insertPolicy(s *server.BgpServer, address string, p *api.Peer, family api.Family_Afi) error {
+	familyType := "v4"
+	if family == api.Family_AFI_IP6 {
+		familyType = "v6"
+	}
+
+	setName := fmt.Sprintf("peer-%s", p.Conf.NeighborAddress)
+	policyName := fmt.Sprintf("%s-%s", setName, familyType)
+
+	policy := &api.Policy{
+		Name: policyName,
+		Statements: []*api.Statement{
+			{
+				Conditions: &api.Conditions{
+					AfiSafiIn: []*api.Family{
+						{
+							Afi:  family,
+							Safi: api.Family_SAFI_UNICAST,
+						},
+					},
+					NeighborSet: &api.MatchSet{
+						Type: api.MatchSet_ANY,
+						Name: setName,
+					},
+				},
+				Actions: &api.Actions{
+					RouteAction: api.RouteAction_ACCEPT,
+					Nexthop: &api.NexthopAction{
+						Address: address,
+					},
+				},
+			},
+			{
+				Conditions: &api.Conditions{
+					NeighborSet: &api.MatchSet{
+						Type: api.MatchSet_ANY,
+						Name: setName,
+					},
+				},
+				Actions: &api.Actions{
+					RouteAction: api.RouteAction_ACCEPT,
+				},
+			},
+		},
+	}
+
+	err := s.AddPolicy(context.Background(), &api.AddPolicyRequest{
+		Policy: policy,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to add policy: %w", err)
+	}
+
+	err = s.AddPolicyAssignment(context.Background(), &api.AddPolicyAssignmentRequest{
+		Assignment: &api.PolicyAssignment{
+			Name:      "global",
+			Direction: api.PolicyDirection_EXPORT,
+			Policies: []*api.Policy{
+				{
+					Name: policy.Name,
+				},
+			},
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("failed to add policy assignment: %v", err)
+	}
+
+	return nil
+}

pkg/bgp/peers_test.go

@@ -0,0 +1,54 @@
+package bgp
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseBGPPeerConfig(t *testing.T) {
+	type args struct {
+		config string
+	}
+	tests := []struct {
+		name         string
+		args         args
+		wantBgpPeers []Peer
+		wantErr      bool
+	}{
+		{
+			name: "IPv4, default port",
+			args: args{config: "192.168.0.10:65000::false,192.168.0.11:65000::false"},
+			wantBgpPeers: []Peer{
+				{Address: "192.168.0.10", Port: 179, AS: 65000, MultiHop: false},
+				{Address: "192.168.0.11", Port: 179, AS: 65000, MultiHop: false},
+			},
+		},
+		{
+			name: "IPv4, different port",
+			args: args{config: "192.168.0.10:65000::false:180,192.168.0.11:65000::false:190"},
+			wantBgpPeers: []Peer{
+				{Address: "192.168.0.10", Port: 180, AS: 65000, MultiHop: false},
+				{Address: "192.168.0.11", Port: 190, AS: 65000, MultiHop: false},
+			},
+		},
+		{
+			name: "IPv6, multi-protocol",
+			args: args{config: "[fd00:1111:2222:3333:c7d9:7235:6bf7:5d52]:65501::false/mpbgp_nexthop=auto_sourceif"},
+			wantBgpPeers: []Peer{
+				{Address: "fd00:1111:2222:3333:c7d9:7235:6bf7:5d52", Port: 179, AS: 65501, MultiHop: false, MpbgpNexthop: "auto_sourceif"},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotBgpPeers, err := ParseBGPPeerConfig(tt.args.config)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseBGPPeerConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotBgpPeers, tt.wantBgpPeers) {
+				t.Errorf("ParseBGPPeerConfig() = %v, want %v", gotBgpPeers, tt.wantBgpPeers)
+			}
+		})
+	}
+}

pkg/bgp/server.go

@@ -3,53 +3,85 @@ package bgp
 import (
 	"context"
 	"fmt"
-	"log"
 	"time"
 
-	api "github.com/osrg/gobgp/api"
-	gobgp "github.com/osrg/gobgp/pkg/server"
+	log "log/slog"
+
+	api "github.com/osrg/gobgp/v3/api"
+	gobgp "github.com/osrg/gobgp/v3/pkg/server"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // NewBGPServer takes a configuration and returns a running BGP server instance
 func NewBGPServer(c *Config) (b *Server, err error) {
 	if c.AS == 0 {
-		return nil, fmt.Errorf("You need to provide AS")
+		return nil, fmt.Errorf("you need to provide AS")
 	}
 
-	// if c.SourceIP != "" && c.SourceIF != "" {
-	// 	return nil, fmt.Errorf("SourceIP and SourceIF are mutually exclusive")
-	// }
+	if c.SourceIP != "" && c.SourceIF != "" {
+		return nil, fmt.Errorf("sourceIP and SourceIF are mutually exclusive")
+	}
 
 	if len(c.Peers) == 0 {
-		return nil, fmt.Errorf("You need to provide at least one peer")
+		return nil, fmt.Errorf("you need to provide at least one peer")
 	}
 
 	b = &Server{
 		s: gobgp.NewBgpServer(),
 		c: c,
+
+		BGPSessionInfoGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "kube_vip",
+			Subsystem: "manager",
+			Name:      "bgp_session_info",
+			Help:      "Display state of session by setting metric for label value with current state to 1",
+		}, []string{"state", "peer"}),
 	}
+	return
+}
+
+// Start starts the BGP server
+func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (err error) {
 	go b.s.Serve()
 
 	if err = b.s.StartBgp(context.Background(), &api.StartBgpRequest{
 		Global: &api.Global{
-			As:         c.AS,
-			RouterId:   c.RouterID,
+			Asn:        b.c.AS,
+			RouterId:   b.c.RouterID,
 			ListenPort: -1,
 		},
 	}); err != nil {
 		return
 	}
 
-	if err = b.s.MonitorPeer(context.Background(), &api.MonitorPeerRequest{}, func(p *api.Peer) { log.Println(p) }); err != nil {
+	if err = b.s.WatchEvent(context.Background(), &api.WatchEventRequest{Peer: &api.WatchEventRequest_Peer{}}, func(r *api.WatchEventResponse) {
+		if p := r.GetPeer(); p != nil && p.Type == api.WatchEventResponse_PeerEvent_STATE {
+			log.Info("[BGP]", "peer", p.String())
+			if peerStateChangeCallback != nil {
+				peerStateChangeCallback(p)
+			}
+		}
+	}); err != nil {
 		return
 	}
 
-	for _, p := range c.Peers {
+	for _, p := range b.c.Peers {
 		if err = b.AddPeer(p); err != nil {
 			return
 		}
 	}
 
+	if b.c.Zebra.Enabled {
+		if err = b.s.EnableZebra(context.Background(), &api.EnableZebraRequest{
+			Url:          b.c.Zebra.URL,
+			Version:      b.c.Zebra.Version,
+			SoftwareName: b.c.Zebra.SoftwareName,
+		}); err != nil {
+			log.Error(err.Error())
+			return
+		}
+	}
+
 	return
 }
 

pkg/bgp/types.go

@@ -1,29 +1,145 @@
 package bgp
 
-import gobgp "github.com/osrg/gobgp/pkg/server"
+import (
+	"fmt"
+	"net"
+
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	api "github.com/osrg/gobgp/v3/api"
+	gobgp "github.com/osrg/gobgp/v3/pkg/server"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/vishvananda/netlink"
+)
 
 // Peer defines a BGP Peer
 type Peer struct {
-	Address  string
-	AS       uint32
-	Password string
-	MultiHop bool
+	Address      string
+	Port         uint16
+	AS           uint32
+	Password     string
+	MultiHop     bool
+	MpbgpNexthop string
+	MpbgpIPv4    string
+	MpbgpIPv6    string
+}
+
+func (p *Peer) setMpbgpOptions(server *Config) {
+	if p.MpbgpNexthop == "" {
+		p.MpbgpNexthop = server.MpbgpNexthop
+	}
+
+	if p.MpbgpIPv4 == "" {
+		p.MpbgpIPv4 = server.MpbgpIPv4
+	}
+
+	if p.MpbgpIPv6 == "" {
+		p.MpbgpIPv6 = server.MpbgpIPv6
+	}
+}
+
+func (p *Peer) findMpbgpAddresses(ap *api.Peer, server *Config) (string, string, error) {
+	var ipv4Address, ipv6Address string
+	switch p.MpbgpNexthop {
+	case "fixed":
+		ap.Transport.LocalAddress = server.SourceIP
+		if p.MpbgpIPv4 == "" && p.MpbgpIPv6 == "" {
+			return "", "", fmt.Errorf("to use MP-BGP with fixed address at least one IPv4 or IPv6 address has to be provided [current - IPv4: %s, IPv6: %s]",
+				p.MpbgpIPv4, p.MpbgpIPv6)
+		}
+
+		if p.MpbgpIPv4 != "" {
+			if net.ParseIP(p.MpbgpIPv4) == nil {
+				return "", "", fmt.Errorf("provided address '%s' is not a valid IPv4 address", p.MpbgpIPv4)
+			}
+		}
+		if p.MpbgpIPv6 != "" {
+			if net.ParseIP(p.MpbgpIPv6) == nil {
+				return "", "", fmt.Errorf("provided address '%s' is not a valid IPv6 address", p.MpbgpIPv6)
+			}
+		}
+
+		ipv4Address = p.MpbgpIPv4
+		ipv6Address = p.MpbgpIPv6
+	case "auto_sourceip":
+		ap.Transport.LocalAddress = server.SourceIP
+
+		// Resolve the local interface by SourceIP
+		iface, err := vip.GetInterfaceByIP(server.SourceIP)
+		if err != nil {
+			return "", "", fmt.Errorf("failed to get interface by IP: %v", err)
+		}
+
+		if vip.IsIPv4(server.SourceIP) {
+			// Get the non link-local IPv6 address on that interface
+			ipv6Address, err = vip.GetNonLinkLocalIP(iface, netlink.FAMILY_V6)
+			if err != nil {
+				return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
+			}
+		} else {
+			// Get the non link-local IPv4 address on that interface
+			ipv4Address, err = vip.GetNonLinkLocalIP(iface, netlink.FAMILY_V4)
+			if err != nil {
+				return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
+			}
+		}
+	case "auto_sourceif":
+		ap.Transport.BindInterface = server.SourceIF
+
+		iface, err := netlink.LinkByName(server.SourceIF)
+		if err != nil {
+			return "", "", fmt.Errorf("failed to get interface by name: %v", err)
+		}
+
+		// Get the non link-local IPv4 address on that interface
+		ipv4Address, err = vip.GetNonLinkLocalIP(&iface, netlink.FAMILY_V4)
+		if err != nil {
+			return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
+		}
+
+		// Get the non link-local IPv6 address on that interface
+		ipv6Address, err = vip.GetNonLinkLocalIP(&iface, netlink.FAMILY_V6)
+		if err != nil {
+			return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
+		}
+	default:
+		return "", "", fmt.Errorf("option %s for MP-BPG nexthop is not supported", server.MpbgpNexthop)
+	}
+
+	return ipv4Address, ipv6Address, nil
 }
 
 // Config defines the BGP server configuration
 type Config struct {
-	AS       uint32
-	RouterID string
-	NextHop  string
-	SourceIP string
-	SourceIF string
+	AS           uint32
+	RouterID     string
+	SourceIP     string
+	SourceIF     string
+	MpbgpNexthop string
+	MpbgpIPv4    string
+	MpbgpIPv6    string
+
+	HoldTime          uint64
+	KeepaliveInterval uint64
 
 	Peers []Peer
-	IPv6  bool
+
+	Zebra ZebraConfig
+}
+
+// Defines Zebra connection configuration. More on the topic - https://github.com/osrg/gobgp/blob/master/docs/sources/zebra.md#configuration
+type ZebraConfig struct {
+	Enabled      bool
+	URL          string
+	Version      uint32
+	SoftwareName string
 }
 
 // Server manages a server object
 type Server struct {
 	s *gobgp.BgpServer
 	c *Config
+
+	// This is a prometheus gauge indicating the state of the sessions.
+	// 1 means "ESTABLISHED", 0 means "NOT ESTABLISHED"
+	BGPSessionInfoGauge *prometheus.GaugeVec
 }

pkg/cluster/cluster.go

@@ -1,52 +1,80 @@
 package cluster
 
 import (
+	"sync"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/networkinterface"
 	"github.com/kube-vip/kube-vip/pkg/vip"
 )
 
-const leaderLogcount = 5
-
 // Cluster - The Cluster object manages the state of the cluster for a particular node
 type Cluster struct {
-	stateMachine FSM
-	stop         chan bool
-	completed    chan bool
-	Network      vip.Network
+	stop      chan bool
+	completed chan bool
+	once      sync.Once
+	Network   []vip.Network
+	arpMgr    *arp.Manager
 }
 
 // InitCluster - Will attempt to initialise all of the required settings for the cluster
-func InitCluster(c *kubevip.Config, disableVIP bool) (*Cluster, error) {
-
-	// TODO - Check for root (needed to netlink)
-	var network vip.Network
+func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Cluster, error) {
+	var networks []vip.Network
 	var err error
 
 	if !disableVIP {
 		// Start the Virtual IP Networking configuration
-		network, err = startNetworking(c)
+		networks, err = startNetworking(c, intfMgr)
 		if err != nil {
 			return nil, err
 		}
 	}
 	// Initialise the Cluster structure
 	newCluster := &Cluster{
-		Network: network,
+		Network: networks,
+		arpMgr:  arpMgr,
 	}
 
+	log.Debug("service security", "enabled", c.EnableServiceSecurity)
+
 	return newCluster, nil
 }
 
-func startNetworking(c *kubevip.Config) (vip.Network, error) {
+func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vip.Network, error) {
 	address := c.VIP
 
 	if c.Address != "" {
 		address = c.Address
 	}
 
-	network, err := vip.NewConfig(address, c.Interface, c.DDNS)
-	if err != nil {
-		return nil, err
+	addresses := vip.Split(address)
+
+	networks := []vip.Network{}
+	for _, addr := range addresses {
+		network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.RoutingTableID,
+			c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend,
+			c.EnableLoadBalancer, c.EnableServiceSecurity, intfMgr)
+		if err != nil {
+			return nil, err
+		}
+		networks = append(networks, network...)
 	}
-	return network, nil
+
+	return networks, nil
+}
+
+// Stop - Will stop the Cluster and release VIP if needed
+func (cluster *Cluster) Stop() {
+	// Close the stop channel, which will shut down the VIP (if needed)
+	if cluster.stop != nil {
+		cluster.once.Do(func() { // Ensure that the close channel can only ever be called once
+			close(cluster.stop)
+		})
+	}
+
+	// Wait until the completed channel is closed, signallign all shutdown tasks completed
+	<-cluster.completed
 }

pkg/cluster/clusterDDNS.go

@@ -13,14 +13,15 @@ import (
 // dnsUpdater already have the functionality to keep trying resolve the IP
 // and update the VIP configuration if it changes
 func (cluster *Cluster) StartDDNS(ctx context.Context) error {
-	ddnsMgr := vip.NewDDNSManager(ctx, cluster.Network)
-	ip, err := ddnsMgr.Start()
-	if err != nil {
-		return err
-	}
-
-	if err = cluster.Network.SetIP(ip); err != nil {
-		return err
+	for i := range cluster.Network {
+		ddnsMgr := vip.NewDDNSManager(ctx, cluster.Network[i])
+		ip, err := ddnsMgr.Start()
+		if err != nil {
+			return err
+		}
+		if err = cluster.Network[i].SetIP(ip); err != nil {
+			return err
+		}
 	}
 
 	return nil

pkg/cluster/clusterLeader.go

@@ -1,469 +0,0 @@
-package cluster
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/signal"
-	"path/filepath"
-	"syscall"
-	"time"
-
-	"github.com/kube-vip/kube-vip/pkg/bgp"
-	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	leaderelection "github.com/kube-vip/kube-vip/pkg/leaderElection"
-	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
-	"github.com/kube-vip/kube-vip/pkg/packet"
-
-	"github.com/kube-vip/kube-vip/pkg/vip"
-
-	"github.com/packethost/packngo"
-
-	log "github.com/sirupsen/logrus"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/tools/leaderelection/resourcelock"
-)
-
-const plunderLock = "plndr-cp-lock"
-
-// Manager degines the manager of the load-balancing services
-type Manager struct {
-	KubernetesClient *kubernetes.Clientset
-}
-
-// NewManager will create a new managing object
-func NewManager(path string, inCluster bool, port int) (*Manager, error) {
-	var clientset *kubernetes.Clientset
-	if inCluster {
-		// This will attempt to load the configuration when running within a POD
-		cfg, err := rest.InClusterConfig()
-		if err != nil {
-			return nil, fmt.Errorf("error creating kubernetes client config: %s", err.Error())
-		}
-		clientset, err = kubernetes.NewForConfig(cfg)
-
-		if err != nil {
-			return nil, fmt.Errorf("error creating kubernetes client: %s", err.Error())
-		}
-		// use the current context in kubeconfig
-	} else {
-		if path == "" {
-			path = filepath.Join(os.Getenv("HOME"), ".kube", "config")
-		}
-		config, err := clientcmd.BuildConfigFromFlags("", path)
-		if err != nil {
-			panic(err.Error())
-		}
-
-		// We modify the config so that we can always speak to the correct host
-		id, err := os.Hostname()
-		if err != nil {
-			return nil, err
-		}
-
-		config.Host = fmt.Sprintf("%s:%v", id, port)
-		clientset, err = kubernetes.NewForConfig(config)
-
-		if err != nil {
-			return nil, fmt.Errorf("error creating kubernetes client: %s", err.Error())
-		}
-	}
-
-	return &Manager{
-		KubernetesClient: clientset,
-	}, nil
-}
-
-// StartLeaderCluster - Begins a running instance of the Raft cluster
-func (cluster *Cluster) StartLeaderCluster(c *kubevip.Config, sm *Manager, bgpServer *bgp.Server) error {
-
-	id, err := os.Hostname()
-	if err != nil {
-		return err
-	}
-
-	log.Infof("Beginning cluster membership, namespace [%s], lock name [%s], id [%s]", c.Namespace, plunderLock, id)
-
-	// we use the Lease lock type since edits to Leases are less common
-	// and fewer objects in the cluster watch "all Leases".
-	lock := &resourcelock.LeaseLock{
-		LeaseMeta: metav1.ObjectMeta{
-			Name:      plunderLock,
-			Namespace: c.Namespace,
-		},
-		Client: sm.KubernetesClient.CoordinationV1(),
-		LockConfig: resourcelock.ResourceLockConfig{
-			Identity: id,
-		},
-	}
-
-	// use a Go context so we can tell the leaderelection code when we
-	// want to step down
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	// use a Go context so we can tell the arp loop code when we
-	// want to step down
-	ctxArp, cancelArp := context.WithCancel(context.Background())
-	defer cancelArp()
-
-	// use a Go context so we can tell the dns loop code when we
-	// want to step down
-	ctxDNS, cancelDNS := context.WithCancel(context.Background())
-	defer cancelDNS()
-
-	// listen for interrupts or the Linux SIGTERM signal and cancel
-	// our context, which the leader election code will observe and
-	// step down
-	signalChan := make(chan os.Signal, 1)
-	// Add Notification for Userland interrupt
-	signal.Notify(signalChan, syscall.SIGINT)
-
-	// Add Notification for SIGTERM (sent from Kubernetes)
-	signal.Notify(signalChan, syscall.SIGTERM)
-
-	// Add Notification for SIGKILL (sent from Kubernetes)
-	signal.Notify(signalChan, syscall.SIGKILL)
-
-	go func() {
-		<-signalChan
-		log.Info("Received termination, signaling shutdown")
-		// Cancel the context, which will in turn cancel the leadership
-		cancel()
-		// Cancel the arp context, which will in turn stop any broadcasts
-	}()
-
-	// (attempt to) Remove the virtual IP, incase it already exists
-	cluster.Network.DeleteIP()
-
-	// Managers for Vip load balancers and none-vip loadbalancers
-	nonVipLB := loadbalancer.LBManager{}
-	VipLB := loadbalancer.LBManager{}
-
-	// Defer a function to check if the bgpServer has been created and if so attempt to close it
-	defer func() {
-		if bgpServer != nil {
-			bgpServer.Close()
-		}
-	}()
-
-	// If Packet is enabled then we can begin our preperation work
-	var packetClient *packngo.Client
-	if c.EnableMetal {
-		packetClient, err = packngo.NewClient()
-		if err != nil {
-			log.Error(err)
-		}
-
-		// We're using Packet with BGP, popuplate the Peer information from the API
-		if c.EnableBGP {
-			log.Infoln("Looking up the BGP configuration from packet")
-			err = packet.BGPLookup(packetClient, c)
-			if err != nil {
-				log.Error(err)
-			}
-		}
-	}
-
-	if c.EnableBGP {
-		// Lets start BGP
-		log.Info("Starting the BGP server to adverise VIP routes to VGP peers")
-		bgpServer, err = bgp.NewBGPServer(&c.BGPConfig)
-		if err != nil {
-			log.Error(err)
-		}
-	}
-
-	if c.EnableLoadBalancer {
-
-		// Iterate through all Configurations
-		if len(c.LoadBalancers) != 0 {
-			for x := range c.LoadBalancers {
-				// If the load balancer doesn't bind to the VIP
-				if c.LoadBalancers[x].BindToVip == false {
-					err = nonVipLB.Add("", &c.LoadBalancers[x])
-					if err != nil {
-						log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-					}
-				}
-			}
-		}
-	}
-	// start the leader election code loop
-	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
-		Lock: lock,
-		// IMPORTANT: you MUST ensure that any code you have that
-		// is protected by the lease must terminate **before**
-		// you call cancel. Otherwise, you could have a background
-		// loop still running and another process could
-		// get elected before your background loop finished, violating
-		// the stated goal of the lease.
-		ReleaseOnCancel: true,
-		LeaseDuration:   time.Duration(c.LeaseDuration) * time.Second,
-		RenewDeadline:   time.Duration(c.RenewDeadline) * time.Second,
-		RetryPeriod:     time.Duration(c.RetryPeriod) * time.Second,
-		Callbacks: leaderelection.LeaderCallbacks{
-			OnStartedLeading: func(ctx context.Context) {
-				// we're notified when we start
-				log.Info("This node is starting with leadership of the cluster")
-				// setup ddns first
-				// for first time, need to wait until IP is allocated from DHCP
-				if cluster.Network.IsDDNS() {
-					if err := cluster.StartDDNS(ctxDNS); err != nil {
-						log.Error(err)
-					}
-				}
-
-				// start the dns updater if address is dns
-				if cluster.Network.IsDNS() {
-					log.Infof("starting the DNS updater for the address %s", cluster.Network.DNSName())
-					ipUpdater := vip.NewIPUpdater(cluster.Network)
-					ipUpdater.Run(ctxDNS)
-				}
-
-				err = cluster.Network.AddIP()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				if c.EnableMetal {
-					// We're not using Packet with BGP
-					if !c.EnableBGP {
-						// Attempt to attach the EIP in the standard manner
-						log.Debugf("Attaching the Packet EIP through the API to this host")
-						err = packet.AttachEIP(packetClient, c, id)
-						if err != nil {
-							log.Error(err)
-						}
-					}
-				}
-
-				if c.EnableBGP {
-					// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
-					cidrVip := fmt.Sprintf("%s/%s", cluster.Network.IP(), c.VIPCIDR)
-					log.Debugf("Attempting to advertise the address [%s] over BGP", cidrVip)
-
-					err = bgpServer.AddHost(cidrVip)
-					if err != nil {
-						log.Error(err)
-					}
-				}
-
-				if c.EnableLoadBalancer {
-					// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-					for x := range c.LoadBalancers {
-
-						if c.LoadBalancers[x].BindToVip == true {
-							err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-							if err != nil {
-								log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-
-								// Stop all load balancers associated with the VIP
-								err = VipLB.StopAll()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-
-								err = cluster.Network.DeleteIP()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-							}
-						}
-					}
-				}
-
-				if c.EnableARP == true {
-					ctxArp, cancelArp = context.WithCancel(context.Background())
-
-					ipString := cluster.Network.IP()
-
-					var ndp *vip.NdpResponder
-					if vip.IsIPv6(ipString) {
-						ndp, err = vip.NewNDPResponder(c.Interface)
-						if err != nil {
-							log.Fatalf("failed to create new NDP Responder")
-						}
-					}
-
-					go func(ctx context.Context) {
-						if ndp != nil {
-							defer ndp.Close()
-						}
-
-						for {
-							select {
-							case <-ctx.Done(): // if cancel() execute
-								return
-							default:
-								// Ensure the address exists on the interface before attempting to ARP
-								set, err := cluster.Network.IsSet()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-								if !set {
-									log.Warnf("Re-applying the VIP configuration [%s] to the interface [%s]", ipString, c.Interface)
-									err = cluster.Network.AddIP()
-									if err != nil {
-										log.Warnf("%v", err)
-									}
-								}
-
-								if vip.IsIPv4(ipString) {
-									// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
-									err := vip.ARPSendGratuitous(ipString, c.Interface)
-									if err != nil {
-										log.Warnf("%v", err)
-									}
-								} else {
-									// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
-									err := ndp.SendGratuitous(ipString)
-									if err != nil {
-										log.Warnf("%v", err)
-									}
-								}
-							}
-							time.Sleep(3 * time.Second)
-						}
-					}(ctxArp)
-				}
-			},
-			OnStoppedLeading: func() {
-				// we can do cleanup here
-				log.Info("This node is becoming a follower within the cluster")
-
-				// Stop the dns context
-				cancelDNS()
-				// Stop the Arp context if it is running
-				cancelArp()
-
-				// Stop the BGP server
-				if bgpServer != nil {
-					err = bgpServer.Close()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-
-				// Stop all load balancers associated with the VIP
-				err = VipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				err = cluster.Network.DeleteIP()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				log.Fatal("lost leadership, restarting kube-vip")
-			},
-			OnNewLeader: func(identity string) {
-				// we're notified when new leader elected
-				log.Infof("Node [%s] is assuming leadership of the cluster", identity)
-
-				if identity == id {
-					// We have the lock
-				}
-			},
-		},
-	})
-
-	return nil
-}
-
-// TODO - refactor an active machine func(), this will replace the singleNode code and have a single code block
-
-// func (cluster *Cluster) active(c *kubevip.Config) error {
-// 	// we're notified when we start
-// 	log.Info("This node is starting with leadership of the cluster")
-// 	// setup ddns first
-// 	// for first time, need to wait until IP is allocated from DHCP
-// 	if cluster.Network.IsDDNS() {
-// 		if err := cluster.StartDDNS(ctxDns); err != nil {
-// 			log.Error(err)
-// 		}
-// 	}
-
-// 	// start the dns updater if address is dns
-// 	if cluster.Network.IsDNS() {
-// 		log.Infof("starting the DNS updater for the address %s", cluster.Network.DNSName())
-// 		ipUpdater := vip.NewIPUpdater(cluster.Network)
-// 		ipUpdater.Run(ctxDns)
-// 	}
-
-// 	err := cluster.Network.AddIP()
-// 	if err != nil {
-// 		log.Warnf("%v", err)
-// 	}
-
-// 	if c.EnablePacket {
-// 		// We're not using Packet with BGP
-// 		if !c.EnableBGP {
-// 			// Attempt to attach the EIP in the standard manner
-// 			log.Debugf("Attaching the Packet EIP through the API to this host")
-// 			err = packet.AttachEIP(packetClient, c, id)
-// 			if err != nil {
-// 				log.Error(err)
-// 			}
-// 		}
-// 	}
-
-// 	if c.EnableBGP {
-// 		// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
-// 		cidrVip := fmt.Sprintf("%s/%s", cluster.Network.IP(), c.VIPCIDR)
-// 		log.Debugf("Attempting to advertise the address [%s] over BGP", cidrVip)
-
-// 		err = bgpServer.AddHost(cidrVip)
-// 		if err != nil {
-// 			log.Error(err)
-// 		}
-// 	}
-
-// 	if c.EnableLoadBalancer {
-// 		// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-// 		for x := range c.LoadBalancers {
-
-// 			if c.LoadBalancers[x].BindToVip == true {
-// 				err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-// 				if err != nil {
-// 					log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-
-// 					// Stop all load balancers associated with the VIP
-// 					err = VipLB.StopAll()
-// 					if err != nil {
-// 						log.Warnf("%v", err)
-// 					}
-
-// 					err = cluster.Network.DeleteIP()
-// 					if err != nil {
-// 						log.Warnf("%v", err)
-// 					}
-// 				}
-// 			}
-// 		}
-// 	}
-
-// 	if c.EnableARP == true {
-// 		ctxArp, cancelArp = context.WithCancel(context.Background())
-
-// 		go func(ctx context.Context) {
-// 			for {
-// 				select {
-// 				case <-ctx.Done(): // if cancel() execute
-// 					return
-// 				default:
-// 					// Gratuitous ARP, will broadcast to new MAC <-> IP
-// 					err = vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-// 					if err != nil {
-// 						log.Warnf("%v", err)
-// 					}
-// 				}
-// 				time.Sleep(3 * time.Second)
-// 			}
-// 		}(ctxArp)
-// 	}
-// }

pkg/cluster/clusterLeaderElection.go

@@ -0,0 +1,390 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/etcd"
+	"github.com/kube-vip/kube-vip/pkg/k8s"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
+
+	log "log/slog"
+
+	clientv3 "go.etcd.io/etcd/client/v3"
+	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	watchtools "k8s.io/client-go/tools/watch"
+)
+
+// Manager degines the manager of the load-balancing services
+type Manager struct {
+	KubernetesClient   *kubernetes.Clientset
+	RetryWatcherClient *kubernetes.Clientset
+	// This channel is used to signal a shutdown
+	SignalChan chan os.Signal
+
+	EtcdClient *clientv3.Client
+}
+
+// NewManager will create a new managing object
+func NewManager(path string, inCluster bool, port int) (*Manager, error) {
+	var hostname string
+
+	// If inCluster is set then it will likely have started as a static pod or won't have the
+	// VIP up before trying to connect to the API server, we set the API endpoint to this machine to
+	// ensure connectivity. Else if the path passed is empty and not running in the cluster,
+	// attempt to look for a kubeconfig in the default HOME dir.
+
+	hostname = fmt.Sprintf("kubernetes:%v", port)
+
+	if len(path) == 0 && !inCluster {
+		path = filepath.Join(os.Getenv("HOME"), ".kube", "config")
+
+		// We modify the config so that we can always speak to the correct host
+		id, err := os.Hostname()
+		if err != nil {
+			return nil, err
+		}
+
+		hostname = fmt.Sprintf("%s:%v", id, port)
+	}
+
+	config, err := k8s.NewRestConfig(path, inCluster, hostname)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create k8s REST config: %w", err)
+	}
+
+	clientset, err := k8s.NewClientset(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating a new k8s clientset: %v", err)
+	}
+
+	rwConfig, err := k8s.NewRestConfig(path, inCluster, hostname)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create k8s REST config for retryClientSet: %w", err)
+	}
+
+	rwConfig.Timeout = 0 // empty value to disable the timeout
+	rwClientSet, err := k8s.NewClientset(rwConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create k8s client for retry watcher: %w", err)
+	}
+
+	return &Manager{
+		KubernetesClient:   clientset,
+		RetryWatcherClient: rwClientSet,
+	}, nil
+}
+
+// StartCluster - Begins a running instance of the Leader Election cluster
+func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *bgp.Server) error {
+	var err error
+
+	log.Info("cluster membership", "namespace", c.Namespace, "lock", c.LeaseName, "id", c.NodeName)
+
+	// use a Go context so we can tell the leaderelection code when we
+	// want to step down
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// use a Go context so we can tell the arp loop code when we
+	// want to step down
+	ctxArp, cancelArp := context.WithCancel(context.Background())
+	defer cancelArp()
+
+	// use a Go context so we can tell the dns loop code when we
+	// want to step down
+	ctxDNS, cancelDNS := context.WithCancel(context.Background())
+	defer cancelDNS()
+
+	// listen for interrupts or the Linux SIGTERM signal and cancel
+	// our context, which the leader election code will observe and
+	// step down
+	signalChan := make(chan os.Signal, 1)
+	// Add Notification for Userland interrupt
+	signal.Notify(signalChan, syscall.SIGINT)
+
+	// Add Notification for SIGTERM (sent from Kubernetes)
+	signal.Notify(signalChan, syscall.SIGTERM)
+
+	go func() {
+		<-signalChan
+		log.Info("Received termination, signaling cluster shutdown")
+		// Cancel the context, which will in turn cancel the leadership
+		cancel()
+		// Cancel the arp context, which will in turn stop any broadcasts
+	}()
+
+	// (attempt to) Remove the virtual IP, in case it already exists
+
+	for i := range cluster.Network {
+		deleted, err := cluster.Network[i].DeleteIP()
+		if err != nil {
+			log.Error("could not delete virtualIP", "err", err)
+		}
+		if deleted {
+			log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+		}
+	}
+
+	// Defer a function to check if the bgpServer has been created and if so attempt to close it
+	defer func() {
+		if bgpServer != nil {
+			bgpServer.Close()
+		}
+	}()
+
+	if c.EnableBGP && bgpServer == nil {
+		// Lets start BGP
+		log.Info("Starting the BGP server to advertise VIP routes to VGP peers")
+		bgpServer, err = bgp.NewBGPServer(&c.BGPConfig)
+		if err != nil {
+			log.Error("new BGP server", "err", err)
+		}
+		if err := bgpServer.Start(nil); err != nil {
+			log.Error("starting BGP server", "err", err)
+		}
+	}
+
+	run := &runConfig{
+		config:  c,
+		leaseID: c.NodeName,
+		sm:      sm,
+		onStartedLeading: func(ctx context.Context) { //nolint TODO: potential clean code
+			// As we're leading lets start the vip service
+			err := cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, cancel)
+			if err != nil {
+				log.Error("starting VIP service on leader", "err", err)
+			}
+		},
+		onStoppedLeading: func() {
+			// we can do cleanup here
+			log.Info("This node is becoming a follower within the cluster")
+
+			// Stop the dns context
+			cancelDNS()
+			// Stop the Arp context if it is running
+			cancelArp()
+
+			// Stop the BGP server
+			if bgpServer != nil {
+				err := bgpServer.Close()
+				if err != nil {
+					log.Warn("close BGP server", "err", err)
+				}
+			}
+
+			for i := range cluster.Network {
+				deleted, err := cluster.Network[i].DeleteIP()
+				if err != nil {
+					log.Warn("delete VIP", "err", err)
+				}
+				if deleted {
+					log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+				}
+			}
+
+			log.Error("lost leadership, restarting kube-vip")
+			panic("") // TODO - we could also return here
+		},
+		onNewLeader: func(identity string) {
+			// we're notified when new leader elected
+			log.Info("New leader", "leader", identity)
+		},
+	}
+
+	switch c.LeaderElectionType {
+	case "kubernetes", "":
+		cluster.runKubernetesLeaderElectionOrDie(ctx, run)
+	case "etcd":
+		cluster.runEtcdLeaderElectionOrDie(ctx, run)
+	default:
+		log.Info(fmt.Sprintf("LeaderElectionMode %s not supported, exiting", c.LeaderElectionType))
+	}
+
+	return nil
+}
+
+type runConfig struct {
+	config  *kubevip.Config
+	leaseID string
+	sm      *Manager
+
+	// onStartedLeading is called when this member starts leading.
+	onStartedLeading func(context.Context)
+	// onStoppedLeading is called when this member stops leading.
+	onStoppedLeading func()
+	// onNewLeader is called when the client observes a leader that is
+	// not the previously observed leader. This includes the first observed
+	// leader when the client starts.
+	onNewLeader func(identity string)
+}
+
+func (cluster *Cluster) runKubernetesLeaderElectionOrDie(ctx context.Context, run *runConfig) {
+	// we use the Lease lock type since edits to Leases are less common
+	// and fewer objects in the cluster watch "all Leases".
+	lock := &resourcelock.LeaseLock{
+		LeaseMeta: metav1.ObjectMeta{
+			Name:        run.config.LeaseName,
+			Namespace:   run.config.Namespace,
+			Annotations: run.config.LeaseAnnotations,
+		},
+		Client: run.sm.KubernetesClient.CoordinationV1(),
+		LockConfig: resourcelock.ResourceLockConfig{
+			Identity: run.leaseID,
+		},
+	}
+
+	// start the leader election code loop
+	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
+		Lock: lock,
+		// IMPORTANT: you MUST ensure that any code you have that
+		// is protected by the lease must terminate **before**
+		// you call cancel. Otherwise, you could have a background
+		// loop still running and another process could
+		// get elected before your background loop finished, violating
+		// the stated goal of the lease.
+		ReleaseOnCancel: true,
+		LeaseDuration:   time.Duration(run.config.LeaseDuration) * time.Second,
+		RenewDeadline:   time.Duration(run.config.RenewDeadline) * time.Second,
+		RetryPeriod:     time.Duration(run.config.RetryPeriod) * time.Second,
+		Callbacks: leaderelection.LeaderCallbacks{
+			OnStartedLeading: run.onStartedLeading,
+			OnStoppedLeading: run.onStoppedLeading,
+			OnNewLeader:      run.onNewLeader,
+		},
+	})
+}
+
+func (cluster *Cluster) runEtcdLeaderElectionOrDie(ctx context.Context, run *runConfig) {
+	etcd.RunElectionOrDie(ctx, &etcd.LeaderElectionConfig{
+		EtcdConfig:           etcd.ClientConfig{Client: run.sm.EtcdClient},
+		Name:                 run.config.LeaseName,
+		MemberID:             run.leaseID,
+		LeaseDurationSeconds: int64(run.config.LeaseDuration),
+		Callbacks: etcd.LeaderCallbacks{
+			OnStartedLeading: run.onStartedLeading,
+			OnStoppedLeading: run.onStoppedLeading,
+			OnNewLeader:      run.onNewLeader,
+		},
+	})
+}
+
+func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
+	// Use a restartable watcher, as this should help in the event of etcd or timeout issues
+	log.Info("Kube-Vip is watching nodes for control-plane labels")
+
+	listOptions := metav1.ListOptions{
+		LabelSelector: "node-role.kubernetes.io/control-plane",
+	}
+
+	rw, err := watchtools.NewRetryWatcherWithContext(ctxArp, "1", &cache.ListWatch{
+		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
+			return sm.RetryWatcherClient.CoreV1().Nodes().Watch(context.Background(), listOptions)
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("error creating label watcher: %s", err.Error())
+	}
+
+	go func() {
+		<-sm.SignalChan
+		log.Info("Received termination, signaling shutdown")
+		// Cancel the context
+		rw.Stop()
+	}()
+
+	ch := rw.ResultChan()
+	// defer rw.Stop()
+
+	for event := range ch {
+		// We need to inspect the event and get ResourceVersion out of it
+		switch event.Type {
+		case watch.Added, watch.Modified:
+			node, ok := event.Object.(*v1.Node)
+			if !ok {
+				return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
+			}
+			// Find the node IP address (this isn't foolproof)
+			for x := range node.Status.Addresses {
+				if node.Status.Addresses[x].Type == v1.NodeInternalIP {
+					if checkIfNodeIsReady(node) {
+						err = lb.AddBackend(node.Status.Addresses[x].Address, port)
+						if err != nil {
+							log.Error("add IPVS backend", "err", err)
+						}
+					} else {
+						err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
+						if err != nil {
+							log.Error("remove IPVS backend", "err", err)
+						}
+					}
+				}
+			}
+		case watch.Deleted:
+			node, ok := event.Object.(*v1.Node)
+			if !ok {
+				return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
+			}
+
+			// Find the node IP address (this isn't foolproof)
+			for x := range node.Status.Addresses {
+				if node.Status.Addresses[x].Type == v1.NodeInternalIP {
+					err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
+					if err != nil {
+						log.Error("Del IPVS backend", "err", err)
+					}
+				}
+			}
+
+			log.Info("Node deleted", "name", node.Name)
+
+		case watch.Bookmark:
+			// Un-used
+		case watch.Error:
+			log.Error("Error attempting to watch Kubernetes Nodes")
+
+			// This round trip allows us to handle unstructured status
+			errObject := apierrors.FromObject(event.Object)
+			statusErr, ok := errObject.(*apierrors.StatusError)
+			if !ok {
+				log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
+			}
+
+			status := statusErr.ErrStatus
+			log.Error("watcher", "status", status)
+		default:
+		}
+	}
+
+	log.Info("Exiting Node watcher")
+	return nil
+}
+
+func checkIfNodeIsReady(node *v1.Node) bool {
+	if node == nil {
+		return false
+	}
+	for _, condition := range node.Status.Conditions {
+		if condition.Type == v1.NodeReady {
+			if condition.Status == v1.ConditionTrue {
+				return true
+			}
+		}
+	}
+	return false
+}

pkg/cluster/clusterRaft.go

@@ -1,296 +0,0 @@
-package cluster
-
-import (
-	"fmt"
-	"net"
-	"time"
-
-	"github.com/hashicorp/raft"
-	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
-	"github.com/kube-vip/kube-vip/pkg/vip"
-	log "github.com/sirupsen/logrus"
-)
-
-// StartRaftCluster - Begins a running instance of the Raft cluster
-func (cluster *Cluster) StartRaftCluster(c *kubevip.Config) error {
-
-	// Create local configuration address
-	localAddress := fmt.Sprintf("%s:%d", c.LocalPeer.Address, c.LocalPeer.Port)
-
-	// Begin the Raft configuration
-	config := raft.DefaultConfig()
-	config.LocalID = raft.ServerID(c.LocalPeer.ID)
-	logger := log.StandardLogger().Writer()
-	config.LogOutput = logger
-
-	// Initialize communication
-	address, err := net.ResolveTCPAddr("tcp", localAddress)
-	if err != nil {
-		return err
-	}
-
-	// Create transport
-	transport, err := raft.NewTCPTransport(localAddress, address, 3, 10*time.Second, logger)
-	if err != nil {
-		return err
-	}
-
-	// Create Raft structures
-	snapshots := raft.NewInmemSnapshotStore()
-	logStore := raft.NewInmemStore()
-	stableStore := raft.NewInmemStore()
-
-	// Cluster configuration
-	configuration := raft.Configuration{}
-
-	// Add Local Peer
-	configuration.Servers = append(configuration.Servers, raft.Server{
-		ID:      raft.ServerID(c.LocalPeer.ID),
-		Address: raft.ServerAddress(fmt.Sprintf("%s:%d", c.LocalPeer.Address, c.LocalPeer.Port))})
-
-	// If we want to start a node as leader then we will not add any remote peers, this will leave this as a cluster of one
-	// The remotePeers will add themselves to the cluster as they're added
-	if c.StartAsLeader != true {
-		for x := range c.RemotePeers {
-			// Make sure that we don't add in this server twice
-			if c.LocalPeer.Address != c.RemotePeers[x].Address {
-
-				// Build the address from the peer configuration
-				peerAddress := fmt.Sprintf("%s:%d", c.RemotePeers[x].Address, c.RemotePeers[x].Port)
-
-				// Set this peer into the raft configuration
-				configuration.Servers = append(configuration.Servers, raft.Server{
-					ID:      raft.ServerID(c.RemotePeers[x].ID),
-					Address: raft.ServerAddress(peerAddress)})
-			}
-		}
-		log.Info("This node will attempt to start as Follower")
-	} else {
-		log.Info("This node will attempt to start as Leader")
-	}
-
-	// Bootstrap cluster
-	if err := raft.BootstrapCluster(config, logStore, stableStore, snapshots, transport, configuration); err != nil {
-		return err
-	}
-
-	// Create RAFT instance
-	raftServer, err := raft.NewRaft(config, cluster.stateMachine, logStore, stableStore, snapshots, transport)
-	if err != nil {
-		return err
-	}
-
-	cluster.stop = make(chan bool, 1)
-	cluster.completed = make(chan bool, 1)
-	ticker := time.NewTicker(time.Second)
-	isLeader := c.StartAsLeader
-
-	// (attempt to) Remove the virtual IP, incase it already exists
-	cluster.Network.DeleteIP()
-
-	// leader log broadcast - this counter is used to stop flooding STDOUT with leader log entries
-	var leaderbroadcast int
-	// Managers for Vip load balancers and none-vip loadbalancers
-	nonVipLB := loadbalancer.LBManager{}
-	VipLB := loadbalancer.LBManager{}
-
-	// Iterate through all Configurations
-	for x := range c.LoadBalancers {
-		// If the load balancer doesn't bind to the VIP
-		if c.LoadBalancers[x].BindToVip == false {
-			err = nonVipLB.Add("", &c.LoadBalancers[x])
-			if err != nil {
-				log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-			}
-		}
-	}
-
-	// On a cold start the node will sleep for 5 seconds to ensure that leader elections are complete
-	log.Infoln("This instance will wait approximately 5 seconds, from cold start to ensure cluster elections are complete")
-	time.Sleep(time.Second * 5)
-
-	go func() {
-		for {
-			if c.AddPeersAsBackends == true {
-				// Get addresses and change backends
-
-				// c.LoadBalancers[0].Backends
-				// for x := range raftServer.GetConfiguration().Configuration().Servers {
-				// 	raftServer.GetConfiguration().Configuration().Servers[x].Address
-				// }
-
-			}
-			// Broadcast the current leader on this node if it's the correct time (every leaderLogcount * time.Second)
-			if leaderbroadcast == leaderLogcount {
-				log.Infof("The Node [%s] is leading", raftServer.Leader())
-				// Reset the timer
-				leaderbroadcast = 0
-
-				// ensure that if this node is the leader, it is set as the leader
-				if localAddress == string(raftServer.Leader()) {
-					// Re-broadcast arp to ensure network stays up to date
-					if c.EnableARP == true {
-						// Gratuitous ARP, will broadcast to new MAC <-> IP
-						err = vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-					}
-					if !isLeader {
-						log.Infoln("This node is leading, but isnt the leader (correcting)")
-						isLeader = true
-					}
-				} else {
-					// (attempt to) Remove the virtual IP, incase it already exists to keep nodes clean
-					cluster.Network.DeleteIP()
-					isLeader = false
-				}
-
-			}
-			leaderbroadcast++
-
-			select {
-			case leader := <-raftServer.LeaderCh():
-				log.Infoln("New Election event")
-				if leader {
-					isLeader = true
-
-					log.Info("This node is assuming leadership of the cluster")
-					err = cluster.Network.AddIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-
-					// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-
-					for x := range c.LoadBalancers {
-
-						if c.LoadBalancers[x].BindToVip == true {
-							err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-							if err != nil {
-								log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-								log.Errorf("Dropping Leadership to another node in the cluster")
-								raftServer.LeadershipTransfer()
-
-								// Stop all load balancers associated with the VIP
-								err = VipLB.StopAll()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-
-								err = cluster.Network.DeleteIP()
-								if err != nil {
-									log.Warnf("%v", err)
-								}
-							}
-						}
-					}
-
-					if c.EnableARP == true {
-						// Gratuitous ARP, will broadcast to new MAC <-> IP
-						err = vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-					}
-				} else {
-					isLeader = false
-
-					log.Info("This node is becoming a follower within the cluster")
-
-					// Stop all load balancers associated with the VIP
-					err = VipLB.StopAll()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-
-					err = cluster.Network.DeleteIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-
-			case <-ticker.C:
-
-				if isLeader {
-
-					result, err := cluster.Network.IsSet()
-					if err != nil {
-						log.WithFields(log.Fields{"error": err, "ip": cluster.Network.IP(), "interface": cluster.Network.Interface()}).Error("Could not check ip")
-					}
-
-					if result == false {
-						log.Error("This node is leader and is adopting the virtual IP")
-
-						err = cluster.Network.AddIP()
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-						// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-
-						for x := range c.LoadBalancers {
-
-							if c.LoadBalancers[x].BindToVip == true {
-								err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-								if err != nil {
-									log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-								}
-							}
-						}
-						if c.EnableARP == true {
-							// Gratuitous ARP, will broadcast to new MAC <-> IP
-							err = vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-							if err != nil {
-								log.Warnf("%v", err)
-							}
-						}
-					}
-				}
-
-			case <-cluster.stop:
-				log.Info("[RAFT] Stopping this node")
-				log.Info("[LOADBALANCER] Stopping load balancers")
-
-				// Stop all load balancers associated with the VIP
-				err = VipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				// Stop all load balancers associated with the Host
-				err = nonVipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				if isLeader {
-					log.Info("[VIP] Releasing the Virtual IP")
-					err = cluster.Network.DeleteIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-
-				close(cluster.completed)
-
-				return
-			}
-		}
-	}()
-
-	log.Info("Started")
-
-	return nil
-}
-
-// Stop - Will stop the Cluster and release VIP if needed
-func (cluster *Cluster) Stop() {
-	// Close the stop chanel, which will shut down the VIP (if needed)
-	close(cluster.stop)
-
-	// Wait until the completed channel is closed, signallign all shutdown tasks completed
-	<-cluster.completed
-
-	log.Info("Stopped")
-}

pkg/cluster/service.go

@@ -0,0 +1,381 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"net"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
+	"github.com/kube-vip/kube-vip/pkg/backend"
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+	"github.com/vishvananda/netlink"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Config, sm *Manager, bgpServer *bgp.Server, cancelLeaderElection context.CancelFunc) error {
+	var err error
+
+	// listen for interrupts or the Linux SIGTERM signal and cancel
+	// our context, which the leader election code will observe and
+	// step down
+	signalChan := make(chan os.Signal, 1)
+	// Add Notification for Userland interrupt
+	signal.Notify(signalChan, syscall.SIGINT)
+
+	// Add Notification for SIGTERM (sent from Kubernetes)
+	signal.Notify(signalChan, syscall.SIGTERM)
+
+	loadbalancers := []*loadbalancer.IPVSLoadBalancer{}
+
+	var arpWG sync.WaitGroup
+
+	for i := range cluster.Network {
+		network := cluster.Network[i]
+		if err := network.SetMask(c.VIPSubnet); err != nil {
+			log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
+			panic("")
+		}
+		if network.IsDDNS() {
+			if err := cluster.StartDDNS(ctxDNS); err != nil {
+				log.Error(err.Error())
+			}
+		}
+
+		// start the dns updater if address is dns
+		if network.IsDNS() {
+			log.Info("starting the DNS updater", "address", network.DNSName())
+			ipUpdater := vip.NewIPUpdater(network)
+			ipUpdater.Run(ctxDNS)
+		}
+
+		if !c.EnableRoutingTable {
+			if _, err = network.AddIP(false); err != nil {
+				log.Error(err.Error())
+			}
+		}
+
+		if c.EnableBGP {
+			// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
+			log.Debug("Attempting to advertise over BGP", "address", network.CIDR())
+			err = bgpServer.AddHost(network.CIDR())
+			if err != nil {
+				log.Error(err.Error())
+			}
+		}
+
+		if c.EnableLoadBalancer {
+			lb, err := loadbalancer.NewIPVSLB(network.IP(), c.LoadBalancerPort, c.LoadBalancerForwardingMethod, c.BackendHealthCheckInterval, c.Interface, cancelLeaderElection, signalChan)
+			if err != nil {
+				log.Error("Error creating IPVS LoadBalancer", "err", err)
+			}
+
+			go func() {
+				err = sm.NodeWatcher(ctxArp, lb, c.Port) //TODO: We're using the ctxARP as the context this will change when rkatz finishes his change
+				if err != nil {
+					log.Error("Error watching node labels", "err", err)
+				}
+			}()
+
+			loadbalancers = append(loadbalancers, lb)
+		}
+
+		if c.EnableARP {
+			arpWG.Add(1)
+			go cluster.layer2Update(ctxArp, network, c, &arpWG)
+		}
+	}
+
+	if c.EnableLoadBalancer {
+		// Shutdown function that will wait on this signal, unless we call it ourselves
+		<-signalChan
+		for _, lb := range loadbalancers {
+			err = lb.RemoveIPVSLB()
+			if err != nil {
+				log.Error("Error stopping IPVS LoadBalancer", "err", err)
+			}
+		}
+	}
+
+	if c.EnableRoutingTable {
+		backendMapV4 := backend.Map{}
+		backendMapV6 := backend.Map{}
+		// only check localhost
+
+		nodename := ""
+		if c.NodeName != "" {
+			nodename = c.NodeName
+		} else {
+			nodename = os.Getenv("HOSTNAME")
+		}
+
+		ips := []string{}
+		if nodename != "" {
+			if ips, err = getNodeIPs(ctxArp, nodename, sm.KubernetesClient); err != nil && !apierrors.IsNotFound(err) {
+				log.Error("failed to get IP of control-plane nod", "err", err)
+			}
+		}
+
+		if len(ips) == 0 {
+			isV6, err := isV6(cluster.Network[0].IP())
+			if err != nil {
+				return fmt.Errorf("failed to parse IP '%s'", cluster.Network[0].IP())
+			}
+			if !isV6 {
+				ips = append(ips, "127.0.0.1")
+			} else {
+				ips = append(ips, "::1")
+			}
+
+			log.Info("no IP address found for node - will fallback to use localhost address", "addresses", ips)
+		}
+
+		for _, ip := range ips {
+			entry := backend.Entry{Addr: ip, Port: c.Port}
+			ipv6, err := isV6(ip)
+			if err != nil {
+				log.Error("failed to check IP type", "IP", ip, "error", err)
+			}
+			if !ipv6 {
+				backendMapV4[entry] = false
+			} else {
+				backendMapV6[entry] = false
+			}
+		}
+
+		stop := make(chan struct{})
+
+		// will wait for system interrupt and will send stop signal to backend watch
+		go func() {
+			<-signalChan
+			stop <- struct{}{}
+		}()
+
+		backend.Watch(func() {
+			for i := range cluster.Network {
+				network := cluster.Network[i]
+				networkIP := network.IP()
+				isNetworkV6, err := isV6(networkIP)
+				if err != nil {
+					log.Error("failed to check IP type", "IP", networkIP, "error", err)
+					continue
+				}
+
+				backendMap := &backendMapV4
+				if isNetworkV6 {
+					backendMap = &backendMapV6
+				}
+
+				for entry := range *backendMap {
+					if entry.Check() {
+						_, err = network.AddIP(true)
+						if err != nil {
+							log.Error("error adding address", "err", err)
+						}
+						if !(*backendMap)[entry] {
+							log.Info("added backend", "ip", network.IP())
+						}
+
+						err = network.AddRoute(true)
+						if err != nil && !errors.Is(err, fs.ErrExist) && !errors.Is(err, syscall.ESRCH) {
+							log.Warn(err.Error())
+						} else if err == nil && !(*backendMap)[entry] {
+							log.Info("added route", "route", network.PrepareRoute().String())
+						}
+
+						(*backendMap)[entry] = true
+						break
+					}
+					(*backendMap)[entry] = false
+				}
+
+				deleteAddress := true
+				for entry := range *backendMap {
+					if (*backendMap)[entry] {
+						deleteAddress = false
+						break
+					}
+				}
+
+				if deleteAddress {
+					err = network.DeleteRoute()
+					if err != nil && !errors.Is(err, fs.ErrNotExist) && !errors.Is(err, syscall.ESRCH) {
+						log.Warn("deleting route", "err", err)
+					} else if err == nil {
+						log.Info("deleted route", "route", network.PrepareRoute().String())
+					}
+
+					deleted, err := network.DeleteIP()
+					if err != nil {
+						log.Error("error deleting IP", "err", err)
+						panic("")
+					}
+					if deleted {
+						log.Info("deleted address", "IP", network.IP(), "interface", network.Interface())
+					}
+				}
+			}
+		}, c.BackendHealthCheckInterval, stop)
+	}
+
+	return nil
+}
+
+func isV6(ip string) (bool, error) {
+	ipaddr := net.ParseIP(ip)
+	if ipaddr == nil {
+		return false, fmt.Errorf("failed to parse IP '%s'", ip)
+	}
+	return ipaddr.To4() == nil, nil
+}
+
+func getNodeIPs(ctx context.Context, nodename string, client *kubernetes.Clientset) ([]string, error) {
+	node, err := client.CoreV1().Nodes().Get(ctx, nodename, metav1.GetOptions{})
+	if err != nil && !apierrors.IsNotFound(err) {
+		return []string{}, fmt.Errorf("failed to get data about '%s' node: %w", nodename, err)
+	}
+	ips := []string{}
+	for _, addr := range node.Status.Addresses {
+		if addr.Type == corev1.NodeInternalIP {
+			ips = append(ips, addr.Address)
+		}
+	}
+	return ips, nil
+}
+
+// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
+func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp *bgp.Server, name string, CountRouteReferences func(*netlink.Route) int) {
+	// use a Go context so we can tell the arp loop code when we
+	// want to step down
+	//nolint
+	ctxArp, cancelArp := context.WithCancel(ctx)
+
+	cluster.stop = make(chan bool, 1)
+	cluster.completed = make(chan bool, 1)
+
+	var arpWG sync.WaitGroup
+
+	for i := range cluster.Network {
+		network := cluster.Network[i]
+		if err := network.SetMask(c.VIPSubnet); err != nil {
+			log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
+			panic("")
+		}
+		_, err := network.DeleteIP()
+		if err != nil {
+			log.Warn("attempted to clean existing VIP", "err", err)
+		}
+
+		if c.EnableRoutingTable && (c.EnableLeaderElection || c.EnableServicesElection) {
+			err = network.AddRoute(false)
+			if err != nil {
+				log.Warn(err.Error())
+			}
+		} else if !c.EnableRoutingTable {
+			if _, err = network.AddIP(false); err != nil {
+				log.Warn(err.Error())
+			}
+		}
+
+		if c.EnableARP {
+			arpWG.Add(1)
+			go cluster.layer2Update(ctxArp, network, c, &arpWG)
+		}
+
+		if c.EnableBGP && (c.EnableLeaderElection || c.EnableServicesElection) {
+			// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
+			log.Debug("(svcs) attempting to advertise over BGP", "address", network.CIDR())
+			err = bgp.AddHost(network.CIDR())
+			if err != nil {
+				log.Error(err.Error())
+			}
+		}
+	}
+
+	go func() {
+		<-cluster.stop
+		// Stop the Arp context if it is running
+		cancelArp()
+
+		arpWG.Wait() // wait for all cluster ARP/NDP to be finished
+
+		log.Info("[LOADBALANCER] Stopping load balancers", "name", name)
+
+		if c.EnableRoutingTable {
+			for i := range cluster.Network {
+				// chek if route is not  referenced by another service
+				r := cluster.Network[i].PrepareRoute()
+				if CountRouteReferences(r) < 1 {
+					log.Info("[VIP] Deleting Route for VIP", "IP", cluster.Network[i].IP())
+					if err := cluster.Network[i].DeleteRoute(); err != nil {
+						log.Warn(err.Error())
+					}
+				}
+			}
+
+			close(cluster.completed)
+			return
+		}
+		for i := range cluster.Network {
+			if c.EnableARP && cluster.arpMgr.Count(cluster.Network[i].ARPName()) > 0 {
+				continue
+			}
+			log.Info("[VIP] Deleting VIP", "ip", cluster.Network[i].IP())
+			deleted, err := cluster.Network[i].DeleteIP()
+			if err != nil {
+				log.Warn(err.Error())
+			}
+			if deleted {
+				log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+			}
+		}
+
+		close(cluster.completed)
+	}()
+}
+
+// Layer2Update, handles the creation of the
+func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config, arpWG *sync.WaitGroup) {
+	defer arpWG.Done()
+	log.Info("layer 2 broadcaster starting")
+	var ndp *vip.NdpResponder
+	var err error
+	ipString := network.IP()
+	if vip.IsIPv6(ipString) {
+		if network.IPisLinkLocal() {
+			log.Error("layer2 is link-local can't use NDP", "address", ipString)
+
+		} else {
+			ndp, err = vip.NewNDPResponder(network.Interface())
+			if err != nil {
+				log.Error("failed to create new NDP Responder", "error", err)
+			} else {
+				if ndp != nil {
+					defer ndp.Close()
+				}
+			}
+		}
+	}
+
+	log.Debug("layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
+
+	arpInstance := arp.NewInstance(network, ndp)
+	cluster.arpMgr.Insert(arpInstance)
+
+	<-ctx.Done() // if cancel() execute
+	log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
+	cluster.arpMgr.Remove(arpInstance)
+}

pkg/cluster/singleNode.go

@@ -2,14 +2,11 @@ package cluster
 
 import (
 	"context"
-	"fmt"
-	"time"
 
-	log "github.com/sirupsen/logrus"
+	log "log/slog"
 
 	"github.com/kube-vip/kube-vip/pkg/bgp"
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
 	"github.com/kube-vip/kube-vip/pkg/vip"
 )
 
@@ -17,203 +14,71 @@ import (
 func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) error {
 	// Start kube-vip as a single node server
 
-	// TODO - Split all this code out as a seperate function
-	log.Infoln("Starting kube-vip as a single node cluster")
+	// TODO - Split all this code out as a separate function
+	log.Info("Starting kube-vip as a single node cluster")
 
 	log.Info("This node is assuming leadership of the cluster")
 
 	cluster.stop = make(chan bool, 1)
 	cluster.completed = make(chan bool, 1)
 
-	// Managers for Vip load balancers and none-vip loadbalancers
-	nonVipLB := loadbalancer.LBManager{}
-	VipLB := loadbalancer.LBManager{}
-
-	// Iterate through all Configurations
-	for x := range c.LoadBalancers {
-		// If the load balancer doesn't bind to the VIP
-		if c.LoadBalancers[x].BindToVip == false {
-			err := nonVipLB.Add("", &c.LoadBalancers[x])
+	for i := range cluster.Network {
+		if !disableVIP {
+			deleted, err := cluster.Network[i].DeleteIP()
 			if err != nil {
-				log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
+				log.Warn("Attempted to clean existing VIP", "err", err)
+			}
+			if deleted {
+				log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+			}
+
+			_, err = cluster.Network[i].AddIP(false)
+			if err != nil {
+				log.Warn(err.Error())
 			}
 
 		}
-	}
 
-	if !disableVIP {
-		err := cluster.Network.DeleteIP()
-		if err != nil {
-			log.Warnf("Attempted to clean existing VIP => %v", err)
-		}
-
-		err = cluster.Network.AddIP()
-		if err != nil {
-			log.Warnf("%v", err)
-		}
-
-		// Once we have the VIP running, start the load balancer(s) that bind to the VIP
-		for x := range c.LoadBalancers {
-
-			if c.LoadBalancers[x].BindToVip == true {
-				err = VipLB.Add(cluster.Network.IP(), &c.LoadBalancers[x])
-				if err != nil {
-					log.Warnf("Error creating loadbalancer [%s] type [%s] -> error [%s]", c.LoadBalancers[x].Name, c.LoadBalancers[x].Type, err)
-				}
+		if c.EnableARP {
+			// Gratuitous ARP, will broadcast to new MAC <-> IP
+			err := vip.ARPSendGratuitous(cluster.Network[i].IP(), c.Interface)
+			if err != nil {
+				log.Warn(err.Error())
 			}
 		}
 	}
 
-	if c.EnableARP == true {
-		// Gratuitous ARP, will broadcast to new MAC <-> IP
-		err := vip.ARPSendGratuitous(cluster.Network.IP(), c.Interface)
-		if err != nil {
-			log.Warnf("%v", err)
-		}
-	}
-
 	go func() {
-		for {
-			select {
-			case <-cluster.stop:
-				log.Info("[LOADBALANCER] Stopping load balancers")
+		<-cluster.stop
 
-				// Stop all load balancers associated with the VIP
-				err := VipLB.StopAll()
+		if !disableVIP {
+			for i := range cluster.Network {
+				log.Info("[VIP] Releasing the VIP", "address", cluster.Network[i].IP())
+				deleted, err := cluster.Network[i].DeleteIP()
 				if err != nil {
-					log.Warnf("%v", err)
+					log.Warn(err.Error())
 				}
-
-				// Stop all load balancers associated with the Host
-				err = nonVipLB.StopAll()
-				if err != nil {
-					log.Warnf("%v", err)
+				if deleted {
+					log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
 				}
-
-				if !disableVIP {
-
-					log.Info("[VIP] Releasing the Virtual IP")
-					err = cluster.Network.DeleteIP()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-				close(cluster.completed)
-				return
 			}
 		}
+		close(cluster.completed)
 	}()
-	log.Infoln("Started Load Balancer and Virtual IP")
+	log.Info("Started Load Balancer and Virtual IP")
 	return nil
 }
 
-// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
-func (cluster *Cluster) StartLoadBalancerService(c *kubevip.Config, bgp *bgp.Server) error {
-	// Start a kube-vip loadbalancer service
-	log.Infof("Starting advertising address [%s] with kube-vip", c.VIP)
-
+func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp.Server) error {
 	// use a Go context so we can tell the arp loop code when we
 	// want to step down
 	ctxArp, cancelArp := context.WithCancel(context.Background())
 	defer cancelArp()
 
-	cluster.stop = make(chan bool, 1)
-	cluster.completed = make(chan bool, 1)
+	// use a Go context so we can tell the dns loop code when we
+	// want to step down
+	ctxDNS, cancelDNS := context.WithCancel(context.Background())
+	defer cancelDNS()
 
-	err := cluster.Network.DeleteIP()
-	if err != nil {
-		log.Warnf("Attempted to clean existing VIP => %v", err)
-	}
-
-	err = cluster.Network.AddIP()
-	if err != nil {
-		log.Warnf("%v", err)
-	}
-
-	if c.EnableARP == true {
-		ctxArp, cancelArp = context.WithCancel(context.Background())
-
-		ipString := cluster.Network.IP()
-
-		var ndp *vip.NdpResponder
-		if vip.IsIPv6(ipString) {
-			ndp, err = vip.NewNDPResponder(c.Interface)
-			if err != nil {
-				log.Fatalf("failed to create new NDP Responder")
-			}
-		}
-		go func(ctx context.Context) {
-			if ndp != nil {
-				defer ndp.Close()
-			}
-
-			for {
-
-				select {
-				case <-ctx.Done(): // if cancel() execute
-					return
-				default:
-					// Ensure the address exists on the interface before attempting to ARP
-					set, err := cluster.Network.IsSet()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-					if !set {
-						log.Warnf("Re-applying the VIP configuration [%s] to the interface [%s]", ipString, c.Interface)
-						err = cluster.Network.AddIP()
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-					}
-
-					if vip.IsIPv4(ipString) {
-						// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
-						err := vip.ARPSendGratuitous(ipString, c.Interface)
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-					} else {
-						// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
-						err := ndp.SendGratuitous(ipString)
-						if err != nil {
-							log.Warnf("%v", err)
-						}
-					}
-				}
-				time.Sleep(3 * time.Second)
-			}
-		}(ctxArp)
-	}
-
-	if c.EnableBGP {
-		// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
-		cidrVip := fmt.Sprintf("%s/%s", cluster.Network.IP(), c.VIPCIDR)
-		log.Debugf("Attempting to advertise the address [%s] over BGP", cidrVip)
-		err = bgp.AddHost(cidrVip)
-		if err != nil {
-			log.Error(err)
-		}
-	}
-
-	go func() {
-		for {
-			select {
-			case <-cluster.stop:
-				// Stop the Arp context if it is running
-				cancelArp()
-
-				log.Info("[LOADBALANCER] Stopping load balancers")
-				log.Infof("[VIP] Releasing the Virtual IP [%s]", c.VIP)
-				err = cluster.Network.DeleteIP()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				close(cluster.completed)
-				return
-			}
-		}
-	}()
-	log.Infoln("Started Load Balancer and Virtual IP")
-	return nil
+	return cluster.vipService(ctxArp, ctxDNS, c, sm, bgp, nil)
 }

pkg/cluster/state.go

@@ -1,39 +0,0 @@
-package cluster
-
-import (
-	"io"
-
-	"github.com/hashicorp/raft"
-)
-
-// FSM - Finite State Machine for Raft
-type FSM struct {
-}
-
-// Apply - TODO
-func (fsm FSM) Apply(log *raft.Log) interface{} {
-	return nil
-}
-
-// Restore - TODO
-func (fsm FSM) Restore(snap io.ReadCloser) error {
-	return nil
-}
-
-// Snapshot - TODO, returns an empty snapshot
-func (fsm FSM) Snapshot() (raft.FSMSnapshot, error) {
-	return Snapshot{}, nil
-}
-
-// Snapshot -
-type Snapshot struct {
-}
-
-// Persist -
-func (snapshot Snapshot) Persist(sink raft.SnapshotSink) error {
-	return nil
-}
-
-// Release -
-func (snapshot Snapshot) Release() {
-}

pkg/egress/egress.go

@@ -0,0 +1,99 @@
+package egress
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/kube-vip/kube-vip/pkg/iptables"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/nftables"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
+
+func Teardown(podIP, vipIP, namespace, serviceUUID string, annotations map[string]string, useNftables bool) error {
+	// Look up the destination ports from the annotations on the service
+	destinationPorts := annotations[kubevip.EgressDestinationPorts]
+	deniedNetworks := annotations[kubevip.EgressDeniedNetworks]
+	allowedNetworks := annotations[kubevip.EgressAllowedNetworks]
+	internalEgress := annotations[kubevip.EgressInternal]
+
+	protocol := iptables.ProtocolIPv4
+	IPv6 := false
+	if vip.IsIPv6(podIP) {
+		protocol = iptables.ProtocolIPv6
+		IPv6 = true
+	}
+
+	// Use the internal egress implementation
+	if internalEgress != "" {
+		return nftables.DeleteSNAT(IPv6, serviceUUID)
+	}
+
+	i, err := vip.CreateIptablesClient(useNftables, namespace, protocol)
+	if err != nil {
+		return fmt.Errorf("error Creating iptables client [%s]", err)
+	}
+
+	if deniedNetworks != "" {
+		networks := strings.Split(deniedNetworks, ",")
+		for x := range networks {
+			err = i.DeleteMangleReturnForNetwork(vip.MangleChainName, networks[x])
+			if err != nil {
+				return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
+			}
+		}
+	}
+
+	if allowedNetworks != "" {
+		networks := strings.Split(allowedNetworks, ",")
+		for x := range networks {
+			err = i.DeleteMangleMarkingForNetwork(podIP, vip.MangleChainName, networks[x])
+			if err != nil {
+				return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
+			}
+		}
+	} else {
+		// Remove the marking of egress packets
+		err = i.DeleteMangleMarking(podIP, vip.MangleChainName)
+		if err != nil {
+			return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+		}
+	}
+
+	// Clear up SNAT rules
+	if destinationPorts != "" {
+		fixedPorts := strings.Split(destinationPorts, ",")
+
+		for _, fixedPort := range fixedPorts {
+			var proto, port string
+
+			data := strings.Split(fixedPort, ":")
+			if len(data) == 0 {
+				continue
+			} else if len(data) == 1 {
+				proto = "tcp"
+				port = data[0]
+			} else {
+				proto = data[0]
+				port = data[1]
+			}
+
+			err = i.DeleteSourceNatForDestinationPort(podIP, vipIP, port, proto)
+			if err != nil {
+				return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+			}
+
+		}
+	} else {
+		err = i.DeleteSourceNat(podIP, vipIP)
+		if err != nil {
+			return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+		}
+	}
+
+	err = vip.DeleteExistingSessions(podIP, false, destinationPorts, "")
+	if err != nil {
+		return fmt.Errorf("error changing iptables rules for egress [%s]", err)
+	}
+	return nil
+}

pkg/endpoints/endpoints.go

@@ -0,0 +1,165 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	"net"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	v1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	"k8s.io/apimachinery/pkg/watch"
+)
+
+type Processor struct {
+	config    *kubevip.Config
+	provider  providers.Provider
+	bgpServer *bgp.Server
+	worker    endpointWorker
+	instances *[]*instance.Instance
+}
+
+func NewEndpointProcessor(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
+	instances *[]*instance.Instance) *Processor {
+	return &Processor{
+		config:    config,
+		provider:  provider,
+		bgpServer: bgpServer,
+		instances: instances,
+		worker:    newEndpointWorker(config, provider, bgpServer, instances),
+	}
+}
+
+func (p *Processor) AddOrModify(ctx *servicecontext.Context, event watch.Event,
+	lastKnownGoodEndpoint *string, service *v1.Service, id string, leaderElectionActive *bool,
+	serviceFunc func(context.Context, *v1.Service) error,
+	leaderCtx *context.Context, cancel *context.CancelFunc) (bool, error) {
+
+	var err error
+	if err = p.provider.LoadObject(event.Object, *cancel); err != nil {
+		return false, fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
+	}
+
+	endpoints, err := p.worker.getEndpoints(service, id)
+	if err != nil {
+		return false, err
+	}
+
+	if err := p.worker.setInstanceEndpointsStatus(service, endpoints); err != nil {
+		log.Error("updating instance", "err", err)
+	}
+
+	// Find out if we have any local endpoints
+	// if out endpoint is empty then populate it
+	// if not, go through the endpoints and see if ours still exists
+	// If we have a local endpoint then begin the leader Election, unless it's already running
+	//
+
+	// Check that we have local endpoints
+	if len(endpoints) != 0 {
+		// Ignore IPv4
+		if service.Annotations[kubevip.EgressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
+			return true, nil
+		}
+
+		p.updateLastKnownGoodEndpoint(lastKnownGoodEndpoint, endpoints, service, leaderElectionActive, *cancel)
+		// start leader election if it's enabled and not already started
+		if !*leaderElectionActive && p.config.EnableServicesElection {
+			go func() {
+				*leaderCtx, *cancel = context.WithCancel(ctx.Ctx)
+				startLeaderElection(*leaderCtx, leaderElectionActive, service, serviceFunc)
+			}()
+		}
+
+		// There are local endpoints available on the node
+		if !p.config.EnableServicesElection && !p.config.EnableLeaderElection {
+			if err := p.worker.processInstance(ctx, service, leaderElectionActive); err != nil {
+				return false, fmt.Errorf("failed to process non-empty instance: %w", err)
+			}
+		}
+	} else {
+		// There are no local endpoints
+		p.worker.clear(ctx, lastKnownGoodEndpoint, service, *cancel, leaderElectionActive)
+	}
+
+	// Set the service accordingly
+	p.updateAnnotations(service, lastKnownGoodEndpoint)
+
+	log.Debug("watcher", "provider",
+		p.provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", *lastKnownGoodEndpoint, "active leader election", *leaderElectionActive)
+
+	return false, nil
+}
+
+func (p *Processor) Delete(service *v1.Service, id string) error {
+	if err := p.worker.delete(service, id); err != nil {
+		return fmt.Errorf("[%s] error deleting service: %w", p.provider.GetLabel(), err)
+	}
+	return nil
+}
+
+func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service, leaderElectionActive *bool, cancel context.CancelFunc) {
+	// if we haven't populated one, then do so
+	if *lastKnownGoodEndpoint == "" {
+		*lastKnownGoodEndpoint = endpoints[0]
+		return
+	}
+
+	// check out previous endpoint exists
+	stillExists := false
+
+	for x := range endpoints {
+		if endpoints[x] == *lastKnownGoodEndpoint {
+			stillExists = true
+		}
+	}
+	// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
+	if !stillExists {
+		p.worker.removeEgress(service, lastKnownGoodEndpoint)
+		if *leaderElectionActive && (p.config.EnableServicesElection || p.config.EnableLeaderElection) {
+			log.Warn("existing endpoint has been removed, restarting leaderElection", "provider", p.provider.GetLabel(), "endpoint", *lastKnownGoodEndpoint)
+			// Stop the existing leaderElection
+			cancel()
+			// disable last leaderElection flag
+			*leaderElectionActive = false
+		}
+		// Set our active endpoint to an existing one
+		*lastKnownGoodEndpoint = endpoints[0]
+	}
+}
+
+func (p *Processor) updateAnnotations(service *v1.Service, lastKnownGoodEndpoint *string) {
+	// Set the service accordingly
+	if service.Annotations[kubevip.Egress] == "true" {
+		activeEndpointAnnotation := kubevip.ActiveEndpoint
+
+		if !p.config.EnableEndpoints && p.provider.GetProtocol() == string(discoveryv1.AddressTypeIPv6) {
+			activeEndpointAnnotation = kubevip.ActiveEndpointIPv6
+		}
+		service.Annotations[activeEndpointAnnotation] = *lastKnownGoodEndpoint
+	}
+}
+
+func startLeaderElection(ctx context.Context, leaderElectionActive *bool, service *v1.Service, serviceFunc func(context.Context, *v1.Service) error) {
+	// This is a blocking function, that will restart (in the event of failure)
+	for {
+		// if the context isn't cancelled restart
+		if ctx.Err() != context.Canceled {
+			*leaderElectionActive = true
+			err := serviceFunc(ctx, service)
+			if err != nil {
+				log.Error(err.Error())
+			}
+			*leaderElectionActive = false
+		} else {
+			*leaderElectionActive = false
+			break
+		}
+	}
+}

pkg/endpoints/endpoints_bgp.go

@@ -0,0 +1,125 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	v1 "k8s.io/api/core/v1"
+)
+
+type BGP struct {
+	generic
+	bgpServer *bgp.Server
+}
+
+func newBGP(generic generic, bgpServer *bgp.Server) endpointWorker {
+	return &BGP{
+		generic:   generic,
+		bgpServer: bgpServer,
+	}
+}
+
+func (b *BGP) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
+	if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
+		for _, cluster := range instance.Clusters {
+			for i := range cluster.Network {
+				if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) {
+					log.Debug("attempting to advertise BGP service", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP())
+					err := b.bgpServer.AddHost(cluster.Network[i].CIDR())
+					if err != nil {
+						log.Error("error adding BGP host", "provider", b.provider.GetLabel(), "err", err)
+					} else {
+						log.Info("added BGP host", "provider",
+							b.provider.GetLabel(), "ip", cluster.Network[i].CIDR(), "service name", service.Name, "namespace", service.Namespace)
+						ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
+						*leaderElectionActive = true
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (b *BGP) clear(ctx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
+	if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
+		// If BGP mode is enabled - routes should be deleted
+		if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
+			for _, cluster := range instance.Clusters {
+				for i := range cluster.Network {
+					err := b.bgpServer.DelHost(cluster.Network[i].CIDR())
+					if err != nil {
+						log.Error("deleting BGP host", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "err", err)
+					} else {
+						log.Info("deleted BGP host", "provider",
+							b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace)
+						ctx.ConfiguredNetworks.Delete(cluster.Network[i])
+						*leaderElectionActive = false
+					}
+				}
+			}
+
+		}
+	}
+
+	b.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
+}
+
+func (b *BGP) getEndpoints(service *v1.Service, id string) ([]string, error) {
+	return b.getAllEndpoints(service, id)
+}
+
+func (b *BGP) delete(service *v1.Service, id string) error {
+	// When no-leader-elecition mode
+	if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
+		// find all existing local endpoints
+		endpoints, err := b.getEndpoints(service, id)
+		if err != nil {
+			return fmt.Errorf("[%s] error getting endpoints: %w", b.provider.GetLabel(), err)
+		}
+
+		// If there were local endpoints deleted
+		if len(endpoints) > 0 {
+			b.deleteAction(service)
+		}
+	}
+
+	return nil
+}
+
+func (b *BGP) deleteAction(service *v1.Service) {
+	b.clearBGPHosts(service)
+}
+
+func (b *BGP) clearBGPHosts(service *v1.Service) {
+	ClearBGPHosts(service, b.instances, b.bgpServer)
+}
+
+func (b *BGP) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
+	return nil
+}
+
+func ClearBGPHosts(service *v1.Service, instances *[]*instance.Instance, bgpServer *bgp.Server) {
+	if instance := instance.FindServiceInstance(service, *instances); instance != nil {
+		ClearBGPHostsByInstance(instance, bgpServer)
+	}
+}
+
+func ClearBGPHostsByInstance(instance *instance.Instance, bgpServer *bgp.Server) {
+	for _, cluster := range instance.Clusters {
+		for i := range cluster.Network {
+			network := cluster.Network[i]
+			err := bgpServer.DelHost(network.CIDR())
+			if err != nil {
+				log.Error("[endpoint] error deleting BGP host", "err", err)
+			} else {
+				log.Debug("[endpoint] deleted BGP host", "ip",
+					network.CIDR(), "service name", instance.ServiceSnapshot.Name, "namespace", instance.ServiceSnapshot.Namespace)
+			}
+		}
+	}
+}

pkg/endpoints/endpoints_generic.go

@@ -0,0 +1,118 @@
+package endpoints
+
+import (
+	"context"
+	"fmt"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/egress"
+	"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	v1 "k8s.io/api/core/v1"
+)
+
+type endpointWorker interface {
+	processInstance(svcCtx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error
+	clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool)
+	getEndpoints(service *v1.Service, id string) ([]string, error)
+	removeEgress(service *v1.Service, lastKnownGoodEndpoint *string)
+	delete(service *v1.Service, id string) error
+	setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error
+}
+
+func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server, instances *[]*instance.Instance) endpointWorker {
+	generic := newGeneric(config, provider, instances)
+
+	if config.EnableRoutingTable {
+		return newRoutingTable(generic)
+	}
+	if config.EnableBGP {
+		return newBGP(generic, bgpServer)
+	}
+
+	return &generic
+}
+
+type generic struct {
+	config    *kubevip.Config
+	provider  providers.Provider
+	instances *[]*instance.Instance
+}
+
+func newGeneric(config *kubevip.Config, provider providers.Provider, instances *[]*instance.Instance) generic {
+	return generic{
+		config:    config,
+		provider:  provider,
+		instances: instances,
+	}
+}
+
+func (g *generic) processInstance(_ *servicecontext.Context, _ *v1.Service, _ *bool) error {
+	return nil
+}
+
+func (g *generic) clear(_ *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
+	g.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
+}
+
+func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
+	if *lastKnownGoodEndpoint != "" {
+		log.Warn("existing  endpoint has been removed, no remaining endpoints for leaderElection", "provider", g.provider.GetLabel(), "endpoint", lastKnownGoodEndpoint)
+		if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP, service.Namespace, string(service.UID), service.Annotations, g.config.EgressWithNftables); err != nil {
+			log.Error("error removing redundant egress rules", "err", err)
+		}
+
+		*lastKnownGoodEndpoint = "" // reset endpoint
+		if g.config.EnableServicesElection || g.config.EnableLeaderElection {
+			cancel() // stop services watcher
+		}
+		*leaderElectionActive = false
+	}
+}
+
+func (g *generic) getEndpoints(_ *v1.Service, id string) ([]string, error) {
+	return g.getLocalEndpoints(id)
+}
+
+func (g *generic) getLocalEndpoints(id string) ([]string, error) {
+	// Build endpoints
+	var endpoints []string
+	var err error
+	if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
+		return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
+	}
+
+	return endpoints, nil
+}
+
+func (g *generic) getAllEndpoints(service *v1.Service, id string) ([]string, error) {
+	// Build endpoints
+	var err error
+	var endpoints []string
+	if !g.config.EnableLeaderElection && !g.config.EnableServicesElection &&
+		service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
+		if endpoints, err = g.provider.GetAllEndpoints(); err != nil {
+			return nil, fmt.Errorf("[%s] error getting all endpoints: %w", g.provider.GetLabel(), err)
+		}
+	} else {
+		if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
+			return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
+		}
+	}
+
+	return endpoints, nil
+}
+
+func (g *generic) removeEgress(_ *v1.Service, _ *string) {
+}
+
+func (g *generic) delete(_ *v1.Service, _ string) error {
+	return nil
+}
+
+func (g *generic) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
+	return nil
+}

pkg/endpoints/endpoints_routing_table.go

@@ -0,0 +1,178 @@
+package endpoints
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"syscall"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/egress"
+	"github.com/kube-vip/kube-vip/pkg/instance"
+	"github.com/kube-vip/kube-vip/pkg/servicecontext"
+	"github.com/vishvananda/netlink"
+	v1 "k8s.io/api/core/v1"
+)
+
+type RoutingTable struct {
+	generic
+}
+
+func newRoutingTable(generic generic) endpointWorker {
+	return &RoutingTable{
+		generic: generic,
+	}
+}
+
+func (rt *RoutingTable) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
+	instance := instance.FindServiceInstance(service, *rt.instances)
+	if instance != nil {
+		for _, cluster := range instance.Clusters {
+			for i := range cluster.Network {
+				if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) && cluster.Network[i].HasEndpoints() {
+					err := cluster.Network[i].AddRoute(false)
+					if err != nil {
+						if errors.Is(err, syscall.EEXIST) {
+							// If route exists, but protocol is not set (e.g. the route was created by the older version
+							// of kube-vip) try to update it if necessary
+							isUpdated, err := cluster.Network[i].UpdateRoutes()
+							if err != nil {
+								return fmt.Errorf("[%s] error updating existing routes: %w", rt.provider.GetLabel(), err)
+							}
+							if isUpdated {
+								log.Info("updated route", "provider",
+									rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
+									service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
+							} else {
+								log.Info("route already present", "provider",
+									rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
+									service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
+							}
+						} else {
+							// If other error occurs, return error
+							return fmt.Errorf("[%s] error adding route: %s", rt.provider.GetLabel(), err.Error())
+						}
+					} else {
+						log.Info("added route", "provider",
+							rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
+							service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
+						ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
+						*leaderElectionActive = true
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
+	if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
+		if errs := ClearRoutes(service, rt.instances); len(errs) == 0 {
+			svcCtx.ConfiguredNetworks.Clear()
+		} else {
+			for _, err := range errs {
+				log.Error("error while clearing routes", "err", err)
+			}
+		}
+	}
+
+	rt.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
+}
+
+func (rt *RoutingTable) getEndpoints(service *v1.Service, id string) ([]string, error) {
+	return rt.getAllEndpoints(service, id)
+}
+
+func (rt *RoutingTable) removeEgress(service *v1.Service, lastKnownGoodEndpoint *string) {
+	if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
+		service.Namespace, string(service.UID), service.Annotations, rt.config.EgressWithNftables); err != nil {
+		log.Warn("removing redundant egress rules", "err", err)
+	}
+}
+
+func (rt *RoutingTable) delete(service *v1.Service, id string) error {
+	// When no-leader-elecition mode
+	if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
+		// find all existing local endpoints
+		endpoints, err := rt.getEndpoints(service, id)
+		if err != nil {
+			return fmt.Errorf("[%s] error getting endpoints: %w", rt.provider.GetLabel(), err)
+		}
+
+		// If there were local endpoints deleted
+		if len(endpoints) > 0 {
+			rt.deleteAction(service)
+		}
+	}
+
+	return nil
+}
+
+func (rt *RoutingTable) deleteAction(service *v1.Service) {
+	ClearRoutes(service, rt.instances)
+}
+
+func (rt *RoutingTable) setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error {
+	instance := instance.FindServiceInstance(service, *rt.instances)
+	if instance == nil {
+		log.Error("failed to find the instance", "service", service.UID, "provider", rt.provider.GetLabel())
+	} else {
+		for _, c := range instance.Clusters {
+			for n := range c.Network {
+				// if there are no endpoints set HasEndpoints false just in case
+				if len(endpoints) < 1 {
+					c.Network[n].SetHasEndpoints(false)
+				}
+				// check if endpoint are available and are of same IP family as service
+				if len(endpoints) > 0 && ((net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(endpoints[0]).To4() == nil)) {
+					c.Network[n].SetHasEndpoints(true)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func ClearRoutes(service *v1.Service, instances *[]*instance.Instance) []error {
+	errs := []error{}
+	if instance := instance.FindServiceInstance(service, *instances); instance != nil {
+		for _, cluster := range instance.Clusters {
+			for i := range cluster.Network {
+				route := cluster.Network[i].PrepareRoute()
+				// check if route we are about to delete is not referenced by more than one service
+				if CountRouteReferences(route, instances) <= 1 {
+					err := cluster.Network[i].DeleteRoute()
+					if err != nil && !errors.Is(err, syscall.ESRCH) {
+						log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
+						errs = append(errs, err)
+					}
+					log.Debug("deleted route", "ip",
+						cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface())
+				}
+			}
+		}
+	}
+	return errs
+}
+
+func CountRouteReferences(route *netlink.Route, instances *[]*instance.Instance) int {
+	cnt := 0
+	for _, instance := range *instances {
+		for _, cluster := range instance.Clusters {
+			for n := range cluster.Network {
+				if cluster.Network[n].HasEndpoints() {
+					r := cluster.Network[n].PrepareRoute()
+					if r.Dst.String() == route.Dst.String() {
+						cnt++
+					}
+				}
+			}
+		}
+	}
+	return cnt
+}

pkg/endpoints/providers/endpoints.go

@@ -0,0 +1,137 @@
+package providers
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	watchtools "k8s.io/client-go/tools/watch"
+	"k8s.io/client-go/util/retry"
+
+	log "log/slog"
+)
+
+type Endpoints struct {
+	label string
+	//nolint:staticcheck // SA1019 endpoints are moving to an opt-in only
+	endpoints *v1.Endpoints
+}
+
+func NewEndpoints() Provider {
+	return &Endpoints{
+		label: "endpoints",
+	}
+}
+
+func (ep *Endpoints) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
+	service *v1.Service) (*watchtools.RetryWatcher, error) {
+	opts := metav1.ListOptions{
+		FieldSelector: fields.OneTermEqualSelector("metadata.name", service.Name).String(),
+	}
+
+	rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
+		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
+			return clientSet.CoreV1().Endpoints(service.Namespace).Watch(ctx, opts)
+		},
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("error creating endpoint watcher: %s", err.Error())
+	}
+
+	return rw, nil
+}
+
+func (ep *Endpoints) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
+	//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
+	eps, ok := endpoints.(*v1.Endpoints)
+	if !ok {
+		cancel()
+		return fmt.Errorf("[%s] unable to parse Kubernetes services from API watcher", ep.GetLabel())
+	}
+	ep.endpoints = eps
+	return nil
+}
+
+func (ep *Endpoints) GetAllEndpoints() ([]string, error) {
+	result := []string{}
+	for subset := range ep.endpoints.Subsets {
+		for address := range ep.endpoints.Subsets[subset].Addresses {
+			addr := strings.Split(ep.endpoints.Subsets[subset].Addresses[address].IP, "/")
+			result = append(result, addr[0])
+		}
+	}
+
+	return result, nil
+}
+
+func (ep *Endpoints) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
+	var localEndpoints []string
+
+	for _, subset := range ep.endpoints.Subsets {
+		for _, address := range subset.Addresses {
+			log.Debug("processing endpoint", "label", ep.label, "ip", address.IP)
+
+			// 1. Compare the Nodename
+			if address.NodeName != nil && id == *address.NodeName {
+				log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname, "nodename", *address.NodeName)
+				localEndpoints = append(localEndpoints, address.IP)
+				continue
+			}
+			// 2. Compare the Hostname (only useful if address.NodeName is not available)
+			if id == address.Hostname {
+				log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname)
+				localEndpoints = append(localEndpoints, address.IP)
+				continue
+			}
+		}
+	}
+	return localEndpoints, nil
+}
+
+func (ep *Endpoints) UpdateServiceAnnotation(endpoint string, _ string, service *v1.Service, clientSet *kubernetes.Clientset) error {
+	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		// Retrieve the latest version of Deployment before attempting update
+		// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
+		currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		currentServiceCopy := currentService.DeepCopy()
+		if currentServiceCopy.Annotations == nil {
+			currentServiceCopy.Annotations = make(map[string]string)
+		}
+
+		currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
+
+		_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
+		if err != nil {
+			log.Error("error updating Service Spec", "label", ep.GetLabel(), "name", currentServiceCopy.Name, "err", err)
+			return err
+		}
+		return nil
+	})
+
+	if retryErr != nil {
+		log.Error("failed to set Services", "label", ep.GetLabel(), "err", retryErr)
+		return retryErr
+	}
+	return nil
+}
+
+func (ep *Endpoints) GetLabel() string {
+	return ep.label
+}
+
+func (ep *Endpoints) GetProtocol() string {
+	return ""
+}

pkg/endpoints/providers/endpointslices.go

@@ -0,0 +1,139 @@
+package providers
+
+import (
+	"context"
+	"fmt"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	v1 "k8s.io/api/core/v1"
+	discoveryv1 "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	watchtools "k8s.io/client-go/tools/watch"
+	"k8s.io/client-go/util/retry"
+)
+
+type Endpointslices struct {
+	label     string
+	endpoints *discoveryv1.EndpointSlice
+}
+
+func NewEndpointslices() Provider {
+	return &Endpointslices{
+		label: "endpointslices",
+	}
+}
+
+func (ep *Endpointslices) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
+	service *v1.Service) (*watchtools.RetryWatcher, error) {
+	labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/service-name": service.Name}}
+
+	opts := metav1.ListOptions{
+		LabelSelector: labels.Set(labelSelector.MatchLabels).String(),
+	}
+
+	rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
+		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
+			return clientSet.DiscoveryV1().EndpointSlices(service.Namespace).Watch(ctx, opts)
+		},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("[%s] error creating endpointslices watcher: %s", ep.label, err.Error())
+	}
+
+	return rw, nil
+}
+
+func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
+	eps, ok := endpoints.(*discoveryv1.EndpointSlice)
+	if !ok {
+		cancel()
+		return fmt.Errorf("[%s] error casting endpoints to v1.Endpoints struct", ep.label)
+	}
+	ep.endpoints = eps
+	return nil
+}
+
+func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
+	result := []string{}
+	for _, ep := range ep.endpoints.Endpoints {
+		result = append(result, ep.Addresses...)
+	}
+	return result, nil
+}
+
+func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
+	var localEndpoints []string
+	for _, endpoint := range ep.endpoints.Endpoints {
+		if !*endpoint.Conditions.Serving {
+			continue
+		}
+		for _, address := range endpoint.Addresses {
+			log.Debug("processing endpoint", "provider", ep.label, "ip", address)
+
+			// 1. Compare the Nodename
+			if endpoint.NodeName != nil && id == *endpoint.NodeName {
+				if endpoint.Hostname != nil {
+					log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname, "nodename", *endpoint.NodeName)
+				} else {
+					log.Debug("found endpoint", "provider", ep.label, "ip", address, "nodename", *endpoint.NodeName)
+				}
+				localEndpoints = append(localEndpoints, address)
+				continue
+			}
+
+			// 2. Compare the Hostname (only useful if endpoint.NodeName is not available)
+			if endpoint.Hostname != nil && id == *endpoint.Hostname {
+				log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname)
+				localEndpoints = append(localEndpoints, address)
+			}
+		}
+	}
+	return localEndpoints, nil
+}
+
+func (ep *Endpointslices) UpdateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, clientSet *kubernetes.Clientset) error {
+	retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		// Retrieve the latest version of Deployment before attempting update
+		// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
+		currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		currentServiceCopy := currentService.DeepCopy()
+		if currentServiceCopy.Annotations == nil {
+			currentServiceCopy.Annotations = make(map[string]string)
+		}
+
+		currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
+		currentServiceCopy.Annotations[kubevip.ActiveEndpointIPv6] = endpointIPv6
+
+		_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
+		if err != nil {
+			log.Error("error updating Service Spec", "provider", ep.label, "service name", currentServiceCopy.Name, "err", err)
+			return err
+		}
+		return nil
+	})
+
+	if retryErr != nil {
+		log.Error("failed to set Services", "provider", ep.label, "err", retryErr)
+		return retryErr
+	}
+	return nil
+}
+
+func (ep *Endpointslices) GetLabel() string {
+	return ep.label
+}
+
+func (ep *Endpointslices) GetProtocol() string {
+	return string(ep.endpoints.AddressType)
+}

pkg/endpoints/providers/interface.go

@@ -0,0 +1,22 @@
+package providers
+
+import (
+	"context"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
+	watchtools "k8s.io/client-go/tools/watch"
+)
+
+type Provider interface {
+	CreateRetryWatcher(context.Context, *kubernetes.Clientset,
+		*v1.Service) (*watchtools.RetryWatcher, error)
+	GetAllEndpoints() ([]string, error)
+	GetLocalEndpoints(string, *kubevip.Config) ([]string, error)
+	GetLabel() string
+	UpdateServiceAnnotation(string, string, *v1.Service, *kubernetes.Clientset) error
+	LoadObject(runtime.Object, context.CancelFunc) error
+	GetProtocol() string
+}

pkg/etcd/client.go

@@ -0,0 +1,26 @@
+package etcd
+
+import (
+	"go.etcd.io/etcd/client/pkg/v3/transport"
+	clientv3 "go.etcd.io/etcd/client/v3"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+)
+
+func NewClient(c *kubevip.Config) (*clientv3.Client, error) {
+	tlsInfo := transport.TLSInfo{
+		TrustedCAFile: c.Etcd.CAFile,
+		CertFile:      c.Etcd.ClientCertFile,
+		KeyFile:       c.Etcd.ClientKeyFile,
+	}
+
+	clientTLS, err := tlsInfo.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	return clientv3.New(clientv3.Config{
+		Endpoints: c.Etcd.Endpoints,
+		TLS:       clientTLS,
+	})
+}

pkg/etcd/election.go

@@ -0,0 +1,227 @@
+package etcd
+
+import (
+	"context"
+	"hash/fnv"
+	"time"
+
+	log "log/slog"
+
+	"github.com/pkg/errors"
+	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.etcd.io/etcd/client/v3/concurrency"
+)
+
+// LeaderElectionConfig allows to configure the leader election params.
+type LeaderElectionConfig struct {
+	// EtcdConfig contains the client to connect to the etcd cluster.
+	EtcdConfig ClientConfig
+
+	// Name uniquely identifies this leader election. All members of the same election
+	// should use the same value here.
+	Name string
+
+	// MemberID identifies uniquely this contestant from other in the leader election.
+	// It will be converted to an int64 using a hash, so theoretically collisions are possible
+	// when using a string. If you want to guarantee safety, us MemberUniqueID to specify a unique
+	// int64 directly.
+	// If two processes start a leader election using the same MemberID, one of them will
+	// fail.
+	MemberID string
+
+	// MemberUniqueID is the int equivalent to MemberID that allows to override the default conversion
+	// from string to int using hashing.
+	MemberUniqueID *uint64
+
+	// LeaseDurationSeconds is the duration that non-leader candidates will
+	// wait to force acquire leadership.
+	// This is just a request to the etcd server but it's not guaranteed, the server
+	// might decide to make the duration longer.
+	LeaseDurationSeconds int64
+
+	// Callbacks are callbacks that are triggered during certain lifecycle
+	// events of the LeaderElector
+	Callbacks LeaderCallbacks
+}
+
+// LeaderCallbacks are callbacks that are triggered during certain
+// lifecycle events of the election.
+type LeaderCallbacks struct {
+	// OnStartedLeading is called when this member starts leading.
+	OnStartedLeading func(context.Context)
+	// OnStoppedLeading is called when this member stops leading.
+	OnStoppedLeading func()
+	// OnNewLeader is called when the client observes a leader that is
+	// not the previously observed leader. This includes the first observed
+	// leader when the client starts.
+	OnNewLeader func(identity string)
+}
+
+// ClientConfig contains the client to connect to the etcd cluster.
+type ClientConfig struct {
+	Client *clientv3.Client
+}
+
+// RunElectionOrDie behaves the same way as RunElection but panics if there is an error.
+func RunElectionOrDie(ctx context.Context, config *LeaderElectionConfig) {
+	if err := RunElection(ctx, config); err != nil {
+		panic(err)
+	}
+}
+
+// RunElection starts a client with the provided config or panics.
+// RunElection blocks until leader election loop is
+// stopped by ctx or it has stopped holding the leader lease.
+func RunElection(ctx context.Context, config *LeaderElectionConfig) error {
+	var memberID uint64
+	if config.MemberUniqueID != nil {
+		memberID = *config.MemberUniqueID
+	} else {
+		h := fnv.New64a()
+		if _, err := h.Write(append([]byte(config.Name), []byte(config.MemberID)...)); err != nil {
+			return err
+		}
+		memberID = h.Sum64()
+	}
+
+	ttl := config.LeaseDurationSeconds
+	r := &pb.LeaseGrantRequest{TTL: ttl, ID: int64(memberID)} //nolint
+	lease, err := clientv3.RetryLeaseClient(
+		config.EtcdConfig.Client,
+	).LeaseGrant(ctx, r)
+	if err != nil {
+		return errors.Wrap(err, "creating lease")
+	}
+
+	leaseID := clientv3.LeaseID(lease.ID)
+
+	s, err := concurrency.NewSession(
+		config.EtcdConfig.Client,
+		concurrency.WithTTL(int(lease.TTL)),
+		concurrency.WithLease(leaseID),
+	)
+	if err != nil {
+		return err
+	}
+
+	election := concurrency.NewElection(s, config.Name)
+
+	m := &member{
+		client:         config.EtcdConfig.Client,
+		election:       election,
+		callbacks:      config.Callbacks,
+		memberID:       config.MemberID,
+		weAreTheLeader: make(chan struct{}, 1),
+		leaseTTL:       lease.TTL,
+	}
+
+	go m.tryToBeLeader(ctx)
+	m.watchLeaderChanges(ctx)
+
+	return nil
+}
+
+type member struct {
+	key              string
+	client           *clientv3.Client
+	election         *concurrency.Election
+	isLeader         bool
+	currentLeaderKey string
+	callbacks        LeaderCallbacks
+	memberID         string
+	weAreTheLeader   chan struct{}
+	leaseTTL         int64
+}
+
+func (m *member) watchLeaderChanges(ctx context.Context) {
+	observeCtx, observeCancel := context.WithCancel(ctx)
+	defer observeCancel()
+	changes := m.election.Observe(observeCtx)
+
+watcher:
+	for {
+		select {
+		case <-ctx.Done():
+			break watcher
+		case <-m.weAreTheLeader:
+
+			m.isLeader = true
+			m.key = m.election.Key() // by this time, this should already be set, since Campaign has already returned
+			log.Debug("Marking self as leader with key", "id", m.memberID, "key", m.key)
+		case response := <-changes:
+			log.Debug("Leader Changes", "id", m.memberID, "response", response)
+			if len(response.Kvs) == 0 {
+				// There is a race condition where just after we stop being the leader
+				// if there are no more leaders, we might get a response with no key-values
+				// just before the response channel is closed or the context is cancel
+				// In that case, just continue and let one of those two things happen
+				continue
+			}
+			newLeaderKey := response.Kvs[0].Key
+			if m.isLeader && m.key != string(newLeaderKey) {
+				// We stopped being leaders
+
+				// exit the loop, so we cancel the observe context so we stop watching
+				// for new leaders. That will close the channel and make this function exit,
+				// which also makes the routine to finish and RunElection returns
+				break watcher
+			}
+
+			if m.currentLeaderKey != string(newLeaderKey) {
+				// we observed a leader, this could be us or someone else
+				m.currentLeaderKey = string(newLeaderKey)
+				m.callbacks.OnNewLeader(string(response.Kvs[0].Value))
+			}
+		}
+	}
+
+	// If we are here, either we have stopped being leaders or we lost the watcher
+	// Make sure we call OnStoppedLeading if we were the leader.
+	if m.isLeader {
+		m.callbacks.OnStoppedLeading()
+	}
+
+	log.Debug("Exiting watcher", "id", m.memberID)
+}
+
+func (m *member) tryToBeLeader(ctx context.Context) {
+	if err := m.election.Campaign(ctx, m.memberID); err != nil {
+		log.Error("Failed trying to become the leader", "err", err)
+		// Resign just in case we acquired leadership just before failing
+		if err := m.election.Resign(m.client.Ctx()); err != nil {
+			log.Warn("Failed to resign after we failed becoming the leader, this might not be a problem if we were never the leader", "err", err)
+		}
+		return
+		// TODO: what to do here?
+		// We probably want watchLeaderChanges to exit as well, since Run
+		// is expecting us to try to become the leader, but if we are here,
+		// we won't. So if we don't panic, we need to signal it somehow
+	}
+
+	// Inform the observer that we are the leader as soon as possible,
+	// so it can detect if we stop being it
+	m.weAreTheLeader <- struct{}{}
+
+	// Once we are the leader, start the routine to resign if context is canceled
+	go m.resignOnCancel(ctx)
+
+	// After becoming the leader, we wait for at least a lease TTL to wait for
+	// the previous leader to detect the new leadership (if there was one) and
+	// stop its processes
+	// TODO: is this too cautious?
+	log.Debug("timeout before OnStartedLeading", "id", m.memberID, "timeout", m.leaseTTL)
+	time.Sleep(time.Second * time.Duration(m.leaseTTL))
+
+	// We are the leader, execute our code
+	m.callbacks.OnStartedLeading(ctx)
+
+	// Here the routine dies if OnStartedLeading doesn't block, there is nothing else to do
+}
+
+func (m *member) resignOnCancel(ctx context.Context) {
+	<-ctx.Done()
+	if err := m.election.Resign(m.client.Ctx()); err != nil {
+		log.Error("Failed to resign after the context was canceled", "err", err)
+	}
+}

pkg/etcd/election_test.go

@@ -0,0 +1,169 @@
+//go:build integration
+// +build integration
+
+package etcd_test
+
+import (
+	"context"
+	"log"
+	"math/rand"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/etcd"
+	. "github.com/onsi/gomega"
+	clientv3 "go.etcd.io/etcd/client/v3"
+	"go.uber.org/zap"
+)
+
+func TestRunElectionWithMemberIDCollision(t *testing.T) {
+	t.Parallel()
+	g := NewWithT(t)
+	ctx := context.Background()
+	cli := client(g)
+	defer cli.Close()
+
+	electionName := randomElectionNameForTest("memberIDConflict")
+	log.Printf("Election name %s\n", electionName)
+	memberCtx, cancelMember1 := context.WithCancel(ctx)
+	config := &etcd.LeaderElectionConfig{
+		EtcdConfig: etcd.ClientConfig{
+			Client: cli,
+		},
+		Name:                 electionName,
+		MemberID:             "my-host",
+		LeaseDurationSeconds: 1,
+		Callbacks: etcd.LeaderCallbacks{
+			OnStartedLeading: func(ctx context.Context) {
+				log.Println("I'm the leader!!!!")
+				log.Println("Renouncing as leader by canceling context")
+				cancelMember1()
+			},
+			OnNewLeader: func(identity string) {
+				log.Printf("New leader: %s\n", identity)
+			},
+			OnStoppedLeading: func() {
+				log.Println("I'm not the leader anymore")
+			},
+		},
+	}
+
+	wg := &sync.WaitGroup{}
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		g.Expect(etcd.RunElection(memberCtx, config)).To(Succeed())
+	}()
+
+	go func() {
+		defer wg.Done()
+		time.Sleep(time.Millisecond * 50) // make sure the first one becomes leader
+		g.Expect(etcd.RunElection(ctx, config)).Should(MatchError(ContainSubstring("creating lease")))
+	}()
+
+	wg.Wait()
+}
+
+func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
+	t.Parallel()
+	g := NewWithT(t)
+	ctx := context.Background()
+	cli := client(g)
+	defer cli.Close()
+
+	cliMember1 := client(g)
+	defer cliMember1.Close()
+
+	electionName := randomElectionNameForTest("steppingDown")
+	configBase := etcd.LeaderElectionConfig{
+		EtcdConfig: etcd.ClientConfig{
+			Client: cli,
+		},
+		Name:                 electionName,
+		LeaseDurationSeconds: 1,
+	}
+
+	member1Ctx, _ := context.WithCancel(ctx)
+	member2Ctx, cancelMember2 := context.WithCancel(ctx)
+
+	config1 := configBase
+	config1.EtcdConfig.Client = cliMember1
+	config1.MemberID = "my-host"
+	uniqueID := rand.Uint64()
+	config1.MemberUniqueID = &uniqueID
+	config1.Callbacks = baseCallbacksForName(config1.MemberID)
+	config1.Callbacks.OnStartedLeading = func(_ context.Context) {
+		log.Println("I'm my-host, the new leader!!!!")
+		log.Println("Losing the leadership on purpose by stopping renewing the lease")
+		g.Expect(cliMember1.Lease.Close()).To(Succeed())
+		log.Println("Member1 leases closed")
+	}
+
+	config2 := configBase
+	config2.MemberID = "my-other-host"
+	config2.Callbacks = baseCallbacksForName(config2.MemberID)
+	config2.Callbacks.OnStartedLeading = func(_ context.Context) {
+		log.Println("I'm my-other-host, the new leader!!!!")
+		log.Println("Renouncing as leader by canceling context")
+		cancelMember2()
+	}
+
+	wg := &sync.WaitGroup{}
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		g.Expect(etcd.RunElection(member1Ctx, &config1)).To(Succeed())
+		log.Println("Member1 routine done")
+	}()
+
+	go func() {
+		defer wg.Done()
+		time.Sleep(time.Millisecond * 50) // Make sure member1 becomes leader
+		g.Expect(etcd.RunElection(member2Ctx, &config2)).To(Succeed())
+		log.Println("Member2 routine done")
+	}()
+
+	wg.Wait()
+}
+
+func baseCallbacksForName(name string) etcd.LeaderCallbacks {
+	return etcd.LeaderCallbacks{
+		OnStartedLeading: func(ctx context.Context) {
+			log.Printf("[%s] I'm the new leader!!!!\n", name)
+		},
+		OnNewLeader: func(identity string) {
+			log.Printf("[%s] New leader: %s\n", name, identity)
+		},
+		OnStoppedLeading: func() {
+			log.Printf("[%s] I'm not the leader anymore\n", name)
+		},
+	}
+}
+
+func randomElectionNameForTest(name string) string {
+	return name + "-" + randomString(6)
+}
+
+const charSet = "0123456789abcdefghijklmnopqrstuvwxyz"
+
+var rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
+
+func randomString(n int) string {
+	result := make([]byte, n)
+	for i := range result {
+		result[i] = charSet[rnd.Intn(len(charSet))]
+	}
+	return string(result)
+}
+
+func client(g Gomega) *clientv3.Client {
+	c, err := clientv3.New(clientv3.Config{
+		Endpoints: []string{"localhost:2379"},
+		Logger:    zap.NewNop(),
+	})
+	g.Expect(err).NotTo(HaveOccurred())
+	return c
+}

pkg/etcd/etcd_suite_test.go

@@ -0,0 +1,153 @@
+//go:build integration
+// +build integration
+
+package etcd_test
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	etcdVersion     = "v3.5.9"
+	etcdBinDir      = "bin"
+	etcdBinPath     = etcdBinDir + "/etcd"
+	downloadURL     = "https://storage.googleapis.com/etcd"
+	tmpDownloadFile = "etcd.tar.gz"
+	pidFile         = "etcd.pid"
+)
+
+func TestMain(m *testing.M) {
+	logrus.SetLevel(logrus.DebugLevel)
+	ctx := context.Background()
+	expectSuccess(startEtcd(ctx), "starting etcd")
+
+	os.Exit(runTestsWithCleanup(m, func() {
+		expectSuccess(stopEtcd(), "stopping etcd")
+	}))
+}
+
+func runTestsWithCleanup(m *testing.M, cleanup func()) int {
+	defer cleanup()
+	return m.Run()
+}
+
+func expectSuccess(err error, msg string) {
+	if err != nil {
+		log.Fatalf("%s: %s\n", msg, err)
+	}
+}
+
+func startEtcd(ctx context.Context) error {
+	if _, err := os.Stat(pidFile); err == nil {
+		log.Println("Etcd already running, reusing")
+		return nil
+	}
+
+	etcdPath, err := installEtcd(ctx)
+	if err != nil {
+		errors.Wrap(err, "installing etcd for tests")
+	}
+
+	etcdCmd := exec.Command(etcdPath, "--data-dir", "./etcd-data")
+	if os.Getenv("ETCD_SERVER_LOGS") == "true" {
+		log.Println("Enabling etcd server logs")
+		etcdCmd.Stdout = os.Stdout
+		etcdCmd.Stderr = os.Stderr
+	}
+	log.Println("Starting etcd")
+	if err := etcdCmd.Start(); err != nil {
+		errors.Wrap(err, "starting etcd for tests")
+	}
+
+	if err := os.WriteFile(pidFile, []byte(strconv.Itoa(etcdCmd.Process.Pid)), 0o600); err != nil {
+		return err
+	}
+
+	log.Println("Waiting for etcd to be up")
+	time.Sleep(time.Second)
+
+	return nil
+}
+
+func installEtcd(ctx context.Context) (string, error) {
+	projectRoot, err := filepath.Abs("../../")
+	if err != nil {
+		return "", err
+	}
+	binDir := filepath.Join(projectRoot, etcdBinDir)
+	etcdPath := filepath.Join(projectRoot, etcdBinPath)
+
+	if _, err := os.Stat(etcdPath); err == nil {
+		log.Println("Etcd already installed, skipping")
+		return etcdPath, nil
+	}
+
+	if err := os.MkdirAll(binDir, 0o755); err != nil {
+		return "", err
+	}
+
+	download := fmt.Sprintf("%s/%s/etcd-%s-linux-amd64.tar.gz", downloadURL, etcdVersion, etcdVersion)
+
+	// Hacky to run bash, but simplifies this code a lot
+	cmd := fmt.Sprintf("curl -sL %s | tar -xzvf - -C %s --strip-components=1", download, binDir)
+	out, err := exec.CommandContext(ctx, "bash", "-c", cmd).CombinedOutput()
+	if err != nil {
+		return "", errors.Wrapf(err, "downloading etcd: %s", string(out))
+	}
+
+	return etcdPath, nil
+}
+
+func stopEtcd() error {
+	if os.Getenv("REUSE_ETCD") == "true" {
+		log.Println("REUSE_ETCD=true, leaving etcd running")
+		return nil
+	}
+
+	if _, err := os.Stat(pidFile); os.IsNotExist(err) {
+		log.Println("Etcd pid file doesn't exit, skipping cleanup")
+		return nil
+	}
+
+	dat, err := os.ReadFile(pidFile)
+	if err != nil {
+		return err
+	}
+	pid, err := strconv.Atoi(string(dat))
+	if err != nil {
+		return err
+	}
+
+	etcdProcess, err := os.FindProcess(pid)
+	if err != nil {
+		return err
+	}
+
+	log.Println("Stopping etcd")
+	if err := etcdProcess.Kill(); err != nil {
+		return errors.Wrap(err, "Failed stopping etcd")
+	}
+
+	log.Println("Deleting etcd data")
+	if err := os.RemoveAll("./etcd-data"); err != nil {
+		return errors.Wrap(err, "deleting etcd data")
+	}
+
+	log.Println("Deleting etcd pid file")
+	if err := os.RemoveAll(pidFile); err != nil {
+		return errors.Wrap(err, "deleting pid file")
+	}
+
+	return nil
+}

pkg/instance/instance.go

@@ -0,0 +1,482 @@
+package instance
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+
+	"log/slog"
+	log "log/slog"
+
+	"github.com/vishvananda/netlink"
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
+	"github.com/kube-vip/kube-vip/pkg/cluster"
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/networkinterface"
+	"github.com/kube-vip/kube-vip/pkg/sysctl"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
+
+// Instance defines an instance of everything needed to manage vips
+type Instance struct {
+	// Virtual IP / Load Balancer configuration
+	VIPConfigs []*kubevip.Config
+
+	// cluster instances
+	Clusters []*cluster.Cluster
+
+	// Service uses DHCP
+	IsDHCP              bool
+	DHCPInterface       string
+	DHCPInterfaceHwaddr string
+	DHCPInterfaceIP     string
+	DHCPHostname        string
+	DHCPClient          *vip.DHCPClient
+
+	// External Gateway IP the service is forwarded from
+	UPNPGatewayIPs []string
+
+	// Kubernetes service mapping
+	ServiceSnapshot *v1.Service
+}
+
+type Port struct {
+	Port uint16
+	Type string
+}
+
+func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Instance, error) {
+	instanceAddresses := FetchServiceAddresses(svc)
+	//instanceUID := string(svc.UID)
+
+	var newVips []*kubevip.Config
+	var link netlink.Link
+	var err error
+
+	for _, address := range instanceAddresses {
+		// Detect if we're using a specific interface for services
+		var svcInterface string
+		svcInterface = svc.Annotations[kubevip.ServiceInterface] // If the service has a specific interface defined, then use it
+		if svcInterface == kubevip.Auto {
+			link, err = autoFindInterface(address)
+			if err != nil {
+				log.Error("automatically discover network interface for annotated IP", "address", address, "err", err)
+			} else {
+				if link == nil {
+					log.Error("automatically discover network interface for annotated IP address", "address", address)
+				}
+			}
+			if link == nil {
+				svcInterface = ""
+			} else {
+				svcInterface = getAutoInterfaceName(link, config.Interface)
+			}
+		}
+		// If it is still blank then use the
+		if svcInterface == "" {
+			switch config.ServicesInterface {
+			case kubevip.Auto:
+				link, err = autoFindInterface(address)
+				if err != nil {
+					log.Error("failed to automatically discover network interface for address", "ip", address, "err", err, "interface", config.Interface)
+				} else if link == nil {
+					log.Error("failed to automatically discover network interface for address", "ip", address, "defaulting to", config.Interface)
+				}
+				svcInterface = getAutoInterfaceName(link, config.Interface)
+			case "":
+				svcInterface = config.Interface
+			default:
+				svcInterface = config.ServicesInterface
+			}
+		}
+
+		if link == nil {
+			if link, err = netlink.LinkByName(svcInterface); err != nil {
+				return nil, fmt.Errorf("failed to get interface %s: %w", svcInterface, err)
+			}
+			if link == nil {
+				return nil, fmt.Errorf("failed to get interface %s", svcInterface)
+			}
+		}
+
+		cidrs := vip.Split(config.VIPSubnet)
+
+		ipv4AutoSubnet := false
+		ipv6AutoSubnet := false
+		if cidrs[0] == kubevip.Auto {
+			ipv4AutoSubnet = true
+		}
+
+		if len(cidrs) > 1 && cidrs[1] == kubevip.Auto {
+			ipv6AutoSubnet = true
+		}
+
+		if (config.Address != "" || config.VIP != "") && (ipv4AutoSubnet || ipv6AutoSubnet) {
+			return nil, fmt.Errorf("auto subnet discovery cannot be used if VIP address was provided")
+		}
+
+		subnet := ""
+		var err error
+		if vip.IsIPv4(address) {
+			if ipv4AutoSubnet {
+				subnet, err = autoFindSubnet(link, address)
+				if err != nil {
+					return nil, fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
+				}
+			} else {
+				if cidrs[0] != "" && cidrs[0] != kubevip.Auto {
+					subnet = cidrs[0]
+				} else {
+					subnet = "32"
+				}
+			}
+		} else {
+			if ipv6AutoSubnet {
+				subnet, err = autoFindSubnet(link, address)
+				if err != nil {
+					return nil, fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
+				}
+			} else {
+				if len(cidrs) > 1 && cidrs[1] != "" && cidrs[1] != kubevip.Auto {
+					subnet = cidrs[1]
+				} else {
+					subnet = "128"
+				}
+			}
+		}
+
+		// Generate new Virtual IP configuration
+		newVips = append(newVips, &kubevip.Config{
+			VIP:                    address,
+			Interface:              svcInterface,
+			SingleNode:             true,
+			EnableARP:              config.EnableARP,
+			EnableBGP:              config.EnableBGP,
+			VIPSubnet:              subnet,
+			EnableRoutingTable:     config.EnableRoutingTable,
+			RoutingTableID:         config.RoutingTableID,
+			RoutingTableType:       config.RoutingTableType,
+			RoutingProtocol:        config.RoutingProtocol,
+			ArpBroadcastRate:       config.ArpBroadcastRate,
+			EnableServiceSecurity:  config.EnableServiceSecurity,
+			DNSMode:                config.DNSMode,
+			DisableServiceUpdates:  config.DisableServiceUpdates,
+			EnableServicesElection: config.EnableServicesElection,
+			KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
+				EnableLeaderElection: config.EnableLeaderElection,
+			},
+		})
+	}
+
+	// Create new service
+	instance := &Instance{
+		//UID:             instanceUID,
+		//VIPs:            instanceAddresses,
+		ServiceSnapshot: svc,
+	}
+	// for _, port := range svc.Spec.Ports {
+	// 	instance.ExternalPorts = append(instance.ExternalPorts, Port{
+	// 		Port: uint16(port.Port), //nolint
+	// 		Type: string(port.Protocol),
+	// 	})
+	// }
+
+	if svc.Annotations != nil {
+		instance.DHCPInterfaceHwaddr = svc.Annotations[kubevip.HwAddrKey]
+		instance.DHCPInterfaceIP = svc.Annotations[kubevip.RequestedIP]
+		instance.DHCPHostname = svc.Annotations[kubevip.LoadbalancerHostname]
+	}
+
+	configPorts := make([]kubevip.Port, 0)
+	for _, p := range svc.Spec.Ports {
+		configPorts = append(configPorts, kubevip.Port{
+			Type: string(p.Protocol),
+			Port: int(p.Port),
+		})
+	}
+	// Generate Load Balancer config
+	newLB := kubevip.LoadBalancer{
+		Name:      fmt.Sprintf("%s-load-balancer", svc.Name),
+		Ports:     configPorts,
+		BindToVip: true,
+	}
+	for _, vip := range newVips {
+		// Add Load Balancer Configuration
+		vip.LoadBalancers = append(vip.LoadBalancers, newLB)
+	}
+	// Create Add configuration to the new service
+	instance.VIPConfigs = newVips
+
+	// If this was purposely created with the address 0.0.0.0,
+	// we will create a macvlan on the main interface and a DHCP client
+	// TODO: Consider how best to handle DHCP with multiple addresses
+	if len(instanceAddresses) == 1 && instanceAddresses[0] == "0.0.0.0" {
+		err := instance.startDHCP()
+		if err != nil {
+			return nil, err
+		}
+		select {
+		case err := <-instance.DHCPClient.ErrorChannel():
+			return nil, fmt.Errorf("error starting DHCP for %s/%s: error: %s",
+				instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
+		case ip := <-instance.DHCPClient.IPChannel():
+			instance.VIPConfigs[0].Interface = instance.DHCPInterface
+			instance.VIPConfigs[0].VIP = ip
+			instance.DHCPInterfaceIP = ip
+		}
+	}
+
+	for _, vipConfig := range instance.VIPConfigs {
+		c, err := cluster.InitCluster(vipConfig, false, intfMgr, arpMgr)
+		if err != nil {
+			log.Error("Failed to add Service %s/%s", svc.Namespace, svc.Name)
+			return nil, err
+		}
+
+		for i := range c.Network {
+			c.Network[i].SetServicePorts(svc)
+		}
+
+		instance.Clusters = append(instance.Clusters, c)
+		log.Info("(svcs) adding VIP", "ip", vipConfig.VIP, "interface", vipConfig.Interface, "namespace", svc.Namespace, "name", svc.Name)
+	}
+
+	return instance, nil
+}
+
+func autoFindInterface(ip string) (netlink.Link, error) {
+	links, err := netlink.LinkList()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list network interfaces: %w", err)
+	}
+
+	address := net.ParseIP(ip)
+
+	family := netlink.FAMILY_V4
+
+	if address.To4() == nil {
+		family = netlink.FAMILY_V6
+	}
+
+	for _, link := range links {
+		addr, err := netlink.AddrList(link, family)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get IP addresses for interface %s: %w", link.Attrs().Name, err)
+		}
+		for _, a := range addr {
+			if a.IPNet.Contains(address) {
+				return link, nil
+			}
+		}
+	}
+
+	return nil, nil
+}
+
+func autoFindSubnet(link netlink.Link, ip string) (string, error) {
+	address := net.ParseIP(ip)
+
+	family := netlink.FAMILY_V4
+	if address.To4() == nil {
+		family = netlink.FAMILY_V6
+	}
+
+	addr, err := netlink.AddrList(link, family)
+	if err != nil {
+		return "", fmt.Errorf("failed to get IP addresses for interface %s: %w", link.Attrs().Name, err)
+	}
+	for _, a := range addr {
+		if a.IPNet.Contains(address) {
+			m, _ := a.IPNet.Mask.Size()
+			return strconv.Itoa(m), nil
+		}
+	}
+	return "", fmt.Errorf("failed to find suitable subnet for address %s", ip)
+}
+
+func getAutoInterfaceName(link netlink.Link, defaultInterface string) string {
+	if link == nil {
+		return defaultInterface
+	}
+	return link.Attrs().Name
+}
+
+func (i *Instance) startDHCP() error {
+	if len(i.VIPConfigs) != 1 {
+		return fmt.Errorf("DHCP requires exactly 1 VIP config, got: %v", len(i.VIPConfigs))
+	}
+	parent, err := netlink.LinkByName(i.VIPConfigs[0].Interface)
+	if err != nil {
+		return fmt.Errorf("error finding VIP Interface, for building DHCP Link : %v", err)
+	}
+
+	// Generate name from UID
+	interfaceName := fmt.Sprintf("vip-%s", i.ServiceSnapshot.UID[0:8])
+
+	// Check if the interface doesn't exist first
+	iface, err := net.InterfaceByName(interfaceName)
+	if err != nil {
+		log.Info("creating new macvlan interface for DHCP", "interface", interfaceName)
+
+		hwaddr, err := net.ParseMAC(i.DHCPInterfaceHwaddr)
+		if i.DHCPInterfaceHwaddr != "" && err != nil {
+			return err
+		} else if hwaddr == nil {
+			hwaddr, err = net.ParseMAC(vip.GenerateMac())
+			if err != nil {
+				return err
+			}
+		}
+
+		log.Info("new macvlan interface", "interface", interfaceName, "hardware address", hwaddr)
+		mac := &netlink.Macvlan{
+			LinkAttrs: netlink.LinkAttrs{
+				Name:         interfaceName,
+				ParentIndex:  parent.Attrs().Index,
+				HardwareAddr: hwaddr,
+			},
+			Mode: netlink.MACVLAN_MODE_DEFAULT,
+		}
+
+		err = netlink.LinkAdd(mac)
+		if err != nil {
+			return fmt.Errorf("could not add %s: %v", interfaceName, err)
+		}
+
+		err = netlink.LinkSetUp(mac)
+		if err != nil {
+			return fmt.Errorf("could not bring up interface [%s] : %v", interfaceName, err)
+		}
+
+		iface, err = net.InterfaceByName(interfaceName)
+		if err != nil {
+			return fmt.Errorf("error finding new DHCP interface by name [%v]", err)
+		}
+	} else {
+		log.Info("Using existing macvlan interface for DHCP", "interface", interfaceName)
+	}
+
+	// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
+	rpfilterSetting := "0"
+
+	// Check if we need to set an override rp_filter value for the interface
+	if i.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
+		// Check the rp_filter value
+		rpFilter, err := strconv.Atoi(i.ServiceSnapshot.Annotations[kubevip.RPFilter])
+		if err != nil {
+			slog.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
+		} else {
+			if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
+				rpfilterSetting = i.ServiceSnapshot.Annotations[kubevip.RPFilter]
+			} else {
+				slog.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
+			}
+		}
+	}
+
+	err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
+	if err != nil {
+		slog.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
+	}
+	var initRebootFlag bool
+	if i.DHCPInterfaceIP != "" {
+		initRebootFlag = true
+	}
+
+	client := vip.NewDHCPClient(iface, initRebootFlag, i.DHCPInterfaceIP)
+
+	// Add hostname to dhcp client if annotated
+	if i.DHCPHostname != "" {
+		log.Info("Hostname specified for dhcp lease", "interface", interfaceName, "hostname", i.DHCPHostname)
+		client.WithHostName(i.DHCPHostname)
+	}
+
+	go client.Start()
+
+	// Set that DHCP is enabled
+	i.IsDHCP = true
+	// Set the name of the interface so that it can be removed on Service deletion
+	i.DHCPInterface = interfaceName
+	i.DHCPInterfaceHwaddr = iface.HardwareAddr.String()
+	// Add the client so that we can call it to stop function
+	i.DHCPClient = client
+
+	return nil
+}
+
+// FetchIngressAddresses tries to get the addresses from status.loadBalancerIP
+func FetchLoadBalancerIngressAddresses(s *v1.Service) []string {
+	// If the service has no status, return empty
+	lbStatusAddresses := []string{}
+
+	if len(s.Status.LoadBalancer.Ingress) == 0 {
+		return lbStatusAddresses
+	}
+
+	for _, ingress := range s.Status.LoadBalancer.Ingress {
+		if ingress.IP != "" {
+			lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
+		}
+		// TODO: Handle hostname if needed
+	}
+
+	return lbStatusAddresses
+}
+
+// FetchServiceAddresses tries to get the addresses from annotations
+// kube-vip.io/loadbalancerIPs, then from spec.loadbalancerIP
+func FetchServiceAddresses(s *v1.Service) []string {
+	annotationAvailable := false
+	if s.Annotations != nil {
+
+		if v, annotationAvailable := s.Annotations[kubevip.LoadbalancerIPAnnotation]; annotationAvailable {
+			ips := strings.Split(v, ",")
+			var trimmedIPs []string
+			for _, ip := range ips {
+				trimmedIPs = append(trimmedIPs, strings.TrimSpace(ip))
+			}
+			return trimmedIPs
+		}
+	}
+
+	lbStatusAddresses := []string{}
+	if !annotationAvailable {
+		if len(s.Status.LoadBalancer.Ingress) > 0 {
+			for _, ingress := range s.Status.LoadBalancer.Ingress {
+				lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
+			}
+		}
+	}
+
+	lbIP := net.ParseIP(s.Spec.LoadBalancerIP)
+	isLbIPv4 := vip.IsIPv4(s.Spec.LoadBalancerIP)
+
+	if len(lbStatusAddresses) > 0 {
+		for _, a := range lbStatusAddresses {
+			if lbStatusIP := net.ParseIP(a); lbStatusIP != nil && lbIP != nil && vip.IsIPv4(a) == isLbIPv4 && !lbIP.Equal(lbStatusIP) {
+				return []string{s.Spec.LoadBalancerIP}
+			}
+		}
+		return lbStatusAddresses
+	}
+
+	if s.Spec.LoadBalancerIP != "" {
+		return []string{s.Spec.LoadBalancerIP}
+	}
+
+	return []string{}
+}
+
+func FindServiceInstance(svc *v1.Service, instances []*Instance) *Instance {
+	log.Debug("finding service", "UID", svc.UID)
+	for i := range instances {
+		log.Debug("saved service", "instance", i, "UID", instances[i].ServiceSnapshot.UID)
+		if instances[i].ServiceSnapshot.UID == svc.UID {
+			return instances[i]
+		}
+	}
+	return nil
+}