Move this error to a warning

Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# Enable GitHub funding
+
+github: [kube-vip]
+

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: gomod
+    directory: /
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: /
+    schedule:
+      interval: weekly

.github/workflows/anchore-syft.yml

@@ -0,0 +1,31 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# This workflow checks out code, builds an image, performs a container image
+# scan with Anchore's Syft tool, and uploads the results to the GitHub Dependency
+# submission API.
+
+# For more information on the Anchore sbom-action usage
+# and parameters, see https://github.com/anchore/sbom-action. For more
+# information about the Anchore SBOM tool, Syft, see
+# https://github.com/anchore/syft
+name: Anchore Syft SBOM scan
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  sbom:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.ref_name }}
+      - name: Anchore SBOM Action
+        uses: anchore/sbom-action@v0.20.10
+        with:
+          format: cyclonedx-json

.github/workflows/ci-pull-request.yaml

@@ -0,0 +1,71 @@
+name: For each PR
+on:
+  pull_request:
+jobs:
+  e2e-tests:
+    runs-on: ubuntu-latest
+    name: E2E tests
+    strategy:
+      matrix:
+        mode: ["arp", "rt", "bgp"]
+      fail-fast: true
+    steps:
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H-%M')"
+      - name: Ensure fs wont cause issues
+        run: sudo sysctl fs.inotify.max_user_instances=8192 &&  sudo sysctl fs.inotify.max_user_watches=524288
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Install Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+      - name: Build image locally
+        run: make dockerx86Local
+      - name: Run Manifest generation tests
+        run: make manifest-test
+      - name: Run ARP mode tests v1.29.0 onwards
+        run: E2E_KEEP_LOGS=true make e2e-tests129-arp
+        if: matrix.mode== 'arp'
+      - name: Run RT mode tests v1.29.0 onwards
+        run: E2E_KEEP_LOGS=true make e2e-tests129-rt
+        if: matrix.mode== 'rt'
+      - name: Get GoBGP binaries
+        run: make get-gobgp
+        if: matrix.mode== 'bgp'
+      - name: Run BGP mode tests v1.29.0 onwards
+        run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true make e2e-tests129-bgp
+        if: matrix.mode== 'bgp'
+      - name: Change log directory permissions
+        run: sudo chmod -R 755 /tmp/kube-vip-test-${{ matrix.mode }}*
+        if: matrix.mode== 'bgp' && always()
+      - name: Save logs
+        uses: actions/upload-artifact@v5
+        with:
+          name: e2e-test-logs-${{ matrix.mode }}-${{ steps.date.outputs.date }}
+          path: /tmp/kube-vip-test-${{ matrix.mode }}*
+        if: always()
+  service-e2e-tests:
+    runs-on: ubuntu-latest
+    name: E2E service tests
+    steps:
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H-%M')"
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Install Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+      - name: Build image with iptables
+        run: make dockerx86ActionIPTables
+      - name: Run tests
+        run: DOCKERTAG=action E2E_KEEP_LOGS=true make service-tests
+      - name: Save logs
+        uses: actions/upload-artifact@v5
+        with:
+          name: services-test-logs-${{ steps.date.outputs.date }}
+          path: /tmp/kube-vip-service-tests*
+        if: always()

.github/workflows/ci.yaml

@@ -1,28 +1,62 @@
-name: For each commit and PR
+name: For each commit
 on:
   push:
   pull_request:
 jobs:
   validation:
     runs-on: ubuntu-latest
-    env:
-      CGO_ENABLED: 0
+    name: Checks and linters
     steps:
-    - name: Init
-      run: sudo apt-get update && sudo apt-get install -y build-essential golint
-    - name: Checkout code
-      uses: actions/checkout@v2
-    - name: Install Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: '1.16'
-    - name: Install golangci-lint
-      run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.42.1
-    - name: checks
-      run: make check
-    - name: test docker build
-      run: make dockerx86Action
-    - name: Manifest generate
-      run: ./testing/testing.sh
-    - name: e2e tests
-      run: DOCKERTAG=action make e2e-tests
+      - name: Init
+        run: sudo apt-get update && sudo apt-get install -y build-essential &&  sudo sysctl fs.inotify.max_user_instances=8192 &&  sudo sysctl fs.inotify.max_user_watches=524288
+      - name: Install golangci-lint
+        run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.64.8
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Install Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+      - name: All checks
+        run: make check
+  unit-tests:
+    runs-on: ubuntu-latest
+    name: Unit tests
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Install Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+      - name: Run tests
+        run: make unit-tests
+  integration-tests:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Install Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+      - name: Run tests
+        run: make integration-tests
+  image-vul-check:
+    runs-on: ubuntu-latest
+    name: Image vulnerability scan
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Build image with iptables
+        run: make dockerx86ActionIPTables
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: "plndr/kube-vip:action"
+          format: "table"
+          exit-code: "1"
+          ignore-unfixed: true
+          vuln-type: "os,library"
+          severity: "CRITICAL,HIGH"

.github/workflows/codeql-analysis.yml

@@ -38,11 +38,16 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
-
+      uses: actions/checkout@v6
+      
+    - name: Install Go
+      uses: actions/setup-go@v6
+      with:
+        go-version-file: go.mod
+        
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v4
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -53,7 +58,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v4
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -67,4 +72,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v4

.github/workflows/main.yaml

@@ -3,37 +3,50 @@ name: Build and publish main image regularly
 on:
   schedule:
     - cron: '25 0 * * *'
+  workflow_dispatch:
+
 
 jobs:
-  docker:
+  nightly_build:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v6
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Login to Github Packages
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}   
-      - name: Build and push main branch
+      - name: Build standard version
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: .
-          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
           push: ${{ github.event_name != 'pull_request' }}
           tags: >-
             plndr/kube-vip:${{ github.ref_name }},
             ghcr.io/kube-vip/kube-vip:${{ github.ref_name }}
+      - name: Build iptables version
+        id: docker_build_iptables
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile_iptables
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: >-
+            plndr/kube-vip-iptables:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip-iptables:${{ github.ref_name }}
       - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
+        run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/release.yaml

@@ -1,41 +1,64 @@
-name: Publish Releases to Docker Hub
+name: Publish Releases to Docker Hub and GitHub Container Registry
 
 on:
   push:
     tags:
       - '*'
+  workflow_dispatch:
+
 jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v6
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Login to Github Packages
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Generate Metadata
+        uses: docker/metadata-action@v5.9.0
+        id: metadata
+        with:
+          labels: |
+            org.opencontainers.image.documentation=https://kube-vip.io/docs/
       - name: Build and push main branch
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: .
-          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
           push: ${{ github.event_name != 'pull_request' }}
+          labels: ${{ steps.metadata.outputs.labels }}
           tags: >-
             plndr/kube-vip:${{ github.ref_name }},
             plndr/kube-vip:latest,
             ghcr.io/kube-vip/kube-vip:${{ github.ref_name }},
             ghcr.io/kube-vip/kube-vip:latest
+      - name: Build iptables version and push main branch
+        id: docker_build_iptables
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile_iptables
+          platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
+          push: ${{ github.event_name != 'pull_request' }}
+          labels: ${{ steps.metadata.outputs.labels }}
+          tags: >-
+            plndr/kube-vip-iptables:${{ github.ref_name }},
+            plndr/kube-vip-iptables:latest,
+            ghcr.io/kube-vip/kube-vip-iptables:${{ github.ref_name }},
+            ghcr.io/kube-vip/kube-vip-iptables:latest
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}

.gitignore

@@ -1 +1,7 @@
-./idea/
+.idea
+kube-vip
+.vscode
+bin
+testing/e2e/etcd/certs
+pkg/etcd/etcd.pid
+pkg/etcd/etcd-data

CHANGELOG.md

@@ -0,0 +1,33 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- SIGUSR1 signal handler for runtime configuration dumps (#1301)
+  - Send SIGUSR1 to kube-vip process to dump current configuration to stdout
+  - Configuration dump includes:
+    - Basic configuration (VIP, interface, port, namespace settings)
+    - BGP configuration (enabled status, AS number, router ID, peers)
+    - ARP/NDP configuration (enabled status, broadcast rate)
+    - Services configuration (enabled status, load balancer settings)
+    - Network interfaces status
+    - Leader election configuration (type, lease details)
+    - Runtime statistics (load balancer, Prometheus, health check settings)
+  - Output format: Human-readable plaintext via fmt.Printf()
+  - Thread-safe implementation using mutex protection
+  - Non-disruptive: Process continues running after configuration dump
+  - Added comprehensive unit tests for all dump methods
+  - Added E2E tests for signal handling
+
+### Changed
+- Updated signal handlers in manager_arp.go, manager_bgp.go, manager_wireguard.go, and manager_table.go to use switch statement pattern for handling multiple signals (SIGUSR1, SIGINT, SIGTERM)
+
+## [v1.0.1] - Previous Release
+
+### Previous changes
+- See git history for changes prior to CHANGELOG.md introduction

CONTRIBUTING.md

@@ -154,7 +154,7 @@ the `$GOPATH`.
 
 To develop locally, you can follow these steps:
 
- 1. [Install Go 1.15](https://golang.org/doc/install)
+ 1. [Install Go 1.19](https://golang.org/doc/install)
  2. Checkout your feature branch and `cd` into it.
  3. To build all Go files and install them under `bin`, run `make bin`
  4. To run all Go unit tests, run `make test-unit`

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:experimental
 
-FROM golang:1.16.12-alpine3.15 as dev
+FROM golang:1.25.4-alpine3.22 as dev
 RUN apk add --no-cache git ca-certificates make
 RUN adduser -D appuser
 COPY . /src/

Dockerfile_iptables

@@ -0,0 +1,21 @@
+# syntax=docker/dockerfile:experimental
+
+FROM golang:1.25.4-alpine3.22 as dev
+RUN apk add --no-cache git make
+RUN adduser -D appuser
+COPY . /src/
+WORKDIR /src
+
+ENV GO111MODULE=on
+RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
+    --mount=type=cache,sharing=locked,id=goroot,target=/root/.cache/go-build \
+    CGO_ENABLED=0 GOOS=linux make build
+
+FROM alpine:3.22.2
+# Update pkgs and add iptables
+RUN apk upgrade && \
+    apk add --no-cache iptables iptables-legacy
+
+# Add kube-vip binary
+COPY --from=dev /src/kube-vip /
+ENTRYPOINT ["/kube-vip"]

Makefile

@@ -2,10 +2,11 @@ SHELL := /bin/sh
 
 # The name of the executable (default is current directory name)
 TARGET := kube-vip
-.DEFAULT_GOAL: $(TARGET)
+.DEFAULT_GOAL := $(TARGET)
 
 # These will be provided to the target
-VERSION := v0.4.2
+VERSION := v1.0.2
+
 BUILD := `git rev-parse HEAD`
 
 # Operating System Default (LINUX)
@@ -14,9 +15,9 @@ TARGETOS=linux
 # Use linker flags to provide version/build settings to the target
 LDFLAGS=-ldflags "-s -w -X=main.Version=$(VERSION) -X=main.Build=$(BUILD) -extldflags -static"
 DOCKERTAG ?= $(VERSION)
-REPOSITORY = plndr
+REPOSITORY ?= docker.io/plndr
 
-.PHONY: all build clean install uninstall fmt simplify check run e2e-tests
+.PHONY: all build clean install uninstall simplify check run e2e-tests
 
 all: check install
 
@@ -36,12 +37,9 @@ install:
 uninstall: clean
 	@rm -f $$(which ${TARGET})
 
-fmt:
-	@gofmt -l -w ./...
-
 demo:
 	@cd demo
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 	@cd ..
 
@@ -53,6 +51,16 @@ dockerx86Dev:
 	@docker buildx build  --platform linux/amd64 --push -t $(REPOSITORY)/$(TARGET):dev .
 	@echo New single x86 Architecture Docker image created
 
+dockerx86Iptables:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 -f ./Dockerfile_iptables --push -t $(REPOSITORY)/$(TARGET):dev .
+	@echo New single x86 Architecture Docker image created
+
+dockerx86IptablesLocal:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 -f ./Dockerfile_iptables -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@echo New single x86 Architecture Docker image created
+
 dockerx86:
 	@-rm ./kube-vip
 	@docker buildx build  --platform linux/amd64 --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
@@ -60,7 +68,7 @@ dockerx86:
 
 docker:
 	@-rm ./kube-vip
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 ## Local (docker load of images)
@@ -75,35 +83,97 @@ dockerx86Action:
 	@docker buildx build  --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):action .
 	@echo New Multi Architecture Docker image created
 
+dockerx86ActionIPTables:
+	@-rm ./kube-vip
+	@docker buildx build  --platform linux/amd64 -f ./Dockerfile_iptables --load -t $(REPOSITORY)/$(TARGET):action .
+	@echo New Multi Architecture Docker image created
+
 dockerLocal:
 	@-rm ./kube-vip
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 simplify:
-	@gofmt -s -l -w ./...
+	@gofmt -s -l -w *.go pkg cmd
 
 check:
 	go mod tidy
 	test -z "$(git status --porcelain)"
-	test -z $(shell gofmt -l main.go | tee /dev/stderr) || echo "[WARN] Fix formatting issues with 'make fmt'"
+	test -z $(shell gofmt -l *.go pkg cmd) || echo "[WARN] Fix formatting issues with 'make simplify'"
 	golangci-lint run
 	go vet ./...
-	
+
 run: install
 	@$(TARGET)
 
 manifests:
 	@make build
 	@mkdir -p ./docs/manifests/$(VERSION)/
-	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-arp.yaml
-	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-lb.yaml
-	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --bgp --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-bgp.yaml
-	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
-	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
-	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
-	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster --provider-config /etc/cloud-sa/cloud-sa.json > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
+	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-arp.yaml
+	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-lb.yaml
+	@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-bgp.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
+	@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
 	@-rm ./kube-vip
 
+manifest-test: 
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --enableLoadBalancer
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --controlplane --services
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer
+	docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster
+
+unit-tests:
+	go test ./...
+
+integration-tests:
+	go test -tags=integration,e2e -v ./pkg/etcd
+
 e2e-tests:
-	E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/ginkgo -tags=e2e -v -p testing/e2e
+	GOMAXPROCS=4 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd
+
+e2e-tests129-arp:
+	GOMAXPROCS=4 TEST_MODE=arp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
+
+e2e-tests129-rt:
+	GOMAXPROCS=4 TEST_MODE=rt V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
+
+e2e-tests129-bgp:
+	GOMAXPROCS=4 TEST_MODE=bgp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
+
+e2e-tests129: e2e-tests129-arp e2e-tests129-rt e2e-tests129-bgp
+
+service-tests:
+	E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/services -Services -simple -deployments -leaderActive -leaderFailover -localDeploy -egress -egressIPv6 -dualStack
+
+trivy: dockerx86ActionIPTables
+	docker run -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy:0.47.0 \
+		image  \
+		--format table \
+		--exit-code  1 \
+		--ignore-unfixed \
+		--vuln-type  'os,library' \
+		--severity  'CRITICAL,HIGH'  \
+		$(REPOSITORY)/$(TARGET):action
+
+kind-quick:
+	echo "Standing up your cluster"
+	kind create cluster --config ./testing/kind/kind.yaml --name kube-vip
+	kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
+	kubectl create configmap --namespace kube-system kubevip --from-literal range-global=172.18.100.10-172.18.100.30
+	kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
+	kind load docker-image --name kube-vip $(REPOSITORY)/$(TARGET):$(DOCKERTAG)
+	docker run --network host --rm $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --services --inCluster --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --servicesElection --interface  eth0 | kubectl apply -f -
+
+kind-reload:
+	kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name services
+	kubectl rollout restart -n kube-system daemonset/kube-vip-ds
+
+get-gobgp:
+	mkdir -p bin
+	wget -nc --directory-prefix=bin https://github.com/osrg/gobgp/releases/download/v3.37.0/gobgp_3.37.0_linux_amd64.tar.gz
+	tar -xvzf bin/gobgp_3.37.0_linux_amd64.tar.gz -C bin
+

README.md

@@ -1,9 +1,11 @@
 # kube-vip
 
-High Availability and Load-Balancing 
+High Availability and Load-Balancing
 
 ![](https://github.com/kube-vip/kube-vip/raw/main/kube-vip.png)
 
+[![Build and publish main image regularly](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml/badge.svg)](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml) [![LFX Health Score](https://img.shields.io/static/v1?label=Health%20Score&message=Healthy&color=A7F3D0&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip) [![LFX Active Contributors](https://img.shields.io/static/v1?label=Active%20contributors%20(1Y)&message=212&color=0094FF&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip)
+
 ## Overview
 Kubernetes Virtual IP and Load-Balancer for both control plane and Kubernetes services
 
@@ -30,7 +32,8 @@ Kube-Vip was originally created to provide a HA solution for the Kubernetes cont
 - Service LoadBalancer address pools per namespace or global
 - Service LoadBalancer address via (existing network DHCP)
 - Service LoadBalancer address exposure to gateway via UPNP
-- ... manifest generation, vendor API integrations and many nore... 
+- Egress! Kube-vip will utilise a service loadbalancer as both the ingress and **egress** for a pod. 
+- ... manifest generation, vendor API integrations and many more...
 
 ## Why?
 
@@ -56,3 +59,13 @@ All of these would require a separate level of configuration and in some infrast
 ## Troubleshooting and Feedback
 
 Please raise issues on the GitHub repository and as mentioned check the documentation at [https://kube-vip.io](https://kube-vip.io/).
+
+## Contributing
+
+Thanks for taking the time to join our community and start contributing! We welcome pull requests. Feel free to dig through the [issues](https://github.com/kube-vip/kube-vip/issues) and jump in.
+
+:warning: This project has issue compiling on MacOS, please compile it on linux distribution
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=kube-vip/kube-vip&type=Date)](https://star-history.com/#kube-vip/kube-vip&Date)

cmd/kube-vip-kubeadm.go

@@ -1,16 +1,13 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
 	"os"
 
-	"github.com/kube-vip/kube-vip/pkg/k8s"
+	log "log/slog"
+
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // kubeadm adds two subcommands for managing a vip during a kubeadm init/join
@@ -26,7 +23,7 @@ func init() {
 var kubeKubeadm = &cobra.Command{
 	Use:   "kubeadm",
 	Short: "Kubeadm functions",
-	Run: func(cmd *cobra.Command, args []string) {
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 		_ = cmd.Help()
 		// TODO - A load of text detailing what's actually happening
 	},
@@ -36,96 +33,84 @@ var kubeKubeadmInit = &cobra.Command{
 	Use:   "init",
 	Short: "kube-vip init",
 	Long:  "The \"init\" subcommand will generate the Kubernetes manifest that will be started by kubeadm through the kubeadm init process",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
 		// TODO - A load of text detailing what's actually happening
 		err := kubevip.ParseEnvironment(&initConfig)
 		if err != nil {
-			log.Fatalf("Error parsing environment from config: %v", err)
+			log.Error("parsing environment", "err", err)
+			return
 		}
 
 		// TODO - check for certain things VIP/interfaces
 		if initConfig.Interface == "" {
 			_ = cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+			log.Error("No interface is specified for kube-vip to bind to")
+			return
 		}
 
 		if initConfig.VIP == "" && initConfig.Address == "" {
 			_ = cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
-		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
 
-		fmt.Println(cfg)
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("generating VIPSubnet", "err", err)
+				return
+			}
+		}
+
+		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
+		fmt.Println(cfg) // output manifest to stdout
 	},
 }
 
 var kubeKubeadmJoin = &cobra.Command{
 	Use:   "join",
 	Short: "kube-vip join",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
 		// TODO - A load of text detailing what's actually happening
 		err := kubevip.ParseEnvironment(&initConfig)
 		if err != nil {
-			log.Fatalf("Error parsing environment from config: %v", err)
+			log.Error("parsing environment", "err", err)
+			return
 		}
 
 		// TODO - check for certain things VIP/interfaces
 		if initConfig.Interface == "" {
 			_ = cmd.Help()
-			log.Fatalln("No interface is specified for kube-vip to bind to")
+			log.Error("No interface is specified for kube-vip to bind to")
+			return
 		}
 
 		if initConfig.VIP == "" && initConfig.Address == "" {
 			_ = cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
 
 		if _, err := os.Stat(kubeConfigPath); os.IsNotExist(err) {
-			log.Fatalf("Unable to find file [%s]", kubeConfigPath)
+			log.Error("kubeConfig not found", "Path", kubeConfigPath)
+			return
 		}
 
-		// We will use kubeconfig in order to find all the master nodes
-		// use the current context in kubeconfig
-		clientset, err := k8s.NewClientset(kubeConfigPath, false, "")
-		if err != nil {
-			log.Fatal(err.Error())
-		}
-
-		opts := metav1.ListOptions{}
-		opts.LabelSelector = "node-role.kubernetes.io/master"
-		nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), opts)
-		if err != nil {
-			log.Fatal(err.Error())
-		}
-		// Iterate over all nodes that are masters and find the details to build a peer list
-		for x := range nodes.Items {
-			// Get hostname and address
-			var nodeAddress, nodeHostname string
-			for y := range nodes.Items[x].Status.Addresses {
-				switch nodes.Items[x].Status.Addresses[y].Type {
-				case corev1.NodeHostName:
-					nodeHostname = nodes.Items[x].Status.Addresses[y].Address
-				case corev1.NodeInternalIP:
-					nodeAddress = nodes.Items[x].Status.Addresses[y].Address
-				}
-			}
-
-			newPeer, err := kubevip.ParsePeerConfig(fmt.Sprintf("%s:%s:%d", nodeHostname, nodeAddress, 10000))
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
 			if err != nil {
-				panic(err.Error())
+				log.Error("generating VIPSubnet", "err", err)
+				return
 			}
-			initConfig.RemotePeers = append(initConfig.RemotePeers, *newPeer)
-
 		}
-		// Generate manifest and print
-		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
-		fmt.Println(cfg)
+
+		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
+		fmt.Println(cfg) // output manifest to stdout
 	},
 }

cmd/kube-vip-manifests.go

@@ -3,8 +3,9 @@ package cmd
 import (
 	"fmt"
 
+	log "log/slog"
+
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
@@ -12,22 +13,26 @@ import (
 // manifests will be used to generate:
 // - Pod spec manifest, mainly used for a static pod (kubeadm)
 // - Daemonset manifest, mainly used to run kube-vip as a deamonset within Kubernetes (k3s/rke)
+// - RBAC manifest, used to generate the RBAC permissions for kube-vip
 
-//var inCluster bool
-var taint bool
+var taint, role, rolebinding bool
 
 func init() {
 	kubeManifest.PersistentFlags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
+	kubeManifest.PersistentFlags().StringVar(&image, "image", "ghcr.io/kube-vip/kube-vip", "Define a hardcoded image with or without tag for the manifest")
 	kubeManifestDaemon.PersistentFlags().BoolVar(&taint, "taint", false, "Taint the manifest for only running on control planes")
+	kubeManifestRbac.PersistentFlags().BoolVar(&role, "role", false, "Generate only a Role inside the serviceNamespace access")
+	kubeManifestRbac.PersistentFlags().BoolVar(&rolebinding, "rolebinding", false, "Generate only a RoleBinding for namespaced access")
 
 	kubeManifest.AddCommand(kubeManifestPod)
 	kubeManifest.AddCommand(kubeManifestDaemon)
+	kubeManifest.AddCommand(kubeManifestRbac)
 }
 
 var kubeManifest = &cobra.Command{
 	Use:   "manifest",
 	Short: "Manifest functions",
-	Run: func(cmd *cobra.Command, args []string) {
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 		_ = cmd.Help()
 		// TODO - A load of text detailing what's actually happening
 	},
@@ -36,47 +41,111 @@ var kubeManifest = &cobra.Command{
 var kubeManifestPod = &cobra.Command{
 	Use:   "pod",
 	Short: "Generate a Pod Manifest",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		var err error
+
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
 		// TODO - A load of text detailing what's actually happening
 		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
-			log.Fatalf("Error parsing environment from config: %v", err)
+			log.Error("parsing environment", "err", err)
+			return
 		}
 
 		// The control plane has a requirement for a VIP being specified
-		if initConfig.EnableControlPane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
 			_ = cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
-		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
 
-		fmt.Println(cfg)
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("config parse", "err", err)
+				return
+			}
+		}
+
+		cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
+		fmt.Println(cfg) // output manifest to stdout
 	},
 }
 
 var kubeManifestDaemon = &cobra.Command{
 	Use:   "daemonset",
 	Short: "Generate a Daemonset Manifest",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		var err error
+
 		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
 		// TODO - A load of text detailing what's actually happening
 		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
-			log.Fatalf("error parsing environment config: %v", err)
+			log.Error("parsing environment", "err", err)
+			return
 		}
-
-		// TODO - check for certain things VIP/interfaces
-
 		// The control plane has a requirement for a VIP being specified
-		if initConfig.EnableControlPane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
 			_ = cmd.Help()
-			log.Fatalln("No address is specified for kube-vip to expose services on")
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
 		}
-		cfg := kubevip.GenerateDaemonsetManifestFromConfig(&initConfig, Release.Version, inCluster, taint)
 
-		fmt.Println(cfg)
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("config parse", "err", err)
+				return
+			}
+		}
+
+		cfg := kubevip.GenerateDaemonsetManifestFromConfig(&initConfig, image, Release.Version, inCluster, taint)
+		fmt.Println(cfg) // output manifest to stdout
+	},
+}
+
+var kubeManifestRbac = &cobra.Command{
+	Use:   "rbac",
+	Short: "Generate an RBAC Manifest",
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		var err error
+
+		initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
+		// TODO - A load of text detailing what's actually happening
+		if err := kubevip.ParseEnvironment(&initConfig); err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+
+		// The control plane has a requirement for a VIP being specified
+		if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
+			_ = cmd.Help()
+			log.Error("No address is specified for kube-vip to expose services on")
+			return
+		}
+
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("generating VIPSubnet", "err", err)
+				return
+			}
+		}
+		saCfg := kubevip.GenerateSA(&initConfig)
+		roleCfg := kubevip.GenerateRole(&initConfig, role)
+		if role {
+			rolebinding = true
+		}
+		roleBindingCfg := kubevip.GenerateRoleBinding(rolebinding, saCfg, roleCfg)
+
+		// Output the YAML manifests to stdout
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(saCfg))
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(roleCfg))
+		fmt.Println("---") // Separator for YAML documents
+		fmt.Println(kubevip.TransformApplyObjectToManifest(roleBindingCfg))
 	},
 }

cmd/kube-vip-start.go

@@ -1,122 +0,0 @@
-package cmd
-
-import (
-	"github.com/kube-vip/kube-vip/pkg/bgp"
-	"github.com/kube-vip/kube-vip/pkg/cluster"
-	"github.com/kube-vip/kube-vip/pkg/kubevip"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-)
-
-// Start as a single node (no cluster), start as a leader in the cluster
-var startConfig kubevip.Config
-var startConfigLB kubevip.LoadBalancer
-var startLocalPeer, startKubeConfigPath string
-var startRemotePeers, startBackends []string
-var inCluster bool
-
-func init() {
-	// Get the configuration file
-	kubeVipStart.Flags().StringVarP(&configPath, "config", "c", "", "Path to a kube-vip configuration")
-	kubeVipStart.Flags().BoolVarP(&disableVIP, "disableVIP", "d", false, "Disable the VIP functionality")
-
-	// Pointers so we can see if they're nil (and not called)
-	kubeVipStart.Flags().StringVar(&startConfig.Interface, "interface", "eth0", "Name of the interface to bind to")
-	kubeVipStart.Flags().StringVar(&startConfig.VIP, "vip", "192.168.0.1", "The Virtual IP address")
-	kubeVipStart.Flags().StringVar(&startConfig.Address, "address", "", "an address (IP or DNS name) to use as a VIP")
-	kubeVipStart.Flags().IntVar(&startConfig.Port, "port", 6443, "listen port for the VIP")
-	kubeVipStart.Flags().BoolVar(&startConfig.DDNS, "ddns", false, "use Dynamic DNS + DHCP to allocate VIP for address")
-	kubeVipStart.Flags().BoolVar(&startConfig.SingleNode, "singleNode", false, "Start this instance as a single node")
-	kubeVipStart.Flags().BoolVar(&startConfig.StartAsLeader, "startAsLeader", false, "Start this instance as the cluster leader")
-	kubeVipStart.Flags().BoolVar(&startConfig.EnableARP, "arp", false, "Use ARP broadcasts to improve VIP re-allocations")
-	kubeVipStart.Flags().StringVar(&startLocalPeer, "localPeer", "server1:192.168.0.1:10000", "Settings for this peer, format: id:address:port")
-	kubeVipStart.Flags().StringSliceVar(&startRemotePeers, "remotePeers", []string{"server2:192.168.0.2:10000", "server3:192.168.0.3:10000"}, "Comma separated remotePeers, format: id:address:port")
-	// Load Balancer flags
-	kubeVipStart.Flags().BoolVar(&startConfigLB.BindToVip, "lbBindToVip", false, "Bind example load balancer to VIP")
-	kubeVipStart.Flags().StringVar(&startConfigLB.Type, "lbType", "tcp", "Type of load balancer instance (TCP/HTTP)")
-	kubeVipStart.Flags().StringVar(&startConfigLB.Name, "lbName", "Example Load Balancer", "The name of a load balancer instance")
-	kubeVipStart.Flags().IntVar(&startConfigLB.Port, "lbPort", 8080, "Port that load balancer will expose on")
-	kubeVipStart.Flags().IntVar(&startConfigLB.BackendPort, "lbBackEndPort", 6443, "A port that all backends may be using (optional)")
-	kubeVipStart.Flags().StringSliceVar(&startBackends, "lbBackends", []string{"192.168.0.1:8080", "192.168.0.2:8080"}, "Comma separated backends, format: address:port")
-	kubeVipStart.Flags().StringVar(&startConfigLB.ForwardingMethod, "lbForwardingMethod", "local", "The forwarding method of a load balancer instance")
-
-	// Cluster configuration
-	kubeVipStart.Flags().StringVar(&startKubeConfigPath, "kubeConfig", "/etc/kubernetes/admin.conf", "The path of a kubernetes configuration file")
-	kubeVipStart.Flags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
-	kubeVipStart.Flags().BoolVar(&startConfig.EnableLeaderElection, "leaderElection", false, "Use the Kubernetes leader election mechanism for clustering")
-
-	// This sets the namespace that the lock should exist in
-	kubeVipStart.Flags().StringVarP(&startConfig.Namespace, "namespace", "n", "kube-system", "The configuration map defined within the cluster")
-}
-
-var kubeVipStart = &cobra.Command{
-	Use:   "start",
-	Short: "Start the Virtual IP / Load balancer",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
-		var err error
-
-		// If a configuration file is loaded, then it will overwrite flags
-
-		if configPath != "" {
-			c, err := kubevip.OpenConfig(configPath)
-			if err != nil {
-				log.Fatalf("%v", err)
-			}
-			startConfig = *c
-		}
-
-		// parse environment variables, these will overwrite anything loaded or flags
-		err = kubevip.ParseEnvironment(&startConfig)
-		if err != nil {
-			log.Fatalln(err)
-		}
-
-		newCluster, err := cluster.InitCluster(&startConfig, disableVIP)
-		if err != nil {
-			log.Fatalf("%v", err)
-		}
-		var bgpServer *bgp.Server
-		if startConfig.SingleNode {
-			// If the Virtual IP isn't disabled then create the netlink configuration
-			// Start a single node cluster
-			if err := newCluster.StartSingleNode(&startConfig, disableVIP); err != nil {
-				log.Errorf("error starting single node: %v", err)
-			}
-		} else {
-			if disableVIP {
-				log.Fatalln("Cluster mode requires the Virtual IP to be enabled, use single node with no VIP")
-			}
-
-			if startConfig.EnableLeaderElection {
-				cm, err := cluster.NewManager(startKubeConfigPath, inCluster, startConfig.Port)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-
-				if startConfig.EnableBGP {
-					log.Info("Starting the BGP server to advertise VIP routes to VGP peers")
-					bgpServer, err = bgp.NewBGPServer(&startConfig.BGPConfig)
-					if err != nil {
-						log.Fatalf("%v", err)
-					}
-
-					// Defer a function to check if the bgpServer has been created and if so attempt to close it
-					defer func() {
-						if bgpServer != nil {
-							bgpServer.Close()
-						}
-					}()
-				}
-
-				// Leader Cluster will block
-				err = newCluster.StartCluster(&startConfig, cm, bgpServer)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-			}
-		}
-
-	},
-}

cmd/kube-vip.go

@@ -3,44 +3,38 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"net"
 	"net/http"
 	"os"
+	"slices"
+	"strings"
 	"time"
 
+	log "log/slog"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
 
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
 	"github.com/kube-vip/kube-vip/pkg/manager"
-	"github.com/kube-vip/kube-vip/pkg/packet"
+	"github.com/kube-vip/kube-vip/pkg/utils"
 	"github.com/kube-vip/kube-vip/pkg/vip"
 )
 
-// Path to the configuration file
-var configPath string
+// Is an option to set the image
+var image string
 
-// Path to the configuration file
-// var namespace string
-
-// Disable the Virtual IP (bind to the existing network stack)
-var disableVIP bool
-
-// Disable the Virtual IP (bind to the existing network stack)
-// var controlPlane bool
-
-// Run as a load balancer service (within a pod / kubernetes)
-// var serviceArp bool
+// Is kube-vip running within cluster
+var inCluster bool
 
 // ConfigMap name within a Kubernetes cluster
 var configMap string
 
-// Configure the level of loggin
-var logLevel uint32
-
-// Provider Config
-var providerConfig string
+// Points to a kubernetes configuration file
+var kubeConfigPath string
 
 // Release - this struct contains the release information populated when building kube-vip
 var Release struct {
@@ -49,11 +43,10 @@ var Release struct {
 }
 
 // Structs used via the various subcommands
-var initConfig kubevip.Config
-var initLoadBalancer kubevip.LoadBalancer
-
-// Points to a kubernetes configuration file
-var kubeConfigPath string
+var (
+	initConfig       kubevip.Config
+	initLoadBalancer kubevip.LoadBalancer
+)
 
 var kubeVipCmd = &cobra.Command{
 	Use:   "kube-vip",
@@ -61,42 +54,44 @@ var kubeVipCmd = &cobra.Command{
 }
 
 func init() {
-
 	// Basic flags
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Interface, "interface", "", "Name of the interface to bind to")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesInterface, "serviceInterface", "", "Name of the interface to bind to (for services)")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIP, "vip", "", "The Virtual IP address")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPSubnet, "vipSubnet", "", "The Virtual IP address subnet e.g. /32 /24 /8 etc.. (Default to 32 for IPv4 and 128 for IPv6)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.NodeName, "nodeName", "", "Name to be used for lease holder. Must be unique for each node/instance")
+
+	// VIP flags
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Address, "address", "", "an address (IP or DNS name) to use as a VIP")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.Port, "port", 6443, "Port for the VIP")
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for Vip changes")
+	kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.Port, "port", 6443, "Port for the VIP")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for VIP changes")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableWireguard, "wireguard", false, "Enable Wireguard for services VIPs")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableRoutingTable, "table", false, "Enable Routing Table for services VIPs")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.PreserveVIPOnLeadershipLoss, "preserveVipOnLeadershipLoss", false, "Preserve ARP VIP addresses on interface when leadership is lost (default: false for backward compatibility)")
 
 	// LoadBalancer flags
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "enableLoadBalancer", false, "enable loadbalancing on the VIP with IPVS")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.LoadBalancerPort, "lbPort", 6443, "loadbalancer port for the VIP")
+	kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.LoadBalancerPort, "lbPort", 6443, "loadbalancer port for the VIP")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerForwardingMethod, "lbForwardingMethod", "local", "loadbalancer forwarding method")
-
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DDNS, "ddns", false, "use Dynamic DNS + DHCP to allocate VIP for address")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MirrorDestInterface, "mirrorDestInterface", "", "network interface where all traffic that traverses the service interface will be mirrored to. Source interface will use default interface is servicesInterface is not set.")
 
 	// Clustering type (leaderElection)
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLeaderElection, "leaderElection", false, "Use the Kubernetes leader election mechanism for clustering")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 5, "Length of time a Kubernetes leader lease can be held for")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time a Kubernetes leader can attempt to renew its lease")
-	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Number of times the host will retry to hold a lease")
-
-	// Packet flags
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableMetal, "metal", false, "This will use the Equinix Metal API (requires the token ENV) to update the EIP <-> VIP")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalAPIKey, "metalKey", "", "The API token for authenticating with the Equinix Metal API")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalProject, "metalProject", "", "The name of project already created within Equinix Metal")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalProjectID, "metalProjectID", "", "The ID of project already created within Equinix Metal")
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ProviderConfig, "provider-config", "", "The path to a provider configuration")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaderElectionType, "leaderElectionType", "kubernetes", "Defines the backend to run the leader election: kubernetes or etcd. Defaults to kubernetes.")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaseName, "leaseName", "plndr-cp-lock", "Name of the lease that is used for leader election")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 5, "Length of time (in seconds) a Kubernetes leader lease can be held for")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time (in seconds) a Kubernetes leader can attempt to renew its lease")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Length of time (in seconds) the LeaderElector clients should wait between tries of actions")
 
 	// BGP flags
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPCIDR, "cidr", "32", "The CIDR range for the virtual IP address")
 	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableBGP, "bgp", false, "This will enable BGP support within kube-vip")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.RouterID, "bgpRouterID", "", "The routerID for the bgp server")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.SourceIF, "sourceIF", "", "The source interface for bgp peering (not to be used with sourceIP)")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.SourceIP, "sourceIP", "", "The source address for bgp peering (not to be used with sourceIF)")
 	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.AS, "localAS", 65000, "The local AS number for the bgp server")
+	kubeVipCmd.PersistentFlags().Uint64Var(&initConfig.BGPConfig.HoldTime, "bgpHoldTimer", 30, "The hold timer for all bgp peers (it defines the time a session is held)")
+	kubeVipCmd.PersistentFlags().Uint64Var(&initConfig.BGPConfig.KeepaliveInterval, "bgpKeepAliveInterval", 10, "The keepalive interval for all bgp peers (it defines the heartbeat of keepalive messages)")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPPeerConfig.Address, "peerAddress", "", "The address of a BGP peer")
 	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPPeerConfig.AS, "peerAS", 65000, "The AS number for a BGP peer")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPPeerConfig.Password, "peerPass", "", "The md5 password for a BGP peer")
@@ -104,29 +99,69 @@ func init() {
 	kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.BGPPeers, "bgppeers", []string{}, "Comma separated BGP Peer, format: address:as:password:multihop")
 	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Annotations, "annotations", "", "Set Node annotations prefix for parsing")
 
-	// Control plane specific flags
-	kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The configuration map defined within the cluster")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPConfig.Zebra.Enabled, "zebra", false, "This will enable Zebra support within kube-vip")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.URL, "zebraUrl", "unix:/var/run/frr/zserv.api", "Path to the unix domain socket for connecting to Zebra daemon")
+	kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.Zebra.Version, "zebraVersion", 6, "Zebra API Version")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.SoftwareName, "zebraSoftwareName", "frr8.3", "Software Name for Zebra")
+
+	// Namespace for kube-vip
+	kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The namespace for the configmap defined within the cluster")
 
 	// Manage logging
-	kubeVipCmd.PersistentFlags().Uint32Var(&logLevel, "log", 4, "Set the level of logging")
+	kubeVipCmd.PersistentFlags().Int32Var(&initConfig.Logging, "log", 0, "Set the level of logging")
 
 	// Service flags
 	kubeVipService.Flags().StringVarP(&configMap, "configMap", "c", "plndr", "The configuration map defined within the cluster")
 
+	// Routing Table flags
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableID, "tableID", 198, "The routing table used for all table entries")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableType, "tableType", unix.RTN_UNICAST, "The type of route that will be added to the routing table")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingProtocol, "routingProtocol", 248, "The routing protocol value used to create routes")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.CleanRoutingTable, "cleanRoutingTable", false, "Clean routing table of redundant routes on start")
+
 	// Behaviour flags
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableControlPane, "controlplane", false, "Enable HA for control plane, hybrid mode")
-	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServices, "services", false, "Enable Kubernetes services, hybrid mode")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableControlPlane, "controlplane", false, "Enable HA for control plane")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DetectControlPlane, "autodetectcp", false, "Determine working address for control plane (from loopback)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServices, "services", false, "Enable Kubernetes services")
+
+	// Extended behaviour flags
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServicesElection, "servicesElection", false, "Enable leader election per kubernetes service")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassOnly, "lbClassOnly", false, fmt.Sprintf("Enable load balancing only for services with LoadBalancerClass %q", kubevip.LBClassName))
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerClassName, "lbClassName", kubevip.LBClassName, fmt.Sprintf("Name of load balancer class for kube-VIP, defaults to %q", kubevip.LBClassName))
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassLegacyHandling, "lbClassNameLegacyHandling", true, "Use legacy LoadBalancer class name handling (e.g. accepting services both with empty and non-empty class)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServiceSecurity, "onlyAllowTrafficServicePorts", false, "Only allow traffic to service ports, others will be dropped, defaults to false")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableNodeLabeling, "enableNodeLabeling", false, fmt.Sprintf("Enable leader node labeling with %q, defaults to false", kubevip.HasIP))
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesLeaseName, "servicesLeaseName", "plndr-svcs-lock", "Name of the lease that is used for leader election for services (in arp mode)")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.DNSMode, "dnsMode", "first", "Name of the mode that DNS lookup will be performed (first, ipv4, ipv6, dual)")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DisableServiceUpdates, "disableServiceUpdates", false, "If true, kube-vip will process services as usual, but will not update service's Status.LoadBalancer.Ingress slice")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpoints, "enableEndpoints", false, "If enabled, kube-vip will only advertise services, but will use the (deprecated since v1.33) endpoints for IP addresses")
+	kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoInterfaceGlobalScope, "loInterfaceGlobalScope", false, "If true, kube-vip will set global scope when using the lo interface, otherwise a host scope will be used by default")
+	kubeVipCmd.PersistentFlags().IntVar(&initConfig.HealthCheckPort, "healthCheckPort", 0, "If set to non-zero (> 1024), then this is the port that the healthcheck will listen on")
 
 	// Prometheus HTTP Server
-	kubeVipCmd.PersistentFlags().StringVar(&initConfig.PrometheusHTTPServer, "promethuesHTTPServer", ":2112", "Host and port used to expose Prometheus metrics via an HTTP server")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.PrometheusHTTPServer, "prometheusHTTPServer", ":2112", "Host and port used to expose Prometheus metrics via an HTTP server")
+
+	// Etcd
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.CAFile, "etcdCACert", "", "Verify certificates of TLS-enabled secure servers using this CA bundle file")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.ClientCertFile, "etcdCert", "", "Identify secure client using this TLS certificate file")
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.Etcd.ClientKeyFile, "etcdKey", "", "Identify secure client using this TLS key file")
+	kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.Etcd.Endpoints, "etcdEndpoints", nil, "Etcd member endpoints")
+
+	// Kubernetes client specific flags
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.K8sConfigFile, "k8sConfigPath", "/etc/kubernetes/admin.conf", "Path to the configuration file used with the Kubernetes client")
+
+	// Configuration file flag
+	kubeVipCmd.PersistentFlags().StringVar(&initConfig.ConfigFile, "config-file", "", "Path to a JSON/YAML configuration file to load settings from")
 
 	kubeVipCmd.AddCommand(kubeKubeadm)
 	kubeVipCmd.AddCommand(kubeManifest)
 	kubeVipCmd.AddCommand(kubeVipManager)
 	kubeVipCmd.AddCommand(kubeVipSample)
 	kubeVipCmd.AddCommand(kubeVipService)
-	kubeVipCmd.AddCommand(kubeVipStart)
 	kubeVipCmd.AddCommand(kubeVipVersion)
+
+	// Set the logging level for all subsequent functions
+	log.SetLogLoggerLevel(log.Level(initConfig.Logging))
 }
 
 // Execute - starts the command parsing process
@@ -140,7 +175,7 @@ func Execute() {
 var kubeVipVersion = &cobra.Command{
 	Use:   "version",
 	Short: "Version and Release information about the Kubernetes Virtual IP Server",
-	Run: func(cmd *cobra.Command, args []string) {
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 		fmt.Printf("Kube-VIP Release Information\n")
 		fmt.Printf("Version:  %s\n", Release.Version)
 		fmt.Printf("Build:    %s\n", Release.Build)
@@ -150,7 +185,7 @@ var kubeVipVersion = &cobra.Command{
 var kubeVipSample = &cobra.Command{
 	Use:   "sample",
 	Short: "Generate a Sample configuration",
-	Run: func(cmd *cobra.Command, args []string) {
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 		_ = cmd.Help()
 	},
 }
@@ -158,18 +193,35 @@ var kubeVipSample = &cobra.Command{
 var kubeVipService = &cobra.Command{
 	Use:   "service",
 	Short: "Start the Virtual IP / Load balancer as a service within a Kubernetes cluster",
-	Run: func(cmd *cobra.Command, args []string) {
-		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
 
-		// parse environment variables, these will overwrite anything loaded or flags
-		err := kubevip.ParseEnvironment(&initConfig)
-		if err != nil {
-			log.Fatalln(err)
+		// Load configuration from file if specified (lowest priority)
+		if initConfig.ConfigFile != "" {
+			err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
+			if err != nil {
+				log.Error("loading config file", "err", err)
+				return
+			}
 		}
 
+		// parse environment variables, these will overwrite anything loaded from config file
+		err := kubevip.ParseEnvironment(&initConfig)
+		if err != nil {
+			log.Error("parsing env", "err", err)
+			return
+		}
+
+		// Change RTN_UNSPEC to default type
+		if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
+			initConfig.RoutingProtocol = unix.RTN_UNICAST
+		}
+
+		// Set the logging level for all subsequent functions
+		log.SetLogLoggerLevel(log.Level(initConfig.Logging))
+
 		if err := initConfig.CheckInterface(); err != nil {
-			log.Fatalln(err)
+			log.Error("checking interface", "err", err)
+			return
 		}
 
 		// User Environment variables as an option to make manifest clearer
@@ -178,16 +230,27 @@ var kubeVipService = &cobra.Command{
 			configMap = envConfigMap
 		}
 
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
+			if err != nil {
+				log.Error("generating CIDR", "err", err)
+				return
+			}
+		}
+
 		// Define the new service manager
 		mgr, err := manager.New(configMap, &initConfig)
 		if err != nil {
-			log.Fatalf("%v", err)
+			log.Error("new manager", "err", err)
+			return
 		}
 
 		// Start the service manager, this will watch the config Map and construct kube-vip services for it
 		err = mgr.Start()
 		if err != nil {
-			log.Fatalf("%v", err)
+			log.Error("manager start", "err", err)
+			return
 		}
 	},
 }
@@ -195,41 +258,144 @@ var kubeVipService = &cobra.Command{
 var kubeVipManager = &cobra.Command{
 	Use:   "manager",
 	Short: "Start the kube-vip manager",
-	Run: func(cmd *cobra.Command, args []string) {
-
-		// parse environment variables, these will overwrite anything loaded or flags
-		err := kubevip.ParseEnvironment(&initConfig)
-		if err != nil {
-			log.Fatalln(err)
+	Run: func(cmd *cobra.Command, args []string) { //nolint TODO
+		// Load configuration from file if specified (lowest priority)
+		if initConfig.ConfigFile != "" {
+			err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
+			if err != nil {
+				log.Error("loading config file", "err", err)
+				return
+			}
 		}
 
-		if err := initConfig.CheckInterface(); err != nil {
-			log.Fatalln(err)
+		// parse environment variables, these will overwrite anything loaded from config file
+		err := kubevip.ParseEnvironment(&initConfig)
+		if err != nil {
+			log.Error("parsing environment", "err", err)
+			return
+		}
+
+		// Change RTN_UNSPEC to default type
+		if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
+			initConfig.RoutingProtocol = unix.RTN_UNICAST
 		}
 
 		// Set the logging level for all subsequent functions
-		log.SetLevel(log.Level(logLevel))
+		log.SetLogLoggerLevel(log.Level(initConfig.Logging))
 
-		if initConfig.Interface == "" {
-			log.Infof("No interface is specified for VIP in config, auto-detecting default Interface")
-			defaultIF, err := vip.GetDefaultGatewayInterface()
+		// Ensure there is an address to generate the CIDR from
+		if initConfig.VIPSubnet == "" && initConfig.Address != "" {
+			initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
 			if err != nil {
-				_ = cmd.Help()
-				log.Fatalf("unable to detect default interface -> [%v]", err)
+				log.Error("No interface is specified for kube-vip to bind to")
+				return
 			}
-			initConfig.Interface = defaultIF.Name
-			log.Infof("kube-vip will bind to interface [%s]", initConfig.Interface)
-
-			go func() {
-				if err := vip.MonitorDefaultInterface(context.TODO(), defaultIF); err != nil {
-					log.Fatalf("crash: %s", err.Error())
-				}
-			}()
 		}
 
-		go servePrometheusHTTPServer(cmd.Context(), PrometheusHTTPServerConfig{
-			Addr: initConfig.PrometheusHTTPServer,
-		})
+		// Welome messages
+		log.Info("kube-vip.io", "version", Release.Version, "build", Release.Build)
+
+		// start prometheus server
+		if initConfig.PrometheusHTTPServer != "" {
+			go servePrometheusHTTPServer(cmd.Context(), PrometheusHTTPServerConfig{
+				Addr: initConfig.PrometheusHTTPServer,
+			})
+		}
+
+		// Determine the kube-vip mode
+		var mode string
+		if initConfig.EnableARP {
+			mode = "ARP"
+		}
+
+		if initConfig.EnableBGP {
+			mode = "BGP"
+		}
+
+		if initConfig.EnableWireguard {
+			mode = "Wireguard"
+		}
+
+		if initConfig.EnableRoutingTable {
+			mode = "Routing Table"
+		}
+
+		// Provide configuration to output/logging
+		log.Info("starting", "namespace", initConfig.Namespace, "Mode", mode, "Control Plane", initConfig.EnableControlPlane, "Services", initConfig.EnableServices)
+
+		// End if nothing is enabled
+		if !initConfig.EnableServices && !initConfig.EnableControlPlane {
+			log.Error("no features are enabled")
+			return
+		}
+
+		if !initConfig.EnableARP && strings.Contains(initConfig.VIPSubnet, kubevip.Auto) {
+			log.Error("auto subnet discovery cannot be used outside ARP mode")
+			return
+		}
+
+		if strings.Contains(initConfig.VIPSubnet, kubevip.Auto) && initConfig.Address != "" {
+			log.Error("auto subnet discovery cannot be used if VIP address was provided")
+			return
+		}
+
+		// If we're using wireguard then all traffic goes through the wg0 interface
+		if initConfig.EnableWireguard {
+			if initConfig.Interface == "" {
+				// Set the vip interface to the wireguard interface
+				initConfig.Interface = "wg0"
+			}
+
+			log.Info("configuring Wireguard networking")
+			l, err := netlink.LinkByName(initConfig.Interface)
+			if err != nil {
+				if strings.Contains(err.Error(), "Link not found") {
+					log.Warn("attempting to create wireguard interface", "interface not found", initConfig.Interface)
+					err = netlink.LinkAdd(&netlink.Wireguard{LinkAttrs: netlink.LinkAttrs{Name: initConfig.Interface}})
+					if err != nil {
+						log.Error("adding link", "err", err)
+						return
+					}
+					l, err = netlink.LinkByName(initConfig.Interface)
+					if err != nil {
+						log.Error("finding link", "err", err)
+						return
+					}
+				}
+			}
+			err = netlink.LinkSetUp(l)
+			if err != nil {
+				log.Error("setting link UP", "err", err)
+				return
+			}
+
+		} else { // if we're not using Wireguard then we'll need to use an actual interface
+			// Check if the interface needs auto-detecting
+			if initConfig.Interface == "" {
+				log.Info("No interface is specified for VIP in config, auto-detecting default Interface")
+				defaultIF, err := vip.GetDefaultGatewayInterface()
+				if err != nil {
+					_ = cmd.Help()
+					log.Error("detecting interface", "err", err)
+					return
+				}
+				initConfig.Interface = defaultIF.Name
+				log.Info("kube-vip bind", "interface", initConfig.Interface)
+
+				go func() {
+					if err := vip.MonitorDefaultInterface(context.TODO(), defaultIF); err != nil {
+
+						log.Error("interface monitor", "err", err)
+						return
+					}
+				}()
+			}
+		}
+		// Perform a check on the state of the interface
+		if err := initConfig.CheckInterface(); err != nil {
+			log.Error("checking interface", "err", err)
+			return
+		}
 
 		// User Environment variables as an option to make manifest clearer
 		envConfigMap := os.Getenv("vip_configmap")
@@ -237,22 +403,11 @@ var kubeVipManager = &cobra.Command{
 			configMap = envConfigMap
 		}
 
-		// If Packet is enabled and there is a provider configuration passed
-		if initConfig.EnableMetal {
-			if providerConfig != "" {
-				providerAPI, providerProject, err := packet.GetPacketConfig(providerConfig)
-				if err != nil {
-					log.Fatalf("%v", err)
-				}
-				initConfig.MetalAPIKey = providerAPI
-				initConfig.MetalProject = providerProject
-			}
-		}
-
 		// Define the new service manager
 		mgr, err := manager.New(configMap, &initConfig)
 		if err != nil {
-			log.Fatalf("configuring new Manager error -> %v", err)
+			log.Error("new manager", "err", err)
+			return
 		}
 
 		prometheus.MustRegister(mgr.PrometheusCollector()...)
@@ -260,7 +415,8 @@ var kubeVipManager = &cobra.Command{
 		// Start the service manager, this will watch the config Map and construct kube-vip services for it
 		err = mgr.Start()
 		if err != nil {
-			log.Fatalf("starting new Manager error -> %v", err)
+			log.Error("start manager", "err", err)
+			return
 		}
 	},
 }
@@ -275,23 +431,34 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
 	var err error
 	mux := http.NewServeMux()
 	mux.Handle("/metrics", promhttp.Handler())
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { //nolint TODO
+		_, _ = w.Write([]byte(`<html>
+			<head><title>kube-vip</title></head>
+			<body>
+			<h1>kube-vip Metrics</h1>
+			<p><a href="` + "/metrics" + `">Metrics</a></p>
+			</body>
+			</html>`))
+	})
 
 	srv := &http.Server{
-		Addr:    config.Addr,
-		Handler: mux,
+		Addr:              config.Addr,
+		Handler:           mux,
+		ReadHeaderTimeout: 2 * time.Second,
 	}
 
 	go func() {
 		if err = srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
-			log.Fatalf("listen:%+s\n", err)
+			log.Error("prometheus HTTP server", "err", err)
+			return
 		}
 	}()
 
-	log.Printf("server started")
+	log.Info("prometheus HTTP server started")
 
 	<-ctx.Done()
 
-	log.Printf("server stopped")
+	log.Info("prometheus HTTP server stopped")
 
 	ctxShutDown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer func() {
@@ -299,11 +466,46 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
 	}()
 
 	if err = srv.Shutdown(ctxShutDown); err != nil {
-		log.Fatalf("server Shutdown Failed:%+s", err)
+		log.Error("shutting down prometheus HTTP server", "err", err)
+		return
 	}
 
 	if err == http.ErrServerClosed {
 		err = nil
 	}
-
+}
+
+func GenerateCidrRange(address string, dnsMode string) (string, error) {
+	var cidrs []string
+
+	addresses := strings.Split(address, ",")
+	for _, a := range addresses {
+		ip := net.ParseIP(a)
+		if ip == nil {
+			// we probably are a DNS name
+			ips, err := utils.LookupHost(a, dnsMode)
+			if len(ips) == 0 || err != nil {
+				return "", fmt.Errorf("invalid IP address: %s from [%s], %v", a, address, err)
+			}
+			for _, addr := range ips {
+				ip = net.ParseIP(addr)
+				if ip.To4() != nil {
+					cidrs = append(cidrs, "32")
+				} else {
+					cidrs = append(cidrs, "128")
+				}
+			}
+		} else {
+			if ip.To4() != nil {
+				cidrs = append(cidrs, "32")
+			} else {
+				cidrs = append(cidrs, "128")
+			}
+		}
+	}
+	// compact as DNS could have a lot of addresses
+	slices.Sort(cidrs)
+	cidrs = slices.Compact(cidrs)
+	slices.Reverse(cidrs)
+	return strings.Join(cidrs, ","), nil
 }

demo/README.md

@@ -0,0 +1,46 @@
+# Demo client-server
+
+This contains some example code to determine how long "failovers" are taking within kube-vip, the server component should live within the cluster and the client should be externally.
+
+## Deploy the server
+
+Simply apply the manifest to a working cluster that has kube-vip deployed:
+
+```
+kubectl apply -f ./demo/server/deploy.yaml
+```
+
+Retrieve the loadBalancer IP that is fronting the service:
+
+```
+kubectl get svc demo-service
+NAME           TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)           AGE
+demo-service   LoadBalancer   10.104.18.147   192.168.0.217   10002:32529/UDP   117m
+```
+
+## Connect the client
+
+From elsewhere, clone the kube-vip repository and connect the client to the server endpoint (loadBalancer IP) with the following command:
+
+```
+go run ./demo/client/main.go -address=<vip>
+```
+
+You will only see output when the client has reconcilled the connection to a pod beneath the service, where it will print the timestamp to reconnection along with the time in milliseconds it took:
+
+```
+15:58:35.916952 3008
+15:58:45.947506 2005
+15:58:57.983151 3007
+15:59:08.013450 2005
+15:59:20.046491 3008
+15:59:30.076341 2507
+15:59:42.110747 3008
+```
+
+## Kill some pods to test
+
+On a machine or control plane that has `kubectl` and has the credentials to speak to the cluster we will run a command to find the demo pod and kill it every 10 seconds:
+
+`while true ; do kubectl delete pod $(kubectl get pods | grep -v NAME | grep vip| awk '{ print $1 }'); sleep 10; done`
+

demo/client/main.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"net"
+	"time"
+)
+
+const udpdata = "a3ViZS12aXAK=kube-vip"
+
+func main() {
+	address := flag.String("address", "127.0.0.1", "The address of the server")
+	port := flag.Int("port", 10002, "the port of the server")
+	interval := flag.Float64("interval", 1000, "Interval in milliseconds")
+	flag.Parse()
+	var errorTime time.Time
+	var errorOccurred bool
+	for {
+		p := make([]byte, 2048)
+		conn, err := net.Dial("udp", fmt.Sprintf("%s:%d", *address, *port))
+		if err != nil {
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			continue
+		}
+
+		err = conn.SetDeadline(time.Now().Add(time.Duration(*interval) * time.Millisecond))
+		if err != nil {
+			//fmt.Printf("Connectivity error [%v]", err)
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			if err = conn.Close(); err != nil {
+				fmt.Printf("Error closing connection [%v]", err)
+			}
+			continue
+		}
+
+		_, err = fmt.Fprint(conn, udpdata)
+		if err != nil {
+			fmt.Printf("Error writing data [%v]", err)
+		}
+
+		_, err = bufio.NewReader(conn).Read(p)
+		if err != nil {
+			//fmt.Printf("read error %v\n", err)
+			if !errorOccurred {
+				errorTime = time.Now()
+				errorOccurred = true
+			}
+			if err = conn.Close(); err != nil {
+				fmt.Printf("Error closing connection [%v]", err)
+			}
+			continue
+		}
+		time.Sleep(time.Duration(*interval) * time.Millisecond)
+		if errorOccurred {
+			finishTime := time.Since(errorTime)
+			//fmt.Printf("connectivity reconciled in %dms\n", finishTime.Milliseconds())
+			//t :=time.Now().Format("15:04:05.000000")
+			fmt.Printf("%s %d\n", time.Now().Format("15:04:05.000000"), finishTime.Milliseconds())
+
+			errorOccurred = false
+		}
+	}
+}

demo/server/Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:experimental
 
-FROM golang:1.13-alpine as dev
+FROM golang:1.19-alpine as dev
 RUN apk add --no-cache git ca-certificates
 RUN adduser -D appuser
 COPY main.go /src/
@@ -13,4 +13,4 @@ RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
 
 FROM scratch
 COPY --from=dev /src/demo /
-CMD ["/demo"]
+CMD ["/demo"]

demo/server/Makefile

@@ -45,7 +45,7 @@ fmt:
 	@gofmt -l -w $(SRC)
 
 docker:
-	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
+	@docker buildx build  --platform linux/amd64,linux/arm64,linux/arm/v7,linux/ppc64le,linux/s390x --push -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
 	@echo New Multi Architecture Docker image created
 
 simplify:

demo/server/deploy.yaml

@@ -6,7 +6,7 @@ metadata:
     app: kube-vip-demo
   name: kube-vip-demo
 spec:
-  replicas: 3
+  replicas: 1
   selector:
     matchLabels:
       app: kube-vip-demo
@@ -30,4 +30,25 @@ spec:
         ports:
             - containerPort: 10001
             - containerPort: 10002
-status: {}
+status: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: demo-service
+  namespace: default
+  labels:
+    app: demo-service
+  annotations:
+    kube-vip.io/egress: "true"
+spec:
+  type: LoadBalancer
+  # "Local" preserves the client source IP and avoids a second hop for
+  # LoadBalancer and NodePort
+  externalTrafficPolicy: Local
+  ports:
+  - name: demo-udp
+    port: 10002
+    protocol: UDP
+  selector:
+    app: kube-vip-demo

demo/server/main.go

@@ -32,7 +32,7 @@ func main() {
 		}
 	}
 
-	if serverType == strings.ToLower("udp") {
+	if serverType == "udp" {
 		// Start the UDP echo server
 
 		ServerAddr, err := net.ResolveUDPAddr("udp", ":10002")

docs/Dockerfile

@@ -1,5 +0,0 @@
-FROM nginx:latest
-RUN apt-get update; apt-get install -y nodejs npm; npm install -g markdown-styles;
-COPY . /docs
-WORKDIR /docs
-RUN generate-md --layout github --input ./ --output /usr/share/nginx/html/

docs/architecture/index.md

@@ -1,157 +0,0 @@
-# Kube-Vip Architecture
-
-This section covers two parts of the architecture:
-
-1. The technical capabilities of `kube-vip`.
-2. The components to build a load balancing service within Kubernetes.
-
-The `kube-vip` project is designed to provide both a highly available networking endpoint and load balancing functionality for underlying networking services. The project was originally designed for the purpose of providing a resilient control plane for Kubernetes but has since expanded to provide the same functionality for Service resources within a Kubernetes cluster.
-
-Additionally, `kube-vip` is designed to be lightweight and multi-architecture. All of the components are built for Linux on `x86`, `armv7`, `armhvf`, and `ppc64le` architectures. This means that `kube-vip` will run fine in bare metal, virtual, and edge (Raspberry Pi or small ARM SoC) use cases.
-
-## Technologies
-
-There are a number of technologies or functional design choices that provide high availability and networking functions as part of a VIP/load balancing solution.
-
-### Cluster
-
-The `kube-vip` service builds a multi-node or multi-pod cluster to provide high availability. In ARP mode, a leader is elected which will inherit the virtual IP and become the leader of the load balancing within the cluster whereas with BGP all nodes will advertise the VIP address.
-
-When using ARP or [Layer 2](https://osi-model.com/data-link-layer/) it will use [leader election](https://godoc.org/k8s.io/client-go/tools/leaderelection).
-
-### Virtual IP
-
-The leader within the cluster will assume the VIP and will have it bound to the selected interface that is declared within the configuration. When the leader changes, it will evacuate the VIP first or in failure scenarios the VIP will be directly assumed by the next elected leader.
-
-When the VIP moves from one host to another, any host that has been using the VIP will retain the previous VIP-to-MAC address mapping until the old ARP entry expires (typically within 30 seconds) and retrieves a new mapping. This can be improved by using [Gratuitous ARP](https://wiki.wireshark.org/Gratuitous_ARP) broadcasts when enabled (detailed below).
-
-### ARP
-
-`kube-vip` can optionally be configured to broadcast a Gratuitous ARP that will typically immediately notify all local hosts that the VIP-to-MAC address mapping has changed.
-
-Below we can see that the failover is typically done within a few seconds as the ARP broadcast is received.
-
-```
-64 bytes from 192.168.0.75: icmp_seq=146 ttl=64 time=0.258 ms
-64 bytes from 192.168.0.75: icmp_seq=147 ttl=64 time=0.240 ms
-92 bytes from 192.168.0.70: Redirect Host(New addr: 192.168.0.75)
-Vr HL TOS  Len   ID Flg  off TTL Pro  cks      Src      Dst
-4  5  00 0054 bc98   0 0000  3f  01 3d16 192.168.0.95  192.168.0.75 
-
-Request timeout for icmp_seq 148
-92 bytes from 192.168.0.70: Redirect Host(New addr: 192.168.0.75)
-Vr HL TOS  Len   ID Flg  off TTL Pro  cks      Src      Dst
-4  5  00 0054 75ff   0 0000  3f  01 83af 192.168.0.95  192.168.0.75 
-
-Request timeout for icmp_seq 149
-92 bytes from 192.168.0.70: Redirect Host(New addr: 192.168.0.75)
-Vr HL TOS  Len   ID Flg  off TTL Pro  cks      Src      Dst
-4  5  00 0054 2890   0 0000  3f  01 d11e 192.168.0.95  192.168.0.75 
-
-Request timeout for icmp_seq 150
-64 bytes from 192.168.0.75: icmp_seq=151 ttl=64 time=0.245 ms
-```
-
-### Load Balancing
-
-`kube-vip` has the capability to provide a high availability address for both the Kubernetes control plane and for a Kubernetes Service. As of v0.4.0, `kube-vip` implements support for true load balancing for the control plane to distribute API requests across control plane nodes.
-
-#### Kubernetes Service Load Balancing
-
-The following is required in the `kube-vip` manifest to enable Service of type `LoadBalancer`:
-
-```yaml
-- name: svc_enable
-  value: "true"
-```
-
-This section details the flow of events in order for `kube-vip` to advertise a Kubernetes Service:
-
-1. An end user exposes an application through Kubernetes as a Service type `LoadBalancer`. For example, imperatively using `kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx`
-2. Within the Kubernetes cluster, a Service object is created with the `spec.type` set to `LoadBalancer`.
-3. A controller (typically a [Cloud Controller](/usage/on-prem)) has a loop that "watches" for Services of the type `LoadBalancer`.
-4. The controller now has the responsibility of providing an IP address for this Service along with doing anything that is network specific for the environment where the cluster is running.
-5. Once the controller has an IP address, it will update the Service field `spec.loadBalancerIP` with the IP address.
-6. `kube-vip` Pods implement a "watcher" for Services that have a `svc.Spec.loadBalancerIP` address attached.
-7. When a new Service appears, `kube-vip` will start advertising this address to the wider network (through BGP/ARP) which will allow traffic to come into the cluster and hit the Service network.
-8. Finally, `kube-vip` will update the Service status so that the API reflects the object is ready. This is done by updating the `status.loadBalancer.ingress` with the VIP address.
-
-#### Control Plane Load-Balancing
-
-As of `kube-vip` v0.4.0, IPVS load balancing is configured for having the VIP in the same subnet as the control plane nodes. NAT-based load balancing will follow later.
-
-To enable control plane load balancing using IPVS, the environment variable `lb_enable` is required in the `kube-vip` manifest:
-
-```yaml
-- name : lb_enable
-  value: "true"
-```
-
-The load balancing is provided through IPVS (IP Virtual Server) and provides a Layer 4 (TCP-based) round-robin across all of the control plane nodes. By default, the load balancer will listen on the default port of 6443 as the Kubernetes API server. The IPVS virtual server lives in kernel space and doesn't create an "actual" service that listens on port 6443. This allows the kernel to parse packets before they're sent to an actual TCP port. This is important to know because it means we don't have any port conflicts having the IPVS load balancer listening on the same port as the API server on the same host.
-
-The load balancer port can be customised by changing the `lb_port` environment variable in the `kube-vip` manifest:
-
-```yaml
-- name: lb_port
-  value: "6443"
-```
-
-##### How it works
-
-Once the `lb_enable` variable is set to `true`, `kube-vip` will do the following:
-
-- In Layer 2 it will create an IPVS service on the leader.
-- In Layer 3 all nodes will create an IPVS service.
-- It will start a Kubernetes node watcher for nodes with the control plane label.
-- It will add/delete them as they're added and removed from the cluster.
-
-#### Debugging control plane load balancing
-
-In order to inspect and debug traffic, install the `ipvsadm` tool.
-
-##### View the configuration
-
-The command `sudo ipvsadm -ln` will display the load balancer configuration.
-
-```sh
-$ sudo ipvsadm -ln
-IP Virtual Server version 1.2.1 (size=4096)
-Prot LocalAddress:Port Scheduler Flags
--> RemoteAddress:Port           Forward Weight ActiveConn InActConn
-TCP  192.168.0.40:6443 rr
--> 192.168.0.41:6443            Local   1      4          0
--> 192.168.0.42:6443            Local   1      3          0
--> 192.168.0.43:6443            Local   1      3          0
-```
-
-##### Watch things interact with the API server
-
-The command `watch sudo ipvsadm -lnc` will auto-refresh the connections to the load balancer.
-
-```sh
-$ watch sudo ipvsadm -lnc
-
-<snip>
-
-sudo ipvsadm -lnc                    k8s01: Tue Nov  9 11:39:39 2021
-
-IPVS connection entries
-pro expire state       source             virtual            destination
-TCP 14:49  ESTABLISHED 192.168.0.42:37090 192.168.0.40:6443  192.168.0.41:6443
-TCP 14:55  ESTABLISHED 192.168.0.45:46510 192.168.0.40:6443  192.168.0.41:6443
-TCP 14:54  ESTABLISHED 192.168.0.43:39602 192.168.0.40:6443  192.168.0.43:6443
-TCP 14:58  ESTABLISHED 192.168.0.44:50458 192.168.0.40:6443  192.168.0.42:6443
-TCP 14:32  ESTABLISHED 192.168.0.43:39648 192.168.0.40:6443  192.168.0.42:6443
-TCP 14:58  ESTABLISHED 192.168.0.40:55944 192.168.0.40:6443  192.168.0.41:6443
-TCP 14:54  ESTABLISHED 192.168.0.42:36950 192.168.0.40:6443  192.168.0.41:6443
-TCP 14:42  ESTABLISHED 192.168.0.44:50488 192.168.0.40:6443  192.168.0.43:6443
-TCP 14:53  ESTABLISHED 192.168.0.45:46528 192.168.0.40:6443  192.168.0.43:6443
-TCP 14:49  ESTABLISHED 192.168.0.40:56040 192.168.0.40:6443  192.168.0.42:6443
-```
-
-## Components within a Kubernetes Cluster
-
-The `kube-vip` Kubernetes load balancer requires a number of components in order to function:
-
-- [Kube-Vip Cloud Provider](https://github.com/kube-vip/kube-vip-cloud-provider)
-- [Kube-Vip Deployment](https://github.com/kube-vip/kube-vip)

docs/control-plane/0.1.4/index.md

@@ -1,147 +0,0 @@
-# Load Balancing a Kubernetes Cluster (Control-Plane) (pre 0.1.5)
-
-This document covers all of the details for using `kube-vip` to build a HA Kubernetes cluster
-
-`tl;dr version`
-- Generate/modify first node `kube-vip` config/manifest
-- `init` first node
-- `join` remaining nodes
-- Add remaining config/manifests
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.75 |
-| controlPlane01 | 10.0.0.70 |
-| controlPlane02 | 10.0.0.71 |
-| controlPlane03 | 10.0.0.72 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.17.0.
-
-### Generate the `kube-vip` configuration
-
-Make sure that the config directory exists: `sudo mkdir -p /etc/kube-vip/`, this directory can be any directory however the `hostPath` in the manifest will need modifying to point to the correct path.
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1 /kube-vip sample config | sudo tee /etc/kube-vip/config.yaml
-```
-
-### Modify the configuration
-
-**Cluster Configuration**
-Modify the `remotePeers` to point to the correct addresses of the other two nodes, ensure that their `id` is unique otherwise this will confuse the raft algorithm. The `localPeer` should be the configuration of the current node (`controlPlane01`), which is where this instance of the cluster will run. 
-
-As this node will be the first node, it will need to elect itself leader as until this occurs the VIP won’t be activated!
-
-`startAsLeader: true`
-
-**VIP Config**
-We will need to set our VIP address to `192.168.0.75` and to ensure all hosts are updated when the VIP moves we will enable ARP broadcasts `gratuitousARP: true`
-
-**Load Balancer**
-We will configure the load balancer to sit on the standard API-Server port `6443` and we will configure the backends to point to the API-servers that will be configured to run on port `6444`. Also for the Kubernetes Control Plane we will configure the load balancer to be of `type: tcp`.
-
-We can also use `6443` for both the VIP and the API-Servers, in order to do this we need to specify that the api-server is bound to it's local IP. To do this we use the `--apiserver-advertise-address` flag as part of the `init`, this means that we can then bind the same port to the VIP and we wont have a port conflict.
-
-**config.yaml**
-
-`user@controlPlane01:/etc/kube-vip$ cat config.yaml`
-
-...
-
-``` 
-remotePeers:
-- id: server2
-  address: 192.168.0.71
-  port: 10000
-- id: server3
-  address: 192.168.0.72
-  port: 10000
-localPeer:
-  id: server1
-  address: 192.168.0.70
-  port: 10000
-vip: 192.168.0.75
-gratuitousARP: true
-singleNode: false
-startAsLeader: true
-interface: ens192
-loadBalancers:
-- name: Kubernetes Control Plane
-  type: tcp
-  port: 6443
-  bindToVip: true
-  backends:
-  - port: 6444
-    address: 192.168.0.70
-  - port: 6444
-    address: 192.168.0.71
-  - port: 6444
-    address: 192.168.0.72
-```
-
-### First Node
-
-To generate the basic Kubernetes static pod `yaml` configuration:
-
-Make sure that the manifest directory exists: `sudo mkdir -p /etc/kubernetes/manifests/`
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1 /kube-vip sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Ensure that `image: plndr/kube-vip:<x>` is modified to point to a specific version (`0.1` at the time of writing), refer to [docker hub](https://hub.docker.com/r/plndr/kube-vip/tags) for details. Also ensure that the `hostPath` points to the correct `kube-vip` configuration, if it isn’t the above path. 
-
-The **vip** is set to `192.168.0.75` and this first node will elect itself as leader, and as part of the `kubeadm init` it will use the VIP in order to speak back to the initialising api-server.
-
-`sudo kubeadm init --control-plane-endpoint “192.168.0.75:6443” --apiserver-bind-port 6444 --upload-certs --kubernetes-version “v1.17.0”`
-
-Once this node is up and running we will be able to see the control-plane pods, including the `kube-vip` pod:
-
-```
-$ kubectl get pods -A
-NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
-<...>
-kube-system   kube-vip-controlplane01                  1/1     Running   0          10m
-```
-
-### Remaining Nodes
-
-We first will need to create the `kube-vip` configuration that resides in `/etc/kube-vip/config.yaml` or we can regenerate it from scratch using the above example. Ensure that the configuration is almost identical with the `localPeer` and `remotePeers` sections are updated for each node. Finally, ensure that the remaining nodes will behave as standard cluster nodes by setting `startAsLeader: false`.
-
-At this point **DON’T** generate the manifests, this is due to some bizarre `kubeadm/kubelet` behaviour.
-
-```
-  kubeadm join 192.168.0.75:6443 --token <tkn> \
-    --discovery-token-ca-cert-hash sha256:<hash> \
-    --control-plane --certificate-key <key> 
-
-```
-
-**After** this node has been added to the cluster, we can add the manifest to also add this node as a `kube-vip` member. (Adding the manifest afterwards doesn’t interfere with `kubeadm`). 
-
-```
-sudo docker run -it --rm plndr/kube-vip:0.1 /kube-vip sample manifest | sudo tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-Once this node is added we will be able to see that the `kube-vip` pod is up and running as expected:
-
-```
-user@controlPlane01:~$ kubectl get pods -A | grep vip
-kube-system   kube-vip-controlplane01                  1/1     Running             1          16m
-kube-system   kube-vip-controlplane02                  1/1     Running             0          18m
-kube-system   kube-vip-controlplane03                  1/1     Running             0          20m
-
-```
-
-If we look at the logs, we can see that the VIP is running on the second node and we’re waiting for our third node to join the cluster:
-
-```
-$ kubectl logs kube-vip-controlplane02  -n kube-system
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-
-```

docs/control-plane/0.1.5/index.md

@@ -1,195 +0,0 @@
-# Load Balancing a Kubernetes Cluster (Control-Plane)
-
-This document covers the newer (post `0.1.5`) method for using `kube-vip` to provide HA for a Kubernetes Cluster. The documentation for older releases can be found [here](./0.1.4/)
-
-This document covers all of the details for using `kube-vip` to build a HA Kubernetes cluster
-
-`tl;dr version`
-- Generate/modify first node `kube-vip` config/manifest
-- `init` first node
-- `join` remaining nodes
-- Add remaining config/manifests
-
-Below are examples of the steps required:
-
-```
-# First Node
-sudo docker run --network host --rm plndr/kube-vip:0.1.5 kubeadm init --interface ens192 --vip 192.168.0.81 --startAsLeader=true | sudo tee /etc/kubernetes/manifests/vip.yaml
-
-sudo kubeadm init --kubernetes-version 1.17.0 --control-plane-endpoint 192.168.0.81 --upload-certs
-
-# Additional Node(s)
-
-sudo kubeadm join 192.168.0.81:6443 --token w5atsr.blahblahblah --control-plane --certificate-key abc123
-
-sudo docker run -v /etc/kubernetes/admin.conf:/etc/kubernetes/admin.conf --network host --rm plndr/kube-vip:0.1.5 kubeadm join --interface ens192 --vip 192.168.0.81 --startAsLeader=false | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.75 |
-| controlPlane01 | 10.0.0.70 |
-| controlPlane02 | 10.0.0.71 |
-| controlPlane03 | 10.0.0.72 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.17.0.
-
-### Generate the `kube-vip` configuration
-
-Kube-Vip no longer requires storing it's configuration in a seperate directory and will now store its configuration in the actual manifest that defines the static pods. 
-
-```
-sudo docker run --network host \
-	--rm plndr/kube-vip:0.1.5 \
-	kubeadm init \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--startAsLeader=true | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-The above command will "initialise" the manifest within the `/etc/kubernetes/manifests` directory, that will be started when we actually initialise our Kubernetes cluster with `kubeadm init`
-
-### Modify the configuration
-
-**Cluster Configuration**
-As this node will be the first node, it will need to elect itself leader as until this occurs the VIP won’t be activated!
-
-`--startAsLeader=true`
-
-**VIP Config**
-We will need to set our VIP address to `192.168.0.75` with `--vip 192.168.0.75` and to ensure all hosts are updated when the VIP moves we will enable ARP broadcasts `--arp` (defaults to `true`)
-
-**Load Balancer**
-We will configure the load balancer to sit on the standard API-Server port `6443` and we will configure the backends to point to the API-servers that will be configured to run on port `6444`. Also for the Kubernetes Control Plane we will configure the load balancer to be of `type: tcp`.
-
-We can also use `6443` for both the VIP and the API-Servers, in order to do this we need to specify that the api-server is bound to it's local IP. To do this we use the `--apiserver-advertise-address` flag as part of the `init`, this means that we can then bind the same port to the VIP and we wont have a port conflict.
-
-**vip.yaml** Static-pod Manifest
-
-`$ sudo cat /etc/kubernetes/manifests/vip.yaml`
-
-
-``` 
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - start
-    env:
-    - name: vip_arp
-      value: "true"
-    - name: vip_interface
-      value: ens192
-    - name: vip_address
-      value: 192.168.0.81
-    - name: vip_startleader
-      value: "true"
-    - name: vip_addpeerstolb
-      value: "true"
-    - name: vip_localpeer
-      value: controlPlane01:192.168.0.70:10000
-    - name: lb_backendport
-      value: "6443"
-    - name: lb_name
-      value: Kubeadm Load Balancer
-    - name: lb_type
-      value: tcp
-    - name: lb_bindtovip
-      value: "true"
-    image: plndr/kube-vip:0.1.5
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - SYS_TIME
-  hostNetwork: true
-status: {}
-```
-
-### First Node
-
-To generate the basic Kubernetes static pod `yaml` configuration:
-
-Make sure that the manifest directory exists: `sudo mkdir -p /etc/kubernetes/manifests/`
-
-```
-sudo docker run --network host \
-	--rm plndr/kube-vip:0.1.5 \
-	kubeadm init \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--startAsLeader=true | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-Ensure that `image: plndr/kube-vip:<x>` is modified to point to a specific version (`0.1.5` at the time of writing), refer to [docker hub](https://hub.docker.com/r/plndr/kube-vip/tags) for details. 
-
-The **vip** is set to `192.168.0.75` and this first node will elect itself as leader, and as part of the `kubeadm init` it will use the VIP in order to speak back to the initialising api-server.
-
-`sudo kubeadm init --control-plane-endpoint “192.168.0.75:6443” --apiserver-bind-port 6444 --upload-certs --kubernetes-version “v1.17.0”`
-
-Once this node is up and running we will be able to see the control-plane pods, including the `kube-vip` pod:
-
-```
-$ kubectl get pods -A
-NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
-<...>
-kube-system   kube-vip-controlplane01                  1/1     Running   0          10m
-```
-
-### Remaining Nodes
-
-
-At this point **DON’T** generate the manifests, this is due to some bizarre `kubeadm/kubelet` behaviour.
-
-```
-  kubeadm join 192.168.0.75:6443 --token <tkn> \
-    --discovery-token-ca-cert-hash sha256:<hash> \
-    --control-plane --certificate-key <key> 
-
-```
-
-**After** this node has been added to the cluster, we can add the manifest to also add this node as a `kube-vip` member. (Adding the manifest afterwards doesn’t interfere with `kubeadm`). 
-
-```
-sudo docker run \
-	-v /etc/kubernetes/admin.conf:/etc/kubernetes/admin.conf \
-	--network host \
-	--rm plndr/kube-vip:0.1.5 \
-	kubeadm join \
-	--interface ens192 \
-	--vip 192.168.0.81 \
-	--startAsLeader=false | sudo tee /etc/kubernetes/manifests/vip.yaml
-
-```
-
-Once this node is added we will be able to see that the `kube-vip` pod is up and running as expected:
-
-```
-user@controlPlane01:~$ kubectl get pods -A | grep vip
-kube-system   kube-vip-controlplane01                  1/1     Running             1          16m
-kube-system   kube-vip-controlplane02                  1/1     Running             0          18m
-kube-system   kube-vip-controlplane03                  1/1     Running             0          20m
-
-```
-
-If we look at the logs, we can see that the VIP is running on the second node and we’re waiting for our third node to join the cluster:
-
-```
-$ kubectl logs kube-vip-controlplane02  -n kube-system
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-time=“2020-02-12T15:33:09Z” level=info msg=“The Node [192.168.0.70:10000] is leading”
-
-```

docs/control-plane/index.md

@@ -1,421 +0,0 @@
-# Load Balancing a Kubernetes Cluster (Control-Plane)
-
-**Note**: The most common deployment currently for HA Kubernetes clusters w/`kube-vip` involved `kubeadm`, however recently we've worked to bring a method of bringing `kube-vip` to other types of Kubernetes cluster. Typically this deployment method makes use of a daemonset that is usually brought up during the cluster instantiation.. So for those wanting to deploy [k3s](https://k3s.io), we now have installation steps available [here](https://kube-vip.io/control-plane/#k3s),
-
-This document covers the newer (post `0.1.6`) method for using `kube-vip` to provide HA for a Kubernetes Cluster. The documentation for older releases can be found [here](./0.1.5/)
-
-From version `0.1.6` we've moved `kube-vip` from raft to leaderElection within the Kubernetes cluster. After a lot of testing it became clear that the leaderElection gave quicker reconciliation when removing nodes etc.. during upgrades and failures.
-
-For **more** configuration around LeaderElection click [here](https://kube-vip.io/control-plane/#leaderelection-configuration).
-
-This document covers all of the details for using `kube-vip` to build a HA Kubernetes cluster
-
-`tl;dr version`
-- Generate/modify first node `kube-vip` config/manifest
-- `init` first node
-- `join` remaining nodes
-- Add remaining config/manifests
-
-Below are examples of the steps required:
-
-```
-# First Node
-sudo docker run --network host --rm ghcr.io/kube-vip/kube-vip:0.3.7 manifest pod \
---interface ens192 \
---vip 192.168.0.75 \
---arp \
---leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-
-sudo kubeadm init --kubernetes-version 1.17.0 --control-plane-endpoint 192.168.0.75 --upload-certs
-
-# Additional Node(s)
-
-sudo kubeadm join 192.168.0.75:6443 --token w5atsr.blahblahblah --control-plane --certificate-key abc123
-
-sudo docker run --network host --rm ghcr.io/kube-vip/kube-vip:0.3.7 manifest pod \
---interface ens192 \
---vip 192.168.0.75 \
---arp \
---leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address    |
-|----------------|------------|
-| VIP            | 10.0.0.75 |
-| controlPlane01 | 10.0.0.70 |
-| controlPlane02 | 10.0.0.71 |
-| controlPlane03 | 10.0.0.72 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.17.0.
-
-### Generate the `kube-vip` configuration
-
-`kube-vip` no longer requires storing its configuration in a separate directory and will now store its configuration in the actual manifest that defines the static pods. 
-
-```
-sudo docker run --network host \
-	--rm ghcr.io/kube-vip/kube-vip:0.3.7 \
-	manifest pod \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--arp \
-	--leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-The above command will "initialise" the manifest within the `/etc/kubernetes/manifests` directory, that will be started when we actually initialise our Kubernetes cluster with `kubeadm init`
-
-### Modify the configuration
-
-**Cluster Configuration**
-To enable Kubernetes leader Election passing the `--leaderElection` flag will enable `kube-vip` to use the Kubernetes leaderElection functionality to work out which member is the leader.
-
-**VIP Config**
-We will need to set our VIP address to `192.168.0.75` with `--vip 192.168.0.75` and to ensure all hosts are updated when the VIP moves we will enable ARP broadcasts `--arp` (defaults to `true`)
-
-**vip.yaml** Static-pod Manifest
-
-`$ sudo cat /etc/kubernetes/manifests/vip.yaml`
-
-
-``` 
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - start
-    env:
-    - name: vip_arp
-      value: "true"
-    - name: vip_interface
-      value: ens160
-    - name: vip_leaderelection
-      value: "true"
-    - name: vip_leaseduration
-      value: "5"
-    - name: vip_renewdeadline
-      value: "3"
-    - name: vip_retryperiod
-      value: "1"
-    - name: vip_address
-      value: 192.168.0.75
-    image: ghcr.io/kube-vip/kube-vip:0.3.7
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - SYS_TIME
-  hostNetwork: true
-status: {}
-```
-
-### First Node
-
-To generate the basic Kubernetes static pod `yaml` configuration:
-
-Make sure that the manifest directory exists: `sudo mkdir -p /etc/kubernetes/manifests/`
-
-```
-sudo docker run --network host \
-	--rm ghcr.io/kube-vip/kube-vip:0.3.7 \
-	manifest pod \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--arp \
-	--leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-Ensure that `image: ghcr.io/kube-vip/kube-vip:<x>` is modified to point to a specific version (`0.3.7` at the time of writing), refer to [GitHyb](https://github.com/kube-vip/kube-vip/pkgs/container/kube-vip) for details. 
-
-The **vip** is set to `192.168.0.75` and this first node will elect itself as leader, and as part of the `kubeadm init` it will use the VIP in order to speak back to the initialising api-server.
-
-`sudo kubeadm init --control-plane-endpoint “192.168.0.75:6443” --upload-certs --kubernetes-version “v1.17.0”`
-
-Once this node is up and running we will be able to see the control-plane pods, including the `kube-vip` pod:
-
-```
-$ kubectl get pods -A
-NAMESPACE     NAME                                     READY   STATUS    RESTARTS   AGE
-<...>
-kube-system   kube-vip-controlplane01                  1/1     Running   0          10m
-```
-
-### Remaining Nodes
-
-
-At this point **DON’T** generate the manifests, this is due to some bizarre `kubeadm/kubelet` behaviour.
-
-```
-  kubeadm join 192.168.0.75:6443 --token <tkn> \
-    --discovery-token-ca-cert-hash sha256:<hash> \
-    --control-plane --certificate-key <key> 
-
-```
-
-**After** this node has been added to the cluster, we can add the manifest to also add this node as a `kube-vip` member. (Adding the manifest afterwards doesn’t interfere with `kubeadm`). 
-
-```
-sudo docker run --network host \
-	--rm ghcr.io/kube-vip/kube-vip:0.3.7 \
-	manifest pod \
-	--interface ens192 \
-	--vip 192.168.0.75 \
-	--arp \
-	--leaderElection | sudo tee /etc/kubernetes/manifests/vip.yaml
-
-```
-
-Once this node is added we will be able to see that the `kube-vip` pod is up and running as expected:
-
-```
-user@controlPlane01:~$ kubectl get pods -A | grep vip
-kube-system   kube-vip-controlplane01                  1/1     Running             1          16m
-kube-system   kube-vip-controlplane02                  1/1     Running             0          18m
-kube-system   kube-vip-controlplane03                  1/1     Running             0          20m
-
-```
-
-## DNS Support
-
-### Static DNS Support (added in 0.2.0)
-
-A new flag `--address` is introduced to support using a DNS record as the control plane endpoint. `kube-vip` will do a dns lookup to retrieve the IP for the DNS record, and use that IP as the VIP. An `dnsUpdater` periodically checks and updates the system if IP changes for the DNS record.
-
-### Dynamic DNS Support (added in 0.2.1)
-
-`kube-vip` was also updated to support DHCP + [Dynamic DNS](https://en.wikipedia.org/wiki/Dynamic_DNS), for the use case where it's not able to reserve a static IP for the control plane endpoint.
-
-A new flag `--ddns` is introduced. Once enabled, `kube-vip` expects the input `--address` will be a FQDN without binding to an IP. Then `kube-vip` will start a dhcp client to allocate an IP for the hostname of FQDN, and maintain the lease for it.
-
-Once DHCP returns an IP for the FQDN, the same `dnsUpdater` runs to periodically checks and updates if IP got changed.
-
-## BGP Support (added in 0.1.8)
-
-In version `0.1.8`+ `kube-vip` was updated to support [BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol) as a VIP failover mechanism. When a node is elected as a leader then it will update it's peers so that they are aware to route traffic to that node in order to access the VIP. 
-
-The following new flags are used:
-
-- `--bgp` This will enable BGP support within kube-vip
-- `--localAS` The local AS number
-- `--bgpRouterID` The local router address
-- `--peerAS` The AS number for a BGP peer
-- `--peerAddress` The address of a BGP peer
-
-### BGP Packet support
-
-If the `--bgp` flag is passed alone with the Packet flags `packet, packetKey and packetProject`, then the Packet API will be used in order to determine the BGP configuration for the nodes being used in the cluster. This automates a lot of the process and makes using BGP within Packet much simpler.
-
-## Packet Support (added in 0.1.7)
-
-Recently in version `0.1.7` of `kube-vip` we added the functionality to use a Packet Elastic IP as the virtual IP fronting the Kubernetes Control plane cluster. In order to first get out virtual IP we will need to use our Packet account and create a EIP (either public (eek) or private). We will only need a single address so a `/32` will suffice, once this is created as part of a Packet project we can now apply this address to the servers that live in the same project. 
-
-In this example we've logged into the UI can created a new EIP of `147.75.1.2`, and we've deployed three small server instances with Ubuntu.
-
-The following new flags are used:
-
-- `--packet` which enables the use of the Packet API
-- `--packetKey` which is our API key
-- `--packetProject`which is the name of our Packet project where our servers and EIP are located.
-
-*Also* the `--arp` flag should NOT be used as it wont work within the Packet network.
-
-### Variables
-
-```
-export EIP=1.1.1.1
-export PACKET_AUTH_TOKEN=XYZ
-```
-
-### First node
-
-```
-# Generate the manifest
-
-sudo docker run --network host --rm ghcr.io/kube-vip/kube-vip:0.3.7 manifest pod \
---arp=false \
---interface lo \
---vip $EIP \
---leaderElection \
---packet \
---packetKey $PACKET_AUTH_TOKEN \
---packetProject vipTest | sudo tee /etc/kubernetes/manifests/vip.yaml\
-
-# Init Kubernetes
-
-sudo kubeadm init --kubernetes-version 1.18.5 --control-plane-endpoint $EIP --upload-certs
-```
-
-### Other nodes
-
-```
-# Join
-kubeadm join $EIP:6443 --token BLAH --control-plane --certificate-key BLAH --discovery-token-ca-cert-hash sha:blah
-
-# Generate Manifest
-
-sudo docker run --network host --rm ghcr.io/kube-vip/kube-vip:0.3.7 manifest pod \
---arp=false \
---interface lo \
---vip $EIP \
---leaderElection \
---packet \
---packetKey $PACKET_AUTH_TOKEN \
---packetProject vipTest | sudo tee /etc/kubernetes/manifests/vip.yaml\
-```
-
-The Elastic IP failover takes some time (30+ seconds) to move from a failed host to a new leader, so in this release it is mainly for testing. 
-
-
-## Upgrades
-
-From above we have a 3 node cluster and the controlPlane01 is leader:
-
-```
-$ kubectl logs -n kube-system kube-vip-controlplane01 -f
-time="2020-07-04T15:12:52Z" level=info msg="Beginning cluster membership, namespace [kube-system], lock name [plunder-lock], id [controlPlane01]"
-I0704 15:12:52.290420       1 leaderelection.go:242] attempting to acquire leader lease  kube-system/plunder-lock...
-I0704 15:12:56.373113       1 leaderelection.go:252] successfully acquired lease kube-system/plunder-lock
-time="2020-07-04T15:12:56Z" level=info msg="This node is assuming leadership of the cluster"
-time="2020-07-04T15:12:56Z" level=error msg="This node is leader and is adopting the virtual IP"
-time="2020-07-04T15:12:56Z" level=info msg="Starting TCP Load Balancer for service [192.168.0.81:0]"
-time="2020-07-04T15:12:56Z" level=info msg="Load Balancer [Kubeadm Load Balancer] started"
-time="2020-07-04T15:12:56Z" level=info msg="Broadcasting ARP update for 192.168.0.81 (00:50:56:a5:69:a1) via ens192"
-time="2020-07-04T15:12:56Z" level=info msg="Starting TCP Load Balancer for service [192.168.0.81:0]"
-time="2020-07-04T15:12:56Z" level=info msg="Load Balancer [Kubeadm Load Balancer] started"
-time="2020-07-04T15:12:56Z" level=info msg="Broadcasting ARP update for 192.168.0.81 (00:50:56:a5:69:a1) via ens192"
-time="2020-07-04T15:12:56Z" level=info msg="new leader elected: controlPlane01"
-```
-
-We will kill this node and watch `kube-vip` logs from another node:
-
-#### Pinging VIP
-
-```
-64 bytes from 192.168.0.81: icmp_seq=667 ttl=64 time=0.387 ms
-Request timeout for icmp_seq 668
-Request timeout for icmp_seq 669
-Request timeout for icmp_seq 670
-Request timeout for icmp_seq 671
-Request timeout for icmp_seq 672
-64 bytes from 192.168.0.81: icmp_seq=673 ttl=64 time=0.453 ms
-```
-
-#### Logs 
-```
-$ kubectl logs -n kube-system kube-vip-controlplane03 -f
-time="2020-07-04T15:17:53Z" level=info msg="Beginning cluster membership, namespace [kube-system], lock name [plunder-lock], id [controlPlane03]"
-I0704 15:17:53.484698       1 leaderelection.go:242] attempting to acquire leader lease  kube-system/plunder-lock...
-time="2020-07-04T15:17:53Z" level=info msg="new leader elected: controlPlane01"
-E0704 15:20:18.864141       1 leaderelection.go:331] error retrieving resource lock kube-system/plunder-lock: etcdserver: request timed out
-time="2020-07-04T15:20:20Z" level=info msg="new leader elected: controlPlane02"
-```
-
-#### Adding `controlPlane04`
-
-A kubeadm join will fail as the `controlPlane01` still exists as an endpoint, so we have two options (manual steps and configmap edit to remove all mention of this node, or we can bring this node up and `kubeadm reset` the node (which we will do)). 
-
-```
-$ kubectl get nodes
-NAME             STATUS     ROLES    AGE   VERSION
-controlplane01   NotReady   master   14m   v1.17.0
-controlplane02   Ready      master   13m   v1.17.2
-controlplane03   Ready      master   13m   v1.17.0
-controlplane04   NotReady   master   9s    v1.17.0
-```
- After this we can add this node into `kube-vip` with the same manifest created by `docker run`.
-
-## LeaderElection configuration
-
-The Kubernetes LeaderElection that is used to manage the election of a new leader now supports having it's settings managed through flags. 
-
- - `--leaseDuration` Length of time a Kubernetes leader lease can be held for
- - `--leaseRenewDuration` Length of time a Kubernetes leader can attempt to renew its lease
- - `--leaseRetry` Number of times the host will retry to hold a lease
-
-For larger clusters the `--leaseDuration` and `--leaseRenewDuration` may need extending due to slower `etcd` performance. (Tested with 2000 nodes)
-
-## k3s
-
-This section details the steps required to deploye `k3s` in a Highly available manner, using kube-vip deployed within k3s as a daemonset on the control plane nodes. As of `k3s` v1 the persistent datastore is back to etcd, however this guide will also include the steps for using `mysql`.
-
-### Example MySQL deployment (optional)
-
-To quickly validate this we can use docker on a host to quickly spin up a mysql database to store the persistent Kubernetes data.
-
-#### Create local directory for BD storage
-`mkdir mysql`
-
-#### Start Docker MySQL container
-`sudo docker run --cap-add SYS_NICE -p 3306:3306 --name k3s-mysql -v /home/dan/mysql:/var/lib/mysql -e MYSQL_ROOT_PASSWORD=k3s-password -d mysql:8`
-
-### Create `kube-vip` manifest
-
-The `kube-vip` manifest contains all the configuration for starting up `kube-vip` within the `k3s` cluster, it runs as a daemonset with affinity/taints for the control-plane nodes. As `k3s` starts it will parse all manifests in the manifests folder and start the highly available VIP across all control plane nodes in the cluster.
-
-#### Create the `k3` manifests directory
-
-Create the manifests directory, this directory is used by `k3s` for all of it's other deployments once it's up and running.
-`sudo mkdir -p /var/lib/rancher/k3s/server/manifests/`
-
-#### Generate the manifest
-
-Modify the `vipAddress` and `vipInterface` to match the floating IP address you'd like to use and the interface it should bind to.
-
-`curl -sL kube-vip.io/k3s | vipAddress=192.168.0.10 vipInterface=ens192 sh | sudo tee /var/lib/rancher/k3s/server/manifests/vip.yaml`
-
-### Start `k3s`
-
-Set the VIP **first**
-
-`export VIP=192.168.0.10`
-
-
-From online `-->`
-
-```
-curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--write-kubeconfig-mode 644 \
--t agent-secret --tls-san $VIP" sh -
-```
-
-From local `-->`
-
-```
-sudo ./k3s server --tls-san $VIP
-```
-
-#### With MySQL
-
-From online `-->`
-
-```
-curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--write-kubeconfig-mode 644 \
---datastore-endpoint mysql://root:k3s-password@tcp(192.168.0.43:3306)/kubernetes \
--t agent-secret --tls-san $VIP" sh -
-```
-
-From local `-->`
-
-```
-sudo ./k3s server --tls-san $VIP \
---datastore-endpoint="mysql://root:k3s-password@tcp(192.168.0.43:3306)/kubernetes"
-```
-
-### Get a `kubeconfig` that uses the vip
-
-````
-mkdir -p $HOME/.kube
-sudo cat /etc/rancher/k3s/k3s.yaml | sed 's/127.0.0.1/'$VIP'/g' > $HOME/.kube/config
-sudo chown $(id -u):$(id -g) $HOME/.kube/config
-```

docs/flags/index.md

@@ -1,101 +0,0 @@
-# Kube-Vip Flag / Environment Variable reference
-
-## Flags
-
-These flags are typically used in the `kube-vip` manifest generation process.
-
-| Category            | Flag                   | Usage                                                              | Notes                                                                           |
-| ------------------- | ---------------------- | ------------------------------------------------------------------ | ------------------------------------------------------------------------------- |
-| **Troubleshooting** |                        |                                                                    |                                                                                 |
-|                     | `--log`                | default 4                                                          | Set to `5` for debugging logs                                                   |
-| **Mode**            |                        |                                                                    |                                                                                 |
-|                     | `--controlplane`       | Enables `kube-vip` control plane functionality                     |                                                                                 |
-|                     | `--services`           | Enables `kube-vip` to watch services of type `LoadBalancer`        |                                                                                 |
-| **VIP Config**      |                        |                                                                    |                                                                                 |
-|                     | `--arp`                | Enables ARP broadcasts from Leader                                 |                                                                                 |
-|                     | `--bgp`                | Enables BGP peering from `kube-vip`                                |                                                                                 |
-|                     | `--vip`                | `<IP Address>`                                                     | (deprecated)                                                                    |
-|                     | `--address`            | `<IP Address>` or `<DNS name>`                                     |                                                                                 |
-|                     | `--interface`          | Linux interface on the node                                        |                                                                                 |
-|                     | `--leaderElection`     | Enables Kubernetes LeaderElection                                  | Used by ARP, as only the leader can broadcast                                   |
-|                     | `--enableLoadBalancer` | Enables IPVS load balancer                                         | `kube-vip` ≥ 0.4.0                                                              |
-|                     | `--lbPort`             | 6443                                                               | The port that the api server will load-balanced on                              |
-|                     | `--lbForwardingMethod` | Select the forwarding method (default local)                       | The IPVS forwarding method (local, masquerade, tunnel, direct, bypass)          |
-| **Services**        |                        |                                                                    |                                                                                 |
-|                     | `--serviceInterface`   | ""                                                          | Defines an optional different interface to bind services too                    |
-|                     | `--cidr`               | Defaults "32"                                                      | Used when advertising BGP addresses (typically as `x.x.x.x/32`)                 |
-| **Kubernetes**      |                        |                                                                    |                                                                                 |
-|                     | `--inCluster`          | Required for `kube-vip` as DaemonSet.                              |  Runs `kube-vip` with a ServiceAccount called `kube-vip`.                       |
-|                     | `--taint`              | Required for `kube-vip` as DaemonSet.                              |  Adds node affinity rules forcing `kube-vip` Pods to run on control plane.      |
-| **LeaderElection**  |                        |                                                                    |                                                                                 |
-|                     | `--leaseDuration`      | default 5                                                          | Seconds a lease is held for                                                     |
-|                     | `--leaseRenewDuration` | default 3                                                          | Seconds a leader can attempt to renew the lease                                 |
-|                     | `--leaseRetry`         | default 1                                                          | Number of times the leader will hold the lease for                              |
-|                     | `--namespace`          | "kube-vip"                                                         | The namespace where the lease will reside                                       |
-| **BGP**             |                        |                                                                    |                                                                                 |
-|                     | `--bgpRouterID`        | `<IP Address>`                                                     | Typically the address of the local node                                         |
-|                     | `--localAS`            | default 65000                                                      | The AS we peer from                                                             |
-|                     | `--bgppeers`           | `<address:AS:password:multihop>`                                   | Comma separated list of BGP peers                                               |
-|                     | `--peerAddress`        | `<IP Address>`                                                     | Address of a single BGP Peer                                                    |
-|                     | `--peerAS`             | default 65000                                                      | AS of a single BGP Peer                                                         |
-|                     | `--peerPass`           | ""                                                                 | Password to work with a single BGP Peer                                         |
-|                     | `--multiHop`           | Enables eBGP MultiHop                                              | Enable multiHop with a single BGP Peer                                          |
-|                     | `--sourceif`           | Source Interface                                                   | Determines which interface BGP should peer _from_                               |
-|                     | `--sourceip`           | Source Address                                                     | Determines which IP address BGP should peer _from_                              |
-|                     | `--annotations`        | `<provider string>`                                                | Startup will be paused until the node annotations contain the BGP configuration |
-| **Equinix Metal**   |                        |                                                                    | (May be deprecated)                                                             |
-|                     | `--metal`              | Enables Equinix Metal API calls                                    |                                                                                 |
-|                     | `--metalKey`           | Equinix Metal API token                                            |                                                                                 |
-|                     | `--metalProject`       | Equinix Metal Project (Name)                                       |                                                                                 |
-|                     | `--metalProjectID`     | Equinix Metal Project (UUID)                                       |                                                                                 |
-|                     | `--provider-config`    | Path to the Equinix Metal provider configuration                   | Requires the Equinix Metal CCM                                                  |
-
-## Environment Variables
-
-These environment variables are usually part of a `kube-vip` manifest and used when running the `kube-vip` Pod.
-
-More environment variables can be read through the `pkg/kubevip/config_envvar.go` file.
-
-| Category            | Environment Variable   | Usage                                                       | Notes                                                                           |
-| ------------------- | ---------------------- | ----------------------------------------------------------- | ------------------------------------------------------------------------------- |
-| **Troubleshooting** |                        |                                                             |                                                                                 |
-|                     | `vip_loglevel`         | default 4                                                   | Set to `5` for debugging logs                                                   |
-| **Mode**            |                        |                                                             |                                                                                 |
-|                     | `cp_enable`            | Enables `kube-vip` control plane functionality              |                                                                                 |
-|                     | `svc_enable`           | Enables `kube-vip` to watch Services of type `LoadBalancer` |                                                                                 |
-| **VIP Config**      |                        |                                                             |                                                                                 |
-|                     | `vip_arp`              | Enables ARP broadcasts from Leader                          |                                                                                 |
-|                     | `bgp_enable`           | Enables BGP peering from `kube-vip`                         |                                                                                 |
-|                     | `vip_address`          | `<IP Address>`                                              | (deprecated)                                                                    |
-|                     | `address`              | `<IP Address>` or `<DNS name>`                              |                                                                                 |
-|                     | `vip_interface`        | `<linux interface>`                                         |                                                                                 |
-|                     | `vip_leaderelection`   | Enables Kubernetes LeaderElection                           | Used by ARP, as only the leader can broadcast                                   |
-|                     | `lb_enable`            | Enables IPVS LoadBalancer                                   | `kube-vip` ≥ 0.4.0. Adds nodes to the IPVS load balancer                        |
-|                     | `lb_port`              | 6443                                                        | The IPVS port that will be used to load-balance control plane requests          |
-|                     | `lb_fwdmethod`         | Select the forwarding method (default local)                | The IPVS forwarding method (local, masquerade, tunnel, direct, bypass)          |
-| **Services**        |                        |                                                             |                                                                                 |
-|                     | `vip_servicesinterface`| ""                                                          | Defines an optional different interface to bind services too                    |
-|                     | `vip_cidr`             | Defaults "32"                                               | Used when advertising BGP addresses (typically as `x.x.x.x/32`)                 |
-| **LeaderElection**  |                        |                                                             |                                                                                 |
-|                     | `vip_leaseduration`    | default 5                                                   | Seconds a lease is held for                                                     |
-|                     | `vip_renewdeadline`    | default 3                                                   | Seconds a leader can attempt to renew the lease                                 |
-|                     | `vip_retryperiod`      | default 1                                                   | Number of times the leader will hold the lease for                              |
-|                     | `cp_namespace`         | "kube-vip"                                                  | The namespace where the lease will reside                                       |
-| **BGP**             |                        |                                                             |                                                                                 |
-|                     | `bgp_routerid`         | `<IP Address>`                                              | Typically the address of the local node                                         |
-|                     | `bgp_routerinterface`  | Interface name                                              | Used to associate the `routerID` with the control plane's interface.            |
-|                     | `bgp_as`               | default 65000                                               | The AS we peer from                                                             |
-|                     | `bgp_peers`            | `<address:AS:password:multihop>`                            | Comma separated list of BGP peers                                               |
-|                     | `bgp_peeraddress`      | `<IP Address>`                                              | Address of a single BGP Peer                                                    |
-|                     | `bgp_peeras`           | default 65000                                               | AS of a single BGP Peer                                                         |
-|                     | `bgp_peerpass`         | ""                                                          | Password to work with a single BGP Peer                                         |
-|                     | `bgp_multihop`         | Enables eBGP MultiHop                                       | Enable multiHop with a single BGP Peer                                          |
-|                     | `bgp_sourceif`         | Source Interface                                            | Determines which interface BGP should peer _from_                               |
-|                     | `bgp_sourceip`         | Source Address                                              | Determines which IP address BGP should peer _from_                              |
-|                     | `annotations`          | `<provider string>`                                         | Startup will be paused until the node annotations contain the BGP configuration |
-| **Equinix Metal**   |                        |                                                             | (May be deprecated)                                                             |
-|                     | `vip_packet`           | Enables Equinix Metal API calls                             |                                                                                 |
-|                     | `PACKET_AUTH_TOKEN`    | Equinix Metal API token                                     |                                                                                 |
-|                     | `vip_packetproject`    | Equinix Metal Project (Name)                                |                                                                                 |
-|                     | `vip_packetprojectid`  | Equinix Metal Project (UUID)                                |                                                                                 |
-|                     | `provider_config`      | Path to the Equinix Metal provider configuration            | Requires the Equinix Metal CCM                                                  |

docs/hybrid/daemonset/index.md

@@ -1,226 +0,0 @@
-# Kube-Vip as a daemonset
-
-In Hybrid mode `kube-vip` will manage a virtual IP address that is passed through it's configuration for a Highly Available Kubernetes cluster, it will also "watch" services of `type:LoadBalancer` and once their `spec.LoadBalancerIP` is updated (typically by a cloud controller) it will advertise this address using BGP/ARP.
-
-
-**Note about Daemonsets**
-
-The "hybrid" mode is now the default mode in `kube-vip` from `0.2.3` onwards, and allows both modes to be enabled at the same time. 
-
-If the Kubernetes installer allows for adding a Virtual IP as an additional [SAN](https://en.wikipedia.org/wiki/Subject_Alternative_Name) to the API server certificate then we can apply `kube-vip` to the cluster once the first node has been brought up. 
-
-Unlike generating the static manifest there are a few more things that may need configuring, this page will cover most scenarios.
-
-## Create the RBAC settings
-
-As a daemonSet runs within the Kubernetes cluster it needs the correct access to be able to watch Kubernetes services and other objects. In order to do this we create a User, Role, and a binding.. we can apply this with the command:
-
-```
-kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
-```
-
-## Generating a Manifest
-
-This section only covers generating a simple *BGP* configuration, as the main focus is will be on additional changes to the manifest. For more examples we can look at [here](/hybrid/static/).
-
-**Note:** Pay attention if using the "static" examples, as the `manifest` subcommand should use `daemonset` and NOT `pod`.
-
-### Set configuration details
-
-`export VIP=192.168.0.40`
-
-`export INTERFACE=<interface>`
-
-### Configure to use a container runtime
-
-The easiest method to generate a manifest is using the container itself, below will create an alias for different container runtimes.
-
-#### containerd
-`alias kube-vip="ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:0.3.7 vip"`
-
-#### Docker
-`alias kube-vip="docker run --network host --rm ghcr.io/kube-vip/kube-vip:0.3.7"`
-
-### BGP Example
-
-This configuration will create a manifest that will start `kube-vip` providing **controlplane** and **services** management. **Unlike** ARP, all nodes in the BGP configuration will advertise virtual IP addresses. 
-
-**Note** we bind the address to `lo` as we don't want multiple devices that have the same address on public interfaces. We can specify all the peers in a comma seperate list in the format of `address:AS:password:multihop`.
-
-**Note 2** we pass the `--inCluster` flag as this is running as a daemonSet within the Kubernetes cluster and therefore will have access to the token inside the running pod.
-
-**Note 2** we pass the `--taint` flag as we're deploying `kube-vip` as both a daemonset and as advertising controlplane, we want to taint this daemonset to only run on the worker nodes.
-
-`export INTERFACE=lo`
-
-```
-kube-vip manifest daemonset \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --inCluster \
-    --taint \
-    --bgp \
-    --bgppeers 192.168.0.10:65000::false,192.168.0.11:65000::false
-```
-
-### Generated Manifest
-
-```
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: lo
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: "true"
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_peers
-          value: "192.168.0.10:65000::false,192.168.0.11:65000::false"
-        - name: vip_address
-          value: 192.168.0.40
-        image: ghcr.io/kube-vip/kube-vip:0.3.7
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      nodeSelector:
-        node-role.kubernetes.io/master: "true"
-      tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
-  updateStrategy: {}
-```
-
-### Manifest Overview
-
-- `nodeSelector` - Ensures that this particular daemonset only runs on control plane nodes
-- `serviceAccountName: kube-vip` - this specifies the user in the `rbac` that will give us the permissions to get/update services.
-- `hostNetwork: true` - This pod will need to modify interfaces (for VIPs)
-- `env {...}` - We pass the configuration into the kube-vip pod through environment variables.
-
-## Equinix Metal Overview (using the [Equinix Metal CCM](https://github.com/packethost/packet-ccm))
-
-The below example is for running `type:LoadBalancer` services on worker nodes only and will create a daemonset that will run `kube-vip`. 
-
-**NOTE** This use-case requires the [Equinix Metal CCM](https://github.com/packethost/packet-ccm) to be installed and that the cluster/kubelet is configured to use an "external" cloud provider.
-
-This is important as the CCM will apply the BGP configuration to the [node annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) making it easy for `kube-vip` to find the networking configuration it needs to expose load balancer addresses. The `--annotations metal.equinix.com` will cause kube-vip to "watch" the annotations of the worker node that it is running on, once all of the configuarion has been applied by the CCM then the `kube-vip` pod is ready to advertise BGP addresses for the service.
-
-```
-kube-vip manifest daemonset \
-  --interface $INTERFACE \
-  --services \
-  --bgp \
-  --annotations metal.equinix.com \
-  --inCluster | k apply -f -
-```
-
-### Troubleshooting
-
-If `kube-vip` has been sat waiting for a long time then you may need to investigate that the annotations have been applied correctly by doing running the `describe` on the node:
-
-```
-kubectl describe node k8s.bgp02
-...
-Annotations:        kubeadm.alpha.kubernetes.io/cri-socket: /var/run/dockershim.sock
-                    node.alpha.kubernetes.io/ttl: 0
-                    metal.equinix.com/node-asn: 65000
-                    metal.equinix.com/peer-asn: 65530
-                    metal.equinix.com/peer-ip: x.x.x.x
-                    metal.equinix.com/src-ip: x.x.x.x
-```
-
-If there are errors regarding `169.254.255.1` or `169.254.255.2` in the `kube-vip` logs then the routes to the ToR switches that provide BGP peering may by missing from the nodes. They can be replaced with the below command:
-
-```
-GATEWAY_IP=$(curl https://metadata.platformequinix.com/metadata | jq -r ".network.addresses[] | select(.public == false) | .gateway")
-ip route add 169.254.255.1 via $GATEWAY_IP
-ip route add 169.254.255.2 via $GATEWAY_IP
-```
-
-Additionally examining the logs of the Packet CCM may reveal why the node is not yet ready.
-
-## K3s overview (on Equinix Metal)
-
-### Step 1: TIDY (best if something was running before)
-`rm -rf /var/lib/rancher /etc/rancher ~/.kube/*; ip addr flush dev lo; ip addr add 127.0.0.1/8 dev lo; mkdir -p /var/lib/rancher/k3s/server/manifests/`
-
-### Step 2: Get rbac
-`curl https://kube-vip.io/manifests/rbac.yaml > /var/lib/rancher/k3s/server/manifests/rbac.yaml`
-
-### Step 3: Generate kube-vip (get EIP from CLI or UI)
-
-```
-export EIP=x.x.x.x
-export INTERFACE=lo
-```
-
-```
-kube-vip manifest daemonset \
-  --interface $INTERFACE \
-  --vip $EIP \
-  --controlplane \
-  --services \
-  --inCluster \
-  --taint \
-  --bgp \
-  --metal \
-  --provider-config /etc/cloud-sa/cloud-sa.json | tee /var/lib/rancher/k3s/server/manifests/vip.yaml
-```
-
-NOTE: the `—provider-config` actually comes from the secret we apply in step 5 (this will leave kube-vip waiting to start)
-
-### Step 4: Up Cluster
-`K3S_TOKEN=SECRET k3s server --cluster-init --tls-san $EIP --no-deploy servicelb --disable-cloud-controller`
-
-### Step 5: Add CCM
-
-`alias k="k3s kubectl"`
-`k apply -f ./secret.yaml`
-
-(^ https://github.com/packethost/packet-ccm/blob/master/deploy/template/secret.yaml)
-
-`k apply -f https://gist.githubusercontent.com/thebsdbox/c86dd970549638105af8d96439175a59/raw/4abf90fb7929ded3f7a201818efbb6164b7081f0/ccm.yaml`
-
-### Step 6: Demo !
-`k apply -f https://k8s.io/examples/application/deployment.yaml`
-`k expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx`
-
-### Step 7 watch and test:
-`k get svc --watch`

docs/hybrid/index.md

@@ -1,142 +0,0 @@
-# Using kube-vip in Hybrid Mode
-
-We can deploy kube-vip in two different methods, which completely depends on your use-case and method for installing Kubernetes:
-
-- Static Pods (hybrid)
-- Daemonset (hybrid, requires taint)
-
-## Prerequisites
-
-In order for `kube-vip` to be able to speak with the Kubernetes API server, we need to be able to resolve the hostname within the pod. In order to ensure this will work as expected the `/etc/hosts` file should have the `hostname` of the server within it. The `/etc/hosts` file is passed into the running container and will ensure that the pod isn't "confused" by any Kubernetes networking.
-
-## Kubernetes Services (`type:LoadBalancer`)
-
-To learn more about how `kube-vip` in hybrid works with the LoadBalancer services within a kubernetes cluster the documentation is [here](./services/). To get `kube-vip` deployed read on!
-
-## Static Pods
-
-Static pods are a Kubernetes pod that is ran by the `kubelet` on a single node, and is **not** managed by the Kubernetes cluster itself. This means that whilst the pod can appear within Kubernetes it can't make use of a variety of kubernetes functionality (such as the kubernetes token or `configMaps`). The static pod approach is primarily required for [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/), this is due to the sequence of actions performed by `kubeadm`. Ideally we want `kube-vip` to be part of the kubernetes cluster, for various bits of functionality we also need `kube-vip` to provide a HA virtual IP as part of the installation.
-
-The sequence of events for this to work follows:
-
-1. Generate a `kube-vip` manifest in the static pods manifest folder
-2. Run `kubeadm init`, this generates the manifests for the control plane and wait to connect to the VIP
-3. The `kubelet` will parse and execute all manifest, including the `kube-vip` manifest
-4. `kube-vip` starts and advertises our VIP
-5. The `kubeadm init` finishes successfully.
-
-## Daemonset
-
-Other Kubernetes distributions can bring up a Kubernetes cluster, without depending on a VIP (BUT they are configured to support one). A prime example of this would be k3s, that can be configured to start and also sign the certificates to allow incoming traffic to a virtual IP. Given we don't need the VIP to exist **before** the cluster, we can bring up the k3s node(s) and then add `kube-vip` as a daemonset for all control plane nodes.
-
-## Deploying `kube-vip`
-
-The simplest method for generating the Kubernetes manifests is with `kube-vip` itself.. The subcommand `manifest pod|daemonset` can be used to generate specific types of Kubernetes manifests for use in a cluster. These subcommands can be configured with additional flags to enable/disable BGP/ARP/LeaderElection and a host of other options.
-
-Both Examples will use the same Architecture:
-
-## Infrastructure architecture
-
-The infrastructure for our example HA Kubernetes cluster is as follows:
-
-| Node           | Address   |
-| -------------- | --------- |
-| VIP            | 10.0.0.40 |
-| controlPlane01 | 10.0.0.41 |
-| controlPlane02 | 10.0.0.42 |
-| controlPlane03 | 10.0.0.43 |
-| worker01       | 10.0.0.44 |
-
-All nodes are running Ubuntu 18.04, Docker CE and will use Kubernetes 1.19.0, we only have one worker as we're going to use our controlPlanes in "hybrid" mode.
-
-## As a static Pod (for kubeadm)
-
-The details for creating a static pod are available [here](./static/)
-
-## As a daemonset
-
-When using `kube-vip` as a daemonset the details are available [here](./daemonset/)
-
-## Kube-Vip flag reference
-
-| Category           | Flag                   | Usage                                                              | Notes                                                                           |
-| ------------------ | ---------------------- | ------------------------------------------------------------------ | ------------------------------------------------------------------------------- |
-| **Mode**           |                        |                                                                    |                                                                                 |
-|                    | `--controlPlane`       | Enables `kube-vip` control-plane functionality                     |                                                                                 |
-|                    | `--services`           | Enables `kube-vip` to watch services of type:LoadBalancer          |                                                                                 |
-| **Vip Config**     |                        |                                                                    |                                                                                 |
-|                    | `--arp`                | Enables ARP broadcasts from Leader                                 |                                                                                 |
-|                    | `--bgp`                | Enables BGP peering from `kube-vip`                                |                                                                                 |
-|                    | `--vip`                | `<IP Address>`                                                     | (deprecated)                                                                    |
-|                    | `--address`            | `<IP Address>` or `<DNS name>`                                     |                                                                                 |
-|                    | `--interface`          | `<linux interface>`                                                |                                                                                 |
-|                    | `--leaderElection`     | Enables Kubernetes LeaderElection                                  | Used by ARP, as only the leader can broadcast                                   |
-| **Services**       |                        |                                                                    |                                                                                 |
-|                    | `--cidr`               | Defaults "32"                                                      | Used when advertising BGP addresses (typically as `x.x.x.x/32`)                 |
-| **Kubernetes**     |                        |                                                                    |                                                                                 |
-|                    | `--inCluster`          | Defaults to looking inside the Pod for the token                   |                                                                                 |
-|                    | `--taint`              | Enables a taint, stopping control plane daemonset being on workers |                                                                                 |
-| **LeaderElection** |                        |                                                                    |                                                                                 |
-|                    | `--leaseDuration`      | default 5                                                          | Seconds a lease is held for                                                     |
-|                    | `--leaseRenewDuration` | default 3                                                          | Seconds a leader can attempt to renew the lease                                 |
-|                    | `--leaseRetry`         | default 1                                                          | Number of times the leader will hold the lease for                              |
-|                    | `--namespace`          | "kube-vip"                                                         | The namespace where the lease will reside                                       |
-| **BGP**            |                        |                                                                    |                                                                                 |
-|                    | `--bgpRouterID`        | `<IP Address>`                                                     | Typically the address of the local node                                         |
-|                    | `--localAS`            | default 65000                                                      | The AS we peer from                                                             |
-|                    | `--bgppeers`           | `<address:AS:password:multihop>`                                   | Comma separated list of BGP peers                                               |
-|                    | `--peerAddress`        | `<IP Address>`                                                     | Address of a single BGP Peer                                                    |
-|                    | `--peerAS`             | default 65000                                                      | AS of a single BGP Peer                                                         |
-|                    | `--peerPass`           | ""                                                                 | Password to work with a single BGP Peer                                         |
-|                    | `--multiHop`           | Enables eBGP MultiHop                                              | Enable multiHop with a single BGP Peer                                          |
-|                    | `--annotations`        | `<provider string>`                                                | Startup will be paused until the node annotations contain the BGP configuration |
-| **Equinix Metal**  |                        |                                                                    | (May be deprecated)                                                             |
-|                    | `--metal`              | Enables Equinix Metal API calls                                    |                                                                                 |
-|                    | `--metalKey`           | Equinix Metal API token                                            |                                                                                 |
-|                    | `--metalProject`       | Equinix Metal Project (Name)                                       |                                                                                 |
-|                    | `--metalProjectID`     | Equinix Metal Project (UUID)                                       |                                                                                 |
-|                    | `--provider-config`    | Path to the Equinix Metal provider configuration                   | Requires the Equinix Metal CCM                                                  |
-
-## Changelog
-
-### Static DNS Support (added in 0.2.0)
-
-A new flag `--address` is introduced to support using a DNS record as the control plane endpoint. `kube-vip` will do a dns lookup to retrieve the IP for the DNS record, and use that IP as the VIP. An `dnsUpdater` periodically checks and updates the system if IP changes for the DNS record.
-
-### Dynamic DNS Support (added in 0.2.1)
-
-`kube-vip` was also updated to support DHCP + [Dynamic DNS](https://en.wikipedia.org/wiki/Dynamic_DNS), for the use case where it's not able to reserve a static IP for the control plane endpoint.
-
-A new flag `--ddns` is introduced. Once enabled, `kube-vip` expects the input `--address` will be a FQDN without binding to an IP. Then `kube-vip` will start a dhcp client to allocate an IP for the hostname of FQDN, and maintain the lease for it.
-
-Once DHCP returns an IP for the FQDN, the same `dnsUpdater` runs to periodically checks and updates if IP got changed.
-
-## BGP Support (added in 0.1.8)
-
-In version `0.1.8` `kube-vip` was updated to support [BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol) as a VIP failover mechanism. When a node is elected as a leader then it will update it's peers so that they are aware to route traffic to that node in order to access the VIP.
-
-The following new flags are used:
-
-- `--bgp` This will enable BGP support within kube-vip
-- `--localAS` The local AS number
-- `--bgpRouterID` The local router address
-- `--peerAS` The AS number for a BGP peer
-- `--peerAddress` The address of a BGP peer
-
-### Equinix Metal BGP support
-
-If the `--bgp` flag is passed along with the Equinix Metal flags `metal, metalKey and metalProject`, then Equinix Metal  API will be used in order to determine the BGP configuration for the nodes being used in the cluster. This automates a lot of the process and makes using BGP within Equinix Metal much simpler.
-
-## Equinix Metal Control Plane Support (added in 0.1.8)
-
-Recently in version `0.1.7` of `kube-vip` we added the functionality to use a Equinix Metal Elastic IP as the virtual IP fronting the Kubernetes Control plane cluster. In order to first get out virtual IP we will need to use our Equinix Metal account and create a EIP (either public or private). We will only need a single address so a `/32` will suffice, once this is created as part of a Equinix Metal project we can now apply this address to the servers that live in the same project.
-
-In this example we've logged into the UI can created a new EIP of `147.75.1.2`, and we've deployed three small server instances with Ubuntu.
-
-The following new flags are used:
-
-- `--metal` which enables the use of the Equinix Metal API
-- `--metalKey` which is our API key
-- `--metalProject`which is the name of our Equinix Metal project where our servers and EIP are located.
-
-*Also* the `--arp` flag should NOT be used as it wont work within the Equinix Metal network.

docs/hybrid/services/index.md

@@ -1,269 +0,0 @@
-# Kube-vip services 
-
-We've designed `kube-vip` to be as de-coupled or agnostic from other components that may exist within a Kubernetes cluster as possible. This has lead to `kube-vip` having a very simplistic but robust approach to advertising Kubernetes services to the outside world and marking these services as ready to use.
-
-## Flow
-
-This section details the flow of events in order for `kube-vip` to advertise a Kubernetes service:
-
-1. An end user exposes a application through Kubernetes as a LoadBalancer => `kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx`
-2. Within the Kubernetes cluster a service object is created with the `svc.Spec.Type = LoadBalancer`
-3. A controller (typically a Cloud Controller) has a loop that "watches" for services of the type `LoadBalancer`.
-4. The controller now has the responsibility of providing an IP address for this service along with doing anything that is network specific for the environment where the cluster is running.
-5. Once the controller has an IP address it will update the service `svc.Spec.LoadBalancerIP` with it's new IP address.
-6. The `kube-vip` pods also implement a "watcher" for services that have a `svc.Spec.LoadBalancerIP` address attached.
-7. When a new service appears `kube-vip` will start advertising this address to the wider network (through BGP/ARP) which will allow traffic to come into the cluster and hit the service network.
-8. Finally `kube-vip` will update the service status so that the API reflects that this LoadBalancer is ready. This is done by updating the `svc.Status.LoadBalancer.Ingress` with the VIP address.
-
-## CCM
-
-We can see from the [flow](#Flow) above that `kube-vip` isn't coupled to anything other than the Kubernetes API, and will only act upon an existing Kubernetes primative (in this case the object of type `Service`). This makes it easy for existing CCMs to simply apply their logic to services of type LoadBalancer and leave `kube-vip` to take the next steps to advertise these load-balancers to the outside world. 
-
-
-## Using the Kube-vip Cloud Provider
-
-The below instructions *should just work* on Kubernetes regardless of architecture (Linux Operating System is the only requirement) - you can quickly install the "latest" components:
-
-**Install the `kube-vip-cloud-provider`**
-
-```
-$ kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
-```
-
-It uses a `statefulSet` and can always be viewed with the following command:
-
-```
-kubectl describe pods -n kube-system kube-vip-cloud-provider-0
-```
-
-**Create a global CIDR or IP Range**
-
-Any `service` in any `namespace` can use an address from the global CIDR `cidr-global` or range `range-global`
-
-```
-kubectl create configmap --namespace kube-system kubevip --from-literal cidr-global=192.168.0.220/29
-```
-or
-```
-kubectl create configmap --namespace kube-system kubevip --from-literal range-global=192.168.1.220-192.168.1.230
-```
-
-Creating services of `type: LoadBalancer` in *any namespace* will now take addresses from the **global** cidr defined in the `configmap` unless a specific 
-
-
-## The Detailed guide
-
-### Deploy the Kube-vip Cloud Provider
-
-**Install the `kube-vip-cloud-provider`**
-
-```
-$ kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
-```
-
-The following output should appear when the manifest is applied: 
-
-```
-serviceaccount/kube-vip-cloud-controller created
-clusterrole.rbac.authorization.k8s.io/system:kube-vip-cloud-controller-role created
-clusterrolebinding.rbac.authorization.k8s.io/system:kube-vip-cloud-controller-binding created
-statefulset.apps/kube-vip-cloud-provider created
-```
-
-We can validate the cloud provider by examining the pods and following the logs:
-
-```
-kubectl describe pods -n kube-system kube-vip-cloud-provider-0
-kubectl logs -n kube-system kube-vip-cloud-provider-0 -f
-```
-
-### The Kube-vip Cloud Provider `configmap`
-
-To manage the IP address ranges for the load balancer instances the `kube-vip-cloud-provider` uses a `configmap` held in the `kube-system` namespace. IP address ranges can be configured using:
-- IP address pools by CIDR
-- IP ranges [start address - end address]
-- Multiple pools by CIDR per namespace
-- Multiple IP ranges per namespace (handles overlapping ranges)
-- Setting of static addresses through --load-balancer-ip=x.x.x.x
-
-To control which IP address range is used for which service the following rules are applied:
-- Global address pools (`cidr-global` or `range-global`) are available for use by *any* `service` in *any* `namespace`
-- Namespace specific address pools (`cidr-<namespace>` or `range-<namespace>`) are *only* available for use by `service` in the *specific* `namespace`
-- Static IP addresses can be applied to a load balancer `service` using the `loadbalancerIP` setting, even outside of the assigned ranges
-
-Example Configmap:
-
-```
-$ kubectl get configmap -n kube-system kubevip -o yaml
-
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kubevip
-  namespace: kube-system
-data:
-  cidr-default: 192.168.0.200/29                      # CIDR-based IP range for use in the default namespace
-  range-development: 192.168.0.210-192.168.0.219      # Range-based IP range for use in the development namespace
-  cidr-finance: 192.168.0.220/29,192.168.0.230/29     # Multiple CIDR-based ranges for use in the finance namespace
-  cidr-global: 192.168.0.240/29                       # CIDR-based range which can be used in any namespace
-```
-
-### Expose a service
-
-We can now expose a service and once the cloud provider has provided an address `kube-vip` will start to advertise that address to the outside world as shown below!
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx
-```
-
-or via a `service` YAML definition
-
-```
-apiVersion: v1
-kind: Service
-metadata:
-  name: nginx
-spec:
-  ports:
-  - name: http
-    port: 80
-    protocol: TCP
-  selector:
-    app: nginx
-  type: LoadBalancer
-  ```
-
-
-We can also expose a specific address by specifying it on the command line:
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```
-
-or including it in the `service` definition:
-
-```
-apiVersion: v1
-kind: Service
-metadata:
-  name: nginx
-spec:
-  ports:
-  - name: http
-    port: 80
-    protocol: TCP
-  selector:
-    app: nginx
-  type: LoadBalancer
-  loadBalancerIP: "1.1.1.1"
-```
-
-### Using DHCP for Load Balancers (experimental)
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to use the local network DHCP server to provide `kube-vip` with a load-balancer address that can be used to access a
- Kubernetes service on the network. 
-
-In order to do this we need to signify to `kube-vip` and the cloud-provider that we don't need one of their managed addresses. We do this by explicitly exposing a service on the 
-address `0.0.0.0`. When `kube-vip` sees a service on this address it will create a `macvlan` interface on the host and request a DHCP address, once this address is provided it will assign it as the VIP and update the Kubernetes service!
-
-```
-$ k expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx-dhcp --load-balancer-ip=0.0.0.0; k get svc
-service/nginx-dhcp exposed
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   0.0.0.0         80:31184/TCP   0s
-
-{ ... a second or so later ... }
-
-$ k get svc
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   192.168.0.155   80:31184/TCP   3s
-```
-
-### Using UPNP to expose a service to the outside world
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to expose a load-balancer on a specific port and using UPNP (on a supported gateway) expose this service to the inte
-rnet.
-
-Most simple networks look something like the following:
-
-`<----- <internal network 192.168.0.0/24> <Gateway / router> <external network address> ----> Internet`
-
-Using UPNP we can create a matching port on the `<external network address>` allowing your service to be exposed to the internet.
-
-#### Enable UPNP
-
-Add the following to the `kube-vip` `env:` section, and the rest should be completely automated. 
-
-**Note** some environments may require (Unifi) will require `Secure mode` being `disabled` (this allows a host with a different address to register a port)
-
-```
-- name: enableUPNP
-  value: "true"
-```
-
-#### Exposing a service
-
-To expose a port successfully we'll need to change the command slightly:
-
-`--target-port=80` the port of the application in the pods (HTT/NGINX)
-`--port=32380` the port the service will be exposed on (and what you should connect to in order to receive traffic from the service)
-
-`kubectl expose deployment plunder-nginx --port=32380 --target-port=80 --type=LoadBalancer --namespace plunder`
-
-The above example should expose a port on your external (internet facing address), that can be tested externally with:
-
-```
-$ curl externalIP:32380
-<!DOCTYPE html>
-<html>
-...
-```
-
-### Expose with Equinix Metal (using the `kube-vip-cloud-provider`)
-
-Either through the CLI or through the UI, create a public IPv4 EIP address.. and this is the address you can expose through BGP!
-
-```
-# packet ip request -p xxx-bbb-ccc -f ams1 -q 1 -t public_ipv4                                                                   
-+-------+---------------+--------+----------------------+
-|   ID  |    ADDRESS    | PUBLIC |       CREATED        |
-+-------+---------------+--------+----------------------+
-| xxxxx |     1.1.1.1   | true   | 2020-11-10T15:57:39Z |
-+-------+---------------+--------+----------------------+
-
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```
-
-## Equinix Metal Overview (using the [Equinix Metal CCM](https://github.com/packethost/packet-ccm))
-
-Below are two examples for running `type:LoadBalancer` services on worker nodes only and will create a daemonset that will run `kube-vip`. 
-
-**NOTE** This use-case requires the [Equinix Metal CCM](https://github.com/packethost/packet-ccm) to be installed and that the cluster/kubelet is configured to use an "external" cloud provider.
-
-### Using Annotations
-
-This is important as the CCM will apply the BGP configuration to the [node annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) making it easy for `kube-vip` to find the networking configuration it needs to expose load balancer addresses. The `--annotations metal.equinix.com` will cause kube-vip to "watch" the annotations of the worker node that it is running on, once all of the configuarion has been applied by the CCM then the `kube-vip` pod is ready to advertise BGP addresses for the service.
-
-```
-kube-vip manifest daemonset \
-  --interface $INTERFACE \
-  --services \
-  --bgp \
-  --annotations metal.equinix.com \
-  --inCluster | k apply -f -
-```
-
-### Using the existing CCM secret 
-
-Alternatively it is possible to create a daemonset that will use the existing CCM secret to do an API lookup, this will allow for discovering the networking configuration needed to advertise loadbalancer addresses through BGP.
-
-```
-kube-vip manifest daemonset --interface $INTERFACE \
---services \
---inCluster \
---bgp \
---metal \
---provider-config /etc/cloud-sa/cloud-sa.json | kubectl apply -f -
-```

docs/hybrid/static/index.md

@@ -1,129 +0,0 @@
-# Kube-vip as a Static Pod
-
-In Hybrid mode `kube-vip` will manage a virtual IP address that is passed through it's configuration for a Highly Available Kubernetes cluster, it will also "watch" services of `type:LoadBalancer` and once their `spec.LoadBalancerIP` is updated (typically by a cloud controller) it will advertise this address using BGP/ARP.
-
-The "hybrid" mode is now the default mode in `kube-vip` from `0.2.3` onwards, and allows both modes to be enabled at the same time. 
-
-## Generating a Manifest
-
-This section details creating a number of manifests for various use cases
-
-### Set configuration details
-
-`export VIP=192.168.0.40`
-
-`export INTERFACE=<interface>`
-
-### Configure to use a container runtime
-
-The easiest method to generate a manifest is using the container itself, below will create an alias for different container runtimes.
-
-#### containerd
-`alias kube-vip="ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:0.3.7 vip /kube-vip"`
-
-#### Docker
-`alias kube-vip="docker run --network host --rm ghcr.io/kube-vip/kube-vip:0.3.7"`
-
-
-### ARP
-
-This configuration will create a manifest that starts `kube-vip` providing **controlplane** and **services** management, using **leaderElection**. When this instance is elected as the leader it will bind the `vip` to the specified `interface`, this is also the same for services of `type:LoadBalancer`.
-
-`export INTERFACE=eth0`
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --arp \
-    --leaderElection | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-### BGP
-
-This configuration will create a manifest that will start `kube-vip` providing **controlplane** and **services** management. **Unlike** ARP, all nodes in the BGP configuration will advertise virtual IP addresses. 
-
-**Note** we bind the address to `lo` as we don't want multiple devices that have the same address on public interfaces. We can specify all the peers in a comma seperate list in the format of `address:AS:password:multihop`.
-
-`export INTERFACE=lo`
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE \
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --localAS 65000 \
-    --bgpRouterID 192.168.0.2 \
-    --bgppeers 192.168.0.10:65000::false,192.168.0.11:65000::false | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-### BGP with Equinix Metal
-
-When deploying Kubernetes with Equinix Metal with the `--controlplane` functionality we need to pre-populate the BGP configuration in order for the control plane to be advertised and work in a HA scenario. Luckily Equinix Metal provides the capability to "look up" the configuration details (for BGP) that we need in order to advertise our virtual IP for HA functionality. We can either make use of the [Equinix Metal API](https://metal.equinix.com/developers/api/) or we can parse the [Equinix Metal Metadata service](https://metal.equinix.com/developers/docs/servers/metadata/).
-
-**Note** If this cluster will be making use of Equinix Metal for `type:LoadBalancer` (by using the [Equinix Metal CCM](https://github.com/packethost/packet-ccm)) then we will need to ensure that nodes are set to use an external cloud-provider. Before doing a `kubeadm init|join` ensure the kubelet has the correct flags by using the following command `echo KUBELET_EXTRA_ARGS=\"--cloud-provider=external\" > /etc/default/kubelet`.
-
-#### Creating a manifest using the API
-
-We can enable `kube-vip` with the capability to discover the required configuration for BGP by passing the `--metal` flag and the API Key and our project ID.
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE\
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --metal \
-    --metalKey xxxxxxx \
-    --metalProjectID xxxxx | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-#### Creating a manifest using the metadata
-
-We can parse the metadata, *however* it requires that the tools `curl` and `jq` are installed. 
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE\
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --peerAS $(curl https://metadata.platformequinix.com/metadata | jq '.bgp_neighbors[0].peer_as') \
-    --peerAddress $(curl https://metadata.platformequinix.com/metadata | jq -r '.bgp_neighbors[0].peer_ips[0]') \
-    --localAS $(curl https://metadata.platformequinix.com/metadata | jq '.bgp_neighbors[0].customer_as') \
-    --bgpRouterID $(curl https://metadata.platformequinix.com/metadata | jq -r '.bgp_neighbors[0].customer_ip') | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-## Deploy your Kubernetes Cluster
-
-### First node
-
-```
-sudo kubeadm init \
-    --kubernetes-version 1.19.0 \
-    --control-plane-endpoint $VIP \
-    --upload-certs
-```
-
-### Additional Node(s)
-
-Due to an oddity with `kubeadm` we can't have our `kube-vip` manifest present **before** joining our additional nodes. So on these control plane nodes we will add them first to the cluster.
-
-```
-sudo kubeadm join $VIP:6443 \
-    --token w5atsr.blahblahblah 
-    --control-plane \
-    --certificate-key abc123
-```
-
-**Once**, joined these nodes can have the same command that we ran on the first node to populate the `/etc/kubernetes/manifests/` folder with the `kube-vip` manifest.
-
-## Services
-
-At this point your `kube-vip` static pods will be up and running and where used with the `--services` flag will also be watching for Kubernetes services that they can advertise. In order for `kube-vip` to advertise a service it needs a CCM or other controller to apply an IP address to the `spec.LoadBalancerIP`, which marks the loadbalancer as defined. 

docs/index.md

@@ -1,70 +0,0 @@
-![kube-vip.png](kube-vip.png)
-
-## Overview
-
-Kube-Vip provides Kubernetes clusters a virtual IP and load balancer for both control plane and Kubernetes Services.
-
-The idea behind `kube-vip` is a small, self-contained, highly-available option for all environments, especially:
-
-- Bare metal
-- On-Premises
-- Edge (ARM / Raspberry Pi)
-- Virtualisation
-- Pretty much anywhere else :)
-
-## Features
-
-Kube-Vip was originally created to provide a HA solution for the Kubernetes control plane, but over time it has evolved to incorporate that same functionality for Kubernetes Services of type [LoadBalancer](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer). Some of the features include:
-
-- VIP addresses can be either IPv4 or IPv6
-- Control Plane with ARP (Layer 2) or BGP (Layer 3)
-- Control Plane using either [leader election](https://godoc.org/k8s.io/client-go/tools/leaderelection) or [raft](https://en.wikipedia.org/wiki/Raft_(computer_science))
-- Control Plane HA with kubeadm (static Pods)
-- Control Plane HA with K3s/and others (DaemonSets)
-- Control Plane LoadBalancing with IPVS (kube-vip ≥ 0.4)
-- Service LoadBalancer using [leader election](https://godoc.org/k8s.io/client-go/tools/leaderelection) for ARP (Layer 2)
-- Service LoadBalancer using multiple nodes with BGP
-- Service LoadBalancer address pools per namespace or global
-- Service LoadBalancer address via (existing network DHCP)
-- Service LoadBalancer address exposure to gateway via UPnP
-- ... manifest generation, vendor API integrations and many more...
-
-## Why?
-
-The "original" purpose of `kube-vip` was to simplify the building of HA Kubernetes clusters, which at the time involved a few components and configurations that all needed to be managed. This was blogged about in detail by [thebsdbox](https://twitter.com/thebsdbox/) [here](https://thebsdbox.co.uk/2020/01/02/Designing-Building-HA-bare-metal-Kubernetes-cluster/#Networking-load-balancing). Since the project has evolved, it can now use those same technologies to provide load balancing capabilities within a Kubernetes Cluster.
-
-## Architecture
-
-The architecture for `kube-vip` (and associated Kubernetes components) is covered in detail [here](/architecture/).
-
-## Installation
-
-There are two main routes for deploying `kube-vip`: either through a [static Pod](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/) when bringing up a Kubernetes cluster with [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/) or as a [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) (typically with distributions like [K3s](https://k3s.io)).
-
-- [Static Pod](/install_static)
-- [DaemonSet](/install_daemonset)
-
-## Usage
-
-- [On-Prem with the kube-vip cloud controller](/usage/on-prem)
-- [KinD](/usage/kind)
-- [Equinix Metal](/usage/EquinixMetal)
-- [k3s](/usage/k3s)
-
-## Flags/Environment Variables
-
-- [Flags and Environment variables](/flags/)
-
-## Links
-
-- [Kube-Vip Cloud Provider Repository](https://github.com/kube-vip/kube-vip-cloud-provider)
-- [Kube-Vip Repository](https://github.com/kube-vip/kube-vip)
-- [Kube-Vip RBAC manifest (required for the DaemonSet)](https://kube-vip.io/manifests/rbac.yaml)
-
-## Copyright
-
-© 2021 [The Linux Foundation](https://www.linuxfoundation.org/). All rights reserved.
-
-The Linux Foundation has registered trademarks and uses trademarks.
-
-For a list trademarks of The Linux Foundation, please see our [Trademark Usage page](https://www.linuxfoundation.org/en/trademark-usage).

docs/install_daemonset/index.md

@@ -1,253 +0,0 @@
-# Kube-Vip as a DaemonSet
-
-## DaemonSet
-
-Some Kubernetes distributions can bring up a Kubernetes cluster without depending on a pre-existing VIP (but they may be configured to support one). A prime example of this would be K3s which can be configured to start and also sign the certificates to allow incoming traffic to a virtual IP. Given we don't need the VIP to exist before the cluster, we can bring up the K3s node(s) and then add `kube-vip` as a DaemonSet for all control plane nodes.
-
-If the Kubernetes installer allows for adding a virtual IP as an additional [SAN](https://en.wikipedia.org/wiki/Subject_Alternative_Name) to the API server certificate, we can apply `kube-vip` to the cluster once the first node has been brought up.
-
-Unlike running `kube-vip` as a [static Pod](/install_static) there are a few more things that may need configuring when running `kube-vip` as a DaemonSet. This page will cover primarily the differences.
-
-## Kube-Vip as HA, Load Balancer, or both
-
-The functionality of `kube-vip` depends on the flags used to create the static Pod manifest. By passing in `--controlplane` we instruct `kube-vip` to provide and advertise a virtual IP to be used by the control plane. By passing in `--services` we tell `kube-vip` to provide load balancing for Kubernetes Service resources created inside the cluster. With both enabled, `kube-vip` will manage a virtual IP address that is passed through its configuration for a highly available Kubernetes cluster. It will also watch Services of type `LoadBalancer` and once their `spec.LoadBalancerIP` is updated (typically by a cloud controller, including (optionally) the one provided by kube-vip in [on-prem](/usage/on-prem) scenarios) it will advertise this address using BGP/ARP. In this example, we will use both when generating the manifest.
-
-## Create the RBAC settings
-
-Since `kube-vip` as a DaemonSet runs as a regular resource instead of a static Pod, it still needs the correct access to be able to watch Kubernetes Services and other objects. In order to do this, RBAC resources must be created which include a ServiceAccount, ClusterRole, and ClusterRoleBinding and can be applied this with the command:
-
-```
-kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
-```
-
-## Generating a Manifest
-
-In order to create an easier experience of consuming the various functionality within `kube-vip`, we can use the `kube-vip` container itself to generate our DaemonSet manifest. We do this by running the `kube-vip` image as a container and passing in the various [flags](/flags/) for the capabilities we want to enable. Generating a `kube-vip` manifest for running as a DaemonSet is almost identical to the process when running `kube-vip` as a [static Pod](/install_static). Only a few flags are different between the two processes. Therefore, refer back to the [Generating a Manifest](/install_static/#generating-a-manifest) section on the [static Pod installation page](/install_static) for the main process steps.
-
-### ARP Example for DaemonSet
-
-When creating the `kube-vip` installation manifest as a DaemonSet, the `manifest` subcommand takes the value `daemonset` as opposed to the `pod` value. The flags `--inCluster` and `--taint` are also needed to configure the DaemonSet to use a ServiceAccount and affine the `kube-vip` Pods to control plane nodes thereby preventing them from running on worker instances.
-
-```
-kube-vip manifest daemonset \
-    --interface $INTERFACE \
-    --address $VIP \
-    --inCluster \
-    --taint \
-    --controlplane \
-    --services \
-    --arp \
-    --leaderElection
-```
-
-#### Example ARP Manifest
-
-```yaml
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-            - matchExpressions:
-              - key: node-role.kubernetes.io/master
-                operator: Exists
-            - matchExpressions:
-              - key: node-role.kubernetes.io/control-plane
-                operator: Exists
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "true"
-        - name: port
-          value: "6443"
-        - name: vip_interface
-          value: ens160
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: address
-          value: 192.168.0.40
-        image: ghcr.io/kube-vip/kube-vip:v0.4.0
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      tolerations:
-      - effect: NoSchedule
-        operator: Exists
-      - effect: NoExecute
-        operator: Exists
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-```
-
-### BGP Example for DaemonSet
-
-This configuration will create a manifest that starts `kube-vip` providing control plane VIP and Kubernetes Service management. Unlike ARP, all nodes in the BGP configuration will advertise virtual IP addresses.
-
-**Note** we bind the address to `lo` as we don't want multiple devices that have the same address on public interfaces. We can specify all the peers in a comma-separated list in the format of `address:AS:password:multihop`.
-
-`export INTERFACE=lo`
-
-```
-kube-vip manifest daemonset \
-    --interface $INTERFACE \
-    --address $VIP \
-    --inCluster \
-    --taint \
-    --controlplane \
-    --services \
-    --bgp \
-    --localAS 65000 \
-    --bgpRouterID 192.168.0.2 \
-    --bgppeers 192.168.0.10:65000::false,192.168.0.11:65000::false
-
-```
-
-#### Example BGP Manifest
-
-```yaml
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-            - matchExpressions:
-              - key: node-role.kubernetes.io/master
-                operator: Exists
-            - matchExpressions:
-              - key: node-role.kubernetes.io/control-plane
-                operator: Exists
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: port
-          value: "6443"
-        - name: vip_interface
-          value: ens160
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-          value: 192.168.0.2
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: bgp_peers
-          value: 192.168.0.10:65000::false,192.168.0.11:65000::false
-        - name: address
-          value: 192.168.0.40
-        image: ghcr.io/kube-vip/kube-vip:v0.4.0
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      tolerations:
-      - effect: NoSchedule
-        operator: Exists
-      - effect: NoExecute
-        operator: Exists
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-```
-
-#### Managing a `routerID` as a DaemonSet
-
-The `routerID` needs to be unique on each node that participates in BGP advertisements. In order to do this, we can modify the manifest so that when `kube-vip` starts it will look up its local address and use that as the `routerID`. Add the following to the `env[]` array of the container:
-
-```yaml
-- name: bgp_routerinterface
-  value: "ens160"
-```
-
-### DaemonSet Manifest Overview
-
-Once the manifest for `kube-vip` as a DaemonSet is generated, these are some of the notable differences over the [static Pod](/install_static) manifest and their significance.
-
-- `nodeSelector`: Ensures that DaemonSet Pods only run on control plane nodes.
-- `serviceAccountName: kube-vip`: Specifies the ServiceAccount name that will be used to get/update Kubernetes Service resources.
-- `tolerations`: Allows scheduling to control plane nodes that normally specify `NoSchedule` or `NoExecute` taints.

docs/install_static/index.md

@@ -1,228 +0,0 @@
-# Kube-Vip as a Static Pod
-
-## Static Pods
-
-[Static Pods](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/) are Kubernetes Pods that are run by the `kubelet` on a single node and are not managed by the Kubernetes cluster itself. This means that whilst the Pod can appear within Kubernetes, it can't make use of a variety of Kubernetes functionality (such as the Kubernetes token or ConfigMap resources). The static Pod approach is primarily required for [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/) as this is due to the sequence of actions performed by `kubeadm`. Ideally, we want `kube-vip` to be part of the Kubernetes cluster, but for various bits of functionality we also need `kube-vip` to provide a HA virtual IP as part of the installation.
-
-The sequence of events for building a highly available Kubernetes cluster with `kubeadm` and `kube-vip` are as follows:
-
-1. Generate a `kube-vip` manifest in the static Pods manifest directory (see the [generating a manifest](#generating-a-manifest) section below).
-2. Run `kubeadm init` with the `--control-plane-endpoint` flag using the VIP address provided when generating the static Pod manifest.
-3. The `kubelet` will parse and execute all manifests, including the `kube-vip` manifest generated in step one and the other control plane components including `kube-apiserver`.
-4. `kube-vip` starts and advertises the VIP address.
-5. The `kubelet` on this first control plane will connect to the VIP advertised in the previous step.
-6. `kubeadm init` finishes successfully on the first control plane.
-7. Using the output from the `kubeadm init` command on the first control plane, run the `kubeadm join` command on the remainder of the control planes.
-8. Copy the generated `kube-vip` manifest to the remainder of the control planes and place in their static Pods manifest directory (default of `/etc/kubernetes/manifests/`).
-
-## Kube-Vip as HA, Load Balancer, or both
-
-The functionality of `kube-vip` depends on the flags used to create the static Pod manifest. By passing in `--controlplane` we instruct `kube-vip` to provide and advertise a virtual IP to be used by the control plane. By passing in `--services` we tell `kube-vip` to provide load balancing for Kubernetes Service resources created inside the cluster. With both enabled, `kube-vip` will manage a virtual IP address that is passed through its configuration for a highly available Kubernetes cluster. It will also watch Services of type `LoadBalancer` and once their `spec.LoadBalancerIP` is updated (typically by a cloud controller, including (optionally) the one provided by kube-vip in [on-prem](/usage/on-prem) scenarios) it will advertise this address using BGP/ARP. In this example, we will use both when generating the manifest.
-
-## Generating a Manifest
-
-In order to create an easier experience of consuming the various functionality within `kube-vip`, we can use the `kube-vip` container itself to generate our static Pod manifest. We do this by running the `kube-vip` image as a container and passing in the various [flags](/flags/) for the capabilities we want to enable.
-
-### Set configuration details
-
-We use environment variables to predefine the values of the inputs to supply to `kube-vip`.
-
-Set the `VIP` address to be used for the control plane:
-
-`export VIP=192.168.0.40`
-
-Set the `INTERFACE` name to the name of the interface on the control plane(s) which will announce the VIP. In many Linux distributions this can be found with the `ip a` command.
-
-`export INTERFACE=ens160`
-
-Get the latest version of the `kube-vip` release by parsing the GitHub API. This step requires that `jq` and `curl` are installed.
-
-`KVVERSION=$(curl -sL https://api.github.com/repos/kube-vip/kube-vip/releases | jq -r ".[0].name")`
-
-To set manually instead, find the desired [release tag](https://github.com/kube-vip/kube-vip/releases):
-
-`export KVVERSION=v0.4.0`
-
-### Creating the manifest
-
-With the input values now set, we can pull and run the `kube-vip` image supplying it the desired flags and values. Once the static Pod manifest is generated for your desired method (ARP or BGP), if running multiple control plane nodes, ensure it is placed in each control plane's static manifest directory (by default, `/etc/kubernetes/manifests`).
-
-Depending on the container runtime, use one of the two aliased commands to create a `kube-vip` command which runs the `kube-vip` image as a container.
-
-For containerd, run the below command:
-
-`alias kube-vip="ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip"`
-
-For Docker, run the below command:
-
-`alias kube-vip="docker run --network host --rm ghcr.io/kube-vip/kube-vip:$KVVERSION"`
-
-### ARP
-
-With the inputs and alias command set, we can run the `kube-vip` container to generate a static Pod manifest which will be directed to a file at `/etc/kubernetes/manifests/kube-vip.yaml`. As such, this is assumed to run on the first control plane node.
-
-This configuration will create a manifest that starts `kube-vip` providing control plane VIP and Kubernetes Service management using the `leaderElection` method and ARP. When this instance is elected as the leader, it will bind the `vip` to the specified `interface`. This is the same behavior for Services of type `LoadBalancer`.
-
-> Note: When running these commands on a to-be control plane node, `sudo` access may be required along with pre-creation of the `/etc/kubernetes/manifests/` directory.
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE \
-    --address $VIP \
-    --controlplane \
-    --services \
-    --arp \
-    --leaderElection | tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-#### Example ARP Manifest
-
-```
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - manager
-    env:
-    - name: vip_arp
-      value: "true"
-    - name: port
-      value: "6443"
-    - name: vip_interface
-      value: ens192
-    - name: vip_cidr
-      value: "32"
-    - name: cp_enable
-      value: "true"
-    - name: cp_namespace
-      value: kube-system
-    - name: vip_ddns
-      value: "false"
-    - name: svc_enable
-      value: "true"
-    - name: vip_leaderelection
-      value: "true"
-    - name: vip_leaseduration
-      value: "5"
-    - name: vip_renewdeadline
-      value: "3"
-    - name: vip_retryperiod
-      value: "1"
-    - name: address
-      value: 192.168.0.40
-    image: ghcr.io/kube-vip/kube-vip:v0.4.0
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_RAW
-        - SYS_TIME
-    volumeMounts:
-    - mountPath: /etc/kubernetes/admin.conf
-      name: kubeconfig
-  hostAliases:
-  - hostnames:
-    - kubernetes
-    ip: 127.0.0.1
-  hostNetwork: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/admin.conf
-    name: kubeconfig
-status: {}
-```
-
-### BGP
-
-This configuration will create a manifest that starts `kube-vip` providing control plane VIP and Kubernetes Service management. Unlike ARP, all nodes in the BGP configuration will advertise virtual IP addresses.
-
-**Note** we bind the address to `lo` as we don't want multiple devices that have the same address on public interfaces. We can specify all the peers in a comma-separated list in the format of `address:AS:password:multihop`.
-
-`export INTERFACE=lo`
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE \
-    --address $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --localAS 65000 \
-    --bgpRouterID 192.168.0.2 \
-    --bgppeers 192.168.0.10:65000::false,192.168.0.11:65000::false | tee /etc/kubernetes/manifests/kube-vip.yaml
-```
-
-#### Example BGP Manifest
-
-```
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - manager
-    env:
-    - name: vip_arp
-      value: "false"
-    - name: port
-      value: "6443"
-    - name: vip_interface
-      value: ens192
-    - name: vip_cidr
-      value: "32"
-    - name: cp_enable
-      value: "true"
-    - name: cp_namespace
-      value: kube-system
-    - name: vip_ddns
-      value: "false"
-    - name: bgp_enable
-      value: "true"
-    - name: bgp_routerid
-      value: 192.168.0.2
-    - name: bgp_as
-      value: "65000"
-    - name: bgp_peeraddress
-    - name: bgp_peerpass
-    - name: bgp_peeras
-      value: "65000"
-    - name: bgp_peers
-      value: 192.168.0.10:65000::false,192.168.0.11:65000::false
-    - name: address
-      value: 192.168.0.40
-    image: ghcr.io/kube-vip/kube-vip:v0.3.9
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_RAW
-        - SYS_TIME
-    volumeMounts:
-    - mountPath: /etc/kubernetes/admin.conf
-      name: kubeconfig
-  hostAliases:
-  - hostnames:
-    - kubernetes
-    ip: 127.0.0.1
-  hostNetwork: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/admin.conf
-    name: kubeconfig
-status: {}
-```

docs/k3s

@@ -1,61 +0,0 @@
-#!/bin/bash
-
-echo "apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: \"true\"
-        - name: vip_interface
-          value: $vipInterface
-        - name: port
-          value: \"6443\"
-        - name: vip_cidr
-          value: \"32\"
-        - name: cp_enable
-          value: \"true\"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: \"false\"
-        - name: vip_address
-          value: $vipAddress
-        image: ghcr.io/kube-vip/kube-vip:v0.3.7
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      nodeSelector:
-        node-role.kubernetes.io/master: \"true\"
-      serviceAccountName: kube-vip
-      tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0"

docs/kubernetes/arp/index.md

@@ -1,181 +0,0 @@
-# Kube-vip (Layer 2 / ARP)
-
-**BEFORE** we begin we should ensure that ipvs has `strict` ARP enabled:
-
-```
-$ kubectl describe configmap -n kube-system kube-proxy | grep ARP
-  strictARP: false
-```
-
-If this is false we can enable it with the command:
-
-```
-$ kubectl get configmap kube-proxy -n kube-system -o yaml | \
- sed -e "s/strictARP: false/strictARP: true/" | \
- kubectl apply -f - -n kube-system
-```
-
-and confirm with:
-
-```
-$ kubectl describe configmap -n kube-system kube-proxy | grep ARP
-  strictARP: true
-```
-
-## Deploy `kube-vip`
-
-To deploy the [latest] then `kubectl apply -f https://kube-vip.io/manifests/kube-vip.yaml`, specific versions should be found in the repository as detailed below:
-
-From the GitHub repository [https://github.com/kube-vip/kube-vip/tree/master/example/deploy](https://github.com/kube-vip/kube-vip/tree/master/example/deploy) find the version of the `kube-vip` to deploy (although typically the highest version number will provider more functionality/stability). The [raw] option in Github will provide the url that can be applied directly with a `kubectl apply -f <url>`.
-
-The following output should appear when the manifest is applied: 
-```
-serviceaccount/vip created
-role.rbac.authorization.k8s.io/vip-role created
-rolebinding.rbac.authorization.k8s.io/vip-role-bind created
-deployment.apps/kube-vip-cluster created
-```
-
-*NOTE* The manifest for the `kube-vip` deployment has rules to ensure affinity (pods are always distributed to different nodes for HA). By default the replicas are set to `3` in the event you have less than `3` worker nodes then those replicas will sit as `pending`. This in itself isn't an issue, it means when new workers are added then they will be scheduled. *However*, tooling such as `kapps` will inspect the manifest before it's applied an error because of issues such as this.
-
-### Editing `kube-vip` configuration
-
-Either download and edit the manifest locally or apply as above and edit the deployment with `kubectl edit deploy/kube-vip-cluster` (change namespace where appropriate `-n`)
-
-```
-		- name: vip_interface
-		  value: ens192
-		- name: vip_configmap
-		  value: plndr
-		- name: vip_arp
-		  value: "true"
-		- name: vip_loglevel
-		  value: "5"
-```
-
-- `vip_interface` - defines the interface that the VIP will bind to
-- `vip_configmap` - defines the `configmap` that `kube-vip` will watch for service configuration
-- `vip_arp` - determines if ARP broadcasts are enabled
-- `vip_loglevel` - determines the verbosity of logging
-
-## Using other namespaces
-
-In this example we'll deploy and load-balance within the namespace `plunder`
-
-### Create the namespace
-
-`kubectl create namespace plunder`
-
-### Add a network range/cidr for this namespace
-
-`kubectl edit -n kube-system configmap/plndr`
-
-We will add the range 192.168.0.210/29 for the namespace plunder underneath the existing range for the namespace default:
-
-```
-apiVersion: v1
-data:
-  cidr-default: 192.168.0.200/29
-  cidr-global: 192.168.0.210/29
-  cidr-plunder: 192.168.0.220/29
-<...>
-```
-
-### Deploy `kube-vip` in the namespace **plunder**
-
-In the same way we deployed `kube-vip` into the default namespace we can deploy the same manifest into a different namespace using `-n namespace` e.g.
-
-**Note** change the version of manifest when actually deploying!
-
-```
-kubectl apply -f https://kube-vip.io/manifests/kube-vip.yaml -n plunder
-```
-
-## Usage
-
-This example will deploy into the namespace `plunder` as mention in the [Using other namespaces](Using other namespaces) example. Remove the `-n plunder` to deploy within the `default` namespace.
-
-### Deploy nginx
-
-```
-kubectl create deployment --image nginx plunder-nginx --namespace plunder
-```
-
-### Create a load balancer
-
-```
-kubectl expose deployment plunder-nginx --port=80 --type=LoadBalancer --namespace plunder
-```
-
-## Using DHCP for Load Balancers (experimental)
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to use the local network DHCP server to provide `kube-vip` with a load-balancer address that can be used to access a Kubernetes service on the network. 
-
-In order to do this we need to signify to `kube-vip` and the cloud-provider that we don't need one of their managed addresses. We do this by explicitly exposing a service on the address `0.0.0.0`. When `kube-vip` sees a service on this address it will create a `macvlan` interface on the host and request a DHCP address, once this address is provided it will assign it as the VIP and update the Kubernetes service!
-
-```
-$ k expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx-dhcp --load-balancer-ip=0.0.0.0; k get svc
-service/nginx-dhcp exposed
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   0.0.0.0         80:31184/TCP   0s
-
-{ ... a second or so later ... }
-
-$ k get svc
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   192.168.0.155   80:31184/TCP   3s
-```
-
-## Using UPNP to expose a service to the outside world
-
-With the latest release of `kube-vip` > 0.2.1, it is possible to expose a load-balancer on a specific port and using UPNP (on a supported gateway) expose this service to the internet.
-
-Most simple networks look something like the following:
-
-`<----- <internal network 192.168.0.0/24> <Gateway / router> <external network address> ----> Internet`
-
-Using UPNP we can create a matching port on the `<external network address>` allowing your service to be exposed to the internet.
-
-### Enable UPNP
-
-Add the following to the `kube-vip` `env:` section, and the rest should be completely automated. 
-
-**Note** some environments may require (Unifi) will require `Secure mode` being `disabled` (this allows a host with a different address to register a port)
-
-```
-- name: enableUPNP
-  value: "true"
-```
-
-### Exposing a service
-
-To expose a port successfully we'll need to change the command slightly:
-
-`--target-port=80` the port of the application in the pods (HTT/NGINX)
-`--port=32380` the port the service will be exposed on (and what you should connect to in order to receive traffic from the service)
-
-`kubectl expose deployment plunder-nginx --port=32380 --target-port=80 --type=LoadBalancer --namespace plunder`
-
-The above example should expose a port on your external (internet facing address), that can be tested externally with:
-
-```
-$ curl externalIP:32380
-<!DOCTYPE html>
-<html>
-...
-```
-
-## Troubleshooting
-
-Typically the logs from the `kube-vip` controller will reveal the most clues as to where a problem may lie.
-
-The `ClusterRoleBinding` is missing will result in the following:
-
-```
-E0229 17:36:38.014351       1 retrywatcher.go:129] Watch failed: unknown (get endpoints)
-E0229 17:36:38.014352       1 retrywatcher.go:129] Watch failed: unknown (get endpoints)
-```
-
-Additionally ensure that the vip_interface matches the correct interface from `ip addr`

docs/kubernetes/bgp/index.md

@@ -1,87 +0,0 @@
-# Kube-vip (Layer 3 / BGP)
-
-## Deploy `kube-vip`
-
-To deploy the [latest] then `kubectl apply -f https://kube-vip.io/manifests/kube-vip.yaml`, specific versions should be found in the repository as detailed below:
-
-From the GitHub repository [https://github.com/kube-vip/kube-vip/tree/master/example/deploy](https://github.com/kube-vip/kube-vip/tree/master/example/deploy) find the version of the `kube-vip` to deploy (although typically the highest version number will provider more functionality/stability). The [raw] option in Github will provide the url that can be applied directly with a `kubectl apply -f <url>`.
-
-The following output should appear when the manifest is applied: 
-```
-serviceaccount/vip created
-role.rbac.authorization.k8s.io/vip-role created
-rolebinding.rbac.authorization.k8s.io/vip-role-bind created
-deployment.apps/kube-vip-cluster created
-```
-
-*NOTE* The manifest for the `kube-vip` deployment has rules to ensure affinity (pods are always distributed to different nodes for HA). By default the replicas are set to `3` in the event you have less than `3` worker nodes then those replicas will sit as `pending`. This in itself isn't an issue, it means when new workers are added then they will be scheduled. *However*, tooling such as `kapps` will inspect the manifest before it's applied an error because of issues such as this.
-
-### Editing `kube-vip` configuration
-
-Either download and edit the manifest locally or apply as above and edit the deployment with `kubectl edit deploy/kube-vip-cluster` (change namespace where appropriate `-n`)
-
-Ensure the `vip_arp` isn't enabled as ARP and BGP can't be used at the same time (today), also that the `vip_interface` is set to localhost (`lo`).
-
-```
-        - name: vip_interface
-		  value: "lo"
-		- name: vip_configmap
-		  value: "plndr" 
-		- name: bgp_enable
-		  value: "true"
-		- name: vip_loglevel
-		  value: "5"
-```
-
-### BGP Specific configuration
-
-Additionally for BGP we'll need some configuration details, your local friendly network admin should be able to help here:
-
-```
-		- name: bgp_routerid
-		  value: "192.168.0.45"
-		- name: bgp_as
-          value: "65000" 
-		- name: bgp_peeraddress
-    	  value: "10.0.0.1"
-		- name: bgp_peeras
-    	  value: "65522"
-```
-
-### BGP on Packet
-
-If you're lucky enough to be running services on Packet then The above BGP information can be found from the API, instead of specifying the above we need to use the following:
-
-```
-		- name: vip_packet
-		  value: "true"
-		- name: vip_packetproject
-		  value: "My Project" 
-		- name: PACKET_AUTH_TOKEN
-		  value: "XXYZZYVVY"
-```
-
-With the above configuration in place, all `kube-vip` pods will start in active mode and when a service is exposed then all nodes will advertise the VIP to the routers.
-
-## Expose a service
-
-Given that `kube-vip` doesn't know your network (at this point) ask your local friendly network OPs for an address you can advertise. That is the address you can expose to the outside world as shown below!
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```
-
-## Expose with packet 
-
-Either through the CLI or through the UI, create a public IPv4 EIP address.. and this is the address you can expose through BGP!
-
-```
-# packet ip request -p xxx-bbb-ccc -f ams1 -q 1 -t public_ipv4                                                                   
-+-------+---------------+--------+----------------------+
-|   ID  |    ADDRESS    | PUBLIC |       CREATED        |
-+-------+---------------+--------+----------------------+
-| xxxxx |     1.1.1.1   | true   | 2020-11-10T15:57:39Z |
-+-------+---------------+--------+----------------------+
-
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```

docs/kubernetes/index.md

@@ -1,75 +0,0 @@
-# Usage
-
-The below instructions *should just work* on Kubernetes regardless of architecture, Linux as the Operating System is the only requirement.
-
-## The `tl;dr` guide
-
-If you just want things to "work", then you can quickly install the "latest" components:
-
-**NOTE** the `kube-vip.yaml` may need customising to set ARP/BGP OR to configure which interface to bind VIPs too.
-
-**Install the `plndr-cloud-provider`, and `kube-vip`**
-
-```
-kubectl apply -f https://kube-vip.io/manifests/controller.yaml
-kubectl apply -f https://kube-vip.io/manifests/kube-vip.yaml
-```
-
-**Create the `cidr` for the `global` namespace**
-
-```
-kubectl create configmap --namespace kube-system plndr --from-literal cidr-global=192.168.0.200/29
-```
-
-Creating services of `type: LoadBalancer` in the default namespace will now take addresses from the **global** cidr defined in the `configmap`.
-
-**Additional namespaces**
-
-Edit the `configmap` and add in the cidr ranges for those namespaces, the key in the cidr should be `cidr-<namespace>`, then ensure that `kube-vip` is deployed into that namespace with the above `apply` command with the `-n namespace` flag.
-
-## The Detailed guide
-
-### Deploy the `plndr-cloud-provider`
-
-To deploy the [latest] then `kubectl apply -f https://kube-vip.io/manifests/controller.yaml`, specific versions should be found in the repository as detailed below:
-
-From the GitHub repository [https://github.com/kube-vip/plndr-cloud-provider/tree/master/example/pod](https://github.com/kube-vip/plndr-cloud-provider/tree/master/example/pod), find the version of the plunder cloud provider manifest (although typically the highest version number will provider more functionality/stability). The [raw] option in Github will provide the url that can be applied directly with a `kubectl apply -f <url>`.
-
-The following output should appear when the manifest is applied: 
-
-```
-serviceaccount/plunder-cloud-controller created
-clusterrole.rbac.authorization.k8s.io/system:plunder-cloud-controller-role created
-clusterrolebinding.rbac.authorization.k8s.io/system:plunder-cloud-controller-binding created
-pod/plndr-cloud-provider created
-```
-
-We can validate the cloud-provider by examining the pods:
-
-`kubectl logs -n kube-system plndr-cloud-provider-0 -f`
-
-#### The `plndr-cloud-provider` `configmap`
-
-The `configmap` details a CIDR range *per* namespace, however as of (`kube-vip 0.2.1` and `plnder-cloud-provider 0.1.4`), there is now the option of having a **global** CIDR range (`cidr-global)`.  
-
-To manage the ranges for the load-balancer instances, the `plndr-cloud-provider` has a `configmap` held in the `kube-system` namespace. The structure for the key/values within the `configmap` should be that the key is in the format `cidr-<namespace>` and the value should be the cidr range.
-
-Example Configmap:
-
-```
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: plndr
-  namespace: kube-system
-data:
-  cidr-default: 192.168.0.200/29
-  cidr-global: 192.168.0.210/29
-```
-
-### Deploying `kube-vip`
-
-To use `kube-vip` in Layer2/ARP the follow this [guide](/kubernetes/arp/)
-
-To use `kube-vip` in Layer3/BGP the follow this [guide](/kubernetes/bgp/)
-

docs/manifests/0.3.1/kube-vip-arp-ds.yaml

@@ -1,63 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "true"
-        - name: vip_interface
-          value: eth0
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: vip_address
-          value: 192.168.0.1
-        image: plndr/kube-vip:0.3.1
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/0.3.1/kube-vip-bgp-ds.yaml

@@ -1,72 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: eth0
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: vip_address
-          value: 192.168.0.1
-        image: plndr/kube-vip:0.3.1
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/0.3.1/kube-vip-bgp-em-ds.yaml

@@ -1,82 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: lo
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "false"
-        - name: cp_namespace
-          value: kube-system
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: provider_config
-          value: /etc/cloud-sa/cloud-sa.json
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: vip_address
-          value: 192.168.0.1
-        image: plndr/kube-vip:0.3.1
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-        volumeMounts:
-        - mountPath: /etc/cloud-sa
-          name: cloud-sa-volume
-          readOnly: true
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      volumes:
-      - name: cloud-sa-volume
-        secret:
-          secretName: packet-cloud-config
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/0.3.1/kube-vip-bgp.yaml

@@ -1,62 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - manager
-    env:
-    - name: vip_arp
-      value: "false"
-    - name: vip_interface
-      value: eth0
-    - name: port
-      value: "6443"
-    - name: vip_cidr
-      value: "32"
-    - name: cp_enable
-      value: "true"
-    - name: cp_namespace
-      value: kube-system
-    - name: svc_enable
-      value: "true"
-    - name: vip_startleader
-      value: "false"
-    - name: vip_addpeerstolb
-      value: "true"
-    - name: vip_localpeer
-      value: code:192.168.0.22:10000
-    - name: bgp_enable
-      value: "true"
-    - name: bgp_routerid
-    - name: bgp_as
-      value: "65000"
-    - name: bgp_peeraddress
-    - name: bgp_peerpass
-    - name: bgp_peeras
-      value: "65000"
-    - name: vip_address
-      value: 192.168.0.1
-    image: plndr/kube-vip:0.3.1
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_RAW
-        - SYS_TIME
-    volumeMounts:
-    - mountPath: /etc/kubernetes/admin.conf
-      name: kubeconfig
-  hostNetwork: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/admin.conf
-    name: kubeconfig
-status: {}
-

docs/manifests/controller.yaml

@@ -1,70 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip-cloud-controller
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-cloud-controller-role
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "create", "update", "list", "put"]
-  - apiGroups: [""]
-    resources: ["configmaps", "endpoints","events","services/status", "leases"]
-    verbs: ["*"]
-  - apiGroups: [""]
-    resources: ["nodes", "services"]
-    verbs: ["list","get","watch","update"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-cloud-controller-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-cloud-controller-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip-cloud-controller
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: kube-vip-cloud-provider
-  namespace: kube-system
-spec:
-  serviceName: kube-vip-cloud-provider
-  podManagementPolicy: OrderedReady
-  replicas: 1
-  revisionHistoryLimit: 10
-  selector:
-    matchLabels:
-      app: kube-vip
-      component: kube-vip-cloud-provider
-  template:
-    metadata:
-      labels:
-        app: kube-vip
-        component: kube-vip-cloud-provider
-    spec:
-      containers:
-      - command:
-        - /kube-vip-cloud-provider
-        - --leader-elect-resource-name=kube-vip-cloud-controller
-        image: kubevip/kube-vip-cloud-provider:latest
-        name: kube-vip-cloud-provider
-        imagePullPolicy: Always
-        resources: {}
-      dnsPolicy: ClusterFirst
-      restartPolicy: Always
-      schedulerName: default-scheduler
-      securityContext: {}
-      terminationGracePeriodSeconds: 30
-      serviceAccountName: kube-vip-cloud-controller

docs/manifests/kube-vip-arp.yaml

@@ -1,85 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "create", "update", "list", "put"]
-  - apiGroups: [""]
-    resources: ["services"]
-    verbs: ["list","get","watch", "update"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-workers
-  name: kube-vip-workers
-  namespace: kube-system
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-workers
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-workers
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-workers
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: ghcr.io/kube-vip/kube-vip:0.3.7
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        env:
-          - name: vip_interface
-            value: "ens160"
-          - name: vip_arp
-            value: "true"
-          - name: vip_loglevel
-            value: "5"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-      serviceAccountName: kube-vip
-status: {}

docs/manifests/kube-vip-bgp.yaml

@@ -1,90 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: [""]
-    resources: ["services"]
-    verbs: ["list","get","watch", "update"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-workers
-  name: kube-vip-workers
-  namespace: kube-system
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-workers
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-workers
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-workers
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: ghcr.io/kube-vip/kube-vip:0.3.7
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        env:
-          - name: vip_interface
-            value: "lo"
-          - name: bgp_enable
-            value: "true"
-          - name: vip_loglevel
-            value: "5"
-          - name: bgp_routerinterface
-            value: "ens160"
-          - name: bgp_as
-            value: "64512"
-          - name: bgp_peeraddress
-            value: "192.168.0.1"
-          - name: bgp_peeras
-            value: "64512"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-      serviceAccountName: kube-vip
-status: {}

docs/manifests/kube-vip-em.yaml

@@ -1,91 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: [""]
-    resources: ["services", "services/status"]
-    verbs: ["list","get","watch", "update"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: lo
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: svc_enable
-          value: "true"
-        - name: provider_config
-          value: /etc/cloud-sa/cloud-sa.json
-        - name: vip_packet
-          value: "true"
-        - name: bgp_enable
-          value: "true"
-        image: ghcr.io/kube-vip/kube-vip:0.3.7
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - SYS_TIME
-        volumeMounts:
-        - mountPath: /etc/cloud-sa
-          name: cloud-sa-volume
-          readOnly: true
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      volumes:
-      - name: cloud-sa-volume
-        secret:
-          secretName: packet-cloud-config
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0

docs/manifests/kube-vip.yaml

@@ -1,83 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: vip
----
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: vip-role
-rules:
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "create", "update", "list", "put"]
-  - apiGroups: [""]
-    resources: ["configmaps", "endpoints"]
-    verbs: ["watch", "get"]
----
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: vip-role-bind
-subjects:
-  - kind: ServiceAccount
-    name: vip 
-    apiGroup: ""
-roleRef:
-  kind: Role
-  name: vip-role
-  apiGroup: "" 
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  creationTimestamp: null
-  labels:
-    app: kube-vip-cluster
-  name: kube-vip-cluster
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: kube-vip-cluster
-  strategy: {}
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        app: kube-vip-cluster
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-                - key: "app"
-                  operator: In
-                  values:
-                  - kube-vip-cluster
-            topologyKey: "kubernetes.io/hostname"
-      containers:
-      - image: ghcr.io/kube-vip/kube-vip:0.3.7
-        imagePullPolicy: Always
-        name: kube-vip
-        command:
-        - /kube-vip
-        - service
-        env:
-          - name: vip_interface
-            value: "ens192"
-          - name: vip_configmap
-            value: "plndr" 
-          - name: vip_arp
-            value: "true"
-          - name: vip_loglevel
-            value: "5"
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-      hostNetwork: true
-      serviceAccountName: vip
-status: {}

docs/manifests/rbac.yaml

@@ -1,32 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-vip
-  namespace: kube-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  annotations:
-    rbac.authorization.kubernetes.io/autoupdate: "true"
-  name: system:kube-vip-role
-rules:
-  - apiGroups: [""]
-    resources: ["services", "services/status", "nodes"]
-    verbs: ["list","get","watch", "update"]
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["list", "get", "watch", "update", "create"]
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: system:kube-vip-binding
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-vip-role
-subjects:
-- kind: ServiceAccount
-  name: kube-vip
-  namespace: kube-system

docs/manifests/v0.3.7/kube-vip-arp-ds.yaml

@@ -1,65 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "true"
-        - name: vip_interface
-          value: eth0
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: vip_address
-          value: 192.168.0.1
-        image: ghcr.io/kube-vip/kube-vip:v0.3.7
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/v0.3.7/kube-vip-arp.yaml

@@ -1,57 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - manager
-    env:
-    - name: vip_arp
-      value: "true"
-    - name: vip_interface
-      value: eth0
-    - name: port
-      value: "6443"
-    - name: vip_cidr
-      value: "32"
-    - name: cp_enable
-      value: "true"
-    - name: cp_namespace
-      value: kube-system
-    - name: vip_ddns
-      value: "false"
-    - name: svc_enable
-      value: "true"
-    - name: vip_leaderelection
-      value: "true"
-    - name: vip_leaseduration
-      value: "5"
-    - name: vip_renewdeadline
-      value: "3"
-    - name: vip_retryperiod
-      value: "1"
-    - name: vip_address
-      value: 192.168.0.1
-    image: ghcr.io/kube-vip/kube-vip:v0.3.7
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_RAW
-        - SYS_TIME
-    volumeMounts:
-    - mountPath: /etc/kubernetes/admin.conf
-      name: kubeconfig
-  hostNetwork: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/admin.conf
-    name: kubeconfig
-status: {}
-

docs/manifests/v0.3.7/kube-vip-bgp-ds.yaml

@@ -1,74 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: eth0
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: vip_address
-          value: 192.168.0.1
-        image: ghcr.io/kube-vip/kube-vip:v0.3.7
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/v0.3.7/kube-vip-bgp-em-ds.yaml

@@ -1,84 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: vip_interface
-          value: eth0
-        - name: port
-          value: "6443"
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: provider_config
-          value: /etc/cloud-sa/cloud-sa.json
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: vip_address
-          value: 192.168.0.1
-        image: ghcr.io/kube-vip/kube-vip:v0.3.7
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-        volumeMounts:
-        - mountPath: /etc/cloud-sa
-          name: cloud-sa-volume
-          readOnly: true
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      volumes:
-      - name: cloud-sa-volume
-        secret:
-          secretName: metal-cloud-config
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/v0.3.7/kube-vip-bgp.yaml

@@ -1,64 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - manager
-    env:
-    - name: vip_arp
-      value: "false"
-    - name: vip_interface
-      value: eth0
-    - name: port
-      value: "6443"
-    - name: vip_cidr
-      value: "32"
-    - name: cp_enable
-      value: "true"
-    - name: cp_namespace
-      value: kube-system
-    - name: vip_ddns
-      value: "false"
-    - name: svc_enable
-      value: "true"
-    - name: vip_startleader
-      value: "false"
-    - name: vip_addpeerstolb
-      value: "true"
-    - name: vip_localpeer
-      value: code:192.168.0.22:10000
-    - name: bgp_enable
-      value: "true"
-    - name: bgp_routerid
-    - name: bgp_as
-      value: "65000"
-    - name: bgp_peeraddress
-    - name: bgp_peerpass
-    - name: bgp_peeras
-      value: "65000"
-    - name: vip_address
-      value: 192.168.0.1
-    image: ghcr.io/kube-vip/kube-vip:v0.3.7
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_RAW
-        - SYS_TIME
-    volumeMounts:
-    - mountPath: /etc/kubernetes/admin.conf
-      name: kubeconfig
-  hostNetwork: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/admin.conf
-    name: kubeconfig
-status: {}
-

docs/manifests/v0.3.9/kube-vip-arp-ds.yaml

@@ -1,65 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "true"
-        - name: port
-          value: "6443"
-        - name: vip_interface
-          value: eth0
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: vip_address
-          value: 192.168.0.1
-        image: ghcr.io/kube-vip/kube-vip:v0.3.9
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/v0.3.9/kube-vip-arp.yaml

@@ -1,61 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - manager
-    env:
-    - name: vip_arp
-      value: "true"
-    - name: port
-      value: "6443"
-    - name: vip_interface
-      value: eth0
-    - name: vip_cidr
-      value: "32"
-    - name: cp_enable
-      value: "true"
-    - name: cp_namespace
-      value: kube-system
-    - name: vip_ddns
-      value: "false"
-    - name: svc_enable
-      value: "true"
-    - name: vip_leaderelection
-      value: "true"
-    - name: vip_leaseduration
-      value: "5"
-    - name: vip_renewdeadline
-      value: "3"
-    - name: vip_retryperiod
-      value: "1"
-    - name: vip_address
-      value: 192.168.0.1
-    image: ghcr.io/kube-vip/kube-vip:v0.3.9
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_RAW
-        - SYS_TIME
-    volumeMounts:
-    - mountPath: /etc/kubernetes/admin.conf
-      name: kubeconfig
-  hostAliases:
-  - hostnames:
-    - kubernetes
-    ip: 127.0.0.1
-  hostNetwork: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/admin.conf
-    name: kubeconfig
-status: {}
-

docs/manifests/v0.3.9/kube-vip-bgp-ds.yaml

@@ -1,74 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: port
-          value: "6443"
-        - name: vip_interface
-          value: eth0
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: vip_address
-          value: 192.168.0.1
-        image: ghcr.io/kube-vip/kube-vip:v0.3.9
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/v0.3.9/kube-vip-bgp-em-ds.yaml

@@ -1,84 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: port
-          value: "6443"
-        - name: vip_interface
-          value: eth0
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: provider_config
-          value: /etc/cloud-sa/cloud-sa.json
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: vip_address
-          value: 192.168.0.1
-        image: ghcr.io/kube-vip/kube-vip:v0.3.9
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-        volumeMounts:
-        - mountPath: /etc/cloud-sa
-          name: cloud-sa-volume
-          readOnly: true
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      volumes:
-      - name: cloud-sa-volume
-        secret:
-          secretName: metal-cloud-config
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/v0.3.9/kube-vip-bgp.yaml

@@ -1,62 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - manager
-    env:
-    - name: vip_arp
-      value: "false"
-    - name: port
-      value: "6443"
-    - name: vip_interface
-      value: eth0
-    - name: vip_cidr
-      value: "32"
-    - name: cp_enable
-      value: "true"
-    - name: cp_namespace
-      value: kube-system
-    - name: vip_ddns
-      value: "false"
-    - name: svc_enable
-      value: "true"
-    - name: bgp_enable
-      value: "true"
-    - name: bgp_routerid
-    - name: bgp_as
-      value: "65000"
-    - name: bgp_peeraddress
-    - name: bgp_peerpass
-    - name: bgp_peeras
-      value: "65000"
-    - name: vip_address
-      value: 192.168.0.1
-    image: ghcr.io/kube-vip/kube-vip:v0.3.9
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_RAW
-        - SYS_TIME
-    volumeMounts:
-    - mountPath: /etc/kubernetes/admin.conf
-      name: kubeconfig
-  hostAliases:
-  - hostnames:
-    - kubernetes
-    ip: 127.0.0.1
-  hostNetwork: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/admin.conf
-    name: kubeconfig
-status: {}
-

docs/manifests/v0.4.0/kube-vip-arp-ds-lb.yaml

@@ -1,69 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "true"
-        - name: port
-          value: "6443"
-        - name: vip_interface
-          value: eth0
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: lb_enable
-          value: "true"
-        - name: lb_port
-          value: "6443"
-        - name: vip_address
-          value: 192.168.0.1
-        image: ghcr.io/kube-vip/kube-vip:v0.4.0
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/v0.4.0/kube-vip-arp-ds.yaml

@@ -1,65 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "true"
-        - name: port
-          value: "6443"
-        - name: vip_interface
-          value: eth0
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: vip_address
-          value: 192.168.0.1
-        image: ghcr.io/kube-vip/kube-vip:v0.4.0
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/v0.4.0/kube-vip-arp-lb.yaml

@@ -1,65 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - manager
-    env:
-    - name: vip_arp
-      value: "true"
-    - name: port
-      value: "6443"
-    - name: vip_interface
-      value: eth0
-    - name: vip_cidr
-      value: "32"
-    - name: cp_enable
-      value: "true"
-    - name: cp_namespace
-      value: kube-system
-    - name: vip_ddns
-      value: "false"
-    - name: svc_enable
-      value: "true"
-    - name: vip_leaderelection
-      value: "true"
-    - name: vip_leaseduration
-      value: "5"
-    - name: vip_renewdeadline
-      value: "3"
-    - name: vip_retryperiod
-      value: "1"
-    - name: lb_enable
-      value: "true"
-    - name: lb_port
-      value: "6443"
-    - name: vip_address
-      value: 192.168.0.1
-    image: ghcr.io/kube-vip/kube-vip:v0.4.0
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_RAW
-        - SYS_TIME
-    volumeMounts:
-    - mountPath: /etc/kubernetes/admin.conf
-      name: kubeconfig
-  hostAliases:
-  - hostnames:
-    - kubernetes
-    ip: 127.0.0.1
-  hostNetwork: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/admin.conf
-    name: kubeconfig
-status: {}
-

docs/manifests/v0.4.0/kube-vip-arp.yaml

@@ -1,61 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - manager
-    env:
-    - name: vip_arp
-      value: "true"
-    - name: port
-      value: "6443"
-    - name: vip_interface
-      value: eth0
-    - name: vip_cidr
-      value: "32"
-    - name: cp_enable
-      value: "true"
-    - name: cp_namespace
-      value: kube-system
-    - name: vip_ddns
-      value: "false"
-    - name: svc_enable
-      value: "true"
-    - name: vip_leaderelection
-      value: "true"
-    - name: vip_leaseduration
-      value: "5"
-    - name: vip_renewdeadline
-      value: "3"
-    - name: vip_retryperiod
-      value: "1"
-    - name: vip_address
-      value: 192.168.0.1
-    image: ghcr.io/kube-vip/kube-vip:v0.4.0
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_RAW
-        - SYS_TIME
-    volumeMounts:
-    - mountPath: /etc/kubernetes/admin.conf
-      name: kubeconfig
-  hostAliases:
-  - hostnames:
-    - kubernetes
-    ip: 127.0.0.1
-  hostNetwork: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/admin.conf
-    name: kubeconfig
-status: {}
-

docs/manifests/v0.4.0/kube-vip-bgp-ds.yaml

@@ -1,74 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: port
-          value: "6443"
-        - name: vip_interface
-          value: eth0
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: vip_address
-          value: 192.168.0.1
-        image: ghcr.io/kube-vip/kube-vip:v0.4.0
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/v0.4.0/kube-vip-bgp-em-ds.yaml

@@ -1,84 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: port
-          value: "6443"
-        - name: vip_interface
-          value: eth0
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: provider_config
-          value: /etc/cloud-sa/cloud-sa.json
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: vip_address
-          value: 192.168.0.1
-        image: ghcr.io/kube-vip/kube-vip:v0.4.0
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-            - SYS_TIME
-        volumeMounts:
-        - mountPath: /etc/cloud-sa
-          name: cloud-sa-volume
-          readOnly: true
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      volumes:
-      - name: cloud-sa-volume
-        secret:
-          secretName: metal-cloud-config
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/v0.4.0/kube-vip-bgp.yaml

@@ -1,62 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - manager
-    env:
-    - name: vip_arp
-      value: "false"
-    - name: port
-      value: "6443"
-    - name: vip_interface
-      value: eth0
-    - name: vip_cidr
-      value: "32"
-    - name: cp_enable
-      value: "true"
-    - name: cp_namespace
-      value: kube-system
-    - name: vip_ddns
-      value: "false"
-    - name: svc_enable
-      value: "true"
-    - name: bgp_enable
-      value: "true"
-    - name: bgp_routerid
-    - name: bgp_as
-      value: "65000"
-    - name: bgp_peeraddress
-    - name: bgp_peerpass
-    - name: bgp_peeras
-      value: "65000"
-    - name: vip_address
-      value: 192.168.0.1
-    image: ghcr.io/kube-vip/kube-vip:v0.4.0
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_RAW
-        - SYS_TIME
-    volumeMounts:
-    - mountPath: /etc/kubernetes/admin.conf
-      name: kubeconfig
-  hostAliases:
-  - hostnames:
-    - kubernetes
-    ip: 127.0.0.1
-  hostNetwork: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/admin.conf
-    name: kubeconfig
-status: {}
-

docs/manifests/v0.4.1/kube-vip-arp-ds-lb.yaml

@@ -1,70 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "true"
-        - name: port
-          value: "6443"
-        - name: vip_interface
-          value: eth0
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: lb_enable
-          value: "true"
-        - name: lb_port
-          value: "6443"
-        - name: lb_fwdmethod
-          value: local
-        - name: vip_address
-          value: 192.168.0.1
-        image: ghcr.io/kube-vip/kube-vip:v0.4.1
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/v0.4.1/kube-vip-arp-ds.yaml

@@ -1,64 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "true"
-        - name: port
-          value: "6443"
-        - name: vip_interface
-          value: eth0
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: vip_address
-          value: 192.168.0.1
-        image: ghcr.io/kube-vip/kube-vip:v0.4.1
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/v0.4.1/kube-vip-arp-lb.yaml

@@ -1,66 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - manager
-    env:
-    - name: vip_arp
-      value: "true"
-    - name: port
-      value: "6443"
-    - name: vip_interface
-      value: eth0
-    - name: vip_cidr
-      value: "32"
-    - name: cp_enable
-      value: "true"
-    - name: cp_namespace
-      value: kube-system
-    - name: vip_ddns
-      value: "false"
-    - name: svc_enable
-      value: "true"
-    - name: vip_leaderelection
-      value: "true"
-    - name: vip_leaseduration
-      value: "5"
-    - name: vip_renewdeadline
-      value: "3"
-    - name: vip_retryperiod
-      value: "1"
-    - name: lb_enable
-      value: "true"
-    - name: lb_port
-      value: "6443"
-    - name: lb_fwdmethod
-      value: local
-    - name: vip_address
-      value: 192.168.0.1
-    image: ghcr.io/kube-vip/kube-vip:v0.4.1
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_RAW
-    volumeMounts:
-    - mountPath: /etc/kubernetes/admin.conf
-      name: kubeconfig
-  hostAliases:
-  - hostnames:
-    - kubernetes
-    ip: 127.0.0.1
-  hostNetwork: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/admin.conf
-    name: kubeconfig
-status: {}
-

docs/manifests/v0.4.1/kube-vip-arp.yaml

@@ -1,60 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - manager
-    env:
-    - name: vip_arp
-      value: "true"
-    - name: port
-      value: "6443"
-    - name: vip_interface
-      value: eth0
-    - name: vip_cidr
-      value: "32"
-    - name: cp_enable
-      value: "true"
-    - name: cp_namespace
-      value: kube-system
-    - name: vip_ddns
-      value: "false"
-    - name: svc_enable
-      value: "true"
-    - name: vip_leaderelection
-      value: "true"
-    - name: vip_leaseduration
-      value: "5"
-    - name: vip_renewdeadline
-      value: "3"
-    - name: vip_retryperiod
-      value: "1"
-    - name: vip_address
-      value: 192.168.0.1
-    image: ghcr.io/kube-vip/kube-vip:v0.4.1
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_RAW
-    volumeMounts:
-    - mountPath: /etc/kubernetes/admin.conf
-      name: kubeconfig
-  hostAliases:
-  - hostnames:
-    - kubernetes
-    ip: 127.0.0.1
-  hostNetwork: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/admin.conf
-    name: kubeconfig
-status: {}
-

docs/manifests/v0.4.1/kube-vip-bgp-ds.yaml

@@ -1,73 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: port
-          value: "6443"
-        - name: vip_interface
-          value: eth0
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: vip_address
-          value: 192.168.0.1
-        image: ghcr.io/kube-vip/kube-vip:v0.4.1
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-      hostNetwork: true
-      serviceAccountName: kube-vip
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/v0.4.1/kube-vip-bgp-em-ds.yaml

@@ -1,83 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  creationTimestamp: null
-  name: kube-vip-ds
-  namespace: kube-system
-spec:
-  selector:
-    matchLabels:
-      name: kube-vip-ds
-  template:
-    metadata:
-      creationTimestamp: null
-      labels:
-        name: kube-vip-ds
-    spec:
-      containers:
-      - args:
-        - manager
-        env:
-        - name: vip_arp
-          value: "false"
-        - name: port
-          value: "6443"
-        - name: vip_interface
-          value: eth0
-        - name: vip_cidr
-          value: "32"
-        - name: cp_enable
-          value: "true"
-        - name: cp_namespace
-          value: kube-system
-        - name: vip_ddns
-          value: "false"
-        - name: svc_enable
-          value: "true"
-        - name: vip_leaderelection
-          value: "true"
-        - name: vip_leaseduration
-          value: "5"
-        - name: vip_renewdeadline
-          value: "3"
-        - name: vip_retryperiod
-          value: "1"
-        - name: provider_config
-          value: /etc/cloud-sa/cloud-sa.json
-        - name: bgp_enable
-          value: "true"
-        - name: bgp_routerid
-        - name: bgp_as
-          value: "65000"
-        - name: bgp_peeraddress
-        - name: bgp_peerpass
-        - name: bgp_peeras
-          value: "65000"
-        - name: vip_address
-          value: 192.168.0.1
-        image: ghcr.io/kube-vip/kube-vip:v0.4.1
-        imagePullPolicy: Always
-        name: kube-vip
-        resources: {}
-        securityContext:
-          capabilities:
-            add:
-            - NET_ADMIN
-            - NET_RAW
-        volumeMounts:
-        - mountPath: /etc/cloud-sa
-          name: cloud-sa-volume
-          readOnly: true
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      volumes:
-      - name: cloud-sa-volume
-        secret:
-          secretName: metal-cloud-config
-  updateStrategy: {}
-status:
-  currentNumberScheduled: 0
-  desiredNumberScheduled: 0
-  numberMisscheduled: 0
-  numberReady: 0
-

docs/manifests/v0.4.1/kube-vip-bgp.yaml

@@ -1,61 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  creationTimestamp: null
-  name: kube-vip
-  namespace: kube-system
-spec:
-  containers:
-  - args:
-    - manager
-    env:
-    - name: vip_arp
-      value: "false"
-    - name: port
-      value: "6443"
-    - name: vip_interface
-      value: eth0
-    - name: vip_cidr
-      value: "32"
-    - name: cp_enable
-      value: "true"
-    - name: cp_namespace
-      value: kube-system
-    - name: vip_ddns
-      value: "false"
-    - name: svc_enable
-      value: "true"
-    - name: bgp_enable
-      value: "true"
-    - name: bgp_routerid
-    - name: bgp_as
-      value: "65000"
-    - name: bgp_peeraddress
-    - name: bgp_peerpass
-    - name: bgp_peeras
-      value: "65000"
-    - name: vip_address
-      value: 192.168.0.1
-    image: ghcr.io/kube-vip/kube-vip:v0.4.1
-    imagePullPolicy: Always
-    name: kube-vip
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_RAW
-    volumeMounts:
-    - mountPath: /etc/kubernetes/admin.conf
-      name: kubeconfig
-  hostAliases:
-  - hostnames:
-    - kubernetes
-    ip: 127.0.0.1
-  hostNetwork: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/admin.conf
-    name: kubeconfig
-status: {}
-

docs/publish.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-echo Deploying updated documentation
-
-# Sleep for dramatic purposes!
-
-sleep 5
-
-generate-md --layout github --input ./ --output /var/www/kube-vip/

docs/usage/EquinixMetal/index.md

@@ -1,147 +0,0 @@
-# Equinix Metal Overview (using the [Equinix Metal CCM](https://github.com/equinix/cloud-provider-equinix-metal))
-
-## BGP with Equinix Metal
-
-When deploying Kubernetes with Equinix Metal with the `--controlplane` functionality we need to pre-populate the BGP configuration in order for the control plane to be advertised and work in a HA scenario. Luckily Equinix Metal provides the capability to "look up" the configuration details (for BGP) that we need in order to advertise our virtual IP (VIP) for HA functionality. We can either make use of the [Equinix Metal API](https://metal.equinix.com/developers/api/) or we can parse the [Equinix Metal Metadata service](https://metal.equinix.com/developers/docs/servers/metadata/).
-
-**Note** If this cluster will be making use of Equinix Metal for `type:LoadBalancer` (by using the [Equinix Metal CCM](https://github.com/equinix/cloud-provider-equinix-metal)) then we will need to ensure that nodes are set to use an external cloud-provider. Before doing a `kubeadm init|join` ensure the kubelet has the correct flags by using the following command `echo KUBELET_EXTRA_ARGS=\"--cloud-provider=external\" > /etc/default/kubelet`.
-
-## Configure to use a container runtime
-
-### Get latest version
-
- We can parse the GitHub API to find the latest version (or we can set this manually)
-
-`KVVERSION=$(curl -sL https://api.github.com/repos/kube-vip/kube-vip/releases | jq -r ".[0].name")`
-
-or manually:
-
-`export KVVERSION=vx.x.x`
-
-The easiest method to generate a manifest is using the container itself, below will create an alias for different container runtimes.
-
-### containerd
-`alias kube-vip="ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip"`
-
-### Docker
-`alias kube-vip="docker run --network host --rm ghcr.io/kube-vip/kube-vip:KVVERSION"`
-
-## Creating HA clusters in Equinix Metal
-    
-### Creating a manifest using the API
-
-We can enable `kube-vip` with the capability to discover the required configuration for BGP by passing the `--metal` flag and the API Key and our project ID.
-
-```
-export VIP= metal_EIP  
-export INTERFACE=<interface>
-```
-where metal_EIP is the Elastic IP (EIP) address your requested via Metal's UI or API. For more informaiton on how to request a Metal's EIP, please see the following [Equinix Metal's EIP document](https://metal.equinix.com/developers/docs/networking/elastic-ips/#elastic-ip-addresses)
-<interface> is the interface you announce your VIP from via BGP. By default it's lo:0 in Equinix Metal.
-    
-```
-kube-vip manifest pod \
-    --interface $INTERFACE\
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --metal \
-    --metalKey xxxxxxx \
-    --metalProjectID xxxxx | tee  /etc/kubernetes/manifests/kube-vip.yaml
-```
-where metalKey is your "personal API key" under "Personal Settings" of your Metal's portal, and MetalProjectID is your Metal's "Project ID" under "Project Settings"
-    
-### Creating a manifest using the metadata
-
-We can parse the metadata, *however* it requires that the tools `curl` and `jq` are installed. 
-
-```
-kube-vip manifest pod \
-    --interface $INTERFACE\
-    --vip $VIP \
-    --controlplane \
-    --services \
-    --bgp \
-    --peerAS $(curl https://metadata.platformequinix.com/metadata | jq '.bgp_neighbors[0].peer_as') \
-    --peerAddress $(curl https://metadata.platformequinix.com/metadata | jq -r '.bgp_neighbors[0].peer_ips[0]') \
-    --localAS $(curl https://metadata.platformequinix.com/metadata | jq '.bgp_neighbors[0].customer_as') \
-    --bgpRouterID $(curl https://metadata.platformequinix.com/metadata | jq -r '.bgp_neighbors[0].customer_ip') | sudo tee /etc/kubernetes/manifests/vip.yaml
-```
-
-## Load Balancing servies on Equinix Metal
-
-Below are two examples for running `type:LoadBalancer` services on worker nodes only and will create a daemonset that will run `kube-vip`. 
-
-**NOTE** This use-case requires the [Equinix Metal CCM](https://github.com/equinix/cloud-provider-equinix-metal) to be installed prior to the kube-vip setup and that the cluster/kubelet is configured to use an "external" cloud provider.
-
-```
-export INTERFACE=<interface>
-```
-where <interface> is the interface you announce your VIP from via BGP. By default it's lo:0 in Equinix Metal.
-    
-### Using Annotations
-
-This is important as the CCM will apply the BGP configuration to the [node annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) making it easy for `kube-vip` to find the networking configuration it needs to expose load balancer addresses. The `--annotations metal.equinix.com` will cause kube-vip to "watch" the annotations of the worker node that it is running on, once all of the configuarion has been applied by the CCM then the `kube-vip` pod is ready to advertise BGP addresses for the service.
-
-```
-kube-vip manifest daemonset \
-  --interface $INTERFACE \
-  --services \
-  --bgp \
-  --annotations metal.equinix.com \
-  --inCluster | k apply -f -
-```
-
-### Using the existing CCM secret 
-
-Alternatively it is possible to create a daemonset that will use the existing CCM secret to do an API lookup, this will allow for discovering the networking configuration needed to advertise loadbalancer addresses through BGP.
-
-```
-kube-vip manifest daemonset --interface $INTERFACE \
---services \
---inCluster \
---bgp \
---metal \
---provider-config /etc/cloud-sa/cloud-sa.json | kubectl apply -f -
-```
-
-### Expose with [Equinix Metal CCM](https://github.com/equinix/cloud-provider-equinix-metal)
-
-Follow the [Equinix Metal's Elastic IP (EIP) document](https://metal.equinix.com/developers/docs/networking/elastic-ips/#elastic-ip-addresses) either through the API, CLI or through the UI, to create a public IPv4 EIP address, for example (145.75.75.1) and this is the address you can expose through BGP as the service loadbalancer.
-
-```
-# metal ip request -p xxx-bbb-ccc -f ams1 -q 1 -t public_ipv4                                                                   
-+-------+---------------+--------+----------------------+
-|   ID  |    ADDRESS    | PUBLIC |       CREATED        |
-+-------+---------------+--------+----------------------+
-| xxxxx | 147.75.75.1   | true   | 2020-11-10T15:57:39Z |
-+-------+---------------+--------+----------------------+
-    
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=147.75.75.1
-```
-
-## Troubleshooting
-
-If `kube-vip` has been sat waiting for a long time then you may need to investigate that the annotations have been applied correctly by doing running the `describe` on the node:
-
-```
-kubectl describe node k8s.bgp02
-...
-Annotations:        kubeadm.alpha.kubernetes.io/cri-socket: /var/run/dockershim.sock
-                    node.alpha.kubernetes.io/ttl: 0
-                    metal.equinix.com/node-asn: 65000
-                    metal.equinix.com/peer-asn: 65530
-                    metal.equinix.com/peer-ip: x.x.x.x
-                    metal.equinix.com/src-ip: x.x.x.x
-```
-
-If there are errors regarding `169.254.255.1` or `169.254.255.2` in the `kube-vip` logs then the routes to the ToR switches that provide BGP peering may by missing from the nodes. They can be replaced with the below command:
-
-```
-GATEWAY_IP=$(curl https://metadata.platformequinix.com/metadata | jq -r ".network.addresses[] | select(.public == false) | .gateway")
-ip route add 169.254.255.1 via $GATEWAY_IP
-ip route add 169.254.255.2 via $GATEWAY_IP
-```
-
-Additionally examining the logs of the Equinix Metal's CCM may reveal why the node is not yet ready.

docs/usage/k3s/index.md

@@ -1,53 +0,0 @@
-# K3s Overview
-
-`kube-vip` works on [K3s environments](https://k3s.io/) similar to most others with the exception of how it gets deployed. Because K3s is able to bootstrap a single server (control plane node) without the availability of the load balancer fronting it, `kube-vip` can be installed as a DaemonSet.
-
-## Prerequisites (on Equinix Metal)
-
-In order to make ARP work on Equinix Metal, follow the [metal-gateway](https://metal.equinix.com/developers/docs/networking/metal-gateway/) guide to have public VLAN subnet which can be used for the load balancer IP.
-
-## Clean Environment
-
-This step is optional but recommended if a K3s installation previously existed.
-
-```
-rm -rf /var/lib/rancher /etc/rancher ~/.kube/*; \ 
-ip addr flush dev lo; \
-ip addr add 127.0.0.1/8 dev lo; 
-```
-
-## Step 1: Create Manifests Folder
-
-K3s has an optional manifests directory that will be searched to [auto-deploy](https://rancher.com/docs/k3s/latest/en/advanced/#auto-deploying-manifests) any manifests found within. Create this directory first in order to later place the `kube-vip` resources inside.
-
-```
-mkdir -p /var/lib/rancher/k3s/server/manifests/
-```
-
-## Step 2: Upload Kube-Vip RBAC Manifest
-
-As `kube-vip` runs as a DaemonSet under K3s and not a static Pod, we will need to ensure that the required permissions exist for it to communicate with the API server. RBAC resources are needed to ensure a ServiceAccount exists with those permissions and bound appropriately.
-
-Get the RBAC manifest and place in the auto-deploy directory:
-
-```
-curl https://kube-vip.io/manifests/rbac.yaml > /var/lib/rancher/k3s/server/manifests/kube-vip-rbac.yaml
-```
-
-## Step 3: Generate a Kube-Vip DaemonSet Manifest
-
-Refer to the [DaemonSet manifest generation documentation](/install_daemonset/#generating-a-manifest) for the process to complete this step.
-
-Either store this generated manifest separately in the `/var/lib/rancher/k3s/server/manifests/` directory, or append to the existing RBAC manifest called `kube-vip-rbac.yaml`. As a general best practice, it is a cleaner approach to place all related resources into a single YAML file.
-
-> Note: Remember to include YAML document delimiters (`---`) when composing multiple documents.
-
-## Step 4: Install a HA K3s Cluster
-
-There are multiple ways to install K3s including `[k3sup](https://k3sup.dev/)` or [running the binary](https://rancher.com/docs/k3s/latest/en/quick-start/) locally. Whichever method you choose, the `--tls-san` flag must be passed with the same IP when generating the `kube-vip` DaemonSet manifest when installing the first server (control plane) instance. This is so that K3s generates an API server certificate with the `kube-vip` virtual IP address.
-
-Once the cluster is installed, you should be able to edit the `kubeconfig` file generated from the process and use the `kube-vip` VIP address to access the control plane.
-
-## Step 5: Service Load Balancing
-
-If wanting to use the `kube-vip` [cloud controller](/usage/on-prem), pass the `--disable servicelb` flag so K3s will not attempt to render Kubernetes Service resources of type `LoadBalancer`. If building with `k3sup`, the flag should be given as an argument to the `--k3s-extra-args` flag itself: `--k3s-extra-args "--disable servicelb"`. To install the `kube-vip` cloud controller, follow the additional steps in the [cloud controller guide](/on-prem/#install-the-kube-vip-cloud-provider).

docs/usage/kind/index.md

@@ -1,80 +0,0 @@
-# Kube-vip on KIND
-
-## Deploying KIND
-
-The documentation for KIND is fantastic and its [quick start](https://kind.sigs.k8s.io/docs/user/quick-start/) guide will have you up and running in no time.
-
-## Find Address Pool for Kube-Vip
-
-We will need to find addresses that can be used by Kube-Vip:
-
-```
-docker network inspect kind -f '{{ range $i, $a := .IPAM.Config }}{{ println .Subnet }}{{ end }}'
-```
-
-This will return a CIDR range such as `172.18.0.0/16` and from here we can select a range.
-
-## Deploy the Kube-Vip Cloud Controller
-
-```
-kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
-```
-
-## Add our Address range
-
-```
-kubectl create configmap --namespace kube-system kubevip --from-literal range-global=172.18.100.10-172.18.100.30
-```
-
-## Install kube-vip
-
-### Create the RBAC settings
-
-Since `kube-vip` as a DaemonSet runs as a regular resource instead of a static Pod, it still needs the correct access to be able to watch Kubernetes Services and other objects. In order to do this, RBAC resources must be created which include a ServiceAccount, ClusterRole, and ClusterRoleBinding and can be applied this with the command:
-
-```
-kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
-```
-
-### Get latest version
-
-We can parse the GitHub API to find the latest version (or we can set this manually)
-
-`KVVERSION=$(curl -sL https://api.github.com/repos/kube-vip/kube-vip/releases | jq -r ".[0].name")`
-
-or manually:
-
-`export KVVERSION=vx.x.x`
-
-The easiest method to generate a manifest is using the container itself, below will create an alias for different container runtimes.
-
-### containerd
-
-`alias kube-vip="ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip /kube-vip"`
-
-### Docker
-
-`alias kube-vip="docker run --network host --rm ghcr.io/kube-vip/kube-vip:$KVVERSION"`
-
-## Deploy Kube-vip as a DaemonSet
-
-```
-kube-vip manifest daemonset --services --inCluster --arp --interface eth0 | kubectl apply -f -
-```
-
-## Test
-
-```
-kubectl apply -f https://k8s.io/examples/application/deployment.yaml
-```
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx
-```
-
-```
-kubectl get svc
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        74m
-nginx        LoadBalancer   10.96.196.235   172.18.100.11   80:31236/TCP   6s
-```

docs/usage/on-prem/index.md

@@ -1,177 +0,0 @@
-# Kube-Vip On-Prem
-
-We've designed `kube-vip` to be as decoupled or agnostic from other components that may exist within a Kubernetes cluster as possible. This has lead to `kube-vip` having a very simplistic but robust approach to advertising Kubernetes Services to the outside world and marking these Services as ready to use.
-
-## Cloud Controller Manager
-
-`kube-vip` isn't coupled to anything other than the Kubernetes API and will only act upon an existing Kubernetes primitive (in this case the object of type `Service`). This makes it easy for existing [cloud controller managers (CCMs)](https://kubernetes.io/docs/concepts/architecture/cloud-controller/) to simply apply their logic to services of type LoadBalancer and leave `kube-vip` to take the next steps to advertise these load balancers to the outside world.
-
-## Using the Kube-Vip Cloud Provider
-
-The `kube-vip` cloud provider can be used to populate an IP address for Services of type `LoadBalancer` similar to what public cloud providers allow through a Kubernetes CCM. The below instructions *should just work* on Kubernetes regardless of the architecture (a Linux OS being the only requirement) and will install the latest components.
-
-## Install the Kube-Vip Cloud Provider
-
-The `kube-vip` cloud provider can be installed from the latest release in the `main` branch by using the following command:
-
-```
-kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
-```
-
-## Create a global CIDR or IP Range
-
-In order for `kube-vip` to set an IP address for a Service of type `LoadBalancer`, it needs to have an availability of IP address to assign. This information is stored in a Kubernetes ConfigMap to which `kube-vip` has access. You control the scope of the IP allocations with the `key` within the ConfigMap. Either CIDR blocks or IP ranges may be specified and scoped either globally (cluster-side) or per-Namespace.
-
-To allow a global (cluster-wide) CIDR block which `kube-vip` can use to allocate an IP to Services of type `LoadBalancer` in any Namespace, create a ConfigMap named `kubevip` with the key `cidr-global` and value equal to a CIDR block available in your environment. For example, the below command creates a global CIDR with value `192.168.0.220/29` from which `kube-vip` will allocate IP addresses.
-
-```
-kubectl create configmap -n kube-system kubevip --from-literal cidr-global=192.168.0.220/29
-```
-
-To use a global range instead, create the key `range-global` with the value set to a valid range of IP addresses. For example, the below command creates a global range using the pool `192.168.1.220-192.168.1.230`.
-
-```
-kubectl create configmap -n kube-system kubevip --from-literal range-global=192.168.1.220-192.168.1.230
-```
-
-Creating services of type `LoadBalancer` in any Namespace will now take addresses from one of the global pools defined in the ConfigMap unless a Namespace-specific pool is created.
-
-### The Kube-Vip Cloud Provider ConfigMap
-
-To manage the IP address ranges for Services of type `LoadBalancer`, the `kube-vip-cloud-provider` uses a ConfigMap held in the `kube-system` Namespace. IP addresses can be configured using one or multiple formats:
-
-- CIDR blocks
-- IP ranges [start address - end address]
-- Multiple pools by CIDR per Namespace
-- Multiple IP ranges per Namespace (handles overlapping ranges)
-- Setting of static addresses through --load-balancer-ip=x.x.x.x (`kubectl expose` command)
-
-To control which IP address range is used for which Service, the following rules are applied:
-
-- Global address pools (`cidr-global` or `range-global`) are available for use by *any* Service in *any* Namespace
-- Namespace specific address pools (`cidr-<namespace>` or `range-<namespace>`) are *only* available for use by a Service in the *specific* Namespace
-- Static IP addresses can be applied to a Service of type `LoadBalancer` using the `spec.loadBalancerIP` field, even outside of the assigned ranges
-
-Example Configmap:
-
-```yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kubevip
-  namespace: kube-system
-data:
-  cidr-default: 192.168.0.200/29                      # CIDR-based IP range for use in the default Namespace
-  range-development: 192.168.0.210-192.168.0.219      # Range-based IP range for use in the development Namespace
-  cidr-finance: 192.168.0.220/29,192.168.0.230/29     # Multiple CIDR-based ranges for use in the finance Namespace
-  cidr-global: 192.168.0.240/29                       # CIDR-based range which can be used in any Namespace
-```
-
-### Expose a Service
-
-We can now expose a Service and once the cloud provider has provided an address, `kube-vip` will start to advertise that address to the outside world as shown below:
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx
-```
-
-or via a Service YAML definition:
-
-```
-apiVersion: v1
-kind: Service
-metadata:
-  name: nginx
-spec:
-  ports:
-  - name: http
-    port: 80
-    protocol: TCP
-  selector:
-    app: nginx
-  type: LoadBalancer
-  ```
-
-We can also expose a specific address by specifying it imperatively on the command line:
-
-```
-kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx --load-balancer-ip=1.1.1.1
-```
-
-or including it in the Service definition:
-
-```
-apiVersion: v1
-kind: Service
-metadata:
-  name: nginx
-spec:
-  ports:
-  - name: http
-    port: 80
-    protocol: TCP
-  selector:
-    app: nginx
-  type: LoadBalancer
-  loadBalancerIP: "1.1.1.1"
-```
-
-### Using DHCP for Load Balancers (experimental)
-
-With `kube-vip` > 0.2.1, it is possible to use the local network DHCP server to provide `kube-vip` with a load balancer address that can be used to access a Kubernetes service on the network.
-
-In order to do this, we need to signify to `kube-vip` and the cloud provider that we don't need one of their managed addresses. We do this by explicitly exposing a Service on the address `0.0.0.0`. When `kube-vip` sees a Service on this address, it will create a `macvlan` interface on the host and request a DHCP address. Once this address is provided, it will assign it as the `LoadBalancer` IP and update the Kubernetes Service.
-
-```
-$ kubectl expose deployment nginx-deployment --port=80 --type=LoadBalancer --name=nginx-dhcp --load-balancer-ip=0.0.0.0; kubectl get svc
-service/nginx-dhcp exposed
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   0.0.0.0         80:31184/TCP   0s
-
-{ ... a second or so later ... }
-
-$ kubectl get svc
-NAME         TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)        AGE
-kubernetes   ClusterIP      10.96.0.1       <none>          443/TCP        17m
-nginx-dhcp   LoadBalancer   10.97.150.208   192.168.0.155   80:31184/TCP   3s
-```
-
-### Using UPnP to expose a Service to the outside world
-
-With `kube-vip` > 0.2.1, it is possible to expose a Service of type `LoadBalancer` on a specific port to the Internet by using UPnP (on a supported gateway).
-
-Most simple networks look something like the following:
-
-`<----- <internal network 192.168.0.0/24> <Gateway / router> <external network address> ----> Internet`
-
-Using UPnP we can create a matching port on the `<external network address>` allowing your Service to be exposed to the Internet.
-
-#### Enable UPnP
-
-Add the following to the `kube-vip` `env:` section of either the static Pod or DaemonSet for `kube-vip`, and the rest should be completely automated.
-
-**Note** some environments may require (Unifi) `Secure mode` being `disabled` (this allows a host with a different address to register a port).
-
-```
-- name: enableUPNP
-  value: "true"
-```
-
-#### Exposing a Service
-
-To expose a port successfully, we'll need to change the command slightly:
-
-`--target-port=80` the port of the application in the pods (HTT/NGINX)
-`--port=32380` the port the Service will be exposed on (and what you should connect to in order to receive traffic from the Service)
-
-`kubectl expose deployment plunder-nginx --port=32380 --target-port=80 --type=LoadBalancer --namespace plunder`
-
-The above example should expose a port on your external (Internet facing) address that can be tested externally with:
-
-```
-$ curl externalIP:32380
-<!DOCTYPE html>
-<html>
-...
-```

example/deployment.yaml

@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:latest
+        ports:
+        - containerPort: 80

example/service-interface-annotation.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-interface-ens192-service
+  annotations:
+    kube-vip.io/serviceInterface: ens192
+spec:
+  selector:
+    app: nginx
+  type: LoadBalancer
+  ports:
+  - name: http
+    port: 80
+    targetPort: 80
+    protocol: TCP

example/service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-service
+spec:
+  selector:
+    app: nginx
+  type: LoadBalancer
+  ports:
+  - name: http
+    port: 80
+    targetPort: 80
+    protocol: TCP

go.mod

@@ -1,36 +1,165 @@
 module github.com/kube-vip/kube-vip
 
-go 1.14
+go 1.24.4
 
 require (
-	github.com/cloudflare/ipvs v0.8.0
-	github.com/davecgh/go-spew v1.1.1
-	github.com/ghodss/yaml v1.0.0
-	github.com/golang/protobuf v1.5.2
-	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
-	github.com/insomniacslk/dhcp v0.0.0-20211214070828-5297eed8f489
+	github.com/cloudflare/ipvs v0.11.0
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
+	github.com/docker/docker v28.5.2+incompatible
+	github.com/florianl/go-conntrack v0.4.0
+	github.com/google/go-cmp v0.7.0
+	github.com/google/go-containerregistry v0.20.6
+	github.com/google/nftables v0.3.0
+	github.com/gookit/slog v0.6.0
+	github.com/huin/goupnp v1.3.0
+	github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d
 	github.com/jpillora/backoff v1.0.0
-	github.com/kamhlos/upnp v0.0.0-20210324072331-5661950dff08
-	github.com/mdlayher/ndp v0.0.0-20210831201139-f982b8766fb5
-	github.com/mdlayher/raw v0.0.0-20211126142749-4eae47f3d54b // indirect
-	github.com/onsi/ginkgo v1.14.2
-	github.com/onsi/gomega v1.10.1
-	github.com/osrg/gobgp v0.0.0-20210901090119-8ab84f8636ee
-	github.com/packethost/packngo v0.20.0
+	github.com/mdlayher/ndp v1.1.0
+	github.com/onsi/ginkgo/v2 v2.27.2
+	github.com/onsi/gomega v1.38.2
+	github.com/osrg/gobgp/v3 v3.37.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.11.0
-	github.com/sirupsen/logrus v1.8.1
-	github.com/spf13/cobra v1.3.0
-	github.com/stretchr/testify v1.7.0
-	github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5
-	github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
-	golang.org/x/net v0.0.0-20211216030914-fe4d6282115f
-	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
-	k8s.io/api v0.22.5
-	k8s.io/apimachinery v0.22.5
-	k8s.io/client-go v0.22.5
-	k8s.io/klog/v2 v2.40.1
-	sigs.k8s.io/kind v0.11.1
+	github.com/prometheus/client_golang v1.23.2
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cobra v1.10.1
+	github.com/stretchr/testify v1.11.1
+	github.com/vishvananda/netlink v1.3.1
+	go.etcd.io/etcd/api/v3 v3.6.6
+	go.etcd.io/etcd/client/pkg/v3 v3.6.6
+	go.etcd.io/etcd/client/v3 v3.6.6
+	go.uber.org/zap v1.27.1
+	golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
+	golang.org/x/sync v0.18.0
+	golang.org/x/sys v0.38.0
+	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
+	google.golang.org/grpc v1.77.0
+	google.golang.org/protobuf v1.36.10
+	k8s.io/api v0.34.2
+	k8s.io/apimachinery v0.34.2
+	k8s.io/client-go v0.34.1
+	k8s.io/klog/v2 v2.130.1
+	sigs.k8s.io/kind v0.30.0
+	sigs.k8s.io/yaml v1.6.0
+)
+
+require (
+	al.essio.dev/pkg/shellescape v1.5.1 // indirect
+	github.com/BurntSushi/toml v1.4.0 // indirect
+	github.com/Masterminds/semver/v3 v3.4.0 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/coreos/go-semver v0.3.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/dlclark/regexp2 v1.11.4 // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/eapache/channels v1.1.0 // indirect
+	github.com/eapache/queue v1.1.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.12.2 // indirect
+	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/gnostic-models v0.7.0 // indirect
+	github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gookit/color v1.6.0 // indirect
+	github.com/gookit/goutil v0.7.1 // indirect
+	github.com/gookit/gsr v0.1.1 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/josharian/native v1.1.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/k-sone/critbitgo v1.4.0 // indirect
+	github.com/magiconair/properties v1.8.9 // indirect
+	github.com/mailru/easyjson v0.9.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mdlayher/genetlink v1.3.2 // indirect
+	github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect
+	github.com/mdlayher/packet v1.1.2 // indirect
+	github.com/mdlayher/socket v0.5.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
+	github.com/moby/term v0.5.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/pelletier/go-toml v1.9.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.66.1 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/sagikazarmark/locafero v0.6.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/spf13/pflag v1.0.9 // indirect
+	github.com/spf13/viper v1.19.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tj/go-spin v1.1.0 // indirect
+	github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/vishvananda/netns v0.0.5 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xlab/c-for-go v1.3.0 // indirect
+	github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
+	go.opentelemetry.io/otel v1.38.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 // indirect
+	go.opentelemetry.io/otel/metric v1.38.0 // indirect
+	go.opentelemetry.io/otel/trace v1.38.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/crypto v0.45.0 // indirect
+	golang.org/x/mod v0.29.0 // indirect
+	golang.org/x/net v0.47.0 // indirect
+	golang.org/x/oauth2 v0.32.0 // indirect
+	golang.org/x/term v0.37.0 // indirect
+	golang.org/x/text v0.31.0 // indirect
+	golang.org/x/time v0.9.0 // indirect
+	golang.org/x/tools v0.38.0 // indirect
+	golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
+	k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect
+	modernc.org/cc/v4 v4.24.4 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/opt v0.1.4 // indirect
+	modernc.org/sortutil v1.2.1 // indirect
+	modernc.org/strutil v1.2.1 // indirect
+	modernc.org/token v1.1.0 // indirect
+	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
 )

main.go

@@ -11,6 +11,7 @@ var Version string
 var Build string
 
 func main() {
+
 	cmd.Release.Version = Version
 	cmd.Release.Build = Build
 	cmd.Execute()

pkg/arp/arp.go

@@ -0,0 +1,207 @@
+package arp
+
+import (
+	"context"
+	"fmt"
+	log "log/slog"
+	"sync"
+	"time"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/kube-vip/kube-vip/pkg/vip"
+)
+
+type Manager struct {
+	instances sync.Map
+	config    *kubevip.Config
+}
+
+type Instance struct {
+	network vip.Network
+	ndp     *vip.NdpResponder
+	mu      sync.Mutex
+	counter int
+}
+
+func NewManager(config *kubevip.Config) *Manager {
+	return &Manager{
+		config: config,
+	}
+}
+
+func NewInstance(network vip.Network, ndp *vip.NdpResponder) *Instance {
+	return &Instance{
+		ndp:     ndp,
+		network: network,
+		counter: 1,
+	}
+}
+
+func (i *Instance) Name() string {
+	return i.network.ARPName()
+}
+
+func (m *Manager) Insert(instance *Instance) {
+	i, err := m.get(instance.Name())
+	if err != nil {
+		log.Error("[ARP manager] unable to insert instance", "err", err)
+		return
+	}
+	if i == nil {
+		log.Info("[ARP manager] inserting ARP/NDP instance", "name", instance.Name())
+		m.instances.Store(instance.Name(), instance)
+	} else {
+		i.mu.Lock()
+		defer i.mu.Unlock()
+		i.counter++
+	}
+}
+
+func (m *Manager) Remove(instance *Instance) {
+	m.RemoveWithIPDelete(instance, true)
+}
+
+// RemoveOnLeadershipLoss removes an ARP instance when leadership is lost
+func (m *Manager) RemoveOnLeadershipLoss(instance *Instance) {
+	// Use the inverse of PreserveVIPOnLeadershipLoss to decide whether to delete the IP
+	// If preserve is true, don't delete IP (deleteIP = false)
+	// If preserve is false, delete IP (deleteIP = true), This is the legacy behavior
+	deleteIP := !m.config.PreserveVIPOnLeadershipLoss
+	m.RemoveWithIPDelete(instance, deleteIP)
+}
+
+func (m *Manager) RemoveWithIPDelete(instance *Instance, deleteIP bool) {
+	i, err := m.get(instance.Name())
+	if err != nil {
+		log.Error("[ARP manager] unable to remove the instance", "err", err)
+		return
+	}
+	if i != nil {
+		i.mu.Lock()
+		defer i.mu.Unlock()
+		if i.counter > 1 {
+			i.counter--
+		} else {
+			log.Info("[ARP manager] removing ARP/NDP instance", "name", instance.Name())
+			if deleteIP {
+				if _, err := instance.network.DeleteIP(); err != nil {
+					log.Error("failed to delete IP", "address", instance.network.IP(), "err", err)
+				}
+			}
+			m.instances.Delete(instance.Name())
+		}
+	} else {
+		log.Warn("[ARP manager] unable to remove the instance - instance not found", "name", instance.Name())
+	}
+}
+
+func (m *Manager) Count(name string) int {
+	i, err := m.get(name)
+	if err != nil {
+		log.Error("[ARP manager] unable to count instance", "err", err)
+		return -1
+	}
+	if i != nil {
+		i.mu.Lock()
+		defer i.mu.Unlock()
+		return i.counter
+	}
+	return 0
+}
+
+func (m *Manager) StartAdvertisement(ctx context.Context) {
+	log.Info("[ARP manager] starting ARP/NDP advertisement")
+	for {
+		select {
+		case <-ctx.Done(): // if cancel() execute
+			return
+		default:
+			m.instances.Range(func(_ any, instance any) bool {
+				if i, ok := instance.(*Instance); ok {
+					if i.counter > 0 {
+						ensureIPAndSendGratuitous(i)
+					} else {
+						// this instance should not be advertised - delete the IP just in case...
+						if _, err := i.network.DeleteIP(); err != nil {
+							log.Error("[ARP manager] failed to delete IP", "address", i.network.IP(), "err", err)
+						}
+					}
+				}
+				return true
+			})
+		}
+		if m.config.ArpBroadcastRate < 500 {
+			log.Warn("[ARP manager] arp broadcast rate is too low", "rate (ms)", m.config.ArpBroadcastRate, "setting to (ms)", "3000")
+			m.config.ArpBroadcastRate = 3000
+		}
+		time.Sleep(time.Duration(m.config.ArpBroadcastRate) * time.Millisecond)
+	}
+}
+
+func (m *Manager) get(name string) (*Instance, error) {
+	i, exists := m.instances.Load(name)
+	if !exists {
+		return nil, nil
+	}
+	inst, ok := i.(*Instance)
+	if !ok {
+		return nil, fmt.Errorf("value for name %q is not of Instance pointer type", name)
+	}
+	return inst, nil
+}
+
+// ensureIPAndSendGratuitous - adds IP to the interface if missing, and send
+// either a gratuitous ARP or gratuitous NDP. Re-adds the interface if it is IPv6
+// and in a dadfailed state.
+func ensureIPAndSendGratuitous(instance *Instance) {
+	iface := instance.network.Interface()
+	ipString := instance.network.IP()
+
+	// Check if IP is dadfailed
+	if instance.network.IsDADFAIL() {
+		log.Warn("IP address is in dadfailed state, removing config", "ip", ipString, "interface", iface)
+		deleted, err := instance.network.DeleteIP()
+		if err != nil {
+			log.Warn(err.Error())
+		}
+		if deleted {
+			log.Info("deleted and recreating address with NODAD flag to skip DAD", "IP", ipString, "interface", iface)
+			// Re-add immediately without DAD check since we're recovering from DADFAILED
+			// The AddIP function will set IFA_F_NODAD flag for IPv6 addresses when skipDAD=true
+			if _, err := instance.network.AddIP(false, true); err != nil {
+				log.Error("failed to recreate address after DADFAILED", "IP", ipString, "interface", iface, "err", err)
+			} else {
+				log.Info("successfully recreated address after DADFAILED recovery", "IP", ipString, "interface", iface)
+			}
+		}
+		// Return early after DADFAILED recovery to avoid double IP addition
+		return
+	}
+
+	// Normal case: add IP with precheck and normal DAD process
+	if added, err := instance.network.AddIP(true, false); err != nil {
+		log.Warn(err.Error())
+	} else if added {
+		log.Warn("Re-applied the VIP configuration", "ip", ipString, "interface", iface)
+	}
+
+	if utils.IsIPv6(ipString) {
+		// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
+		if instance.ndp == nil {
+			log.Error("NDP responder was not created")
+		} else {
+			err := instance.ndp.SendGratuitous(ipString)
+			if err != nil {
+				log.Warn(err.Error())
+			}
+		}
+
+	} else {
+		// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
+		err := vip.ARPSendGratuitous(ipString, iface)
+		if err != nil {
+			log.Warn(err.Error())
+		}
+	}
+}

pkg/backend/backend.go

@@ -0,0 +1,87 @@
+package backend
+
+import (
+	"fmt"
+	"time"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/k8s"
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+type Entry struct {
+	Addr    string
+	Port    uint16
+	IsLocal bool
+}
+
+type Map map[Entry]bool
+
+func (e *Entry) Check() bool {
+	var client *kubernetes.Clientset
+	var err error
+	var config *rest.Config
+
+	adminConfigPath := "/etc/kubernetes/admin.conf"
+	// TODO: add one more switch case of homeConfigPath if there is such scenario in future
+	// homeConfigPath := filepath.Join(os.Getenv("HOME"), ".kube", "config")
+
+	var k8sAddr string
+	if utils.IsIPv4(e.Addr) {
+		k8sAddr = fmt.Sprintf("%s:%v", e.Addr, e.Port)
+	} else {
+		k8sAddr = fmt.Sprintf("[%s]:%v", e.Addr, e.Port)
+	}
+
+	switch {
+	case utils.FileExists(adminConfigPath):
+		config, err = k8s.NewRestConfig(adminConfigPath, false, k8sAddr)
+		if err != nil {
+			log.Error("create k8s REST config", "path", adminConfigPath, "err", err)
+			return false
+		}
+	default:
+		config, err = k8s.NewRestConfig("", true, k8sAddr)
+		if err != nil {
+			log.Error("create k8s REST config", "err", err)
+			return false
+		}
+	}
+
+	client, err = k8s.NewClientset(config)
+	if err != nil {
+		log.Error("create k8s client", "err", err)
+		return false
+	}
+
+	_, err = client.DiscoveryClient.ServerVersion()
+	if err != nil {
+		log.Error("discover k8s version", "err", err)
+		return false
+	}
+	return true
+}
+
+func Watch(tickAction func(), interval int, stop chan struct{}) {
+	if interval <= 0 {
+		interval = 5
+	}
+
+	ticker := time.NewTicker(time.Second * time.Duration(interval))
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-stop:
+			ticker.Stop()
+			return
+		case <-ticker.C:
+			ticker.Stop()
+			tickAction()
+			ticker.Reset(time.Second * time.Duration(interval))
+		}
+	}
+}

pkg/bgp/hosts.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"net"
 
-	api "github.com/osrg/gobgp/api"
+	api "github.com/osrg/gobgp/v3/api"
 )
 
 // AddHost will update peers of a host

pkg/bgp/peers.go

@@ -4,36 +4,31 @@ import (
 	"context"
 	"fmt"
 	"net"
-	"strconv"
-	"strings"
 
-	"github.com/golang/protobuf/ptypes" //nolint
-	"github.com/golang/protobuf/ptypes/any"
-	api "github.com/osrg/gobgp/api"
+	//nolint
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	api "github.com/osrg/gobgp/v3/api"
+
+	"github.com/kube-vip/kube-vip/pkg/utils"
+	"github.com/osrg/gobgp/v3/pkg/server"
+	"google.golang.org/protobuf/types/known/anypb"
 )
 
-//  AddPeer will add peers to the BGP configuration
-func (b *Server) AddPeer(peer Peer) (err error) {
-	port := 179
-
-	if t := strings.SplitN(peer.Address, ":", 2); len(t) == 2 {
-		peer.Address = t[0]
-
-		if port, err = strconv.Atoi(t[1]); err != nil {
-			return fmt.Errorf("unable to parse port '%s' as int: %s", t[1], err)
-		}
-	}
-
+// AddPeer will add peers to the BGP configuration
+func (b *Server) AddPeer(peer kubevip.BGPPeer) (err error) {
 	p := &api.Peer{
 		Conf: &api.PeerConf{
 			NeighborAddress: peer.Address,
-			PeerAs:          peer.AS,
+			PeerAsn:         peer.AS,
 			AuthPassword:    peer.Password,
 		},
 
 		Timers: &api.Timers{
 			Config: &api.TimersConfig{
-				ConnectRetry: 10,
+				ConnectRetry:      10,
+				HoldTime:          b.c.HoldTime,
+				KeepaliveInterval: b.c.KeepaliveInterval,
 			},
 		},
 
@@ -46,98 +41,204 @@ func (b *Server) AddPeer(peer Peer) (err error) {
 		Transport: &api.Transport{
 			MtuDiscovery:  true,
 			RemoteAddress: peer.Address,
-			RemotePort:    uint32(port),
+			RemotePort:    uint32(179),
 		},
 	}
 
-	if b.c.SourceIP != "" {
-		p.Transport.LocalAddress = b.c.SourceIP
-	}
-
-	if b.c.SourceIF != "" {
-		p.Transport.BindInterface = b.c.SourceIF
-	}
-
-	return b.s.AddPeer(context.Background(), &api.AddPeerRequest{
-		Peer: p,
-	})
-}
-
-func (b *Server) getPath(ip net.IP) *api.Path {
-	var pfxLen uint32 = 32
-	if ip.To4() == nil {
-		if !b.c.IPv6 {
-			return nil
+	if b.c.MpbgpNexthop != "" {
+		p.AfiSafis = []*api.AfiSafi{
+			{
+				Config: &api.AfiSafiConfig{
+					Family: &api.Family{
+						Afi:  api.Family_AFI_IP,
+						Safi: api.Family_SAFI_UNICAST,
+					},
+					Enabled: true,
+				},
+			},
+			{
+				Config: &api.AfiSafiConfig{
+					Family: &api.Family{
+						Afi:  api.Family_AFI_IP6,
+						Safi: api.Family_SAFI_UNICAST,
+					},
+					Enabled: true,
+				},
+			},
 		}
 
-		pfxLen = 128
+		peer.SetMpbgpOptions(b.c)
+
+		ipv4Address, ipv6Address, err := peer.FindMpbgpAddresses(p, b.c)
+		if err != nil {
+			return fmt.Errorf("failed to get MP-BGP addresses: %w", err)
+		}
+
+		mask := "128"
+		address := ipv4Address
+		family := api.Family_AFI_IP
+		if utils.IsIPv4(p.Conf.NeighborAddress) {
+			mask = "32"
+			address = ipv6Address
+			family = api.Family_AFI_IP6
+		}
+
+		err = b.s.AddDefinedSet(context.Background(), &api.AddDefinedSetRequest{
+			DefinedSet: &api.DefinedSet{
+				DefinedType: api.DefinedType_NEIGHBOR,
+				Name:        fmt.Sprintf("peer-%s", p.Conf.NeighborAddress),
+				List:        []string{fmt.Sprintf("%s/%s", p.Conf.NeighborAddress, mask)},
+			},
+		})
+		if err != nil {
+			return fmt.Errorf("failed to add defined set: %v", err)
+		}
+
+		if address != "" {
+			if err := insertPolicy(b.s, address, p, family); err != nil {
+				return fmt.Errorf("failed to add policy: %w", err)
+			}
+		}
+	} else {
+		if b.c.SourceIP != "" {
+			p.Transport.LocalAddress = b.c.SourceIP
+		}
+
+		if b.c.SourceIF != "" {
+			p.Transport.BindInterface = b.c.SourceIF
+		}
 	}
 
-	//nolint
-	nlri, _ := ptypes.MarshalAny(&api.IPAddressPrefix{
-		Prefix:    ip.String(),
-		PrefixLen: pfxLen,
-	})
+	if err := b.s.AddPeer(context.Background(), &api.AddPeerRequest{Peer: p}); err != nil {
+		return fmt.Errorf("failed to add peer: %v", err)
+	}
+
+	return nil
+}
+
+func (b *Server) getPath(ip net.IP) (path *api.Path) {
+	isV6 := ip.To4() == nil
 
 	//nolint
-	a1, _ := ptypes.MarshalAny(&api.OriginAttribute{
+	originAttr, _ := anypb.New(&api.OriginAttribute{
 		Origin: 0,
 	})
 
-	var nh string
-	if b.c.NextHop != "" {
-		nh = b.c.NextHop
-	} else if b.c.SourceIP != "" {
-		nh = b.c.SourceIP
+	if !isV6 {
+		//nolint
+		nlri, _ := anypb.New(&api.IPAddressPrefix{
+			Prefix:    ip.String(),
+			PrefixLen: 32,
+		})
+
+		//nolint
+		nhAttr, _ := anypb.New(&api.NextHopAttribute{
+			NextHop: "0.0.0.0", // gobgp will fill this
+		})
+
+		path = &api.Path{
+			Family: &api.Family{
+				Afi:  api.Family_AFI_IP,
+				Safi: api.Family_SAFI_UNICAST,
+			},
+			Nlri:   nlri,
+			Pattrs: []*anypb.Any{originAttr, nhAttr},
+		}
 	} else {
-		nh = b.c.RouterID
-	}
+		//nolint
+		nlri, _ := anypb.New(&api.IPAddressPrefix{
+			Prefix:    ip.String(),
+			PrefixLen: 128,
+		})
 
-	//nolint
-	a2, _ := ptypes.MarshalAny(&api.NextHopAttribute{
-		NextHop: nh,
-	})
-
-	return &api.Path{
-		Family: &api.Family{
-			Afi:  api.Family_AFI_IP,
+		v6Family := &api.Family{
+			Afi:  api.Family_AFI_IP6,
 			Safi: api.Family_SAFI_UNICAST,
-		},
-		Nlri:   nlri,
-		Pattrs: []*any.Any{a1, a2},
-	}
-}
-
-// ParseBGPPeerConfig - take a string and parses it into an array of peers
-func ParseBGPPeerConfig(config string) (bgpPeers []Peer, err error) {
-	peers := strings.Split(config, ",")
-	if len(peers) == 0 {
-		return nil, fmt.Errorf("No BGP Peer configurations found")
-	}
-
-	for x := range peers {
-		peer := strings.Split(peers[x], ":")
-		if len(peer) != 4 {
-			return nil, fmt.Errorf("BGP Peer configuration format error <host>:<AS>:<password>:<multihop>")
-		}
-		ASNumber, err := strconv.Atoi(peer[1])
-		if err != nil {
-			return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
-
-		}
-		multiHop, err := strconv.ParseBool(peer[3])
-		if err != nil {
-			return nil, fmt.Errorf("BGP MultiHop format error (true/false) [%s]", peer[1])
 		}
 
-		peerConfig := Peer{
-			Address:  peer[0],
-			AS:       uint32(ASNumber),
-			Password: peer[2],
-			MultiHop: multiHop,
-		}
+		//nolint
+		mpAttr, _ := anypb.New(&api.MpReachNLRIAttribute{
+			Family:   v6Family,
+			NextHops: []string{"::"}, // gobgp will fill this
+			Nlris:    []*anypb.Any{nlri},
+		})
 
-		bgpPeers = append(bgpPeers, peerConfig)
+		path = &api.Path{
+			Family: v6Family,
+			Nlri:   nlri,
+			Pattrs: []*anypb.Any{originAttr, mpAttr},
+		}
 	}
 	return
 }
+
+func insertPolicy(s *server.BgpServer, address string, p *api.Peer, family api.Family_Afi) error {
+	familyType := "v4"
+	if family == api.Family_AFI_IP6 {
+		familyType = "v6"
+	}
+
+	setName := fmt.Sprintf("peer-%s", p.Conf.NeighborAddress)
+	policyName := fmt.Sprintf("%s-%s", setName, familyType)
+
+	policy := &api.Policy{
+		Name: policyName,
+		Statements: []*api.Statement{
+			{
+				Conditions: &api.Conditions{
+					AfiSafiIn: []*api.Family{
+						{
+							Afi:  family,
+							Safi: api.Family_SAFI_UNICAST,
+						},
+					},
+					NeighborSet: &api.MatchSet{
+						Type: api.MatchSet_ANY,
+						Name: setName,
+					},
+				},
+				Actions: &api.Actions{
+					RouteAction: api.RouteAction_ACCEPT,
+					Nexthop: &api.NexthopAction{
+						Address: address,
+					},
+				},
+			},
+			{
+				Conditions: &api.Conditions{
+					NeighborSet: &api.MatchSet{
+						Type: api.MatchSet_ANY,
+						Name: setName,
+					},
+				},
+				Actions: &api.Actions{
+					RouteAction: api.RouteAction_ACCEPT,
+				},
+			},
+		},
+	}
+
+	err := s.AddPolicy(context.Background(), &api.AddPolicyRequest{
+		Policy: policy,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to add policy: %w", err)
+	}
+
+	err = s.AddPolicyAssignment(context.Background(), &api.AddPolicyAssignmentRequest{
+		Assignment: &api.PolicyAssignment{
+			Name:      "global",
+			Direction: api.PolicyDirection_EXPORT,
+			Policies: []*api.Policy{
+				{
+					Name: policy.Name,
+				},
+			},
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("failed to add policy assignment: %v", err)
+	}
+
+	return nil
+}

pkg/bgp/peers_test.go

@@ -0,0 +1,56 @@
+package bgp
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+)
+
+func TestParseBGPPeerConfig(t *testing.T) {
+	type args struct {
+		config string
+	}
+	tests := []struct {
+		name         string
+		args         args
+		wantBgpPeers []kubevip.BGPPeer
+		wantErr      bool
+	}{
+		{
+			name: "IPv4, default port",
+			args: args{config: "192.168.0.10:65000::false,192.168.0.11:65000::false"},
+			wantBgpPeers: []kubevip.BGPPeer{
+				{Address: "192.168.0.10", Port: 179, AS: 65000, MultiHop: false},
+				{Address: "192.168.0.11", Port: 179, AS: 65000, MultiHop: false},
+			},
+		},
+		{
+			name: "IPv4, different port",
+			args: args{config: "192.168.0.10:65000::false:180,192.168.0.11:65000::false:190"},
+			wantBgpPeers: []kubevip.BGPPeer{
+				{Address: "192.168.0.10", Port: 180, AS: 65000, MultiHop: false},
+				{Address: "192.168.0.11", Port: 190, AS: 65000, MultiHop: false},
+			},
+		},
+		{
+			name: "IPv6, multi-protocol",
+			args: args{config: "[fd00:1111:2222:3333:c7d9:7235:6bf7:5d52]:65501::false/mpbgp_nexthop=auto_sourceif"},
+			wantBgpPeers: []kubevip.BGPPeer{
+				{Address: "fd00:1111:2222:3333:c7d9:7235:6bf7:5d52", Port: 179, AS: 65501, MultiHop: false, MpbgpNexthop: "auto_sourceif"},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotBgpPeers, err := kubevip.ParseBGPPeerConfig(tt.args.config)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseBGPPeerConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(gotBgpPeers, tt.wantBgpPeers) {
+				t.Errorf("ParseBGPPeerConfig() = %v, want %v", gotBgpPeers, tt.wantBgpPeers)
+			}
+		})
+	}
+}

pkg/bgp/server.go

@@ -3,53 +3,96 @@ package bgp
 import (
 	"context"
 	"fmt"
-	"log"
 	"time"
 
-	api "github.com/osrg/gobgp/api"
-	gobgp "github.com/osrg/gobgp/pkg/server"
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	api "github.com/osrg/gobgp/v3/api"
+	gobgp "github.com/osrg/gobgp/v3/pkg/server"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
+// Server manages a server object
+type Server struct {
+	s *gobgp.BgpServer
+	c *kubevip.BGPConfig
+
+	// This is a prometheus gauge indicating the state of the sessions.
+	// 1 means "ESTABLISHED", 0 means "NOT ESTABLISHED"
+	BGPSessionInfoGauge *prometheus.GaugeVec
+}
+
 // NewBGPServer takes a configuration and returns a running BGP server instance
-func NewBGPServer(c *Config) (b *Server, err error) {
+func NewBGPServer(c kubevip.BGPConfig) (b *Server, err error) {
 	if c.AS == 0 {
-		return nil, fmt.Errorf("You need to provide AS")
+		return nil, fmt.Errorf("you need to provide AS")
 	}
 
 	if c.SourceIP != "" && c.SourceIF != "" {
-		return nil, fmt.Errorf("SourceIP and SourceIF are mutually exclusive")
+		return nil, fmt.Errorf("sourceIP and SourceIF are mutually exclusive")
 	}
 
 	if len(c.Peers) == 0 {
-		return nil, fmt.Errorf("You need to provide at least one peer")
+		return nil, fmt.Errorf("you need to provide at least one peer")
 	}
 
 	b = &Server{
 		s: gobgp.NewBgpServer(),
-		c: c,
+		c: &c,
+
+		BGPSessionInfoGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "kube_vip",
+			Subsystem: "manager",
+			Name:      "bgp_session_info",
+			Help:      "Display state of session by setting metric for label value with current state to 1",
+		}, []string{"state", "peer"}),
 	}
+	return
+}
+
+// Start starts the BGP server
+func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (err error) {
 	go b.s.Serve()
 
 	if err = b.s.StartBgp(context.Background(), &api.StartBgpRequest{
 		Global: &api.Global{
-			As:         c.AS,
-			RouterId:   c.RouterID,
+			Asn:        b.c.AS,
+			RouterId:   b.c.RouterID,
 			ListenPort: -1,
 		},
 	}); err != nil {
 		return
 	}
 
-	if err = b.s.MonitorPeer(context.Background(), &api.MonitorPeerRequest{}, func(p *api.Peer) { log.Println(p) }); err != nil {
+	if err = b.s.WatchEvent(context.Background(), &api.WatchEventRequest{Peer: &api.WatchEventRequest_Peer{}}, func(r *api.WatchEventResponse) {
+		if p := r.GetPeer(); p != nil && p.Type == api.WatchEventResponse_PeerEvent_STATE {
+			log.Info("[BGP]", "peer", p.String())
+			if peerStateChangeCallback != nil {
+				peerStateChangeCallback(p)
+			}
+		}
+	}); err != nil {
 		return
 	}
 
-	for _, p := range c.Peers {
+	for _, p := range b.c.Peers {
 		if err = b.AddPeer(p); err != nil {
 			return
 		}
 	}
 
+	if b.c.Zebra.Enabled {
+		if err = b.s.EnableZebra(context.Background(), &api.EnableZebraRequest{
+			Url:          b.c.Zebra.URL,
+			Version:      b.c.Zebra.Version,
+			SoftwareName: b.c.Zebra.SoftwareName,
+		}); err != nil {
+			log.Error(err.Error())
+			return
+		}
+	}
+
 	return
 }
 

pkg/bgp/types.go

@@ -1,29 +0,0 @@
-package bgp
-
-import gobgp "github.com/osrg/gobgp/pkg/server"
-
-// Peer defines a BGP Peer
-type Peer struct {
-	Address  string
-	AS       uint32
-	Password string
-	MultiHop bool
-}
-
-// Config defines the BGP server configuration
-type Config struct {
-	AS       uint32
-	RouterID string
-	NextHop  string
-	SourceIP string
-	SourceIF string
-
-	Peers []Peer
-	IPv6  bool
-}
-
-// Server manages a server object
-type Server struct {
-	s *gobgp.BgpServer
-	c *Config
-}

pkg/cluster/cluster.go

@@ -1,61 +1,80 @@
 package cluster
 
 import (
+	"sync"
+
+	log "log/slog"
+
+	"github.com/kube-vip/kube-vip/pkg/arp"
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
+	"github.com/kube-vip/kube-vip/pkg/networkinterface"
 	"github.com/kube-vip/kube-vip/pkg/vip"
-	log "github.com/sirupsen/logrus"
 )
 
 // Cluster - The Cluster object manages the state of the cluster for a particular node
 type Cluster struct {
 	stop      chan bool
 	completed chan bool
-	Network   vip.Network
+	once      sync.Once
+	Network   []vip.Network
+	arpMgr    *arp.Manager
 }
 
 // InitCluster - Will attempt to initialise all of the required settings for the cluster
-func InitCluster(c *kubevip.Config, disableVIP bool) (*Cluster, error) {
-
-	// TODO - Check for root (needed to netlink)
-	var network vip.Network
+func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Cluster, error) {
+	var networks []vip.Network
 	var err error
 
 	if !disableVIP {
 		// Start the Virtual IP Networking configuration
-		network, err = startNetworking(c)
+		networks, err = startNetworking(c, intfMgr)
 		if err != nil {
 			return nil, err
 		}
 	}
 	// Initialise the Cluster structure
 	newCluster := &Cluster{
-		Network: network,
+		Network: networks,
+		arpMgr:  arpMgr,
 	}
 
+	log.Debug("service security", "enabled", c.EnableServiceSecurity)
+
 	return newCluster, nil
 }
 
-func startNetworking(c *kubevip.Config) (vip.Network, error) {
+func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vip.Network, error) {
 	address := c.VIP
 
 	if c.Address != "" {
 		address = c.Address
 	}
 
-	network, err := vip.NewConfig(address, c.Interface, c.DDNS)
-	if err != nil {
-		return nil, err
+	addresses := vip.Split(address)
+
+	networks := []vip.Network{}
+	for _, addr := range addresses {
+		network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.RoutingTableID,
+			c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend,
+			c.EnableLoadBalancer, c.EnableServiceSecurity, intfMgr)
+		if err != nil {
+			return nil, err
+		}
+		networks = append(networks, network...)
 	}
-	return network, nil
+
+	return networks, nil
 }
 
 // Stop - Will stop the Cluster and release VIP if needed
 func (cluster *Cluster) Stop() {
-	// Close the stop chanel, which will shut down the VIP (if needed)
-	close(cluster.stop)
+	// Close the stop channel, which will shut down the VIP (if needed)
+	if cluster.stop != nil {
+		cluster.once.Do(func() { // Ensure that the close channel can only ever be called once
+			close(cluster.stop)
+		})
+	}
 
 	// Wait until the completed channel is closed, signallign all shutdown tasks completed
 	<-cluster.completed
-
-	log.Info("Stopped")
 }

pkg/cluster/clusterDDNS.go

@@ -12,14 +12,13 @@ import (
 // during runtime if IP changes, startDDNS don't have to do reconfigure because
 // dnsUpdater already have the functionality to keep trying resolve the IP
 // and update the VIP configuration if it changes
-func (cluster *Cluster) StartDDNS(ctx context.Context) error {
-	ddnsMgr := vip.NewDDNSManager(ctx, cluster.Network)
+func (cluster *Cluster) StartDDNS(ctx context.Context, network vip.Network) error {
+	ddnsMgr := vip.NewDDNSManager(ctx, network)
 	ip, err := ddnsMgr.Start()
 	if err != nil {
 		return err
 	}
-
-	if err = cluster.Network.SetIP(ip); err != nil {
+	if err = network.SetIP(ip); err != nil {
 		return err
 	}
 

pkg/cluster/clusterLeaderElection.go

@@ -11,14 +11,15 @@ import (
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/kube-vip/kube-vip/pkg/bgp"
+	"github.com/kube-vip/kube-vip/pkg/etcd"
 	"github.com/kube-vip/kube-vip/pkg/k8s"
 	"github.com/kube-vip/kube-vip/pkg/kubevip"
 	"github.com/kube-vip/kube-vip/pkg/loadbalancer"
-	"github.com/kube-vip/kube-vip/pkg/packet"
+	"github.com/kube-vip/kube-vip/pkg/utils"
 
-	"github.com/packethost/packngo"
+	log "log/slog"
 
-	log "github.com/sirupsen/logrus"
+	clientv3 "go.etcd.io/etcd/client/v3"
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -30,21 +31,27 @@ import (
 	watchtools "k8s.io/client-go/tools/watch"
 )
 
-const plunderLock = "plndr-cp-lock"
-
 // Manager degines the manager of the load-balancing services
 type Manager struct {
-	KubernetesClient *kubernetes.Clientset
+	KubernetesClient   *kubernetes.Clientset
+	RetryWatcherClient *kubernetes.Clientset
 	// This channel is used to signal a shutdown
 	SignalChan chan os.Signal
+
+	EtcdClient *clientv3.Client
 }
 
 // NewManager will create a new managing object
 func NewManager(path string, inCluster bool, port int) (*Manager, error) {
 	var hostname string
 
-	// If the path passed is empty and not running in the cluster,
+	// If inCluster is set then it will likely have started as a static pod or won't have the
+	// VIP up before trying to connect to the API server, we set the API endpoint to this machine to
+	// ensure connectivity. Else if the path passed is empty and not running in the cluster,
 	// attempt to look for a kubeconfig in the default HOME dir.
+
+	hostname = fmt.Sprintf("kubernetes:%v", port)
+
 	if len(path) == 0 && !inCluster {
 		path = filepath.Join(os.Getenv("HOME"), ".kube", "config")
 
@@ -57,38 +64,38 @@ func NewManager(path string, inCluster bool, port int) (*Manager, error) {
 		hostname = fmt.Sprintf("%s:%v", id, port)
 	}
 
-	clientset, err := k8s.NewClientset(path, inCluster, hostname)
+	config, err := k8s.NewRestConfig(path, inCluster, hostname)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create k8s REST config: %w", err)
+	}
+
+	clientset, err := k8s.NewClientset(config)
 	if err != nil {
 		return nil, fmt.Errorf("error creating a new k8s clientset: %v", err)
 	}
 
+	rwConfig, err := k8s.NewRestConfig(path, inCluster, hostname)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create k8s REST config for retryClientSet: %w", err)
+	}
+
+	rwConfig.Timeout = 0 // empty value to disable the timeout
+	rwClientSet, err := k8s.NewClientset(rwConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create k8s client for retry watcher: %w", err)
+	}
+
 	return &Manager{
-		KubernetesClient: clientset,
+		KubernetesClient:   clientset,
+		RetryWatcherClient: rwClientSet,
 	}, nil
 }
 
 // StartCluster - Begins a running instance of the Leader Election cluster
 func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *bgp.Server) error {
+	var err error
 
-	id, err := os.Hostname()
-	if err != nil {
-		return err
-	}
-
-	log.Infof("Beginning cluster membership, namespace [%s], lock name [%s], id [%s]", c.Namespace, plunderLock, id)
-
-	// we use the Lease lock type since edits to Leases are less common
-	// and fewer objects in the cluster watch "all Leases".
-	lock := &resourcelock.LeaseLock{
-		LeaseMeta: metav1.ObjectMeta{
-			Name:      plunderLock,
-			Namespace: c.Namespace,
-		},
-		Client: sm.KubernetesClient.CoordinationV1(),
-		LockConfig: resourcelock.ResourceLockConfig{
-			Identity: id,
-		},
-	}
+	log.Info("cluster membership", "namespace", c.Namespace, "lock", c.LeaseName, "id", c.NodeName)
 
 	// use a Go context so we can tell the leaderelection code when we
 	// want to step down
@@ -117,16 +124,22 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
 
 	go func() {
 		<-signalChan
-		log.Info("Received termination, signaling shutdown")
+		log.Info("Received termination, signaling cluster shutdown")
 		// Cancel the context, which will in turn cancel the leadership
 		cancel()
 		// Cancel the arp context, which will in turn stop any broadcasts
 	}()
 
-	// (attempt to) Remove the virtual IP, incase it already exists
-	err = cluster.Network.DeleteIP()
-	if err != nil {
-		log.Errorf("could not delete virtualIP: %v", err)
+	// (attempt to) Remove the virtual IP, in case it already exists
+
+	for i := range cluster.Network {
+		deleted, err := cluster.Network[i].DeleteIP()
+		if err != nil {
+			log.Error("could not delete virtualIP", "err", err)
+		}
+		if deleted {
+			log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+		}
 	}
 
 	// Defer a function to check if the bgpServer has been created and if so attempt to close it
@@ -136,42 +149,165 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
 		}
 	}()
 
-	// If Packet is enabled then we can begin our preparation work
-	var packetClient *packngo.Client
-	if c.EnableMetal {
-		if c.ProviderConfig != "" {
-			key, project, err := packet.GetPacketConfig(c.ProviderConfig)
-			if err != nil {
-				log.Error(err)
-			} else {
-				// Set the environment variable with the key for the project
-				os.Setenv("PACKET_AUTH_TOKEN", key)
-				// Update the configuration with the project key
-				c.MetalProjectID = project
-			}
-		}
-		packetClient, err = packngo.NewClient()
+	if c.EnableBGP && bgpServer == nil {
+		// Lets start BGP
+		log.Info("Starting the BGP server to advertise VIP routes to VGP peers")
+		bgpServer, err = bgp.NewBGPServer(c.BGPConfig)
 		if err != nil {
-			log.Error(err)
+			log.Error("new BGP server", "err", err)
 		}
-
-		// We're using Packet with BGP, popuplate the Peer information from the API
-		if c.EnableBGP {
-			log.Infoln("Looking up the BGP configuration from packet")
-			err = packet.BGPLookup(packetClient, c)
-			if err != nil {
-				log.Error(err)
-			}
+		if err := bgpServer.Start(nil); err != nil {
+			log.Error("starting BGP server", "err", err)
 		}
 	}
 
-	if c.EnableBGP {
-		// Lets start BGP
-		log.Info("Starting the BGP server to advertise VIP routes to VGP peers")
-		bgpServer, err = bgp.NewBGPServer(&c.BGPConfig)
-		if err != nil {
-			log.Error(err)
-		}
+	run := &runConfig{
+		config:  c,
+		leaseID: c.NodeName,
+		sm:      sm,
+		onStartedLeading: func(ctx context.Context) { //nolint TODO: potential clean code
+			// When we become leader, ensure we can take over VIPs even if they're preserved on other nodes
+			if c.PreserveVIPOnLeadershipLoss {
+				log.Info("Becoming leader with VIP preservation enabled - ensuring VIP takeover")
+				// Force add the VIPs (this will work even if they exist due to the precheck logic)
+				for i := range cluster.Network {
+					added, err := cluster.Network[i].AddIP(true, false)
+					if err != nil {
+						log.Error("failed to ensure VIP on leader takeover", "vip", cluster.Network[i].IP(), "err", err)
+					} else if added {
+						log.Info("took over VIP as new leader", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+					} else {
+						log.Info("VIP already configured on interface", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+					}
+				}
+			}
+
+			// Start ARP advertisements now that we have leadership
+			log.Info("Start ARP/NDP advertisement")
+			go cluster.arpMgr.StartAdvertisement(ctxArp)
+
+			// As we're leading lets start the vip service
+			err := cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, cancel)
+			if err != nil {
+				log.Error("starting VIP service on leader", "err", err)
+			}
+		},
+		onStoppedLeading: func() {
+			// we can do cleanup here
+			log.Info("This node is becoming a follower within the cluster")
+
+			// Stop the dns context
+			cancelDNS()
+			// Stop the Arp context if it is running
+			cancelArp()
+
+			// Stop the BGP server
+			if bgpServer != nil {
+				err := bgpServer.Close()
+				if err != nil {
+					log.Warn("close BGP server", "err", err)
+				}
+			}
+
+			// Handle VIP cleanup based on configuration
+			if c.PreserveVIPOnLeadershipLoss {
+				// For IPv6, we must remove VIPs immediately to avoid DAD failures on the new leader
+				// IPv6 Duplicate Address Detection will fail if the new leader tries to add an IP that is still present on this node's interface
+				// We need to check each VIP individually and only remove IPv6 VIPs
+				for i := range cluster.Network {
+					if utils.IsIPv6(cluster.Network[i].IP()) {
+						log.Info("Removing IPv6 VIP immediately (required to prevent DAD failures on new leader)", "ip", cluster.Network[i].IP())
+						deleted, err := cluster.Network[i].DeleteIP()
+						if err != nil {
+							log.Warn("delete VIP", "err", err)
+						}
+						if deleted {
+							log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+						}
+					} else {
+						log.Info("Preserving IPv4 VIP address on interface, only stopped ARP broadcasting", "ip", cluster.Network[i].IP())
+					}
+				}
+			} else {
+				// Legacy behavior: delete VIP addresses on leadership loss
+				log.Info("Deleting VIP addresses on leadership loss (legacy behavior)")
+				for i := range cluster.Network {
+					deleted, err := cluster.Network[i].DeleteIP()
+					if err != nil {
+						log.Warn("delete VIP", "err", err)
+					}
+					if deleted {
+						log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+					}
+				}
+			}
+
+			log.Error("lost leadership, restarting kube-vip")
+			panic("") // TODO - we could also return here
+		},
+		onNewLeader: func(identity string) {
+			// we're notified when new leader elected
+			log.Info("New leader", "leader", identity)
+
+			// If we're not the new leader and we have VIPs preserved from previous leadership,
+			// we need to clean them up to avoid conflicts.
+			if identity != c.NodeName && c.PreserveVIPOnLeadershipLoss {
+				log.Info("Cleaning up preserved VIPs as another node became leader", "new_leader", identity)
+				for i := range cluster.Network {
+					deleted, err := cluster.Network[i].DeleteIP()
+					if err != nil {
+						log.Warn("failed to cleanup preserved VIP", "vip", cluster.Network[i].IP(), "err", err)
+					}
+					if deleted {
+						log.Info("cleaned up preserved VIP to avoid conflict", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface(), "new_leader", identity)
+					} else {
+						log.Debug("VIP was not present on this node", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
+					}
+				}
+			}
+		},
+	}
+
+	switch c.LeaderElectionType {
+	case "kubernetes", "":
+		cluster.runKubernetesLeaderElectionOrDie(ctx, run)
+	case "etcd":
+		cluster.runEtcdLeaderElectionOrDie(ctx, run)
+	default:
+		log.Info(fmt.Sprintf("LeaderElectionMode %s not supported, exiting", c.LeaderElectionType))
+	}
+
+	return nil
+}
+
+type runConfig struct {
+	config  *kubevip.Config
+	leaseID string
+	sm      *Manager
+
+	// onStartedLeading is called when this member starts leading.
+	onStartedLeading func(context.Context)
+	// onStoppedLeading is called when this member stops leading.
+	onStoppedLeading func()
+	// onNewLeader is called when the client observes a leader that is
+	// not the previously observed leader. This includes the first observed
+	// leader when the client starts.
+	onNewLeader func(identity string)
+}
+
+func (cluster *Cluster) runKubernetesLeaderElectionOrDie(ctx context.Context, run *runConfig) {
+	// we use the Lease lock type since edits to Leases are less common
+	// and fewer objects in the cluster watch "all Leases".
+	lock := &resourcelock.LeaseLock{
+		LeaseMeta: metav1.ObjectMeta{
+			Name:        run.config.LeaseName,
+			Namespace:   run.config.Namespace,
+			Annotations: run.config.LeaseAnnotations,
+		},
+		Client: run.sm.KubernetesClient.CoordinationV1(),
+		LockConfig: resourcelock.ResourceLockConfig{
+			Identity: run.leaseID,
+		},
 	}
 
 	// start the leader election code loop
@@ -184,63 +320,42 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
 		// get elected before your background loop finished, violating
 		// the stated goal of the lease.
 		ReleaseOnCancel: true,
-		LeaseDuration:   time.Duration(c.LeaseDuration) * time.Second,
-		RenewDeadline:   time.Duration(c.RenewDeadline) * time.Second,
-		RetryPeriod:     time.Duration(c.RetryPeriod) * time.Second,
+		LeaseDuration:   time.Duration(run.config.LeaseDuration) * time.Second,
+		RenewDeadline:   time.Duration(run.config.RenewDeadline) * time.Second,
+		RetryPeriod:     time.Duration(run.config.RetryPeriod) * time.Second,
 		Callbacks: leaderelection.LeaderCallbacks{
-			OnStartedLeading: func(ctx context.Context) {
-				// As we're leading lets start the vip service
-				err = cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, packetClient)
-				if err != nil {
-					log.Errorf("Error starting the VIP service on the leader [%s]", err)
-				}
-
-			},
-			OnStoppedLeading: func() {
-				// we can do cleanup here
-				log.Info("This node is becoming a follower within the cluster")
-
-				// Stop the dns context
-				cancelDNS()
-				// Stop the Arp context if it is running
-				cancelArp()
-
-				// Stop the BGP server
-				if bgpServer != nil {
-					err = bgpServer.Close()
-					if err != nil {
-						log.Warnf("%v", err)
-					}
-				}
-
-				err = cluster.Network.DeleteIP()
-				if err != nil {
-					log.Warnf("%v", err)
-				}
-
-				log.Fatal("lost leadership, restarting kube-vip")
-			},
-			OnNewLeader: func(identity string) {
-				// we're notified when new leader elected
-				log.Infof("Node [%s] is assuming leadership of the cluster", identity)
-			},
+			OnStartedLeading: run.onStartedLeading,
+			OnStoppedLeading: run.onStoppedLeading,
+			OnNewLeader:      run.onNewLeader,
 		},
 	})
-
-	return nil
 }
 
-func (sm *Manager) NodeWatcher(lb *loadbalancer.IPVSLoadBalancer, port int) error {
+func (cluster *Cluster) runEtcdLeaderElectionOrDie(ctx context.Context, run *runConfig) {
+	etcd.RunElectionOrDie(ctx, &etcd.LeaderElectionConfig{
+		EtcdConfig:           etcd.ClientConfig{Client: run.sm.EtcdClient},
+		Name:                 run.config.LeaseName,
+		MemberID:             run.leaseID,
+		LeaseDurationSeconds: int64(run.config.LeaseDuration),
+		Callbacks: etcd.LeaderCallbacks{
+			OnStartedLeading: run.onStartedLeading,
+			OnStoppedLeading: run.onStoppedLeading,
+			OnNewLeader:      run.onNewLeader,
+		},
+	})
+}
+
+func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
 	// Use a restartable watcher, as this should help in the event of etcd or timeout issues
-	log.Infof("Kube-Vip is watching nodes for control-plane labels")
+	log.Info("Kube-Vip is watching nodes for control-plane labels")
 
 	listOptions := metav1.ListOptions{
 		LabelSelector: "node-role.kubernetes.io/control-plane",
 	}
 
-	rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
-		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
-			return sm.KubernetesClient.CoreV1().Nodes().Watch(context.Background(), listOptions)
+	rw, err := watchtools.NewRetryWatcherWithContext(ctxArp, "1", &cache.ListWatch{
+		WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
+			return sm.RetryWatcherClient.CoreV1().Nodes().Watch(context.Background(), listOptions)
 		},
 	})
 	if err != nil {
@@ -255,7 +370,7 @@ func (sm *Manager) NodeWatcher(lb *loadbalancer.IPVSLoadBalancer, port int) erro
 	}()
 
 	ch := rw.ResultChan()
-	//defer rw.Stop()
+	// defer rw.Stop()
 
 	for event := range ch {
 		// We need to inspect the event and get ResourceVersion out of it
@@ -263,36 +378,41 @@ func (sm *Manager) NodeWatcher(lb *loadbalancer.IPVSLoadBalancer, port int) erro
 		case watch.Added, watch.Modified:
 			node, ok := event.Object.(*v1.Node)
 			if !ok {
-				return fmt.Errorf("Unable to parse Kubernetes Node from Annotation watcher")
+				return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
 			}
-			//Find the node IP address (this isn't foolproof)
+			// Find the node IP address (this isn't foolproof)
 			for x := range node.Status.Addresses {
-
 				if node.Status.Addresses[x].Type == v1.NodeInternalIP {
-					err = lb.AddBackend(node.Status.Addresses[x].Address, port)
-					if err != nil {
-						log.Errorf("Add IPVS backend [%v]", err)
+					if checkIfNodeIsReady(node) {
+						err = lb.AddBackend(node.Status.Addresses[x].Address, port)
+						if err != nil {
+							log.Error("add IPVS backend", "err", err)
+						}
+					} else {
+						err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
+						if err != nil {
+							log.Error("remove IPVS backend", "err", err)
+						}
 					}
 				}
 			}
 		case watch.Deleted:
 			node, ok := event.Object.(*v1.Node)
 			if !ok {
-				return fmt.Errorf("Unable to parse Kubernetes Node from Annotation watcher")
+				return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
 			}
 
-			//Find the node IP address (this isn't foolproof)
+			// Find the node IP address (this isn't foolproof)
 			for x := range node.Status.Addresses {
-
 				if node.Status.Addresses[x].Type == v1.NodeInternalIP {
 					err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
 					if err != nil {
-						log.Errorf("Del IPVS backend [%v]", err)
+						log.Error("Del IPVS backend", "err", err)
 					}
 				}
 			}
 
-			log.Infof("Node [%s] has been deleted", node.Name)
+			log.Info("Node deleted", "name", node.Name)
 
 		case watch.Bookmark:
 			// Un-used
@@ -303,17 +423,29 @@ func (sm *Manager) NodeWatcher(lb *loadbalancer.IPVSLoadBalancer, port int) erro
 			errObject := apierrors.FromObject(event.Object)
 			statusErr, ok := errObject.(*apierrors.StatusError)
 			if !ok {
-				log.Errorf(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
-
+				log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
 			}
 
 			status := statusErr.ErrStatus
-			log.Errorf("%v", status)
+			log.Error("watcher", "status", status)
 		default:
 		}
 	}
 
-	log.Infoln("Exiting Annotations watcher")
+	log.Info("Exiting Node watcher")
 	return nil
-
+}
+
+func checkIfNodeIsReady(node *v1.Node) bool {
+	if node == nil {
+		return false
+	}
+	for _, condition := range node.Status.Conditions {
+		if condition.Type == v1.NodeReady {
+			if condition.Status == v1.ConditionTrue {
+				return true
+			}
+		}
+	}
+	return false
 }