Compare commits

..

1 Commits

Author SHA1 Message Date
Dan Finneran
296c9c1580 Unintentionally created duplicate functions that can't be used
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-08-12 12:17:20 +00:00
160 changed files with 4785 additions and 18755 deletions

View File

@@ -22,10 +22,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@v4
with:
ref: ${{ github.ref_name }}
- name: Anchore SBOM Action
uses: anchore/sbom-action@v0.24.0
uses: anchore/sbom-action@v0.20.4
with:
format: cyclonedx-json

View File

@@ -9,63 +9,43 @@ jobs:
matrix:
mode: ["arp", "rt", "bgp"]
fail-fast: true
max-parallel: 1
steps:
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H-%M')"
- name: Ensure fs wont cause issues
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image locally
run: make dockerx86Local
- name: Run Manifest generation tests
run: make manifest-test
- name: Run ARP mode tests
run: DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true make e2e-tests-arp
- name: Run ARP mode tests v1.29.0 onwards
run: make e2e-tests129-arp
if: matrix.mode== 'arp'
- name: Run RT mode tests
run: DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true make e2e-tests-rt
- name: Run RT mode tests v1.29.0 onwards
run: make e2e-tests129-rt
if: matrix.mode== 'rt'
- name: Get GoBGP binaries
run: make get-gobgp
if: matrix.mode== 'bgp'
- name: Run BGP mode tests
run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true make e2e-tests-bgp
- name: Run BGP mode tests v1.29.0 onwards
run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 make e2e-tests129-bgp
if: matrix.mode== 'bgp'
- name: Change log directory permissions
run: sudo chmod -R 755 /tmp/kube-vip-test*
if: matrix.mode== 'bgp' && always()
- name: Save logs
uses: actions/upload-artifact@v7
with:
name: e2e-test-logs-${{ matrix.mode }}-${{ steps.date.outputs.date }}
path: /tmp/kube-vip-test*
if: always()
service-e2e-tests:
runs-on: ubuntu-latest
name: E2E service tests
steps:
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H-%M')"
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image with iptables
run: make dockerx86ActionIPTables
- name: Run tests
run: DOCKER_API_VERSION=1.48 DOCKERTAG=action E2E_KEEP_LOGS=true make service-tests
- name: Save logs
uses: actions/upload-artifact@v7
with:
name: services-test-logs-${{ steps.date.outputs.date }}
path: /tmp/kube-vip-services*
if: always()
run: DOCKERTAG=action make service-tests

View File

@@ -10,14 +10,11 @@ jobs:
- name: Init
run: sudo apt-get update && sudo apt-get install -y build-essential && sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Install golangci-lint
uses: golangci/golangci-lint-action@v9
with:
version: v2.12
install-only: true
run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.64.8
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: All checks
@@ -27,9 +24,9 @@ jobs:
name: Unit tests
steps:
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Run tests
@@ -39,9 +36,9 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Run tests
@@ -51,7 +48,7 @@ jobs:
name: Image vulnerability scan
steps:
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Build image with iptables
run: make dockerx86ActionIPTables
- name: Run Trivy vulnerability scanner

View File

@@ -38,16 +38,16 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version-file: go.mod
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v4
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@@ -58,7 +58,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v4
uses: github/codeql-action/autobuild@v3
# Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
@@ -72,4 +72,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v4
uses: github/codeql-action/analyze@v3

View File

@@ -11,25 +11,25 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to Github Packages
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build standard version
id: docker_build
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
@@ -39,7 +39,7 @@ jobs:
ghcr.io/kube-vip/kube-vip:${{ github.ref_name }}
- name: Build iptables version
id: docker_build_iptables
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: .
file: Dockerfile_iptables

View File

@@ -11,31 +11,31 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to Github Packages
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Generate Metadata
uses: docker/metadata-action@v6.1.0
uses: docker/metadata-action@v5.7.0
id: metadata
with:
labels: |
org.opencontainers.image.documentation=https://kube-vip.io/docs/
- name: Build and push main branch
id: docker_build
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
@@ -48,7 +48,7 @@ jobs:
ghcr.io/kube-vip/kube-vip:latest
- name: Build iptables version and push main branch
id: docker_build_iptables
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: .
file: Dockerfile_iptables

View File

@@ -1,38 +1,13 @@
version: "2"
run:
timeout: 10m
linters:
enable:
- bodyclose
- gosec
- misspell
- unconvert
- unparam
settings:
misspell:
ignore-rules:
- creater
staticcheck:
checks:
- all
# Disable QF1008 to retain embedded fields for better readability.
- "-QF1008"
exclusions:
generated: lax
presets:
- comments
- common-false-positives
- legacy
- std-error-handling
paths:
- third_party$
- builtin$
- examples$
formatters:
enable:
- gofmt
- goimports
exclusions:
generated: lax
paths:
- third_party$
- builtin$
- examples$
- bodyclose
- gofmt
- goimports
- revive
- gosec
- misspell
- unconvert
- unparam

View File

@@ -1,54 +0,0 @@
# Changelog
All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
### Fixed
- Retry on 403 Forbidden and 401 Unauthorized in `ServicesWatcher` at startup with exponential backoff. Fixes #1464.
- Reintroduce BGP config via node annotations. Fixes #1488.
- Fail fast in runtime `manager` and `service` paths when legacy `vip_address` is used without `vip_subnet` in control-plane ARP, BGP, or Routing Table mode.
- Cancel the mode context on init or configuration failure before waiting on goroutines during shutdown.
### Added
- Configurable control-plane health check for BGP mode without leader election
- Polls a configurable HTTP(S) endpoint (e.g. `https://localhost:6443/livez`) to verify the exposed service is healthy (usually the local kube-apiserver)
- Withdraws the BGP route after a configurable number of consecutive failures, removing the unhealthy node from the ECMP set
- Re-announces the route automatically once the endpoint recovers
- Gracefully withdraws the route on shutdown (SIGTERM)
- Supports custom CA certificates for TLS verification
- Configuration via environment variables or CLI flags:
- `control_plane_health_check_address` / `--controlPlaneHealthCheckAddress`: URL to poll
- `control_plane_health_check_period_seconds` / `--controlPlaneHealthCheckPeriodSeconds`: interval between checks (default: 5)
- `control_plane_health_check_timeout_seconds` / `--controlPlaneHealthCheckTimeoutSeconds`: per-request timeout (default: 3)
- `control_plane_health_check_failure_threshold` / `--controlPlaneHealthCheckFailureThreshold`: consecutive failures before withdrawal (default: 3)
- `control_plane_health_check_ca_path` / `--controlPlaneHealthCheckCAPath`: CA cert for HTTPS verification
- SIGUSR1 signal handler for runtime configuration dumps (#1301)
- Send SIGUSR1 to kube-vip process to dump current configuration to stdout
- Configuration dump includes:
- Basic configuration (VIP, interface, port, namespace settings)
- BGP configuration (enabled status, AS number, router ID, peers)
- ARP/NDP configuration (enabled status, broadcast rate)
- Services configuration (enabled status, load balancer settings)
- Network interfaces status
- Leader election configuration (type, lease details)
- Runtime statistics (load balancer, Prometheus, health check settings)
- Output format: Human-readable plaintext via fmt.Printf()
- Thread-safe implementation using mutex protection
- Non-disruptive: Process continues running after configuration dump
- Added comprehensive unit tests for all dump methods
- Added E2E tests for signal handling
### Changed
- Updated signal handlers in manager_arp.go, manager_bgp.go, manager_wireguard.go, and manager_table.go to use switch statement pattern for handling multiple signals (SIGUSR1, SIGINT, SIGTERM)
- wireguard.go now manages a complete wireguard interface on the current network namespace
- manager_wireguard.go uses the new wireguard.go implementation
## [v1.0.1] - Previous Release
### Previous changes
- See git history for changes prior to CHANGELOG.md introduction

View File

@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:experimental
FROM golang:1.26.3-alpine3.23 as dev
FROM golang:1.24.5-alpine3.22 as dev
RUN apk add --no-cache git ca-certificates make
RUN adduser -D appuser
COPY . /src/

View File

@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:experimental
FROM golang:1.26.3-alpine3.23 as dev
FROM golang:1.24.5-alpine3.22 as dev
RUN apk add --no-cache git make
RUN adduser -D appuser
COPY . /src/
@@ -11,7 +11,7 @@ RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
--mount=type=cache,sharing=locked,id=goroot,target=/root/.cache/go-build \
CGO_ENABLED=0 GOOS=linux make build
FROM alpine:3.23.4
FROM alpine:3.22.1
# Update pkgs and add iptables
RUN apk upgrade && \
apk add --no-cache iptables iptables-legacy

View File

@@ -5,7 +5,7 @@ TARGET := kube-vip
.DEFAULT_GOAL := $(TARGET)
# These will be provided to the target
VERSION := v1.2.0
VERSION := v1.0.0
BUILD := `git rev-parse HEAD`
@@ -15,11 +15,9 @@ TARGETOS=linux
# Use linker flags to provide version/build settings to the target
LDFLAGS=-ldflags "-s -w -X=main.Version=$(VERSION) -X=main.Build=$(BUILD) -extldflags -static"
DOCKERTAG ?= $(VERSION)
REPOSITORY ?= docker.io/plndr
GO_VERSION := 1.25.6
K8S_VERSION ?= v1.35.0
REPOSITORY ?= plndr
.PHONY: all build clean install uninstall simplify check run e2e-tests unit-tests integration-tests unit-tests-docker integration-tests-docker
.PHONY: all build clean install uninstall simplify check run e2e-tests
all: check install
@@ -111,45 +109,44 @@ run: install
manifests:
@make build
@mkdir -p ./docs/manifests/$(VERSION)/
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-arp.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-lb.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-bgp.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-arp.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-lb.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --bgp --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-bgp.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
@-rm ./kube-vip
manifest-test:
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --enableLoadBalancer
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --controlplane --services
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --enableLoadBalancer
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --bgp --controlplane --services
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster
unit-tests:
go test -race ./...
unit-tests-docker:
docker run --rm -w /kube-vip -v $$(pwd):/kube-vip -v kube-vip-gomod-cache:/go/pkg/mod -v kube-vip-gobuild-cache:/root/.cache/go-build golang:$(GO_VERSION) make unit-tests
go test ./...
integration-tests:
go test -tags=integration,e2e -v ./pkg/etcd
e2e-tests-arp: get-whoami
GOMAXPROCS=4 TEST_MODE=arp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests:
GOMAXPROCS=4 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd
e2e-tests-rt: get-whoami
GOMAXPROCS=4 TEST_MODE=rt K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129-arp:
GOMAXPROCS=4 TEST_MODE=arp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests-bgp: get-whoami get-gobgp
GOMAXPROCS=4 TEST_MODE=bgp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129-rt:
GOMAXPROCS=4 TEST_MODE=rt V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests: e2e-tests-arp e2e-tests-rt e2e-tests-bgp
e2e-tests129-bgp:
GOMAXPROCS=4 TEST_MODE=bgp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129: e2e-tests129-arp e2e-tests129-rt e2e-tests129-bgp
service-tests:
$(MAKE) -C testing/e2e/e2e dockerLocal
E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/services -Services -simple -deployments -leaderActive -leaderFailover -localDeploy -egress -egressIPv6 -dualStack
trivy: dockerx86ActionIPTables
@@ -169,10 +166,10 @@ kind-quick:
kubectl create configmap --namespace kube-system kubevip --from-literal range-global=172.18.100.10-172.18.100.30
kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
kind load docker-image --name kube-vip $(REPOSITORY)/$(TARGET):$(DOCKERTAG)
docker run --network host --rm $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --services --inCluster --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --servicesElection --interface eth0 | kubectl apply -f -
docker run --network host --rm $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --services --inCluster --arp --servicesElection --interface eth0 | kubectl apply -f -
kind-reload:
kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name services
kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name kube-vip
kubectl rollout restart -n kube-system daemonset/kube-vip-ds
get-gobgp:
@@ -180,5 +177,3 @@ get-gobgp:
wget -nc --directory-prefix=bin https://github.com/osrg/gobgp/releases/download/v3.37.0/gobgp_3.37.0_linux_amd64.tar.gz
tar -xvzf bin/gobgp_3.37.0_linux_amd64.tar.gz -C bin
get-whoami:
docker pull ghcr.io/traefik/whoami:v1.11

View File

@@ -58,56 +58,14 @@ All of these would require a separate level of configuration and in some infrast
## Troubleshooting and Feedback
### SELinux and IPVS kernel modules
When using IPVS load balancing on nodes with SELinux enforcing, kube-vip may be
blocked from requesting kernel modules from inside the container. Symptoms can
include the kube-vip pod entering `Error` or `CrashLoopBackOff`, logs that show
`ensure IPVS kernel modules are loaded`, or audit denials for `module_request`
from `container_t`.
Load the required IPVS modules on every node that can run kube-vip before
deploying it:
```shell
sudo modprobe ip_vs
sudo modprobe ip_vs_rr
```
To persist this across reboots, add the modules to a file such as
`/etc/modules-load.d/kube-vip-ipvs.conf`:
```text
ip_vs
ip_vs_rr
```
Preloading only the required modules is preferred to enabling the SELinux
`domain_kernel_load_modules` boolean for containers.
Please raise issues on the GitHub repository and as mentioned check the documentation at [https://kube-vip.io](https://kube-vip.io/).
## Community Tools
- **[KubeStellar Console — Guided kube-vip Install](https://console.kubestellar.io/missions/install-kube-vip)** — A step-by-step guided installation experience for kube-vip with pre-flight checks, validation, troubleshooting, and rollback support.
## Contributing
Thanks for taking the time to join our community and start contributing! We welcome pull requests. Feel free to dig through the [issues](https://github.com/kube-vip/kube-vip/issues) and jump in.
:warning: This project has issue compiling on MacOS, please compile it on linux distribution
Additionally it is now relatively easy and quick to develop with [skaffold](https://skaffold.dev/), and the `skaffold.yaml` exists within the root folder of the gir repository.
### Set up a kind development environment
1. `kind create cluster --config ./testing/kind.yaml`
2. `kubectl apply -f https://kube-vip.io/manifests/rbac.yaml`
3. Create a load balancer range `configMap` from the kind cluster
4. Apply the CCM manifest
5. Start skaffold with `skaffold dev`
6. Start developing and see changes applied in real-time.
## Star History
[![Star History Chart](https://api.star-history.com/svg?repos=kube-vip/kube-vip&type=Date)](https://star-history.com/#kube-vip/kube-vip&Date)

View File

@@ -42,10 +42,6 @@ var kubeKubeadmInit = &cobra.Command{
log.Error("parsing environment", "err", err)
return
}
if err := initConfig.Validate(); err != nil {
log.Error("validating configuration", "err", err)
return
}
// TODO - check for certain things VIP/interfaces
if initConfig.Interface == "" {
@@ -62,18 +58,14 @@ var kubeKubeadmInit = &cobra.Command{
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating VIPSubnet", "err", err)
return
}
}
cfg, err := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
if err != nil {
log.Error("unable to create manifest", "err", err)
return
}
cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
fmt.Println(cfg) // output manifest to stdout
},
}
@@ -90,10 +82,6 @@ var kubeKubeadmJoin = &cobra.Command{
log.Error("parsing environment", "err", err)
return
}
if err := initConfig.Validate(); err != nil {
log.Error("validating configuration", "err", err)
return
}
// TODO - check for certain things VIP/interfaces
if initConfig.Interface == "" {
@@ -115,18 +103,14 @@ var kubeKubeadmJoin = &cobra.Command{
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating VIPSubnet", "err", err)
return
}
}
cfg, err := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
if err != nil {
log.Error("unable to create manifest", "err", err)
return
}
cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
fmt.Println(cfg) // output manifest to stdout
},
}

View File

@@ -19,7 +19,6 @@ var taint, role, rolebinding bool
func init() {
kubeManifest.PersistentFlags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
kubeManifest.PersistentFlags().StringVar(&image, "image", "ghcr.io/kube-vip/kube-vip", "Define a hardcoded image with or without tag for the manifest")
kubeManifestDaemon.PersistentFlags().BoolVar(&taint, "taint", false, "Taint the manifest for only running on control planes")
kubeManifestRbac.PersistentFlags().BoolVar(&role, "role", false, "Generate only a Role inside the serviceNamespace access")
kubeManifestRbac.PersistentFlags().BoolVar(&rolebinding, "rolebinding", false, "Generate only a RoleBinding for namespaced access")
@@ -50,32 +49,24 @@ var kubeManifestPod = &cobra.Command{
log.Error("parsing environment", "err", err)
return
}
if err := initConfig.Validate(); err != nil {
log.Error("validating configuration", "err", err)
return
}
// The control plane has a requirement for a VIP being specified
if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
_ = cmd.Help()
log.Error("no address is specified for kube-vip to expose services on")
log.Error("No address is specified for kube-vip to expose services on")
return
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("config parse", "err", err)
return
}
}
cfg, err := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
if err != nil {
log.Error("unable to create manifest", "err", err)
return
}
cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
fmt.Println(cfg) // output manifest to stdout
},
}
@@ -92,31 +83,23 @@ var kubeManifestDaemon = &cobra.Command{
log.Error("parsing environment", "err", err)
return
}
if err := initConfig.Validate(); err != nil {
log.Error("validating configuration", "err", err)
return
}
// The control plane has a requirement for a VIP being specified
if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
_ = cmd.Help()
log.Error("no address is specified for kube-vip to expose services on")
log.Error("No address is specified for kube-vip to expose services on")
return
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("config parse", "err", err)
return
}
}
cfg, err := kubevip.GenerateDaemonsetManifestFromConfig(&initConfig, image, Release.Version, inCluster, taint)
if err != nil {
log.Error("unable to create manifest", "err", err)
return
}
cfg := kubevip.GenerateDaemonsetManifestFromConfig(&initConfig, Release.Version, inCluster, taint)
fmt.Println(cfg) // output manifest to stdout
},
}
@@ -133,21 +116,17 @@ var kubeManifestRbac = &cobra.Command{
log.Error("parsing environment", "err", err)
return
}
if err := initConfig.Validate(); err != nil {
log.Error("validating configuration", "err", err)
return
}
// The control plane has a requirement for a VIP being specified
if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
_ = cmd.Help()
log.Error("no address is specified for kube-vip to expose services on")
log.Error("No address is specified for kube-vip to expose services on")
return
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating VIPSubnet", "err", err)
return

View File

@@ -6,14 +6,12 @@ import (
"net"
"net/http"
"os"
"slices"
"strconv"
"strings"
"sync"
"time"
log "log/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/spf13/cobra"
"github.com/vishvananda/netlink"
@@ -21,14 +19,9 @@ import (
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/manager"
"github.com/kube-vip/kube-vip/pkg/metrics"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
// Is an option to set the image
var image string
// Is kube-vip running within cluster
var inCluster bool
@@ -69,7 +62,6 @@ func init() {
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for VIP changes")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableWireguard, "wireguard", false, "Enable Wireguard for services VIPs")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableRoutingTable, "table", false, "Enable Routing Table for services VIPs")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.PreserveVIPOnLeadershipLoss, "preserveVipOnLeadershipLoss", false, "Preserve ARP VIP addresses on interface when leadership is lost (default: false for backward compatibility)")
// LoadBalancer flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "enableLoadBalancer", false, "enable loadbalancing on the VIP with IPVS")
@@ -82,9 +74,9 @@ func init() {
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLeaderElection, "leaderElection", false, "Use the Kubernetes leader election mechanism for clustering")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaderElectionType, "leaderElectionType", "kubernetes", "Defines the backend to run the leader election: kubernetes or etcd. Defaults to kubernetes.")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaseName, "leaseName", "plndr-cp-lock", "Name of the lease that is used for leader election")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 15, "Length of time (in seconds) a Kubernetes leader lease can be held for")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 10, "Length of time (in seconds) a Kubernetes leader can attempt to renew its lease")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 2, "Length of time (in seconds) the LeaderElector clients should wait between tries of actions")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 5, "Length of time (in seconds) a Kubernetes leader lease can be held for")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time (in seconds) a Kubernetes leader can attempt to renew its lease")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Length of time (in seconds) the LeaderElector clients should wait between tries of actions")
// BGP flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableBGP, "bgp", false, "This will enable BGP support within kube-vip")
@@ -105,11 +97,6 @@ func init() {
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.URL, "zebraUrl", "unix:/var/run/frr/zserv.api", "Path to the unix domain socket for connecting to Zebra daemon")
kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.Zebra.Version, "zebraVersion", 6, "Zebra API Version")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.SoftwareName, "zebraSoftwareName", "frr8.3", "Software Name for Zebra")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ControlPlaneHealthCheck.Address, "controlPlaneHealthCheckAddress", "", "URL to poll for the control-plane health check when using BGP without leader election")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.ControlPlaneHealthCheck.PeriodSeconds, "controlPlaneHealthCheckPeriodSeconds", 5, "Seconds between control-plane health checks")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.ControlPlaneHealthCheck.TimeoutSeconds, "controlPlaneHealthCheckTimeoutSeconds", 3, "Timeout for each control-plane health check request")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.ControlPlaneHealthCheck.FailureThreshold, "controlPlaneHealthCheckFailureThreshold", 3, "Consecutive control-plane health check failures before withdrawing the BGP route")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ControlPlaneHealthCheck.CAPath, "controlPlaneHealthCheckCAPath", "", "Path to CA certificate for TLS verification when the control-plane health check URL is HTTPS")
// Namespace for kube-vip
kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The namespace for the configmap defined within the cluster")
@@ -133,16 +120,13 @@ func init() {
// Extended behaviour flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServicesElection, "servicesElection", false, "Enable leader election per kubernetes service")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassOnly, "lbClassOnly", false, fmt.Sprintf("Enable load balancing only for services with LoadBalancerClass %q", kubevip.LBClassName))
kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerClassName, "lbClassName", kubevip.LBClassName, fmt.Sprintf("Name of load balancer class for kube-VIP, defaults to %q", kubevip.LBClassName))
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassOnly, "lbClassOnly", false, "Enable load balancing only for services with LoadBalancerClass \"kube-vip.io/kube-vip-class\"")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerClassName, "lbClassName", "kube-vip.io/kube-vip-class", "Name of load balancer class for kube-VIP, defaults to \"kube-vip.io/kube-vip-class\"")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassLegacyHandling, "lbClassNameLegacyHandling", true, "Use legacy LoadBalancer class name handling (e.g. accepting services both with empty and non-empty class)")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServiceSecurity, "onlyAllowTrafficServicePorts", false, "Only allow traffic to service ports, others will be dropped, defaults to false")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableNodeLabeling, "enableNodeLabeling", false, fmt.Sprintf("Enable leader node labeling with %q, defaults to false", kubevip.HasIP))
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableNodeLabeling, "enableNodeLabeling", false, "Enable leader node labeling with \"kube-vip.io/has-ip=<VIP address>\", defaults to false")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesLeaseName, "servicesLeaseName", "plndr-svcs-lock", "Name of the lease that is used for leader election for services (in arp mode)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.DNSMode, "dnsMode", "first", "Name of the mode that DNS lookup will be performed (first, ipv4, ipv6, dual)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.DHCPMode, "dhcpMode", "", "Mode DHCP resolving will use to obtain IP addresses (ipv4, ipv6, dual)")
kubeVipCmd.PersistentFlags().UintVar(&initConfig.DHCPBackoffAttempts, "dhcpBackoffAttempts", kubevip.DefaultDHCPBackoffAttempts,
fmt.Sprintf("number of times DHCP client will try to obtain an IP address (defaults to: %d, 0 for unlimited retries)", kubevip.DefaultDHCPBackoffAttempts))
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DisableServiceUpdates, "disableServiceUpdates", false, "If true, kube-vip will process services as usual, but will not update service's Status.LoadBalancer.Ingress slice")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpoints, "enableEndpoints", false, "If enabled, kube-vip will only advertise services, but will use the (deprecated since v1.33) endpoints for IP addresses")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoInterfaceGlobalScope, "loInterfaceGlobalScope", false, "If true, kube-vip will set global scope when using the lo interface, otherwise a host scope will be used by default")
@@ -160,9 +144,6 @@ func init() {
// Kubernetes client specific flags
kubeVipCmd.PersistentFlags().StringVar(&initConfig.K8sConfigFile, "k8sConfigPath", "/etc/kubernetes/admin.conf", "Path to the configuration file used with the Kubernetes client")
// Configuration file flag
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ConfigFile, "config-file", "", "Path to a JSON/YAML configuration file to load settings from")
kubeVipCmd.AddCommand(kubeKubeadm)
kubeVipCmd.AddCommand(kubeManifest)
kubeVipCmd.AddCommand(kubeVipManager)
@@ -205,25 +186,12 @@ var kubeVipService = &cobra.Command{
Short: "Start the Virtual IP / Load balancer as a service within a Kubernetes cluster",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
// Load configuration from file if specified (lowest priority)
if initConfig.ConfigFile != "" {
err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
if err != nil {
log.Error("loading config file", "err", err)
return
}
}
// parse environment variables, these will overwrite anything loaded from config file
// parse environment variables, these will overwrite anything loaded or flags
err := kubevip.ParseEnvironment(&initConfig)
if err != nil {
log.Error("parsing env", "err", err)
return
}
if err := initConfig.Validate(); err != nil {
log.Error("validating configuration", "err", err)
return
}
// Change RTN_UNSPEC to default type
if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
@@ -244,36 +212,24 @@ var kubeVipService = &cobra.Command{
configMap = envConfigMap
}
// Legacy vip_address requires vip_subnet for control-plane ARP, BGP, and Routing Table modes.
if initConfig.EnableControlPlane &&
(initConfig.EnableARP || initConfig.EnableBGP || initConfig.EnableRoutingTable) {
if err := initConfig.CheckSubnetExists(); err != nil {
log.Error("checking subnet exists if vip_address defined", "err", err)
return
}
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating CIDR", "err", err)
return
}
}
ctx, cancel := context.WithCancel(cmd.Context())
defer cancel()
// Define the new service manager
mgr, err := manager.New(ctx, configMap, &initConfig)
mgr, err := manager.New(configMap, &initConfig)
if err != nil {
log.Error("new manager", "err", err)
return
}
// Start the service manager, this will watch the config Map and construct kube-vip services for it
err = mgr.Start(ctx)
err = mgr.Start()
if err != nil {
log.Error("manager start", "err", err)
return
@@ -285,25 +241,12 @@ var kubeVipManager = &cobra.Command{
Use: "manager",
Short: "Start the kube-vip manager",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
// Load configuration from file if specified (lowest priority)
if initConfig.ConfigFile != "" {
err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
if err != nil {
log.Error("loading config file", "err", err)
return
}
}
// parse environment variables, these will overwrite anything loaded from config file
// parse environment variables, these will overwrite anything loaded or flags
err := kubevip.ParseEnvironment(&initConfig)
if err != nil {
log.Error("parsing environment", "err", err)
return
}
if err := initConfig.Validate(); err != nil {
log.Error("validating configuration", "err", err)
return
}
// Change RTN_UNSPEC to default type
if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
@@ -313,18 +256,9 @@ var kubeVipManager = &cobra.Command{
// Set the logging level for all subsequent functions
log.SetLogLoggerLevel(log.Level(initConfig.Logging))
// Legacy vip_address requires vip_subnet for control-plane ARP, BGP, and Routing Table modes.
if initConfig.EnableControlPlane &&
(initConfig.EnableARP || initConfig.EnableBGP || initConfig.EnableRoutingTable) {
if err := initConfig.CheckSubnetExists(); err != nil {
log.Error("checking subnet exists if vip_address defined", "err", err)
return
}
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("No interface is specified for kube-vip to bind to")
return
@@ -334,19 +268,10 @@ var kubeVipManager = &cobra.Command{
// Welome messages
log.Info("kube-vip.io", "version", Release.Version, "build", Release.Build)
wg := sync.WaitGroup{}
defer wg.Wait()
// create main manager context
ctx, cancel := context.WithCancel(cmd.Context())
defer cancel()
// start prometheus server
if initConfig.PrometheusHTTPServer != "" {
wg.Go(func() {
servePrometheusHTTPServer(ctx, PrometheusHTTPServerConfig{
Addr: initConfig.PrometheusHTTPServer,
})
go servePrometheusHTTPServer(cmd.Context(), PrometheusHTTPServerConfig{
Addr: initConfig.PrometheusHTTPServer,
})
}
@@ -430,13 +355,13 @@ var kubeVipManager = &cobra.Command{
initConfig.Interface = defaultIF.Name
log.Info("kube-vip bind", "interface", initConfig.Interface)
wg.Go(func() {
if err := vip.MonitorDefaultInterface(ctx, defaultIF); err != nil {
go func() {
if err := vip.MonitorDefaultInterface(context.TODO(), defaultIF); err != nil {
log.Error("interface monitor", "err", err)
return
}
})
}()
}
}
// Perform a check on the state of the interface
@@ -452,17 +377,16 @@ var kubeVipManager = &cobra.Command{
}
// Define the new service manager
mgr, err := manager.New(ctx, configMap, &initConfig)
mgr, err := manager.New(configMap, &initConfig)
if err != nil {
log.Error("new manager", "err", err)
return
}
metrics.RegisterPrometheusMetrics()
metrics.BuildInfo.WithLabelValues(Release.Version, Release.Build, initConfig.NodeName)
prometheus.MustRegister(mgr.PrometheusCollector()...)
// Start the service manager, this will watch the config Map and construct kube-vip services for it
err = mgr.Start(ctx)
err = mgr.Start()
if err != nil {
log.Error("start manager", "err", err)
return
@@ -496,20 +420,19 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
ReadHeaderTimeout: 2 * time.Second,
}
wg := sync.WaitGroup{}
wg.Go(func() {
go func() {
if err = srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Error("prometheus HTTP server", "err", err)
return
}
})
}()
log.Info("prometheus HTTP server started")
<-ctx.Done()
// create prometheus shutdown context (independent of other contexts)
log.Info("prometheus HTTP server stopped")
ctxShutDown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer func() {
cancel()
@@ -523,43 +446,28 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
if err == http.ErrServerClosed {
err = nil
}
log.Info("prometheus HTTP server stopped")
wg.Wait()
}
func GenerateCidrRange(address string, dnsMode string) (string, error) {
func GenerateCidrRange(address string) (string, error) {
var cidrs []string
addresses := strings.Split(address, ",")
for _, a := range addresses {
ip := net.ParseIP(a)
if ip == nil {
// we probably are a DNS name
ips, err := utils.LookupHost(a, dnsMode, true)
ips, err := net.LookupIP(a)
if len(ips) == 0 || err != nil {
return "", fmt.Errorf("invalid IP address: %s from [%s], %v", a, address, err)
}
for _, addr := range ips {
ip = net.ParseIP(addr)
if ip.To4() != nil {
cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv4))
} else {
cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv6))
}
}
ip = ips[0]
}
if ip.To4() != nil {
cidrs = append(cidrs, "32")
} else {
if ip.To4() != nil {
cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv4))
} else {
cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv6))
}
cidrs = append(cidrs, "128")
}
}
// compact as DNS could have a lot of addresses
slices.Sort(cidrs)
cidrs = slices.Compact(cidrs)
slices.Reverse(cidrs)
return strings.Join(cidrs, ","), nil
}

View File

@@ -19,7 +19,7 @@ func main() {
var errorOccurred bool
for {
p := make([]byte, 2048)
conn, err := net.Dial("udp", net.JoinHostPort(*address, fmt.Sprint(port)))
conn, err := net.Dial("udp", fmt.Sprintf("%s:%d", *address, *port))
if err != nil {
if !errorOccurred {
errorTime = time.Now()

146
go.mod
View File

@@ -1,53 +1,50 @@
module github.com/kube-vip/kube-vip
go 1.26.0
go 1.24.4
require (
github.com/cloudflare/ipvs v0.12.0
github.com/cloudflare/ipvs v0.11.0
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/docker/docker v28.5.2+incompatible
github.com/florianl/go-conntrack v0.7.0
github.com/docker/docker v28.3.3+incompatible
github.com/florianl/go-conntrack v0.4.0
github.com/google/go-cmp v0.7.0
github.com/google/go-containerregistry v0.21.6
github.com/google/nftables v0.3.0
github.com/gookit/slog v0.6.0
github.com/gookit/slog v0.5.8
github.com/huin/goupnp v1.3.0
github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d
github.com/jpillora/backoff v1.0.0
github.com/mdlayher/ndp v1.1.0
github.com/onsi/ginkgo/v2 v2.29.0
github.com/onsi/gomega v1.41.0
github.com/onsi/ginkgo/v2 v2.23.4
github.com/onsi/gomega v1.37.0
github.com/osrg/gobgp/v3 v3.37.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.23.2
github.com/sirupsen/logrus v1.9.4
github.com/spf13/cobra v1.10.2
github.com/stretchr/testify v1.11.1
github.com/prometheus/client_golang v1.22.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.9.1
github.com/stretchr/testify v1.10.0
github.com/vishvananda/netlink v1.3.1
go.etcd.io/etcd/api/v3 v3.6.11
go.etcd.io/etcd/client/pkg/v3 v3.6.11
go.etcd.io/etcd/client/v3 v3.6.11
go.uber.org/zap v1.28.0
go.etcd.io/etcd/api/v3 v3.6.1
go.etcd.io/etcd/client/pkg/v3 v3.6.1
go.etcd.io/etcd/client/v3 v3.6.1
go.uber.org/zap v1.27.0
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
golang.org/x/sync v0.20.0
golang.org/x/sys v0.45.0
golang.org/x/sync v0.15.0
golang.org/x/sys v0.33.0
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
google.golang.org/grpc v1.81.1
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af
gopkg.in/yaml.v3 v3.0.1
k8s.io/api v0.36.1
k8s.io/apimachinery v0.36.1
k8s.io/client-go v0.36.1
k8s.io/klog/v2 v2.140.0
sigs.k8s.io/kind v0.31.0
sigs.k8s.io/yaml v1.6.0
google.golang.org/grpc v1.73.0
google.golang.org/protobuf v1.36.6
k8s.io/api v0.33.3
k8s.io/apimachinery v0.33.3
k8s.io/client-go v0.33.3
k8s.io/klog/v2 v2.130.1
sigs.k8s.io/kind v0.29.0
sigs.k8s.io/yaml v1.4.0
)
require (
al.essio.dev/pkg/shellescape v1.5.1 // indirect
github.com/BurntSushi/toml v1.4.0 // indirect
github.com/Masterminds/semver/v3 v3.4.0 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/Microsoft/go-winio v0.4.14 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/containerd/errdefs v1.0.0 // indirect
@@ -57,15 +54,16 @@ require (
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/go-connections v0.7.0 // indirect
github.com/dlclark/regexp2 v1.11.4 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/eapache/channels v1.1.0 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/emicklei/go-restful/v3 v3.13.0 // indirect
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.8.0 // indirect
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
@@ -74,13 +72,13 @@ require (
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.7.0 // indirect
github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect
github.com/google/gnostic-models v0.6.9 // indirect
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gookit/color v1.6.0 // indirect
github.com/gookit/goutil v0.7.1 // indirect
github.com/gookit/gsr v0.1.1 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
github.com/gookit/color v1.5.4 // indirect
github.com/gookit/goutil v0.6.18 // indirect
github.com/gookit/gsr v0.1.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
@@ -91,7 +89,7 @@ require (
github.com/mailru/easyjson v0.9.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mdlayher/genetlink v1.3.2 // indirect
github.com/mdlayher/netlink v1.8.0 // indirect
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect
github.com/mdlayher/packet v1.1.2 // indirect
github.com/mdlayher/socket v0.5.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
@@ -99,8 +97,8 @@ require (
github.com/moby/sys/atomicwriter v0.1.0 // indirect
github.com/moby/term v0.5.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
github.com/morikuni/aec v1.1.0 // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
@@ -108,48 +106,60 @@ require (
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/pierrec/lz4/v4 v4.1.22 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.62.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/sagikazarmark/locafero v0.6.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.7.1 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/spf13/pflag v1.0.6 // indirect
github.com/spf13/viper v1.19.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tj/go-spin v1.1.0 // indirect
github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/vishvananda/netns v0.0.5 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xlab/c-for-go v1.3.0 // indirect
github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
go.opentelemetry.io/otel v1.43.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect
go.opentelemetry.io/otel/metric v1.43.0 // indirect
go.opentelemetry.io/otel/trace v1.43.0 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.37.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 // indirect
go.opentelemetry.io/otel/metric v1.37.0 // indirect
go.opentelemetry.io/otel/sdk v1.37.0 // indirect
go.opentelemetry.io/otel/trace v1.37.0 // indirect
go.uber.org/automaxprocs v1.6.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.51.0 // indirect
golang.org/x/mod v0.36.0 // indirect
golang.org/x/net v0.54.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/term v0.43.0 // indirect
golang.org/x/text v0.37.0 // indirect
golang.org/x/time v0.14.0 // indirect
golang.org/x/tools v0.45.0 // indirect
golang.org/x/crypto v0.39.0 // indirect
golang.org/x/mod v0.25.0 // indirect
golang.org/x/net v0.41.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/term v0.32.0 // indirect
golang.org/x/text v0.26.0 // indirect
golang.org/x/time v0.9.0 // indirect
golang.org/x/tools v0.33.0 // indirect
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect
k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect
modernc.org/cc/v4 v4.24.4 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/opt v0.1.4 // indirect
modernc.org/sortutil v1.2.1 // indirect
modernc.org/strutil v1.2.1 // indirect
modernc.org/token v1.1.0 // indirect
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
sigs.k8s.io/randfill v1.0.0 // indirect
sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
)

457
go.sum
View File

@@ -2,20 +2,22 @@ al.essio.dev/pkg/shellescape v1.5.1 h1:86HrALUujYS/h+GtqoB26SBEdkWfmMI6FubjXlsXy
al.essio.dev/pkg/shellescape v1.5.1/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8=
github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cloudflare/ipvs v0.12.0 h1:UeKRM4q82F+XQjw1sGRwsH3IOy6keBYUwlqTUqJhy7Y=
github.com/cloudflare/ipvs v0.12.0/go.mod h1:SvzXyDX2E33X/Cokbgts7rFXueiSqp6NlEMGDT7Bu24=
github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
github.com/cloudflare/ipvs v0.11.0 h1:niLcbqfv8+RSYk+yI+jhiCIHobEdEhO++mcwCX0Znsw=
github.com/cloudflare/ipvs v0.11.0/go.mod h1:XsbuKcQpqb3rpjYPtsFsM8BNZlgLw+Z0iC6RhocqDd8=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
@@ -35,36 +37,35 @@ github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa5
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c=
github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k=
github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0=
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes=
github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU=
github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/florianl/go-conntrack v0.7.0 h1:kWbRhLRUqBlmi2ncASmygN5yI281wlpnTUJs69tcs7g=
github.com/florianl/go-conntrack v0.7.0/go.mod h1:7kzfTjtQCRsAVkuornpI8KiwFfa79QmcZXZQex1IXhU=
github.com/florianl/go-conntrack v0.4.0 h1:TlYkxytdwgVayfU0cKwkHurQA0Rd1ZSEBRckRYDUu18=
github.com/florianl/go-conntrack v0.4.0/go.mod h1:iPDx4oIats2T7X7Jm3PFyRCJM1GfZhJaSHOWROYOrE8=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs=
github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo=
github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M=
github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk=
github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE=
github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc=
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -80,42 +81,42 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw=
github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-containerregistry v0.21.6 h1:T+yqQIlJXKrM98Om4DlW3GoWQAmhZuLMwoDOvVrtiUM=
github.com/google/go-containerregistry v0.21.6/go.mod h1:U7MMSBIJynke2MVQrQk19NP9k/uQsGz/h0amIFSHMbo=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg=
github.com/google/nftables v0.3.0/go.mod h1:BCp9FsrbF1Fn/Yu6CLUc9GGZFw/+hsxfluNXXmxBfRM=
github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg=
github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gookit/assert v0.1.1 h1:lh3GcawXe/p+cU7ESTZ5Ui3Sm/x8JWpIis4/1aF0mY0=
github.com/gookit/assert v0.1.1/go.mod h1:jS5bmIVQZTIwk42uXl4lyj4iaaxx32tqH16CFj0VX2E=
github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA=
github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs=
github.com/gookit/goutil v0.7.1 h1:AaFJPN9mrdeYBv8HOybri26EHGCC34WJVT7jUStGJsI=
github.com/gookit/goutil v0.7.1/go.mod h1:vJS9HXctYTCLtCsZot5L5xF+O1oR17cDYO9R0HxBmnU=
github.com/gookit/gsr v0.1.1 h1:TaHD3M7qa6lcAf9D2J4mGNg+QjgDtD1bw7uctF8RXOM=
github.com/gookit/gsr v0.1.1/go.mod h1:7wv4Y4WCnil8+DlDYHBjidzrEzfHhXEoFjEA0pPPWpI=
github.com/gookit/slog v0.6.0 h1:KEQxOJxbTtk7oyqah6nJOEKjOdI0z5qoqkX7I6G65g4=
github.com/gookit/slog v0.6.0/go.mod h1:hPlpNi/WIcGmkEjHzQTS7s5JZkHmmnGy9sYo6csa08s=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c=
github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
github.com/gookit/goutil v0.6.18 h1:MUVj0G16flubWT8zYVicIuisUiHdgirPAkmnfD2kKgw=
github.com/gookit/goutil v0.6.18/go.mod h1:AY/5sAwKe7Xck+mEbuxj0n/bc3qwrGNe3Oeulln7zBA=
github.com/gookit/gsr v0.1.0 h1:0gadWaYGU4phMs0bma38t+Do5OZowRMEVlHv31p0Zig=
github.com/gookit/gsr v0.1.0/go.mod h1:7wv4Y4WCnil8+DlDYHBjidzrEzfHhXEoFjEA0pPPWpI=
github.com/gookit/slog v0.5.8 h1:XZCeHLQvvOZWcSUDZcqxXITsL9+d1ESsKZoASBmK1lI=
github.com/gookit/slog v0.5.8/go.mod h1:s0ViFOY/IgUuT4MDPF0l9x5/npcciy8pL4xwWZadnoc=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8CrSJVmaolDVOxTfS9kc36uB6H40kdbQq8=
@@ -128,23 +129,34 @@ github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d h1:VkCNWh6tuQLgD
github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d/go.mod h1:VvGYjkZoJyKqlmT1yzakUs4mfKMNB0XdODP0+rdml6k=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE=
github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung=
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ=
github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok=
github.com/jsimonetti/rtnetlink v0.0.0-20201216134343-bde56ed16391/go.mod h1:cR77jAZG3Y3bsb8hF6fHJbFoyFukLFOkQ98S0pQz3xw=
github.com/jsimonetti/rtnetlink v0.0.0-20201220180245-69540ac93943/go.mod h1:z4c53zj6Eex712ROyh8WI0ihysb5j2ROyV42iNogmAs=
github.com/jsimonetti/rtnetlink v0.0.0-20210122163228-8d122574c736/go.mod h1:ZXpIyOK59ZnN7J0BV99cZUPmsqDRZ3eq5X+st7u/oSA=
github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b/go.mod h1:8w9Rh8m+aHZIG69YPGGem1i5VzoyRC8nw2kA8B+ik5U=
github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190/go.mod h1:NmKSdU4VGSiv1bMsdqNALI4RSvvjtz65tTMCnD05qLo=
github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/k-sone/critbitgo v1.4.0 h1:l71cTyBGeh6X5ATh6Fibgw3+rtNT80BA0uNNWgkPrbE=
github.com/k-sone/critbitgo v1.4.0/go.mod h1:7E6pyoyADnFxlUBEKcnfS49b7SUAQGMK+OAp/UQvo0s=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
@@ -153,24 +165,35 @@ github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a
github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo=
github.com/mdlayher/ethtool v0.0.0-20211028163843-288d040e9d60/go.mod h1:aYbhishWc4Ai3I2U4Gaa2n3kHWSwzme6EsG/46HRQbE=
github.com/mdlayher/genetlink v1.0.0/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc=
github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw=
github.com/mdlayher/genetlink v1.3.2/go.mod h1:tcC3pkCrPUGIKKsCsp0B3AdaaKuHtaxoJRz3cc+528o=
github.com/mdlayher/ndp v1.1.0 h1:QylGKGVtH60sKZUE88+IW5ila1Z/M9/OXhWdsVKuscs=
github.com/mdlayher/ndp v1.1.0/go.mod h1:FmgESgemgjl38vuOIyAHWUUL6vQKA/pQNkvXdWsdQFM=
github.com/mdlayher/netlink v1.6.2/go.mod h1:O1HXX2sIWSMJ3Qn1BYZk1yZM+7iMki/uYGGiwGyq/iU=
github.com/mdlayher/netlink v1.8.0 h1:e7XNIYJKD7hUct3Px04RuIGJbBxy1/c4nX7D5YyvvlM=
github.com/mdlayher/netlink v1.8.0/go.mod h1:UhgKXUlDQhzb09DrCl2GuRNEglHmhYoWAHid9HK3594=
github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA=
github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M=
github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY=
github.com/mdlayher/netlink v1.1.1/go.mod h1:WTYpFb/WTvlRJAyKhZL5/uy69TDDpHHu2VZmb2XgV7o=
github.com/mdlayher/netlink v1.2.0/go.mod h1:kwVW1io0AZy9A1E2YYgaD4Cj+C+GPkU6klXCMzIJ9p8=
github.com/mdlayher/netlink v1.2.1/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU=
github.com/mdlayher/netlink v1.2.2-0.20210123213345-5cc92139ae3e/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU=
github.com/mdlayher/netlink v1.3.0/go.mod h1:xK/BssKuwcRXHrtN04UBkwQ6dY9VviGGuriDdoPSWys=
github.com/mdlayher/netlink v1.4.0/go.mod h1:dRJi5IABcZpBD2A3D0Mv/AiX8I9uDEu5oGkAVrekmf8=
github.com/mdlayher/netlink v1.4.1/go.mod h1:e4/KuJ+s8UhfUpO9z00/fDZZmhSrs+oxyqAS9cNgn6Q=
github.com/mdlayher/netlink v1.5.0/go.mod h1:1Kr8BBFxGyUyNmztC9WLOayqYVAd2wsgOZm18nqGuzQ=
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 h1:A1Cq6Ysb0GM0tpKMbdCXCIfBclan4oHk1Jb+Hrejirg=
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42/go.mod h1:BB4YCPDOzfy7FniQ/lxuYQ3dgmM2cZumHbK8RpTjN2o=
github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY=
github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4=
github.com/mdlayher/socket v0.2.3/go.mod h1:bz12/FozYNH/VbvC3q7TRIK/Y6dH1kCKsXaUeXi/FmY=
github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00/go.mod h1:GAFlyu4/XV68LkQKYzKhIo/WW7j3Zi0YRAz/BOoanUc=
github.com/mdlayher/socket v0.0.0-20211007213009-516dcbdf0267/go.mod h1:nFZ1EtZYK8Gi/k6QNu7z7CgO20i/4ExeQswwWuPmG/g=
github.com/mdlayher/socket v0.1.0/go.mod h1:mYV5YIZAfHh4dzDVzI8x8tWLWCliuX8Mon5Awbj+qDs=
github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos=
github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ=
github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE=
github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A=
github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws=
github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
@@ -186,80 +209,83 @@ github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFL
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/morikuni/aec v1.1.0 h1:vBBl0pUnvi/Je71dsRrhMBtreIqNMYErSAbEeb8jrXQ=
github.com/morikuni/aec v1.1.0/go.mod h1:xDRgiq/iw5l+zkao76YTKzKttOp2cwPEne25HDkJnBw=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/onsi/ginkgo/v2 v2.29.0 h1:rfh+ZFjgJhYWRoIqVf3Uwx/W20yLrcrE2h2GmYVRaag=
github.com/onsi/ginkgo/v2 v2.29.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
github.com/onsi/gomega v1.41.0 h1:OwKp4pXNgVxf6sCplzYo794OFNuoL2q2SBMU5NSWOjA=
github.com/onsi/gomega v1.41.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A=
github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus=
github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8=
github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y=
github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
github.com/osrg/gobgp/v3 v3.37.0 h1:+ObuOdvj7G7nxrT0fKFta+EAupdWf/q1WzbXydr8IOY=
github.com/osrg/gobgp/v3 v3.37.0/go.mod h1:kVHVFy1/fyZHJ8P32+ctvPeJogn9qKwa1YCeMRXXrP0=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk=
github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds=
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 h1:pyC9PaHYZFgEKFdlp3G8RaCKgVpHZnecvArXvPXcFkM=
github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
@@ -270,101 +296,144 @@ github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zd
github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/xlab/c-for-go v1.3.0 h1:WDc+/E59g1OcZ4dYB4K9V4Doh4NnVBMYxaZFhNi77Lg=
github.com/xlab/c-for-go v1.3.0/go.mod h1:0qVnGIfoNg7c5gABptM38Y7nnXgAMo5KYWVQPycu63Y=
github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 h1:Sw125DKxZhPUI4JLlWugkzsrlB50jR9v2khiD9FxuSo=
github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245/go.mod h1:C+diUUz7pxhNY6KAoLgrTYARGWnt82zWTylZlxT92vk=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.etcd.io/etcd/api/v3 v3.6.11 h1:XFGTgrJ8nak3kB4NgMG8t7NT+lEeuuvKQAqUHKVgkWQ=
go.etcd.io/etcd/api/v3 v3.6.11/go.mod h1:HYfTh0jyh+uFgp6gMbxJteIDYY97yMuYz85Rnw6Gy9o=
go.etcd.io/etcd/client/pkg/v3 v3.6.11 h1:e41mp315Yn3QMGPmEzCyLsMINgJXTY/dX8kM++1csxU=
go.etcd.io/etcd/client/pkg/v3 v3.6.11/go.mod h1:DysuMe/inqRyC/1tjRR6hReH/VV9Lufs27YKSKBWWJg=
go.etcd.io/etcd/client/v3 v3.6.11 h1:LAByD96VmmeuairkvdAcE0RZnrmGz/q3ceeWePo9bwc=
go.etcd.io/etcd/client/v3 v3.6.11/go.mod h1:vOTDMCo+fGPEClJqcFEFSqZ+8e7WKV7AyqJjX//HR2w=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak=
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g=
go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.etcd.io/etcd/api/v3 v3.6.1 h1:yJ9WlDih9HT457QPuHt/TH/XtsdN2tubyxyQHSHPsEo=
go.etcd.io/etcd/api/v3 v3.6.1/go.mod h1:lnfuqoGsXMlZdTJlact3IB56o3bWp1DIlXPIGKRArto=
go.etcd.io/etcd/client/pkg/v3 v3.6.1 h1:CxDVv8ggphmamrXM4Of8aCC8QHzDM4tGcVr9p2BSoGk=
go.etcd.io/etcd/client/pkg/v3 v3.6.1/go.mod h1:aTkCp+6ixcVTZmrJGa7/Mc5nMNs59PEgBbq+HCmWyMc=
go.etcd.io/etcd/client/v3 v3.6.1 h1:KelkcizJGsskUXlsxjVrSmINvMMga0VWwFF0tSPGEP0=
go.etcd.io/etcd/client/v3 v3.6.1/go.mod h1:fCbPUdjWNLfx1A6ATo9syUmFVxqHH9bCnPLBZmnLmMY=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 h1:bDMKF3RUSxshZ5OjOTi8rsHGaPKsAt76FaqgvIUySLc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0/go.mod h1:dDT67G/IkA46Mr2l9Uj7HsQVwsjASyV9SjGofsiUZDA=
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo=
go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q=
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329 h1:9kj3STMvgqy3YA4VQXBrN7925ICMxD5wzMRcgA30588=
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20220923203811-8be639271d50/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210110051926-789bb1bd4061/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210123111255-9b0068b26619/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -373,50 +442,68 @@ golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uI
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA=
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10 h1:3GDAcqdIg1ozBNLgPy4SLT84nfcBjr6rhGtXYtrkWLU=
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10/go.mod h1:T97yPqesLiNrOYxkwmhMI0ZIlJDm+p0PMR8eRVeR5tQ=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA=
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ=
google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I=
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY=
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok=
google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc=
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo=
gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4=
gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
k8s.io/api v0.36.1 h1:XbL/EMj8K2aJpJtePmqUyQMsM0D4QI2pvl7YKJ20FTY=
k8s.io/api v0.36.1/go.mod h1:KOWo4ey3TINlXjeHVuwB3i+tXXnu+UcwFBHlI/9dvEo=
k8s.io/apimachinery v0.36.1 h1:G63Gjx2W+q0YD+72Vo8oY0nDnePVwnuzTmmy5ENrVSA=
k8s.io/apimachinery v0.36.1/go.mod h1:ibYOR00vW/I1kzvi5SF0dRuJ52BvKtfvRdOn35GPQ+8=
k8s.io/client-go v0.36.1 h1:FN/K8QIT2CEDt+2WB2HnWrUANZ50AP5GII43/SP2JR0=
k8s.io/client-go v0.36.1/go.mod h1:s6rAnCtTGYDQnpNjEhSaISV+2O8jwruZ6m3QOYBFbtU=
k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc=
k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0=
k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg=
k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0=
k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU=
k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk=
honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8=
k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE=
k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA=
k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM=
k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA=
k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg=
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4=
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8=
k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0=
k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
modernc.org/cc/v4 v4.24.4 h1:TFkx1s6dCkQpd6dKurBNmpo+G8Zl4Sq/ztJ+2+DEsh0=
modernc.org/cc/v4 v4.24.4/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
modernc.org/ccorpus2 v1.5.2 h1:Ui+4tc58mf/W+2arcYCJR903y3zl3ecsI7Fpaaqozyw=
modernc.org/ccorpus2 v1.5.2/go.mod h1:Wifvo4Q/qS/h1aRoC2TffcHsnxwTikmi1AuLANuucJQ=
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
sigs.k8s.io/kind v0.31.0 h1:UcT4nzm+YM7YEbqiAKECk+b6dsvc/HRZZu9U0FolL1g=
sigs.k8s.io/kind v0.31.0/go.mod h1:FSqriGaoTPruiXWfRnUXNykF8r2t+fHtK0P0m1AbGF8=
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
sigs.k8s.io/kind v0.29.0 h1:3TpCsyh908IkXXpcSnsMjWdwdWjIl7o9IMZImZCWFnI=
sigs.k8s.io/kind v0.29.0/go.mod h1:ldWQisw2NYyM6k64o/tkZng/1qQW7OlzcN5a8geJX3o=
sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8=
sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc=
sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps=
sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

View File

@@ -2,18 +2,16 @@ package arp
import (
"context"
"fmt"
log "log/slog"
"sync"
"time"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
type Manager struct {
instances sync.Map
instances map[string]*Instance
config *kubevip.Config
}
@@ -25,12 +23,9 @@ type Instance struct {
}
func NewManager(config *kubevip.Config) *Manager {
if config.ArpBroadcastRate < 500 {
log.Warn("[ARP manager] arp broadcast rate is too low", "rate (ms)", config.ArpBroadcastRate, "setting to (ms)", "3000")
config.ArpBroadcastRate = 3000
}
return &Manager{
config: config,
instances: make(map[string]*Instance),
config: config,
}
}
@@ -47,14 +42,10 @@ func (i *Instance) Name() string {
}
func (m *Manager) Insert(instance *Instance) {
i, err := m.get(instance.Name())
if err != nil {
log.Error("[ARP manager] unable to insert instance", "err", err)
return
}
if i == nil {
log.Info("[ARP manager] inserting ARP/NDP instance", "name", instance.Name())
m.instances.Store(instance.Name(), instance)
i, ok := m.instances[instance.Name()]
if !ok {
log.Info("inserting ARP/NDP instance", "name", instance.Name())
m.instances[instance.Name()] = instance
} else {
i.mu.Lock()
defer i.mu.Unlock()
@@ -63,49 +54,20 @@ func (m *Manager) Insert(instance *Instance) {
}
func (m *Manager) Remove(instance *Instance) {
m.RemoveWithIPDelete(instance, true)
}
// RemoveOnLeadershipLoss removes an ARP instance when leadership is lost
func (m *Manager) RemoveOnLeadershipLoss(instance *Instance) {
// Use the inverse of PreserveVIPOnLeadershipLoss to decide whether to delete the IP
// If preserve is true, don't delete IP (deleteIP = false)
// If preserve is false, delete IP (deleteIP = true), This is the legacy behavior
deleteIP := !m.config.PreserveVIPOnLeadershipLoss
m.RemoveWithIPDelete(instance, deleteIP)
}
func (m *Manager) RemoveWithIPDelete(instance *Instance, deleteIP bool) {
i, err := m.get(instance.Name())
if err != nil {
log.Error("[ARP manager] unable to remove the instance", "err", err)
return
}
if i != nil {
if i, ok := m.instances[instance.Name()]; ok {
i.mu.Lock()
defer i.mu.Unlock()
i.counter--
if i.counter == 0 {
log.Info("[ARP manager] removing ARP/NDP instance", "name", instance.Name())
if deleteIP {
if _, err := instance.network.DeleteIP(); err != nil {
log.Error("failed to delete IP", "address", instance.network.IP(), "err", err)
}
}
m.instances.Delete(instance.Name())
if i.counter > 1 {
i.counter--
} else {
log.Info("removing ARP/NDP instance", "name", instance.Name())
delete(m.instances, instance.Name())
}
} else {
log.Warn("[ARP manager] unable to remove the instance - instance not found", "name", instance.Name())
}
}
func (m *Manager) Count(name string) int {
i, err := m.get(name)
if err != nil {
log.Error("[ARP manager] unable to count instance", "err", err)
return -1
}
if i != nil {
if i, ok := m.instances[name]; ok {
i.mu.Lock()
defer i.mu.Unlock()
return i.counter
@@ -114,47 +76,26 @@ func (m *Manager) Count(name string) int {
}
func (m *Manager) StartAdvertisement(ctx context.Context) {
log.Info("[ARP manager] starting ARP/NDP advertisement")
ticker := time.NewTicker(time.Duration(m.config.ArpBroadcastRate) * time.Millisecond)
defer ticker.Stop()
log.Info("Starting ARP/NDP advertisement")
for {
select {
case <-ctx.Done(): // if cancel() execute
return
case <-ticker.C: // send gratuitous ARP/NDP on each tick
m.instances.Range(func(_ any, instance any) bool {
if i, ok := instance.(*Instance); ok {
i.mu.Lock()
defer i.mu.Unlock()
if i.counter > 0 {
ensureIPAndSendGratuitous(i)
} else {
// this instance should not be advertised - delete the IP just in case...
if _, err := i.network.DeleteIP(); err != nil {
log.Error("[ARP manager] failed to delete IP", "address", i.network.IP(), "err", err)
}
}
default:
for _, instance := range m.instances {
if instance.counter > 0 {
ensureIPAndSendGratuitous(instance)
}
return true
})
}
}
if m.config.ArpBroadcastRate < 500 {
log.Error("arp broadcast rate is too low", "rate (ms)", m.config.ArpBroadcastRate, "setting to (ms)", "3000")
m.config.ArpBroadcastRate = 3000
}
time.Sleep(time.Duration(m.config.ArpBroadcastRate) * time.Millisecond)
}
}
func (m *Manager) get(name string) (*Instance, error) {
i, exists := m.instances.Load(name)
if !exists {
return nil, nil
}
inst, ok := i.(*Instance)
if !ok {
return nil, fmt.Errorf("value for name %q is not of Instance pointer type", name)
}
return inst, nil
}
// ensureIPAndSendGratuitous - adds IP to the interface if missing, and send
// either a gratuitous ARP or gratuitous NDP. Re-adds the interface if it is IPv6
// and in a dadfailed state.
@@ -170,27 +111,23 @@ func ensureIPAndSendGratuitous(instance *Instance) {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted and recreating address with NODAD flag to skip DAD", "IP", ipString, "interface", iface)
// Re-add immediately without DAD check since we're recovering from DADFAILED
// The AddIP function will set IFA_F_NODAD flag for IPv6 addresses when skipDAD=true
if _, err := instance.network.AddIP(false, true); err != nil {
log.Error("failed to recreate address after DADFAILED", "IP", ipString, "interface", iface, "err", err)
} else {
log.Info("successfully recreated address after DADFAILED recovery", "IP", ipString, "interface", iface)
}
log.Info("deleted and recreating address", "IP", ipString, "interface", iface)
// if _, err := instance.network.AddIP(false); err != nil {
// log.Error("failed to recreate address", "IP", ipString, "interface", iface)
// }
}
// Return early after DADFAILED recovery to avoid double IP addition
return
}
// Normal case: add IP with precheck and normal DAD process
if added, err := instance.network.AddIP(true, false); err != nil {
// Ensure the address exists on the interface before attempting to ARP
// if instance.network.HasEndpoints() {
if added, err := instance.network.AddIP(true); err != nil {
log.Warn(err.Error())
} else if added {
log.Warn("Re-applied the VIP configuration", "ip", ipString, "interface", iface)
}
// }
if utils.IsIPv6(ipString) {
if vip.IsIPv6(ipString) {
// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
if instance.ndp == nil {
log.Error("NDP responder was not created")
@@ -208,4 +145,5 @@ func ensureIPAndSendGratuitous(instance *Instance) {
log.Warn(err.Error())
}
}
}

View File

@@ -1,7 +1,6 @@
package backend
import (
"context"
"fmt"
"time"
@@ -9,6 +8,7 @@ import (
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
@@ -31,7 +31,7 @@ func (e *Entry) Check() bool {
// homeConfigPath := filepath.Join(os.Getenv("HOME"), ".kube", "config")
var k8sAddr string
if utils.IsIPv4(e.Addr) {
if vip.IsIPv4(e.Addr) {
k8sAddr = fmt.Sprintf("%s:%v", e.Addr, e.Port)
} else {
k8sAddr = fmt.Sprintf("[%s]:%v", e.Addr, e.Port)
@@ -66,7 +66,7 @@ func (e *Entry) Check() bool {
return true
}
func Watch(ctx context.Context, interval int, tickAction func()) {
func Watch(tickAction func(), interval int, stop chan struct{}) {
if interval <= 0 {
interval = 5
}
@@ -76,10 +76,13 @@ func Watch(ctx context.Context, interval int, tickAction func()) {
for {
select {
case <-ctx.Done():
case <-stop:
ticker.Stop()
return
case <-ticker.C:
ticker.Stop()
tickAction()
ticker.Reset(time.Second * time.Duration(interval))
}
}
}

View File

@@ -3,78 +3,46 @@ package bgp
import (
"context"
"fmt"
log "log/slog"
"net"
api "github.com/osrg/gobgp/v3/api"
)
// AddHost will update peers of a host
func (b *Server) AddHost(ctx context.Context, addr string, object string) error {
b.mtx.Lock()
defer b.mtx.Unlock()
objects, exists := b.tracker[addr]
if !exists {
b.tracker[addr] = make(map[string]bool)
objects = b.tracker[addr]
ip, _, err := net.ParseCIDR(addr)
if err != nil {
return err
}
p := b.getPath(ip)
if p == nil {
return fmt.Errorf("failed to get path for %v", ip)
}
if _, err := b.s.AddPath(ctx, &api.AddPathRequest{
Path: p,
}); err != nil {
return err
}
log.Debug("[BGP] added host", "addr", addr, "cnt", len(objects)+1)
}
objects[object] = true
return nil
}
// DelHost will inform peers to remove a host
func (b *Server) DelHost(ctx context.Context, addr string, object string) error {
b.mtx.Lock()
defer b.mtx.Unlock()
objects, exists := b.tracker[addr]
if !exists {
log.Debug("[BGP] deleting host - nothing to delete", "addr", addr)
return nil
}
func (b *Server) AddHost(addr string) (err error) {
ip, _, err := net.ParseCIDR(addr)
if err != nil {
return err
}
delete(objects, object)
if len(objects) == 0 {
p := b.getPath(ip)
if p == nil {
return nil
}
if err := b.s.DeletePath(ctx, &api.DeletePathRequest{
Path: p,
}); err != nil {
return err
}
delete(b.tracker, addr)
log.Debug("[BGP] deleted host", "addr", addr, "cnt", len(objects))
p := b.getPath(ip)
if p == nil {
return fmt.Errorf("failed to get path for %v", ip)
}
return nil
_, err = b.s.AddPath(context.Background(), &api.AddPathRequest{
Path: p,
})
if err != nil {
return err
}
return
}
// DelHost will inform peers to remove a host
func (b *Server) DelHost(addr string) (err error) {
ip, _, err := net.ParseCIDR(addr)
if err != nil {
return err
}
p := b.getPath(ip)
if p == nil {
return
}
return b.s.DeletePath(context.Background(), &api.DeletePathRequest{
Path: p,
})
}

View File

@@ -5,29 +5,22 @@ import (
"fmt"
"net"
"strconv"
"time"
"strings"
"github.com/jpillora/backoff"
"github.com/kube-vip/kube-vip/pkg/kubevip"
//nolint
"github.com/kube-vip/kube-vip/pkg/vip"
api "github.com/osrg/gobgp/v3/api"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/osrg/gobgp/v3/pkg/config/oc"
"github.com/osrg/gobgp/v3/pkg/server"
"google.golang.org/protobuf/types/known/anypb"
)
const defaultBGPPort uint32 = 179
// AddPeer will add peers to the BGP configuration
func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error) {
func (b *Server) AddPeer(peer Peer) (err error) {
p := &api.Peer{
Conf: &api.PeerConf{
NeighborAddress: peer.Address,
PeerAsn: peer.AS,
NeighborInterface: peer.Interface,
AuthPassword: peer.Password,
NeighborAddress: peer.Address,
PeerAsn: peer.AS,
AuthPassword: peer.Password,
},
Timers: &api.Timers{
@@ -47,21 +40,10 @@ func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error)
Transport: &api.Transport{
MtuDiscovery: true,
RemoteAddress: peer.Address,
RemotePort: defaultBGPPort,
RemotePort: uint32(179),
},
}
if peer.Interface != "" {
neighborAddress, err := getIPv6LinkLocalNeighborAddress(ctx, peer.Interface)
if err != nil {
return fmt.Errorf("failed to get link-local address of interface %s: %w", peer.Interface, err)
}
p.State = &api.PeerState{
NeighborAddress: neighborAddress,
}
}
if b.c.MpbgpNexthop != "" {
p.AfiSafis = []*api.AfiSafi{
{
@@ -84,23 +66,23 @@ func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error)
},
}
peer.SetMpbgpOptions(b.c)
peer.setMpbgpOptions(b.c)
ipv4Address, ipv6Address, err := peer.FindMpbgpAddresses(p, b.c)
ipv4Address, ipv6Address, err := peer.findMpbgpAddresses(p, b.c)
if err != nil {
return fmt.Errorf("failed to get MP-BGP addresses: %w", err)
}
mask := strconv.Itoa(vip.DefaultMaskIPv6)
mask := "128"
address := ipv4Address
family := api.Family_AFI_IP
if utils.IsIPv4(p.Conf.NeighborAddress) {
mask = strconv.Itoa(vip.DefaultMaskIPv4)
if vip.IsIPv4(p.Conf.NeighborAddress) {
mask = "32"
address = ipv6Address
family = api.Family_AFI_IP6
}
err = b.s.AddDefinedSet(ctx, &api.AddDefinedSetRequest{
err = b.s.AddDefinedSet(context.Background(), &api.AddDefinedSetRequest{
DefinedSet: &api.DefinedSet{
DefinedType: api.DefinedType_NEIGHBOR,
Name: fmt.Sprintf("peer-%s", p.Conf.NeighborAddress),
@@ -112,7 +94,7 @@ func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error)
}
if address != "" {
if err := insertPolicy(ctx, b.s, address, p, family); err != nil {
if err := insertPolicy(b.s, address, p, family); err != nil {
return fmt.Errorf("failed to add policy: %w", err)
}
}
@@ -126,7 +108,7 @@ func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error)
}
}
if err := b.s.AddPeer(ctx, &api.AddPeerRequest{Peer: p}); err != nil {
if err := b.s.AddPeer(context.Background(), &api.AddPeerRequest{Peer: p}); err != nil {
return fmt.Errorf("failed to add peer: %v", err)
}
@@ -145,7 +127,7 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
//nolint
nlri, _ := anypb.New(&api.IPAddressPrefix{
Prefix: ip.String(),
PrefixLen: vip.DefaultMaskIPv4,
PrefixLen: 32,
})
//nolint
@@ -165,7 +147,7 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
//nolint
nlri, _ := anypb.New(&api.IPAddressPrefix{
Prefix: ip.String(),
PrefixLen: vip.DefaultMaskIPv6,
PrefixLen: 128,
})
v6Family := &api.Family{
@@ -189,7 +171,105 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
return
}
func insertPolicy(ctx context.Context, s *server.BgpServer, address string, p *api.Peer, family api.Family_Afi) error {
// ParseBGPPeerConfig - take a string and parses it into an array of peers
func ParseBGPPeerConfig(config string) (bgpPeers []Peer, err error) {
peers := strings.Split(config, ",")
if len(peers) == 0 {
return nil, fmt.Errorf("no BGP Peer configurations found")
}
for x := range peers {
peerStr := peers[x]
config := strings.Split(peerStr, "/")
peerStr = config[0]
if peerStr == "" {
continue
}
isV6Peer := peerStr[0] == '['
address := ""
if isV6Peer {
addressEndPos := strings.IndexByte(peerStr, ']')
if addressEndPos == -1 {
return nil, fmt.Errorf("no matching ] found for IPv6 BGP Peer")
}
address = peerStr[1:addressEndPos]
peerStr = peerStr[addressEndPos+1:]
}
peer := strings.Split(peerStr, ":")
if len(peer) < 2 {
return nil, fmt.Errorf("mandatory peering params <host>:<AS> incomplete")
}
if !isV6Peer {
address = peer[0]
}
ASNumber, err := strconv.ParseUint(peer[1], 10, 32)
if err != nil {
return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
}
password := ""
if len(peer) >= 3 {
password = peer[2]
}
multiHop := false
if len(peer) >= 4 {
multiHop, err = strconv.ParseBool(peer[3])
if err != nil {
return nil, fmt.Errorf("BGP MultiHop format error (true/false) [%s]", peer[1])
}
}
var port uint64
if len(peer) >= 5 {
port, err = strconv.ParseUint(peer[4], 10, 16)
if err != nil {
return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
}
} else {
port = 179
}
var mpbgpNexthop, mpbgpIPv4, mpbgpIPv6 string
if len(config) > 1 {
configData := strings.Split(config[1], ";")
for _, cfg := range configData {
c := strings.Split(cfg, "=")
switch c[0] {
case "mpbgp_nexthop":
mpbgpNexthop = c[1]
case "mpbgp_ipv4":
mpbgpIPv4 = c[1]
case "mpbgp_ipv6":
mpbgpIPv6 = c[1]
default:
return nil, fmt.Errorf("peer configuration parameter '%s' is not supported", c[0])
}
}
}
peerConfig := Peer{
Address: address,
AS: uint32(ASNumber),
Port: uint16(port),
Password: password,
MultiHop: multiHop,
MpbgpNexthop: mpbgpNexthop,
MpbgpIPv4: mpbgpIPv4,
MpbgpIPv6: mpbgpIPv6,
}
bgpPeers = append(bgpPeers, peerConfig)
}
return
}
func insertPolicy(s *server.BgpServer, address string, p *api.Peer, family api.Family_Afi) error {
familyType := "v4"
if family == api.Family_AFI_IP6 {
familyType = "v6"
@@ -235,14 +315,14 @@ func insertPolicy(ctx context.Context, s *server.BgpServer, address string, p *a
},
}
err := s.AddPolicy(ctx, &api.AddPolicyRequest{
err := s.AddPolicy(context.Background(), &api.AddPolicyRequest{
Policy: policy,
})
if err != nil {
return fmt.Errorf("failed to add policy: %w", err)
}
err = s.AddPolicyAssignment(ctx, &api.AddPolicyAssignmentRequest{
err = s.AddPolicyAssignment(context.Background(), &api.AddPolicyAssignmentRequest{
Assignment: &api.PolicyAssignment{
Name: "global",
Direction: api.PolicyDirection_EXPORT,
@@ -259,44 +339,3 @@ func insertPolicy(ctx context.Context, s *server.BgpServer, address string, p *a
return nil
}
func getIPv6LinkLocalNeighborAddress(ctx context.Context, peerInterface string) (string, error) {
neighCtx, neighCancel := context.WithTimeout(ctx, time.Minute)
defer neighCancel()
bo := backoff.Backoff{
Factor: 2,
Jitter: true,
Min: 1 * time.Second,
Max: 5 * time.Second,
}
maxAttempts := 20.0
var err error
for {
select {
case <-neighCtx.Done():
if err != nil {
return "", fmt.Errorf("failed to get link-local address of interface %s: %w", peerInterface, err)
}
return "", fmt.Errorf("failed to get link-local address of interface %s: %w", peerInterface, neighCtx.Err())
default:
dur := bo.Duration()
neighborAddress, err := oc.GetIPv6LinkLocalNeighborAddress(peerInterface)
if err != nil && bo.Attempt() >= maxAttempts {
return "", fmt.Errorf("failed to get link-local address of interface %s: %w", peerInterface, err)
}
if neighborAddress != "" {
return neighborAddress, nil
}
t := time.NewTimer(dur)
select {
case <-neighCtx.Done():
t.Stop()
case <-t.C:
}
}
}
}

View File

@@ -1,4 +1,4 @@
package kubevip
package bgp
import (
"reflect"
@@ -12,13 +12,13 @@ func TestParseBGPPeerConfig(t *testing.T) {
tests := []struct {
name string
args args
wantBgpPeers []BGPPeer
wantBgpPeers []Peer
wantErr bool
}{
{
name: "IPv4, default port",
args: args{config: "192.168.0.10:65000::false,192.168.0.11:65000::false"},
wantBgpPeers: []BGPPeer{
wantBgpPeers: []Peer{
{Address: "192.168.0.10", Port: 179, AS: 65000, MultiHop: false},
{Address: "192.168.0.11", Port: 179, AS: 65000, MultiHop: false},
},
@@ -26,7 +26,7 @@ func TestParseBGPPeerConfig(t *testing.T) {
{
name: "IPv4, different port",
args: args{config: "192.168.0.10:65000::false:180,192.168.0.11:65000::false:190"},
wantBgpPeers: []BGPPeer{
wantBgpPeers: []Peer{
{Address: "192.168.0.10", Port: 180, AS: 65000, MultiHop: false},
{Address: "192.168.0.11", Port: 190, AS: 65000, MultiHop: false},
},
@@ -34,47 +34,10 @@ func TestParseBGPPeerConfig(t *testing.T) {
{
name: "IPv6, multi-protocol",
args: args{config: "[fd00:1111:2222:3333:c7d9:7235:6bf7:5d52]:65501::false/mpbgp_nexthop=auto_sourceif"},
wantBgpPeers: []BGPPeer{
wantBgpPeers: []Peer{
{Address: "fd00:1111:2222:3333:c7d9:7235:6bf7:5d52", Port: 179, AS: 65501, MultiHop: false, MpbgpNexthop: "auto_sourceif"},
},
},
{
name: "IPv6 bracketed, with password and multihop",
args: args{config: "[fd00:100:64::2]:65000:secret:true"},
wantBgpPeers: []BGPPeer{
{Address: "fd00:100:64::2", Port: 179, AS: 65000, Password: "secret", MultiHop: true},
},
},
{
name: "IPv6 bracketed, empty fields",
args: args{config: "[fd00:100:64::2]:65000::false"},
wantBgpPeers: []BGPPeer{
{Address: "fd00:100:64::2", Port: 179, AS: 65000, MultiHop: false},
},
},
{
name: "Unnumbered",
args: args{config: "unnumbered:eth0,unnumbered:eth1:65000::true/mpbgp_nexthop=auto_sourceif"},
wantBgpPeers: []BGPPeer{
{Interface: "eth0", MultiHop: false},
{Interface: "eth1", AS: 65000, MultiHop: true, MpbgpNexthop: "auto_sourceif"},
},
},
{
name: "Malformed parameter (no value)",
args: args{config: "1.2.3.4:65000/mpbgp_nexthop"},
wantErr: true,
},
{
name: "Unsupported parameter",
args: args{config: "1.2.3.4:65000/unknown=value"},
wantErr: true,
},
{
name: "Malformed IPv6 (no matching bracket)",
args: args{config: "[fd00:100:64::2:65000"},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {

View File

@@ -3,26 +3,17 @@ package bgp
import (
"context"
"fmt"
"sync"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
api "github.com/osrg/gobgp/v3/api"
gobgp "github.com/osrg/gobgp/v3/pkg/server"
"github.com/prometheus/client_golang/prometheus"
)
// Server manages a server object
type Server struct {
s *gobgp.BgpServer
c *kubevip.BGPConfig
mtx sync.Mutex
tracker map[string]map[string]bool
}
// NewBGPServer takes a configuration and returns a running BGP server instance
func NewBGPServer(c kubevip.BGPConfig) (b *Server, err error) {
func NewBGPServer(c *Config) (b *Server, err error) {
if c.AS == 0 {
return nil, fmt.Errorf("you need to provide AS")
}
@@ -36,18 +27,24 @@ func NewBGPServer(c kubevip.BGPConfig) (b *Server, err error) {
}
b = &Server{
s: gobgp.NewBgpServer(),
c: &c,
tracker: make(map[string]map[string]bool),
s: gobgp.NewBgpServer(),
c: c,
BGPSessionInfoGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "kube_vip",
Subsystem: "manager",
Name: "bgp_session_info",
Help: "Display state of session by setting metric for label value with current state to 1",
}, []string{"state", "peer"}),
}
return
}
// Start starts the BGP server
func (b *Server) Start(ctx context.Context, peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (err error) {
func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (err error) {
go b.s.Serve()
if err = b.s.StartBgp(ctx, &api.StartBgpRequest{
if err = b.s.StartBgp(context.Background(), &api.StartBgpRequest{
Global: &api.Global{
Asn: b.c.AS,
RouterId: b.c.RouterID,
@@ -57,7 +54,7 @@ func (b *Server) Start(ctx context.Context, peerStateChangeCallback func(*api.Wa
return
}
if err = b.s.WatchEvent(ctx, &api.WatchEventRequest{Peer: &api.WatchEventRequest_Peer{}}, func(r *api.WatchEventResponse) {
if err = b.s.WatchEvent(context.Background(), &api.WatchEventRequest{Peer: &api.WatchEventRequest_Peer{}}, func(r *api.WatchEventResponse) {
if p := r.GetPeer(); p != nil && p.Type == api.WatchEventResponse_PeerEvent_STATE {
log.Info("[BGP]", "peer", p.String())
if peerStateChangeCallback != nil {
@@ -69,13 +66,13 @@ func (b *Server) Start(ctx context.Context, peerStateChangeCallback func(*api.Wa
}
for _, p := range b.c.Peers {
if err = b.AddPeer(ctx, p); err != nil {
if err = b.AddPeer(p); err != nil {
return
}
}
if b.c.Zebra.Enabled {
if err = b.s.EnableZebra(ctx, &api.EnableZebraRequest{
if err = b.s.EnableZebra(context.Background(), &api.EnableZebraRequest{
Url: b.c.Zebra.URL,
Version: b.c.Zebra.Version,
SoftwareName: b.c.Zebra.SoftwareName,
@@ -90,8 +87,7 @@ func (b *Server) Start(ctx context.Context, peerStateChangeCallback func(*api.Wa
// Close will stop a running BGP Server
func (b *Server) Close() error {
// create new BGP stop context (independent)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
ctx, cf := context.WithTimeout(context.Background(), 5*time.Second)
defer cf()
return b.s.StopBgp(ctx, &api.StopBgpRequest{})
}

145
pkg/bgp/types.go Normal file
View File

@@ -0,0 +1,145 @@
package bgp
import (
"fmt"
"net"
"github.com/kube-vip/kube-vip/pkg/vip"
api "github.com/osrg/gobgp/v3/api"
gobgp "github.com/osrg/gobgp/v3/pkg/server"
"github.com/prometheus/client_golang/prometheus"
"github.com/vishvananda/netlink"
)
// Peer defines a BGP Peer
type Peer struct {
Address string
Port uint16
AS uint32
Password string
MultiHop bool
MpbgpNexthop string
MpbgpIPv4 string
MpbgpIPv6 string
}
func (p *Peer) setMpbgpOptions(server *Config) {
if p.MpbgpNexthop == "" {
p.MpbgpNexthop = server.MpbgpNexthop
}
if p.MpbgpIPv4 == "" {
p.MpbgpIPv4 = server.MpbgpIPv4
}
if p.MpbgpIPv6 == "" {
p.MpbgpIPv6 = server.MpbgpIPv6
}
}
func (p *Peer) findMpbgpAddresses(ap *api.Peer, server *Config) (string, string, error) {
var ipv4Address, ipv6Address string
switch p.MpbgpNexthop {
case "fixed":
ap.Transport.LocalAddress = server.SourceIP
if p.MpbgpIPv4 == "" && p.MpbgpIPv6 == "" {
return "", "", fmt.Errorf("to use MP-BGP with fixed address at least one IPv4 or IPv6 address has to be provided [current - IPv4: %s, IPv6: %s]",
p.MpbgpIPv4, p.MpbgpIPv6)
}
if p.MpbgpIPv4 != "" {
if net.ParseIP(p.MpbgpIPv4) == nil {
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv4 address", p.MpbgpIPv4)
}
}
if p.MpbgpIPv6 != "" {
if net.ParseIP(p.MpbgpIPv6) == nil {
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv6 address", p.MpbgpIPv6)
}
}
ipv4Address = p.MpbgpIPv4
ipv6Address = p.MpbgpIPv6
case "auto_sourceip":
ap.Transport.LocalAddress = server.SourceIP
// Resolve the local interface by SourceIP
iface, err := vip.GetInterfaceByIP(server.SourceIP)
if err != nil {
return "", "", fmt.Errorf("failed to get interface by IP: %v", err)
}
if vip.IsIPv4(server.SourceIP) {
// Get the non link-local IPv6 address on that interface
ipv6Address, err = vip.GetNonLinkLocalIP(iface, netlink.FAMILY_V6)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
}
} else {
// Get the non link-local IPv4 address on that interface
ipv4Address, err = vip.GetNonLinkLocalIP(iface, netlink.FAMILY_V4)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
}
}
case "auto_sourceif":
ap.Transport.BindInterface = server.SourceIF
iface, err := netlink.LinkByName(server.SourceIF)
if err != nil {
return "", "", fmt.Errorf("failed to get interface by name: %v", err)
}
// Get the non link-local IPv4 address on that interface
ipv4Address, err = vip.GetNonLinkLocalIP(&iface, netlink.FAMILY_V4)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
}
// Get the non link-local IPv6 address on that interface
ipv6Address, err = vip.GetNonLinkLocalIP(&iface, netlink.FAMILY_V6)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
}
default:
return "", "", fmt.Errorf("option %s for MP-BPG nexthop is not supported", server.MpbgpNexthop)
}
return ipv4Address, ipv6Address, nil
}
// Config defines the BGP server configuration
type Config struct {
AS uint32
RouterID string
SourceIP string
SourceIF string
MpbgpNexthop string
MpbgpIPv4 string
MpbgpIPv6 string
HoldTime uint64
KeepaliveInterval uint64
Peers []Peer
Zebra ZebraConfig
}
// Defines Zebra connection configuration. More on the topic - https://github.com/osrg/gobgp/blob/master/docs/sources/zebra.md#configuration
type ZebraConfig struct {
Enabled bool
URL string
Version uint32
SoftwareName string
}
// Server manages a server object
type Server struct {
s *gobgp.BgpServer
c *Config
// This is a prometheus gauge indicating the state of the sessions.
// 1 means "ESTABLISHED", 0 means "NOT ESTABLISHED"
BGPSessionInfoGauge *prometheus.GaugeVec
}

View File

@@ -1,39 +1,28 @@
package cluster
import (
"crypto/tls"
"crypto/x509"
"fmt"
"net/http"
"os"
"time"
"sync"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/node"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/vip"
)
// Cluster - The Cluster object manages the state of the cluster for a particular node
type Cluster struct {
stop chan bool
Network []vip.Network
arpMgr *arp.Manager
routeMgr *route.Manager
nodeLabelMgr node.Labeler
labelAdded bool
healthCheckHTTPClient *http.Client
stop chan bool
completed chan bool
once sync.Once
Network []vip.Network
arpMgr *arp.Manager
}
// InitCluster - Will attempt to initialise all of the required settings for the cluster
func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.Manager, arpMgr *arp.Manager,
routeMgr *route.Manager, nodeLabelMgr node.Labeler) (*Cluster, error) {
func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Cluster, error) {
var networks []vip.Network
var healthCheckHTTPClient *http.Client
var err error
if !disableVIP {
@@ -43,22 +32,10 @@ func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.M
return nil, err
}
}
if c.ControlPlaneHealthCheck.Address != "" {
healthCheckHTTPClient, err = newHealthCheckHTTPClient(c)
if err != nil {
return nil, fmt.Errorf("initializing BGP health check client: %w", err)
}
}
// Initialise the Cluster structure
newCluster := &Cluster{
Network: networks,
arpMgr: arpMgr,
stop: make(chan bool),
routeMgr: routeMgr,
nodeLabelMgr: nodeLabelMgr,
healthCheckHTTPClient: healthCheckHTTPClient,
Network: networks,
arpMgr: arpMgr,
}
log.Debug("service security", "enabled", c.EnableServiceSecurity)
@@ -77,9 +54,9 @@ func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vi
networks := []vip.Network{}
for _, addr := range addresses {
network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.DHCPMode,
c.RequireDualStack, c.IsDualStack, c.RoutingTableID, c.RoutingTableType, c.RoutingProtocol, c.DNSMode,
c.LoadBalancerForwardingMethod, c.IptablesBackend, c.EnableLoadBalancer, c.EnableServiceSecurity, intfMgr)
network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.RoutingTableID,
c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend,
c.EnableLoadBalancer, c.EnableServiceSecurity, intfMgr)
if err != nil {
return nil, err
}
@@ -93,42 +70,11 @@ func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vi
func (cluster *Cluster) Stop() {
// Close the stop channel, which will shut down the VIP (if needed)
if cluster.stop != nil {
close(cluster.stop)
cluster.stop = make(chan bool) // recreate channel for future use
cluster.once.Do(func() { // Ensure that the close channel can only ever be called once
close(cluster.stop)
})
}
}
func newHealthCheckHTTPClient(c *kubevip.Config) (*http.Client, error) {
defaultTransport, ok := http.DefaultTransport.(*http.Transport)
if !ok {
return nil, fmt.Errorf("unexpected default HTTP transport type %T", http.DefaultTransport)
}
transport := defaultTransport.Clone()
if c.ControlPlaneHealthCheck.CAPath != "" {
caCert, err := os.ReadFile(c.ControlPlaneHealthCheck.CAPath)
if err != nil {
return nil, fmt.Errorf("reading health check CA cert %q: %w", c.ControlPlaneHealthCheck.CAPath, err)
}
rootCAs, err := x509.SystemCertPool()
if err != nil || rootCAs == nil {
rootCAs = x509.NewCertPool()
}
if !rootCAs.AppendCertsFromPEM(caCert) {
return nil, fmt.Errorf("health check CA cert %q contains no valid certificates", c.ControlPlaneHealthCheck.CAPath)
}
tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12}
if transport.TLSClientConfig != nil {
tlsConfig = transport.TLSClientConfig.Clone()
}
tlsConfig.RootCAs = rootCAs
transport.TLSClientConfig = tlsConfig
}
return &http.Client{
Timeout: time.Duration(c.ControlPlaneHealthCheck.TimeoutSeconds) * time.Second,
Transport: transport,
}, nil
// Wait until the completed channel is closed, signallign all shutdown tasks completed
<-cluster.completed
}

View File

@@ -2,7 +2,6 @@ package cluster
import (
"context"
"sync"
"github.com/kube-vip/kube-vip/pkg/vip"
)
@@ -13,14 +12,16 @@ import (
// during runtime if IP changes, startDDNS don't have to do reconfigure because
// dnsUpdater already have the functionality to keep trying resolve the IP
// and update the VIP configuration if it changes
func (cluster *Cluster) StartDDNS(ctx context.Context, network vip.Network, backoffAttempts uint, wg *sync.WaitGroup) error {
ddnsMgr := vip.NewDDNSManager(network, backoffAttempts)
ip, err := ddnsMgr.Start(ctx, wg)
if err != nil {
return err
}
if err = network.SetIP(ip); err != nil {
return err
func (cluster *Cluster) StartDDNS(ctx context.Context) error {
for i := range cluster.Network {
ddnsMgr := vip.NewDDNSManager(ctx, cluster.Network[i])
ip, err := ddnsMgr.Start()
if err != nil {
return err
}
if err = cluster.Network[i].SetIP(ip); err != nil {
return err
}
}
return nil

View File

@@ -3,59 +3,131 @@ package cluster
import (
"context"
"fmt"
"sync"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/etcd"
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/loadbalancer"
log "log/slog"
clientv3 "go.etcd.io/etcd/client/v3"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
watchtools "k8s.io/client-go/tools/watch"
)
// StartCluster - Begins a running instance of the Leader Election cluster
func (cluster *Cluster) StartCluster(ctx context.Context, c *kubevip.Config,
em *election.Manager, bgpServer *bgp.Server, leaseMgr *lease.Manager, killFunc func()) error {
// Manager degines the manager of the load-balancing services
type Manager struct {
KubernetesClient *kubernetes.Clientset
RetryWatcherClient *kubernetes.Clientset
// This channel is used to signal a shutdown
SignalChan chan os.Signal
ns, leaseName := lease.NamespaceName(c.LeaseName, c)
EtcdClient *clientv3.Client
}
leaseID := lease.NewID(c.LeaderElectionType, ns, leaseName)
// NewManager will create a new managing object
func NewManager(path string, inCluster bool, port int) (*Manager, error) {
var hostname string
log.Info("cluster membership", "namespace", leaseID.Namespace(), "lock", leaseID.Name(), "id", c.NodeName)
// If inCluster is set then it will likely have started as a static pod or won't have the
// VIP up before trying to connect to the API server, we set the API endpoint to this machine to
// ensure connectivity. Else if the path passed is empty and not running in the cluster,
// attempt to look for a kubeconfig in the default HOME dir.
objectName := lease.ObjectName(leaseID, "cp")
objLease := leaseMgr.Add(ctx, leaseID)
isNew := objLease.Add(objectName)
hostname = fmt.Sprintf("kubernetes:%v", port)
wg := sync.WaitGroup{}
defer wg.Wait()
if len(path) == 0 && !inCluster {
path = filepath.Join(os.Getenv("HOME"), ".kube", "config")
// Start a goroutine that will delete the lease when the service context is cancelled.
// This is important for proper cleanup when a service is deleted - it ensures that
// the lease context (svcLease.Ctx) gets cancelled, which causes RunOrDie to return.
// Without this, RunOrDie would continue running until leadership is naturally lost.
wg.Go(func() {
<-objLease.Ctx.Done()
leaseMgr.Delete(leaseID, objectName)
})
if !isNew {
log.Debug("this election was already done, waiting for it to finish", "lease", leaseName)
<-objLease.Ctx.Done()
return nil
}
wg.Go(func() {
select {
case <-cluster.stop:
case <-ctx.Done():
// We modify the config so that we can always speak to the correct host
id, err := os.Hostname()
if err != nil {
return nil, err
}
hostname = fmt.Sprintf("%s:%v", id, port)
}
config, err := k8s.NewRestConfig(path, inCluster, hostname)
if err != nil {
return nil, fmt.Errorf("failed to create k8s REST config: %w", err)
}
clientset, err := k8s.NewClientset(config)
if err != nil {
return nil, fmt.Errorf("error creating a new k8s clientset: %v", err)
}
rwConfig, err := k8s.NewRestConfig(path, inCluster, hostname)
if err != nil {
return nil, fmt.Errorf("failed to create k8s REST config for retryClientSet: %w", err)
}
rwConfig.Timeout = 0 // empty value to disable the timeout
rwClientSet, err := k8s.NewClientset(rwConfig)
if err != nil {
return nil, fmt.Errorf("failed to create k8s client for retry watcher: %w", err)
}
return &Manager{
KubernetesClient: clientset,
RetryWatcherClient: rwClientSet,
}, nil
}
// StartCluster - Begins a running instance of the Leader Election cluster
func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *bgp.Server) error {
var err error
log.Info("cluster membership", "namespace", c.Namespace, "lock", c.LeaseName, "id", c.NodeName)
// use a Go context so we can tell the leaderelection code when we
// want to step down
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// use a Go context so we can tell the arp loop code when we
// want to step down
ctxArp, cancelArp := context.WithCancel(context.Background())
defer cancelArp()
// use a Go context so we can tell the dns loop code when we
// want to step down
ctxDNS, cancelDNS := context.WithCancel(context.Background())
defer cancelDNS()
// listen for interrupts or the Linux SIGTERM signal and cancel
// our context, which the leader election code will observe and
// step down
signalChan := make(chan os.Signal, 1)
// Add Notification for Userland interrupt
signal.Notify(signalChan, syscall.SIGINT)
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(signalChan, syscall.SIGTERM)
go func() {
<-signalChan
log.Info("Received termination, signaling cluster shutdown")
// Cancel the leader context, which will in turn cancel the leadership
objLease.Cancel()
})
// Cancel the context, which will in turn cancel the leadership
cancel()
// Cancel the arp context, which will in turn stop any broadcasts
}()
// (attempt to) Remove the virtual IP, in case it already exists
@@ -69,124 +141,54 @@ func (cluster *Cluster) StartCluster(ctx context.Context, c *kubevip.Config,
}
}
objLease.Lock()
// Defer a function to check if the bgpServer has been created and if so attempt to close it
defer func() {
objLease.Unlock()
if bgpServer != nil {
bgpServer.Close()
}
}()
// this object is sharing lease with another object
if objLease.Elected.Load() {
log.Debug("this election was already done, shared lease", "lease", leaseName)
// wait for leader election to start or context to be done
select {
case <-objLease.Started:
case <-objLease.Ctx.Done():
// Lease was cancelled (e.g., leader election ended), return immediately
// This allows the restart loop to create a fresh lease
log.Debug("lease context cancelled before leader election started", "lease", leaseName)
return fmt.Errorf("lease %q context cancelled before leader election started", leaseName)
if c.EnableBGP && bgpServer == nil {
// Lets start BGP
log.Info("Starting the BGP server to advertise VIP routes to VGP peers")
bgpServer, err = bgp.NewBGPServer(&c.BGPConfig)
if err != nil {
log.Error("new BGP server", "err", err)
}
if err := bgpServer.Start(nil); err != nil {
log.Error("starting BGP server", "err", err)
}
cluster.OnStartedLeading(c, objLease, em, bgpServer, killFunc, true)
log.Debug("cluster waiting for leader context done", "lease", leaseName)
// wait for leaderelection to be finished
<-objLease.Ctx.Done()
cluster.OnStoppedLeading(c, objLease, bgpServer)
return nil
}
run := &election.RunConfig{
Config: c,
LeaseID: leaseID,
LeaseAnnotations: c.LeaseAnnotations,
Mgr: em,
OnStartedLeading: func(context.Context) { //nolint TODO: potential clean code
cluster.OnStartedLeading(c, objLease, em, bgpServer, killFunc, false)
},
OnStoppedLeading: func() {
objLease.Elected.Store(false)
cluster.OnStoppedLeading(c, objLease, bgpServer)
},
OnNewLeader: func(identity string) {
cluster.OnNewLeader(identity, c)
},
}
if err := election.RunOrDie(objLease.Ctx, run, c); err != nil {
objLease.Cancel()
return fmt.Errorf("leaderelection failed: %w", err)
}
return nil
}
func (cluster *Cluster) OnStartedLeading(c *kubevip.Config, objLease *lease.Lease,
em *election.Manager, bgpServer *bgp.Server, killFunc func(), isShared bool) {
objLease.Elected.Store(true)
objLease.Unlock()
// When we become leader, ensure we can take over VIPs even if they're preserved on other nodes
if !isShared {
close(objLease.Started)
}
labels := generateLabelsFromConfig(c.Address, kubevip.HasIP)
if err := cluster.nodeLabelMgr.AddLabel(labels); err != nil {
log.Error("error adding label to node", "err", err)
}
cluster.labelAdded = true
if c.PreserveVIPOnLeadershipLoss {
log.Info("Becoming leader with VIP preservation enabled - ensuring VIP takeover")
// Force add the VIPs (this will work even if they exist due to the precheck logic)
for i := range cluster.Network {
added, err := cluster.Network[i].AddIP(true, false)
run := &runConfig{
config: c,
leaseID: c.NodeName,
sm: sm,
onStartedLeading: func(ctx context.Context) { //nolint TODO: potential clean code
// As we're leading lets start the vip service
err := cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, cancel)
if err != nil {
log.Error("failed to ensure VIP on leader takeover", "vip", cluster.Network[i].IP(), "err", err)
} else if added {
log.Info("took over VIP as new leader", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
} else {
log.Info("VIP already configured on interface", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
log.Error("starting VIP service on leader", "err", err)
}
}
}
},
onStoppedLeading: func() {
// we can do cleanup here
log.Info("This node is becoming a follower within the cluster")
// As we're leading lets start the vip service
err := cluster.StartVipService(objLease.Ctx, c, em, bgpServer, killFunc)
if err != nil {
log.Error("starting VIP service on leader", "err", err)
killFunc()
}
}
// Stop the dns context
cancelDNS()
// Stop the Arp context if it is running
cancelArp()
func (cluster *Cluster) OnStoppedLeading(c *kubevip.Config, objLease *lease.Lease,
bgpServer *bgp.Server) {
// we can do cleanup here
log.Info("This node is becoming a follower within the cluster")
// Stop the BGP server
if bgpServer != nil {
err := bgpServer.Close()
if err != nil {
log.Warn("close BGP server", "err", err)
}
}
if cluster.labelAdded {
labels := generateLabelsFromConfig(c.Address, kubevip.HasIP)
if err := cluster.nodeLabelMgr.RemoveLabel(labels); err != nil {
log.Error("error removing label from node", "err", err)
}
cluster.labelAdded = false
}
// Stop the cluster context if it is running
objLease.Cancel()
// Handle VIP cleanup based on configuration
if c.PreserveVIPOnLeadershipLoss {
// For IPv6, we must remove VIPs immediately to avoid DAD failures on the new leader
// IPv6 Duplicate Address Detection will fail if the new leader tries to add an IP that is still present on this node's interface
// We need to check each VIP individually and only remove IPv6 VIPs
for i := range cluster.Network {
if utils.IsIPv6(cluster.Network[i].IP()) {
log.Info("Removing IPv6 VIP immediately (required to prevent DAD failures on new leader)", "ip", cluster.Network[i].IP())
for i := range cluster.Network {
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("delete VIP", "err", err)
@@ -194,53 +196,195 @@ func (cluster *Cluster) OnStoppedLeading(c *kubevip.Config, objLease *lease.Leas
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
} else {
log.Info("Preserving IPv4 VIP address on interface, only stopped ARP broadcasting", "ip", cluster.Network[i].IP())
}
}
} else {
// Legacy behavior: delete VIP addresses on leadership loss
log.Info("Deleting VIP addresses on leadership loss (legacy behavior)")
for i := range cluster.Network {
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("delete VIP", "err", err)
log.Error("lost leadership, restarting kube-vip")
panic("") // TODO - we could also return here
},
onNewLeader: func(identity string) {
// we're notified when new leader elected
log.Info("New leader", "leader", identity)
},
}
switch c.LeaderElectionType {
case "kubernetes", "":
cluster.runKubernetesLeaderElectionOrDie(ctx, run)
case "etcd":
cluster.runEtcdLeaderElectionOrDie(ctx, run)
default:
log.Info(fmt.Sprintf("LeaderElectionMode %s not supported, exiting", c.LeaderElectionType))
}
return nil
}
type runConfig struct {
config *kubevip.Config
leaseID string
sm *Manager
// onStartedLeading is called when this member starts leading.
onStartedLeading func(context.Context)
// onStoppedLeading is called when this member stops leading.
onStoppedLeading func()
// onNewLeader is called when the client observes a leader that is
// not the previously observed leader. This includes the first observed
// leader when the client starts.
onNewLeader func(identity string)
}
func (cluster *Cluster) runKubernetesLeaderElectionOrDie(ctx context.Context, run *runConfig) {
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: run.config.LeaseName,
Namespace: run.config.Namespace,
Annotations: run.config.LeaseAnnotations,
},
Client: run.sm.KubernetesClient.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: run.leaseID,
},
}
// start the leader election code loop
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(run.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(run.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(run.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: run.onStartedLeading,
OnStoppedLeading: run.onStoppedLeading,
OnNewLeader: run.onNewLeader,
},
})
}
func (cluster *Cluster) runEtcdLeaderElectionOrDie(ctx context.Context, run *runConfig) {
etcd.RunElectionOrDie(ctx, &etcd.LeaderElectionConfig{
EtcdConfig: etcd.ClientConfig{Client: run.sm.EtcdClient},
Name: run.config.LeaseName,
MemberID: run.leaseID,
LeaseDurationSeconds: int64(run.config.LeaseDuration),
Callbacks: etcd.LeaderCallbacks{
OnStartedLeading: run.onStartedLeading,
OnStoppedLeading: run.onStoppedLeading,
OnNewLeader: run.onNewLeader,
},
})
}
func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
log.Info("Kube-Vip is watching nodes for control-plane labels")
listOptions := metav1.ListOptions{
LabelSelector: "node-role.kubernetes.io/control-plane",
}
rw, err := watchtools.NewRetryWatcherWithContext(ctxArp, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.RetryWatcherClient.CoreV1().Nodes().Watch(context.Background(), listOptions)
},
})
if err != nil {
return fmt.Errorf("error creating label watcher: %s", err.Error())
}
go func() {
<-sm.SignalChan
log.Info("Received termination, signaling shutdown")
// Cancel the context
rw.Stop()
}()
ch := rw.ResultChan()
// defer rw.Stop()
for event := range ch {
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
node, ok := event.Object.(*v1.Node)
if !ok {
return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
// Find the node IP address (this isn't foolproof)
for x := range node.Status.Addresses {
if node.Status.Addresses[x].Type == v1.NodeInternalIP {
if checkIfNodeIsReady(node) {
err = lb.AddBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("add IPVS backend", "err", err)
}
} else {
err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("remove IPVS backend", "err", err)
}
}
}
}
case watch.Deleted:
node, ok := event.Object.(*v1.Node)
if !ok {
return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
}
// Find the node IP address (this isn't foolproof)
for x := range node.Status.Addresses {
if node.Status.Addresses[x].Type == v1.NodeInternalIP {
err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("Del IPVS backend", "err", err)
}
}
}
log.Info("Node deleted", "name", node.Name)
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes Nodes")
// This round trip allows us to handle unstructured status
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Error("watcher", "status", status)
default:
}
}
log.Error("lost leadership, restarting kube-vip")
log.Info("Exiting Node watcher")
return nil
}
func (cluster *Cluster) OnNewLeader(identity string, c *kubevip.Config) {
// we're notified when new leader elected
log.Info("New leader", "leader", identity)
// If we're not the new leader and we have VIPs preserved from previous leadership,
// we need to clean them up to avoid conflicts.
if identity != c.NodeName && c.PreserveVIPOnLeadershipLoss {
log.Info("Cleaning up preserved VIPs as another node became leader", "new_leader", identity)
for i := range cluster.Network {
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("failed to cleanup preserved VIP", "vip", cluster.Network[i].IP(), "err", err)
}
if deleted {
log.Info("cleaned up preserved VIP to avoid conflict", "IP", cluster.Network[i].IP(),
"interface", cluster.Network[i].Interface(), "new_leader", identity)
} else {
log.Debug("VIP was not present on this node", "IP", cluster.Network[i].IP(),
"interface", cluster.Network[i].Interface())
func checkIfNodeIsReady(node *v1.Node) bool {
if node == nil {
return false
}
for _, condition := range node.Status.Conditions {
if condition.Type == v1.NodeReady {
if condition.Status == v1.ConditionTrue {
return true
}
}
}
}
func generateLabelsFromConfig(addr, labelKey string) map[string]string {
return map[string]string{
labelKey: utils.SanitizeIPForLabel(addr),
}
return false
}

View File

@@ -1,125 +0,0 @@
package cluster_test
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"encoding/pem"
"math/big"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/kubevip"
)
func TestInitCluster_HealthCheckClientNoCA(t *testing.T) {
t.Parallel()
cfg := &kubevip.Config{
EnableBGP: true,
ControlPlaneHealthCheck: kubevip.HealthCheck{
Address: "http://localhost:6443/livez",
TimeoutSeconds: 5,
},
}
_, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
func TestInitCluster_HealthCheckClientValidCA(t *testing.T) {
t.Parallel()
caPEM := generateTestCACert(t)
caFile := filepath.Join(t.TempDir(), "ca.crt")
if err := os.WriteFile(caFile, caPEM, 0o600); err != nil {
t.Fatal(err)
}
cfg := &kubevip.Config{
EnableBGP: true,
ControlPlaneHealthCheck: kubevip.HealthCheck{
Address: "https://localhost:6443/livez",
TimeoutSeconds: 3,
CAPath: caFile,
},
}
_, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
func TestInitCluster_HealthCheckClientInvalidCAPath(t *testing.T) {
t.Parallel()
cfg := &kubevip.Config{
EnableBGP: true,
ControlPlaneHealthCheck: kubevip.HealthCheck{
Address: "https://localhost:6443/livez",
CAPath: "/nonexistent/ca.crt",
},
}
_, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
if err == nil {
t.Fatal("expected error for invalid CA path")
}
if !strings.Contains(err.Error(), "reading health check CA cert") {
t.Errorf("expected error about reading CA cert, got: %v", err)
}
}
func TestInitCluster_HealthCheckClientInvalidCAContent(t *testing.T) {
t.Parallel()
caFile := filepath.Join(t.TempDir(), "bad-ca.crt")
if err := os.WriteFile(caFile, []byte("not a certificate"), 0o600); err != nil {
t.Fatal(err)
}
cfg := &kubevip.Config{
EnableBGP: true,
ControlPlaneHealthCheck: kubevip.HealthCheck{
Address: "https://localhost:6443/livez",
CAPath: caFile,
},
}
_, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
if err == nil {
t.Fatal("expected error for invalid CA content")
}
if !strings.Contains(err.Error(), "contains no valid certificates") {
t.Errorf("expected error about invalid certificates, got: %v", err)
}
}
// generateTestCACert creates a self-signed CA certificate in PEM format for testing.
func generateTestCACert(t *testing.T) []byte {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
template := &x509.Certificate{
SerialNumber: big.NewInt(1),
NotBefore: time.Now(),
NotAfter: time.Now().Add(time.Hour),
IsCA: true,
BasicConstraintsValid: true,
}
certDER, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
}

View File

@@ -6,132 +6,102 @@ import (
"fmt"
"io/fs"
"net"
"net/http"
"strings"
"os"
"os/signal"
"sync"
"syscall"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/backend"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/loadbalancer"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
// BGPRouteManager allows to manage the routes announced by the BGP server.
type BGPRouteManager interface {
AddHost(ctx context.Context, addr string, object string) error
DelHost(ctx context.Context, addr string, object string) error
}
func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config, em *election.Manager,
bgpServer BGPRouteManager, killFunc func()) error {
func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Config, sm *Manager, bgpServer *bgp.Server, cancelLeaderElection context.CancelFunc) error {
var err error
var wg sync.WaitGroup
defer wg.Wait()
// listen for interrupts or the Linux SIGTERM signal and cancel
// our context, which the leader election code will observe and
// step down
signalChan := make(chan os.Signal, 1)
// Add Notification for Userland interrupt
signal.Notify(signalChan, syscall.SIGINT)
wg.Go(func() {
<-ctx.Done()
killFunc()
})
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(signalChan, syscall.SIGTERM)
loadbalancers := []*loadbalancer.IPVSLoadBalancer{}
var arpWG sync.WaitGroup
for i := range cluster.Network {
network := cluster.Network[i]
if network.IsDDNS() {
if err := cluster.StartDDNS(ctx, cluster.Network[i], c.DHCPBackoffAttempts, &wg); err != nil {
log.Error("failed to start DDNS", "err", err)
}
}
if err := network.SetMask(c.VIPSubnet); err != nil {
return fmt.Errorf("failed to set mask for subnet %q: %w", c.VIPSubnet, err)
log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
panic("")
}
if network.IsDDNS() {
if err := cluster.StartDDNS(ctxDNS); err != nil {
log.Error(err.Error())
}
}
// start the dns updater if address is dns
if network.IsDNS() {
log.Info("starting the DNS updater", "address", network.DNSName())
ipUpdater := vip.NewIPUpdater(network)
wg.Go(func() {
ipUpdater.Run(ctx)
})
ipUpdater.Run(ctxDNS)
}
if !c.EnableRoutingTable {
// Normal VIP addition, use skipDAD=false for normal DAD process
if _, err = network.AddIP(false, false); err != nil {
return fmt.Errorf("failed to add IP address %s: %w", network.IP(), err)
if _, err = network.AddIP(false); err != nil {
log.Error(err.Error())
}
}
if c.EnableBGP {
if c.ControlPlaneHealthCheck.Address != "" {
// The health check loop owns route advertisement/withdrawal when configured.
wg.Go(func() {
cluster.bgpHealthCheckLoop(ctx, c, bgpServer, network.CIDR())
})
} else {
// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation.
log.Debug("Attempting to advertise over BGP", "address", network.CIDR())
err = bgpServer.AddHost(ctx, network.CIDR(), c.NodeName)
if err != nil {
log.Error(err.Error())
}
// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
log.Debug("Attempting to advertise over BGP", "address", network.CIDR())
err = bgpServer.AddHost(network.CIDR())
if err != nil {
log.Error(err.Error())
}
}
if c.EnableLoadBalancer {
lb, err := loadbalancer.NewIPVSLB(ctx, network.IP(), c.LoadBalancerPort, c.LoadBalancerForwardingMethod,
c.BackendHealthCheckInterval, killFunc, &wg)
lb, err := loadbalancer.NewIPVSLB(network.IP(), c.LoadBalancerPort, c.LoadBalancerForwardingMethod, c.BackendHealthCheckInterval, c.Interface, cancelLeaderElection, signalChan)
if err != nil {
return fmt.Errorf("creating IPVS LoadBalance: %w", err)
log.Error("Error creating IPVS LoadBalancer", "err", err)
}
wg.Go(func() {
for {
select {
case <-ctx.Done():
return
default:
err = em.NodeWatcher(ctx, lb, c.Port)
if err != nil {
log.Error("Error watching node labels", "err", err)
if errors.Is(err, &utils.PanicError{}) {
killFunc()
return
}
}
}
go func() {
err = sm.NodeWatcher(ctxArp, lb, c.Port) //TODO: We're using the ctxARP as the context this will change when rkatz finishes his change
if err != nil {
log.Error("Error watching node labels", "err", err)
}
})
}()
loadbalancers = append(loadbalancers, lb)
}
if c.EnableARP {
wg.Go(func() {
cluster.layer2Update(ctx, network, c)
})
arpWG.Add(1)
go cluster.layer2Update(ctxArp, network, c, &arpWG)
}
}
if c.EnableLoadBalancer {
// Shutdown function that will wait on this signal, unless we call it ourselves
<-ctx.Done()
<-signalChan
for _, lb := range loadbalancers {
err = lb.RemoveIPVSLB()
if err != nil {
@@ -145,10 +115,17 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
backendMapV6 := backend.Map{}
// only check localhost
ips := []string{}
nodename := ""
if c.NodeName != "" {
if ips, err = getNodeIPs(ctx, c.NodeName, em.KubernetesClient); err != nil && !apierrors.IsNotFound(err) {
log.Error("failed to get IP of control-plane node", "err", err)
nodename = c.NodeName
} else {
nodename = os.Getenv("HOSTNAME")
}
ips := []string{}
if nodename != "" {
if ips, err = getNodeIPs(ctxArp, nodename, sm.KubernetesClient); err != nil && !apierrors.IsNotFound(err) {
log.Error("failed to get IP of control-plane nod", "err", err)
}
}
@@ -179,7 +156,15 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
}
}
backend.Watch(ctx, c.BackendHealthCheckInterval, func() {
stop := make(chan struct{})
// will wait for system interrupt and will send stop signal to backend watch
go func() {
<-signalChan
stop <- struct{}{}
}()
backend.Watch(func() {
for i := range cluster.Network {
network := cluster.Network[i]
networkIP := network.IP()
@@ -188,7 +173,6 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
log.Error("failed to check IP type", "IP", networkIP, "error", err)
continue
}
log.Debug("current ip to process", "ip", networkIP)
backendMap := &backendMapV4
if isNetworkV6 {
@@ -196,11 +180,8 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
}
for entry := range *backendMap {
log.Debug("entry.Check() for entry", "entry", entry)
if entry.Check() {
log.Debug("entry.Check() true")
// Normal VIP addition with precheck, use skipDAD=false for normal DAD process
_, err = network.AddIP(true, false)
_, err = network.AddIP(true)
if err != nil {
log.Error("error adding address", "err", err)
}
@@ -208,11 +189,11 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
log.Info("added backend", "ip", network.IP())
}
err = cluster.routeMgr.Add(c.NodeName, network, true, false)
err = network.AddRoute(true)
if err != nil && !errors.Is(err, fs.ErrExist) && !errors.Is(err, syscall.ESRCH) {
log.Warn(err.Error())
} else if err == nil && !(*backendMap)[entry] {
log.Info("added route", "route", network.PrepareRoute())
log.Info("added route", "route", network.PrepareRoute().String())
}
(*backendMap)[entry] = true
@@ -230,108 +211,29 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
}
if deleteAddress {
err = cluster.routeMgr.Delete(c.NodeName, network)
if err != nil {
err = network.DeleteRoute()
if err != nil && !errors.Is(err, fs.ErrNotExist) && !errors.Is(err, syscall.ESRCH) {
log.Warn("deleting route", "err", err)
} else if err == nil {
log.Info("deleted route", "route", network.PrepareRoute().String())
}
deleted, err := network.DeleteIP()
if err != nil {
log.Error("error deleting IP", "err", err)
killFunc()
return
panic("")
}
if deleted {
log.Info("deleted address", "IP", network.IP(), "interface", network.Interface())
}
}
}
})
}
if c.EnableBGP {
<-ctx.Done()
}, c.BackendHealthCheckInterval, stop)
}
return nil
}
func (cluster *Cluster) bgpHealthCheckLoop(ctx context.Context, c *kubevip.Config, bgpServer BGPRouteManager, vipCIDR string) {
period := time.Duration(c.ControlPlaneHealthCheck.PeriodSeconds) * time.Second
consecutiveFailures := 0
routeAnnounced := false
ticker := time.NewTicker(period)
defer ticker.Stop()
log.Info("Starting BGP health check",
"address", c.ControlPlaneHealthCheck.Address,
"cidr", vipCIDR,
"period", period,
"timeout", cluster.healthCheckHTTPClient.Timeout,
"threshold", c.ControlPlaneHealthCheck.FailureThreshold,
)
for {
statusCode := 0
var healthErr error
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.ControlPlaneHealthCheck.Address, nil)
if err != nil {
healthErr = err
} else {
resp, err := cluster.healthCheckHTTPClient.Do(req)
if err != nil {
healthErr = err
} else {
defer resp.Body.Close()
statusCode = resp.StatusCode
}
}
healthy := healthErr == nil && statusCode == http.StatusOK
if healthy {
consecutiveFailures = 0
if !routeAnnounced {
log.Info("BGP health check passed, announcing route", "cidr", vipCIDR)
if err := bgpServer.AddHost(ctx, vipCIDR, c.NodeName); err != nil {
log.Error("BGP health check: failed to announce route", "cidr", vipCIDR, "err", err)
} else {
routeAnnounced = true
}
}
} else {
consecutiveFailures++
if healthErr != nil {
log.Warn("BGP health check failed", "address", c.ControlPlaneHealthCheck.Address, "consecutive", consecutiveFailures, "err", healthErr)
} else {
log.Warn("BGP health check failed", "address", c.ControlPlaneHealthCheck.Address, "consecutive", consecutiveFailures, "status", statusCode)
}
if consecutiveFailures >= c.ControlPlaneHealthCheck.FailureThreshold && routeAnnounced {
log.Warn("BGP health check threshold reached, withdrawing route", "failureThreshold", c.ControlPlaneHealthCheck.FailureThreshold, "cidr", vipCIDR)
if err := bgpServer.DelHost(ctx, vipCIDR, c.NodeName); err != nil {
log.Error("BGP health check: failed to withdraw route", "cidr", vipCIDR, "err", err)
} else {
routeAnnounced = false
}
}
}
select {
case <-ctx.Done():
if routeAnnounced {
if err := bgpServer.DelHost(ctx, vipCIDR, c.NodeName); err != nil {
log.Error("BGP health check: failed to withdraw route", "cidr", vipCIDR, "err", err)
}
}
return
case <-ticker.C:
}
}
}
func isV6(ip string) (bool, error) {
ipaddr := net.ParseIP(ip)
if ipaddr == nil {
@@ -355,160 +257,109 @@ func getNodeIPs(ctx context.Context, nodename string, client *kubernetes.Clients
}
// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp *bgp.Server, name string, wg *sync.WaitGroup) error {
func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp *bgp.Server, name string, CountRouteReferences func(*netlink.Route) int) {
// use a Go context so we can tell the arp loop code when we
// want to step down
//nolint
lbCtx, lbCancel := context.WithCancel(ctx)
ctxArp, cancelArp := context.WithCancel(ctx)
var lbWg sync.WaitGroup
cluster.stop = make(chan bool, 1)
cluster.completed = make(chan bool, 1)
var arpWG sync.WaitGroup
for i := range cluster.Network {
network := cluster.Network[i]
if network.IsDDNS() {
ddnsReady := make(chan struct{})
lbWg.Go(func() {
// start the DDNS if requested
log.Debug("(svcs) start DDNS", "name", network.DNSName())
if err := cluster.StartDDNS(lbCtx, cluster.Network[i], c.DHCPBackoffAttempts, &lbWg); err != nil {
log.Error("failed to start DDNS", "err", err)
}
close(ddnsReady)
<-lbCtx.Done()
})
<-ddnsReady
}
log.Debug("current ip to process", "ip", network.IP(), "mask", c.VIPSubnet)
if err := network.SetMask(c.VIPSubnet); err != nil {
log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
lbCancel()
return utils.NewPanicError(fmt.Sprintf("failed to set mask for subnet %q: %s", c.VIPSubnet, err.Error()))
panic("")
}
_, err := network.DeleteIP()
if err != nil {
log.Warn("attempted to clean existing VIP", "err", err)
}
log.Debug("config flags", "enable_routing_table", c.EnableRoutingTable, "enable_leader_election", c.EnableLeaderElection, "enable_services_election", c.EnableServicesElection)
if c.EnableRoutingTable && (c.EnableLeaderElection || c.EnableServicesElection) {
err = cluster.routeMgr.Add(name, network, false, false)
err = network.AddRoute(false)
if err != nil {
log.Warn(err.Error())
} else {
log.Info("successful add Route")
}
}
if !c.EnableRoutingTable && !c.EnableBGP && !c.EnableWireguard {
// Normal VIP addition, use skipDAD=false for normal DAD process
// Note: When WireGuard is enabled, the VIP is added to the tunnel interface
// instead of lo, so we skip adding it here.
if _, err = network.AddIP(false, false); err != nil {
} else if !c.EnableRoutingTable {
if _, err = network.AddIP(false); err != nil {
log.Warn(err.Error())
} else {
log.Info("successful add IP")
}
}
if c.EnableARP {
lbWg.Go(func() {
cluster.layer2Update(lbCtx, network, c)
})
arpWG.Add(1)
go cluster.layer2Update(ctxArp, network, c, &arpWG)
}
if c.EnableBGP && (c.EnableLeaderElection || c.EnableServicesElection) {
// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
log.Debug("(svcs) attempting to advertise over BGP", "address", network.CIDR())
err = bgp.AddHost(lbCtx, network.CIDR(), name)
err = bgp.AddHost(network.CIDR())
if err != nil {
log.Error(err.Error())
}
}
}
wg.Go(func() {
for i := range cluster.Network {
network := cluster.Network[i]
// start the dns updater if address is dns
if network.IsDNS() {
log.Info("(svcs) starting the DNS updater", "address", network.DNSName(), "ip", network.IP())
ipUpdater := vip.NewIPUpdater(network)
wg.Go(func() {
ipUpdater.Run(lbCtx)
})
}
}
go func() {
<-cluster.stop
// Stop the loadbalancer context if it is running
lbCancel()
// Stop the Arp context if it is running
cancelArp()
lbWg.Wait() // wait for all cluster ARP/NDP to be finished
arpWG.Wait() // wait for all cluster ARP/NDP to be finished
log.Info("[LOADBALANCER] Stopping load balancers", "name", name)
if c.EnableRoutingTable {
for i := range cluster.Network {
if err := cluster.routeMgr.Delete(name, cluster.Network[i]); err != nil {
log.Warn(err.Error())
// chek if route is not referenced by another service
r := cluster.Network[i].PrepareRoute()
if CountRouteReferences(r) < 1 {
log.Info("[VIP] Deleting Route for VIP", "IP", cluster.Network[i].IP())
if err := cluster.Network[i].DeleteRoute(); err != nil {
log.Warn(err.Error())
}
}
}
close(cluster.completed)
return
}
for i := range cluster.Network {
if c.EnableARP && cluster.arpMgr.Count(cluster.Network[i].ARPName()) > 1 {
if c.EnableARP && cluster.arpMgr.Count(cluster.Network[i].ARPName()) > 0 {
continue
}
// Handle VIP cleanup based on configuration
if c.PreserveVIPOnLeadershipLoss {
// For IPv6, we must remove VIPs immediately to avoid DAD failures on the new leader
// IPv6 Duplicate Address Detection will fail if the new leader tries to add an IP
// that is still present on this node's interface
if utils.IsIPv6(cluster.Network[i].IP()) {
log.Info("[VIP] Removing IPv6 VIP immediately (required to prevent DAD failures on new leader)", "ip", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
} else {
log.Info("[VIP] Preserving IPv4 VIP address on interface, only stopped ARP broadcasting", "ip", cluster.Network[i].IP())
}
} else {
// Legacy behavior: delete VIP addresses on leadership loss
log.Info("[VIP] Deleting VIP", "ip", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
log.Info("[VIP] Deleting VIP", "ip", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
}
})
return nil
close(cluster.completed)
}()
}
// Layer2Update, handles the creation of the
func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config) {
func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config, arpWG *sync.WaitGroup) {
defer arpWG.Done()
log.Info("layer 2 broadcaster starting")
var ndp *vip.NdpResponder
var err error
ipString := network.IP()
if utils.IsIPv6(ipString) {
if vip.IsIPv6(ipString) {
if network.IPisLinkLocal() {
log.Error("layer2 is link-local can't use NDP", "address", ipString)
} else {
ndp, err = waitNDPResponder(ctx, network.Interface())
ndp, err = vip.NewNDPResponder(network.Interface())
if err != nil {
log.Error("failed to create new NDP Responder", "error", err)
} else {
@@ -519,7 +370,6 @@ func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c
}
}
log.Info("layer 2 broadcaster starting", "IP", network.IP(), "device", network.Interface())
log.Debug("layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
arpInstance := arp.NewInstance(network, ndp)
@@ -527,32 +377,5 @@ func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c
<-ctx.Done() // if cancel() execute
log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
cluster.arpMgr.RemoveOnLeadershipLoss(arpInstance)
}
func waitNDPResponder(ctx context.Context, ifaceName string) (*vip.NdpResponder, error) {
ndp, err := vip.NewNDPResponder(ifaceName)
if err != nil && strings.Contains(err.Error(), "no such device") {
log.Warn("unable to create NDP responder at first try", "interface", ifaceName, "err", err)
ndpCreateCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
ticker := time.NewTicker(time.Second)
for {
select {
case <-ndpCreateCtx.Done():
return nil, fmt.Errorf("failed to create NDP responder for interface %q: %w", ifaceName, ndpCreateCtx.Err())
case <-ticker.C:
ndp, err = vip.NewNDPResponder(ifaceName)
if err != nil {
log.Warn("unable to create NDP responder on retry", "interface", ifaceName, "err", err)
} else {
return ndp, nil
}
}
}
} else if err != nil {
return nil, fmt.Errorf("unable to create NDP responder for interface %q: %w", ifaceName, err)
}
return ndp, nil
cluster.arpMgr.Remove(arpInstance)
}

View File

@@ -1,347 +0,0 @@
package cluster_test
import (
"context"
"encoding/pem"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
corev1 "k8s.io/api/core/v1"
)
const testCIDR = "10.0.0.34/32"
func TestBGPHealthCheckLoop_AnnouncesOnHealthy(t *testing.T) {
t.Parallel()
healthcheck := newTestHealthServer(t, http.StatusOK)
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
startVipService(t, newTestConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be announced")
}
func TestBGPHealthCheckLoop_NoAnnouncementUntilHealthy(t *testing.T) {
t.Parallel()
healthcheck := newTestHealthServer(t, http.StatusInternalServerError)
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
startVipService(t, newTestConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
expectConsistently(t, func() bool { return !bgpManager.isAnnounced() },
2*time.Second, "route should not be announced while unhealthy")
healthcheck.setStatus(http.StatusOK)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be announced after recovery")
}
func TestBGPHealthCheckLoop_WithdrawsAfterThreshold(t *testing.T) {
t.Parallel()
healthcheck := newTestHealthServer(t, http.StatusOK)
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
cfg := newTestConfig(healthcheck.server.URL, healthcheck.caPath)
cfg.ControlPlaneHealthCheck.FailureThreshold = 3
startVipService(t, cfg, bgpManager)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be announced")
healthcheck.setStatus(http.StatusServiceUnavailable)
expectConsistently(t, func() bool { return bgpManager.isAnnounced() },
1500*time.Millisecond, "route should stay announced before threshold is reached")
expectEventually(t, func() bool { return !bgpManager.isAnnounced() },
"route should be withdrawn after threshold")
}
func TestBGPHealthCheckLoop_ReAnnouncesOnRecovery(t *testing.T) {
t.Parallel()
healthcheck := newTestHealthServer(t, http.StatusOK)
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
cfg := newTestConfig(healthcheck.server.URL, healthcheck.caPath)
cfg.ControlPlaneHealthCheck.FailureThreshold = 1
startVipService(t, cfg, bgpManager)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be announced")
healthcheck.setStatus(http.StatusServiceUnavailable)
expectEventually(t, func() bool { return !bgpManager.isAnnounced() },
"route should be withdrawn")
healthcheck.setStatus(http.StatusOK)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be re-announced")
}
func TestBGPHealthCheckLoop_StopsOnContextCancel(t *testing.T) {
t.Parallel()
healthcheck := newTestHealthServer(t, http.StatusOK)
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
cancelContext, vipServiceDone := startVipService(t, newTestConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be announced")
cancelContext()
select {
case <-vipServiceDone:
case <-time.After(5 * time.Second):
t.Fatal("vipService did not stop after context cancellation")
}
}
func TestBGPHealthCheckLoop_RetriesAddHostOnFailure(t *testing.T) {
t.Parallel()
healthcheck := newTestHealthServer(t, http.StatusOK)
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
bgpManager.setAddErr(errTestAddHost)
startVipService(t, newTestConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
expectConsistently(t, func() bool { return !bgpManager.isAnnounced() },
2*time.Second, "route should not be announced while AddHost errors")
bgpManager.setAddErr(nil)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be announced after clearing AddHost error")
}
func TestBGPHealthCheckLoop_RetriesDelHostOnFailure(t *testing.T) {
t.Parallel()
healthcheck := newTestHealthServer(t, http.StatusOK)
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
cfg := newTestConfig(healthcheck.server.URL, healthcheck.caPath)
cfg.ControlPlaneHealthCheck.FailureThreshold = 1
startVipService(t, cfg, bgpManager)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be announced")
bgpManager.setDelErr(errTestDelHost)
healthcheck.setStatus(http.StatusServiceUnavailable)
expectConsistently(t, func() bool { return bgpManager.isAnnounced() },
1500*time.Millisecond, "route should stay announced while DelHost errors")
bgpManager.setDelErr(nil)
expectEventually(t, func() bool { return !bgpManager.isAnnounced() },
"route should be withdrawn after clearing DelHost error")
}
var (
errTestAddHost = &testError{msg: "mock AddHost error"}
errTestDelHost = &testError{msg: "mock DelHost error"}
)
type testError struct{ msg string }
func (e *testError) Error() string { return e.msg }
// startVipService launches vipService in a goroutine with a mock network and
// registers a cleanup to cancel the context and wait for it to finish.
// Uses InitCluster so the real code parses certs for the BGP health check client.
func startVipService(t *testing.T, cfg *kubevip.Config, bgpManager *mockBGPRouteManager) (context.CancelFunc, <-chan struct{}) {
t.Helper()
c, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
if err != nil {
t.Fatalf("InitCluster: %v", err)
}
c.Network = []vip.Network{&mockNetwork{ip: "10.0.0.1", cidr: testCIDR}}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
_ = c.StartVipService(ctx, cfg, nil, bgpManager, func() {})
close(done)
}()
t.Cleanup(func() {
cancel()
<-done
})
return cancel, done
}
func newTestConfig(url, caPath string) *kubevip.Config {
return &kubevip.Config{
EnableBGP: true,
ControlPlaneHealthCheck: kubevip.HealthCheck{
Address: url,
CAPath: caPath,
PeriodSeconds: 1,
TimeoutSeconds: 2,
FailureThreshold: 1,
},
}
}
// mockBGPRouteManager tracks announced addresses as a set.
// AddHost adds, DelHost removes. Errors prevent state changes.
type mockBGPRouteManager struct {
mu sync.Mutex
announced map[string]bool
addErr error
delErr error
}
func newMockBGPRouteManager() *mockBGPRouteManager {
return &mockBGPRouteManager{announced: make(map[string]bool)}
}
func (m *mockBGPRouteManager) AddHost(_ context.Context, addr string, _ string) error {
m.mu.Lock()
defer m.mu.Unlock()
if m.addErr != nil {
return m.addErr
}
m.announced[addr] = true
return nil
}
func (m *mockBGPRouteManager) DelHost(_ context.Context, addr string, _ string) error {
m.mu.Lock()
defer m.mu.Unlock()
if m.delErr != nil {
return m.delErr
}
delete(m.announced, addr)
return nil
}
func (m *mockBGPRouteManager) isAnnounced() bool {
m.mu.Lock()
defer m.mu.Unlock()
return m.announced[testCIDR]
}
func (m *mockBGPRouteManager) setAddErr(err error) {
m.mu.Lock()
m.addErr = err
m.mu.Unlock()
}
func (m *mockBGPRouteManager) setDelErr(err error) {
m.mu.Lock()
m.delErr = err
m.mu.Unlock()
}
// mockNetwork implements vip.Network with no-op operations.
type mockNetwork struct {
ip string
cidr string
}
func (m *mockNetwork) AddIP(bool, bool, ...int) (bool, error) { return true, nil }
func (m *mockNetwork) AddRoute(bool) (bool, error) { return false, nil }
func (m *mockNetwork) DeleteIP() (bool, error) { return false, nil }
func (m *mockNetwork) DeleteRoute() error { return nil }
func (m *mockNetwork) UpdateRoutes() (bool, error) { return false, nil }
func (m *mockNetwork) IsSet() (*netlink.Addr, error) { return nil, nil }
func (m *mockNetwork) IP() string { return m.ip }
func (m *mockNetwork) CIDR() string { return m.cidr }
func (m *mockNetwork) IPisLinkLocal() bool { return false }
func (m *mockNetwork) PrepareRoute() *netlink.Route { return nil }
func (m *mockNetwork) RouteHash() string { return "" }
func (m *mockNetwork) SetIP(string) error { return nil }
func (m *mockNetwork) SetServicePorts(*corev1.Service) {}
func (m *mockNetwork) Interface() string { return "eth0" }
func (m *mockNetwork) IsDADFAIL() bool { return false }
func (m *mockNetwork) IsDNS() bool { return false }
func (m *mockNetwork) IsDDNS() bool { return false }
func (m *mockNetwork) DDNSHostName() string { return "" }
func (m *mockNetwork) DNSName() string { return "" }
func (m *mockNetwork) SetMask(string) error { return nil }
func (m *mockNetwork) SetHasEndpoints(bool) {}
func (m *mockNetwork) HasEndpoints() bool { return false }
func (m *mockNetwork) ARPName() string { return "" }
func (m *mockNetwork) GetPossibleSubnets() string { return "" }
func (m *mockNetwork) DHCPFamily() string { return "" }
// testHealthServer wraps an HTTPS httptest.Server with an atomic status code.
// caPath is the path to the server's CA cert for client verification.
type testHealthServer struct {
server *httptest.Server
statusCode atomic.Int64
caPath string
}
func newTestHealthServer(t *testing.T, status int) *testHealthServer {
t.Helper()
healthcheck := &testHealthServer{}
healthcheck.statusCode.Store(int64(status))
healthcheck.server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(int(healthcheck.statusCode.Load()))
}))
cert := healthcheck.server.Certificate()
if cert == nil {
t.Fatal("TLS server has no certificate")
}
caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})
caFile := filepath.Join(t.TempDir(), "ca.crt")
if err := os.WriteFile(caFile, caPEM, 0o600); err != nil {
t.Fatal(err)
}
healthcheck.caPath = caFile
return healthcheck
}
func (ths *testHealthServer) setStatus(code int) {
ths.statusCode.Store(int64(code))
}
// expectConsistently continuously checks that condition remains true for the given duration.
// Fails immediately if the condition becomes false at any point.
func expectConsistently(t *testing.T, condition func() bool, duration time.Duration, msg string) {
t.Helper()
deadline := time.Now().Add(duration)
for time.Now().Before(deadline) {
if !condition() {
t.Fatalf("condition violated: %s", msg)
}
time.Sleep(50 * time.Millisecond)
}
}
// expectEventually polls condition until it returns true or 5s timeout is reached.
func expectEventually(t *testing.T, condition func() bool, msg string) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if condition() {
return
}
time.Sleep(50 * time.Millisecond)
}
t.Fatalf("timeout: %s", msg)
}

84
pkg/cluster/singleNode.go Normal file
View File

@@ -0,0 +1,84 @@
package cluster
import (
"context"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/vip"
)
// StartSingleNode will start a single node cluster
func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) error {
// Start kube-vip as a single node server
// TODO - Split all this code out as a separate function
log.Info("Starting kube-vip as a single node cluster")
log.Info("This node is assuming leadership of the cluster")
cluster.stop = make(chan bool, 1)
cluster.completed = make(chan bool, 1)
for i := range cluster.Network {
if !disableVIP {
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("Attempted to clean existing VIP", "err", err)
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
_, err = cluster.Network[i].AddIP(false)
if err != nil {
log.Warn(err.Error())
}
}
if c.EnableARP {
// Gratuitous ARP, will broadcast to new MAC <-> IP
err := vip.ARPSendGratuitous(cluster.Network[i].IP(), c.Interface)
if err != nil {
log.Warn(err.Error())
}
}
}
go func() {
<-cluster.stop
if !disableVIP {
for i := range cluster.Network {
log.Info("[VIP] Releasing the VIP", "address", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
}
}
close(cluster.completed)
}()
log.Info("Started Load Balancer and Virtual IP")
return nil
}
func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp.Server) error {
// use a Go context so we can tell the arp loop code when we
// want to step down
ctxArp, cancelArp := context.WithCancel(context.Background())
defer cancelArp()
// use a Go context so we can tell the dns loop code when we
// want to step down
ctxDNS, cancelDNS := context.WithCancel(context.Background())
defer cancelDNS()
return cluster.vipService(ctxArp, ctxDNS, c, sm, bgp, nil)
}

View File

@@ -27,7 +27,7 @@ func FindIPAddress(addrName string) (string, string, error) {
// If we're not searching for a specific adapter return the first one
if addrName == "" {
return iface.Name, address, nil
}
} else
// If this is the correct adapter return the details
if iface.Name == addrName {
return iface.Name, address, nil
@@ -37,5 +37,5 @@ func FindIPAddress(addrName string) (string, string, error) {
}
}
return "", "", fmt.Errorf("unknown interface [%s]", addrName)
return "", "", fmt.Errorf("Unknown interface [%s]", addrName)
}

View File

@@ -7,7 +7,6 @@ import (
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
@@ -20,7 +19,7 @@ func Teardown(podIP, vipIP, namespace, serviceUUID string, annotations map[strin
protocol := iptables.ProtocolIPv4
IPv6 := false
if utils.IsIPv6(podIP) {
if vip.IsIPv6(podIP) {
protocol = iptables.ProtocolIPv6
IPv6 = true
}

View File

@@ -1,267 +0,0 @@
package election
import (
"context"
"errors"
"fmt"
"sync"
"time"
log "log/slog"
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/etcd"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/loadbalancer"
"github.com/kube-vip/kube-vip/pkg/utils"
clientv3 "go.etcd.io/etcd/client/v3"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
watchtools "k8s.io/client-go/tools/watch"
)
type Manager struct {
KubernetesClient *kubernetes.Clientset
RetryWatcherClient *kubernetes.Clientset
// This channel is used to signal a shutdown
EtcdClient *clientv3.Client
}
// NewManager will create a new managing object
func NewManager(config *kubevip.Config, k8sClientset, rwClientset *kubernetes.Clientset) (*Manager, error) {
m := &Manager{}
switch config.LeaderElectionType {
case "kubernetes", "":
if k8sClientset == nil || rwClientset == nil {
return nil, fmt.Errorf("provided nil clientset")
}
m.KubernetesClient = k8sClientset
m.RetryWatcherClient = rwClientset
case "etcd":
client, err := etcd.NewClient(config)
if err != nil {
return nil, err
}
m.EtcdClient = client
default:
return nil, fmt.Errorf("invalid LeaderElectionMode %s not supported", config.LeaderElectionType)
}
return m, nil
}
func RunOrDie(ctx context.Context, run *RunConfig, c *kubevip.Config) error {
switch c.LeaderElectionType {
case "kubernetes", "":
runKubernetesLeaderElectionOrDie(ctx, run)
case "etcd":
if err := runEtcdLeaderElectionOrDie(ctx, run); err != nil {
return err
}
default:
log.Info("LeaderElectionMode not supported, exiting", "mode", c.LeaderElectionType)
}
return nil
}
func runKubernetesLeaderElectionOrDie(ctx context.Context, run *RunConfig) {
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: run.LeaseID.Name(),
Namespace: run.LeaseID.Namespace(),
Annotations: run.LeaseAnnotations,
},
Client: run.Mgr.KubernetesClient.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: run.Config.NodeName,
},
}
// start the leader election code loop
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(run.Config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(run.Config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(run.Config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: run.OnStartedLeading,
OnStoppedLeading: run.OnStoppedLeading,
OnNewLeader: run.OnNewLeader,
},
})
}
func runEtcdLeaderElectionOrDie(ctx context.Context, run *RunConfig) error {
if err := etcd.RunElectionOrDie(ctx, &etcd.LeaderElectionConfig{
EtcdConfig: etcd.ClientConfig{Client: run.Mgr.EtcdClient},
Name: run.LeaseID.NamespacedName(),
MemberID: run.Config.NodeName,
LeaseDurationSeconds: int64(run.Config.LeaseDuration),
Callbacks: etcd.LeaderCallbacks{
OnStartedLeading: run.OnStartedLeading,
OnStoppedLeading: run.OnStoppedLeading,
OnNewLeader: run.OnNewLeader,
},
}); err != nil {
return fmt.Errorf("etcd leaderelection: %w", err)
}
return nil
}
type Actions interface {
OnStartedLeading(ctx context.Context)
OnStoppedLeading()
OnNewLeader(identity string)
}
type RunConfig struct {
Config *kubevip.Config
LeaseID lease.ID
Mgr *Manager
LeaseAnnotations map[string]string
// onStartedLeading is called when this member starts leading.
OnStartedLeading func(context.Context)
// onStoppedLeading is called when this member stops leading.
OnStoppedLeading func()
// onNewLeader is called when the client observes a leader that is
// not the previously observed leader. This includes the first observed
// leader when the client starts.
OnNewLeader func(identity string)
}
func (em *Manager) NodeWatcher(ctx context.Context, lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
log.Info("Kube-Vip is watching nodes for control-plane labels")
listOptions := metav1.ListOptions{
LabelSelector: "node-role.kubernetes.io/control-plane",
}
wg := sync.WaitGroup{}
defer wg.Wait()
watchCtx, watchCancel := context.WithCancel(ctx)
defer watchCancel()
rw, err := watchtools.NewRetryWatcherWithContext(watchCtx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return utils.WatchWithAuthRetry(ctx, func(ctx context.Context) (watch.Interface, error) {
return em.RetryWatcherClient.CoreV1().Nodes().Watch(watchCtx, listOptions)
})
},
})
if err != nil {
return fmt.Errorf("error creating label watcher: %w", err)
}
wg.Go(func() {
<-watchCtx.Done()
log.Info("Node watcher context cancelled, stopping")
// Stop the retrywatcher
rw.Stop()
})
ch := rw.ResultChan()
var watchErr error
for event := range ch {
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
node, ok := event.Object.(*v1.Node)
if !ok {
return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
}
// Find the node IP address (this isn't foolproof)
for x := range node.Status.Addresses {
if node.Status.Addresses[x].Type == v1.NodeInternalIP {
if checkIfNodeIsReady(node) {
err = lb.AddBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("adding node to load balancer", "node", node.Name, "ip", node.Status.Addresses[x].Address, "err", err)
if errors.Is(err, &utils.PanicError{}) {
return fmt.Errorf("add IPVS backend: %w", err)
}
}
} else {
err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("removing node from load balancer", "node", node.Name, "ip", node.Status.Addresses[x].Address, "err", err)
}
}
}
}
case watch.Deleted:
node, ok := event.Object.(*v1.Node)
if !ok {
return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
}
// Find the node IP address (this isn't foolproof)
for x := range node.Status.Addresses {
if node.Status.Addresses[x].Type == v1.NodeInternalIP {
err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("removing node from load balancer", "node", node.Name, "ip", node.Status.Addresses[x].Address, "err", err)
}
}
}
log.Info("Node deleted", "name", node.Name)
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes Nodes")
// This round trip allows us to handle unstructured status
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Error("watcher", "status", status)
watchErr = fmt.Errorf("node watcher error, status: %s", status.String())
default:
}
}
log.Info("Exiting Node watcher")
return watchErr
}
func checkIfNodeIsReady(node *v1.Node) bool {
if node == nil {
return false
}
for _, condition := range node.Status.Conditions {
if condition.Type == v1.NodeReady {
if condition.Status == v1.ConditionTrue {
return true
}
}
}
return false
}

View File

@@ -4,9 +4,6 @@ import (
"context"
"fmt"
"net"
"sync"
"sync/atomic"
"time"
log "log/slog"
@@ -14,14 +11,10 @@ import (
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/wireguard"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
)
type Processor struct {
@@ -30,29 +23,26 @@ type Processor struct {
bgpServer *bgp.Server
worker endpointWorker
instances *[]*instance.Instance
leaseMgr *lease.Manager
}
func NewEndpointProcessor(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
instances *[]*instance.Instance, leaseMgr *lease.Manager, tunnelMgr *wireguard.TunnelManager, routeMgr *route.Manager) *Processor {
instances *[]*instance.Instance) *Processor {
return &Processor{
config: config,
provider: provider,
bgpServer: bgpServer,
instances: instances,
leaseMgr: leaseMgr,
worker: newEndpointWorker(config, provider, bgpServer, instances, leaseMgr, tunnelMgr, routeMgr),
worker: newEndpointWorker(config, provider, bgpServer, instances),
}
}
func (p *Processor) AddOrModify(svcCtx *servicecontext.Context, event watch.Event,
lastKnownGoodEndpoint *string, service *v1.Service, id string,
serviceFunc func(*servicecontext.Context, *v1.Service, *sync.WaitGroup, bool) error, wg *sync.WaitGroup,
clientSet *kubernetes.Clientset,
egressUpdateFunc func(context.Context, *v1.Service) error) (bool, error) {
func (p *Processor) AddOrModify(ctx *servicecontext.Context, event watch.Event,
lastKnownGoodEndpoint *string, service *v1.Service, id string, leaderElectionActive *bool,
serviceFunc func(context.Context, *v1.Service) error,
leaderCtx *context.Context, cancel *context.CancelFunc) (bool, error) {
var err error
if err = p.provider.LoadObject(event.Object, svcCtx.Cancel); err != nil {
if err = p.provider.LoadObject(event.Object, *cancel); err != nil {
return false, fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
}
@@ -61,7 +51,7 @@ func (p *Processor) AddOrModify(svcCtx *servicecontext.Context, event watch.Even
return false, err
}
if err := p.worker.setInstanceEndpointsStatus(svcCtx.Ctx, service, endpoints); err != nil {
if err := p.worker.setInstanceEndpointsStatus(service, endpoints); err != nil {
log.Error("updating instance", "err", err)
}
@@ -71,90 +61,53 @@ func (p *Processor) AddOrModify(svcCtx *servicecontext.Context, event watch.Even
// If we have a local endpoint then begin the leader Election, unless it's already running
//
les := atomic.Int64{}
// Check that we have local endpoints
if len(endpoints) != 0 {
// Ignore IPv4
if service.Annotations[kubevip.EgressIPv6] == "true" && !hasV6(endpoints) {
if service.Annotations[kubevip.EgressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
return true, nil
}
p.updateLastKnownGoodEndpoint(lastKnownGoodEndpoint, endpoints, service)
if p.config.EnableServicesElection {
wg.Go(func() {
les.Add(1)
p.startLeaderElection(svcCtx, service, serviceFunc, wg)
})
} else if p.config.EnableARP || (p.config.EnableRoutingTable && p.config.EnableLeaderElection) {
if !svcCtx.Signalled.Load() {
inst := instance.FindServiceInstance(service, *p.instances)
if inst == nil {
return true, fmt.Errorf("[%s] failed to find an instance for service %s/%s", p.provider.GetLabel(), service.Namespace, service.Name)
}
// global leader election is enabled for services
for x := range inst.VIPConfigs {
log.Debug("starting loadbalancer for service", "name", service.Name, "namespace", service.Namespace, "uid", service.UID)
if err := inst.Clusters[x].StartLoadBalancerService(svcCtx.Ctx, inst.VIPConfigs[x], p.bgpServer, lease.ServiceNamespacedName(service), wg); err != nil {
return true, fmt.Errorf("failed to start lb: %w", err)
}
}
}
p.updateLastKnownGoodEndpoint(lastKnownGoodEndpoint, endpoints, service, leaderElectionActive, *cancel)
// start leader election if it's enabled and not already started
if !*leaderElectionActive && p.config.EnableServicesElection {
go func() {
*leaderCtx, *cancel = context.WithCancel(ctx.Ctx)
startLeaderElection(*leaderCtx, leaderElectionActive, service, serviceFunc)
}()
}
svcCtx.SignalReadiness()
// There are local endpoints available on the node
// Process immediately if:
// - No services/leader election is enabled, OR
// - WireGuard is enabled (it always needs immediate DNAT rule updates)
if (!p.config.EnableServicesElection && !p.config.EnableLeaderElection) || p.config.EnableWireguard {
if err := p.worker.processInstance(svcCtx, service); err != nil {
if !p.config.EnableServicesElection && !p.config.EnableLeaderElection {
if err := p.worker.processInstance(ctx, service, leaderElectionActive); err != nil {
return false, fmt.Errorf("failed to process non-empty instance: %w", err)
}
}
} else {
// There are no local endpoints
if svcCtx.Signalled.Load() {
svcCtx.ResetReadiness()
p.worker.clear(svcCtx, lastKnownGoodEndpoint, service)
if p.config.EnableARP && !p.config.EnableServicesElection {
i := instance.FindServiceInstance(service, *p.instances)
for _, c := range i.Clusters {
c.Stop()
}
}
}
p.worker.clear(ctx, lastKnownGoodEndpoint, service, *cancel, leaderElectionActive)
}
// Set the service accordingly
p.updateAnnotations(service, lastKnownGoodEndpoint, clientSet, egressUpdateFunc)
p.updateAnnotations(service, lastKnownGoodEndpoint)
log.Debug("watcher", "provider",
p.provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", *lastKnownGoodEndpoint)
p.provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", *lastKnownGoodEndpoint, "active leader election", *leaderElectionActive)
return false, nil
}
func (p *Processor) Delete(ctx context.Context, service *v1.Service, id string) error {
if err := p.worker.delete(ctx, service, id); err != nil {
func (p *Processor) Delete(service *v1.Service, id string) error {
if err := p.worker.delete(service, id); err != nil {
return fmt.Errorf("[%s] error deleting service: %w", p.provider.GetLabel(), err)
}
return nil
}
func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service) {
func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service, leaderElectionActive *bool, cancel context.CancelFunc) {
// if we haven't populated one, then do so
family := utils.IPv4Family
if service.Annotations[kubevip.EgressIPv6] == "true" {
family = utils.IPv6Family
}
ep := getEndpoint(endpoints, family)
if *lastKnownGoodEndpoint == "" {
*lastKnownGoodEndpoint = ep
*lastKnownGoodEndpoint = endpoints[0]
return
}
@@ -168,131 +121,45 @@ func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, e
}
// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
if !stillExists {
ip := net.ParseIP(*lastKnownGoodEndpoint)
if (ip.To4() != nil && service.Annotations[kubevip.Egress] == "true") ||
(ip.To4() == nil && service.Annotations[kubevip.EgressIPv6] == "true") {
p.worker.removeEgress(service, lastKnownGoodEndpoint)
p.worker.removeEgress(service, lastKnownGoodEndpoint)
if *leaderElectionActive && (p.config.EnableServicesElection || p.config.EnableLeaderElection) {
log.Warn("existing endpoint has been removed, restarting leaderElection", "provider", p.provider.GetLabel(), "endpoint", *lastKnownGoodEndpoint)
// Stop the existing leaderElection
cancel()
// disable last leaderElection flag
*leaderElectionActive = false
}
// Set our active endpoint to an existing one
*lastKnownGoodEndpoint = ep
*lastKnownGoodEndpoint = endpoints[0]
}
}
func (p *Processor) updateAnnotations(service *v1.Service, lastKnownGoodEndpoint *string,
clientSet *kubernetes.Clientset,
egressUpdateFunc func(context.Context, *v1.Service) error) {
func (p *Processor) updateAnnotations(service *v1.Service, lastKnownGoodEndpoint *string) {
// Set the service accordingly
if service.Annotations[kubevip.Egress] == "true" {
ip := net.ParseIP(*lastKnownGoodEndpoint)
activeEndpointAnnotation := kubevip.ActiveEndpoint
// Store old values from ServiceSnapshot to detect if annotation actually changed
// We use the ServiceSnapshot instead of the service parameter because the service parameter
// may have stale annotations if the last update failed
var oldEndpoint, oldEndpointIPv6 string
if p.instances != nil {
serviceInstance := instance.FindServiceInstance(service, *p.instances)
if serviceInstance != nil {
oldEndpoint = serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint]
oldEndpointIPv6 = serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpointIPv6]
}
}
// Fall back to service annotations if we couldn't find the instance
if oldEndpoint == "" && oldEndpointIPv6 == "" {
oldEndpoint = service.Annotations[kubevip.ActiveEndpoint]
oldEndpointIPv6 = service.Annotations[kubevip.ActiveEndpointIPv6]
}
// Determine which annotation to update based on IP version
var endpoint, endpointIPv6 string
if ip.To4() == nil && !p.config.EnableEndpoints {
// IPv6
endpointIPv6 = *lastKnownGoodEndpoint
endpoint = oldEndpoint // Preserve existing IPv4 if any
} else {
// IPv4
endpoint = *lastKnownGoodEndpoint
endpointIPv6 = oldEndpointIPv6 // Preserve existing IPv6 if any
}
// Check if annotation actually changed
annotationChanged := (oldEndpoint != endpoint) || (oldEndpointIPv6 != endpointIPv6)
if !annotationChanged {
return // Nothing to do
}
// Persist to Kubernetes
ctx := context.Background()
if err := p.provider.UpdateServiceAnnotation(ctx, endpoint, endpointIPv6, service, clientSet); err != nil {
log.Warn("failed to update service annotation", "service", service.Name, "namespace", service.Namespace, "err", err)
return
}
log.Debug("updated active endpoint annotation", "service", service.Name, "namespace", service.Namespace, "endpoint", *lastKnownGoodEndpoint)
// Trigger egress reconfiguration
// For services with leader election, the service watcher doesn't process Modified events
// after initial setup, so we need to directly call the update function
if egressUpdateFunc != nil {
// Create a copy of service with updated annotations
svcCopy := service.DeepCopy()
svcCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
svcCopy.Annotations[kubevip.ActiveEndpointIPv6] = endpointIPv6
if err := egressUpdateFunc(ctx, svcCopy); err != nil {
log.Error("failed to reconfigure egress", "service", service.Name, "namespace", service.Namespace, "err", err)
}
if !p.config.EnableEndpoints && p.provider.GetProtocol() == string(discoveryv1.AddressTypeIPv6) {
activeEndpointAnnotation = kubevip.ActiveEndpointIPv6
}
service.Annotations[activeEndpointAnnotation] = *lastKnownGoodEndpoint
}
}
func (p *Processor) startLeaderElection(svcCtx *servicecontext.Context, service *v1.Service, serviceFunc func(*servicecontext.Context, *v1.Service, *sync.WaitGroup, bool) error, wg *sync.WaitGroup) {
func startLeaderElection(ctx context.Context, leaderElectionActive *bool, service *v1.Service, serviceFunc func(context.Context, *v1.Service) error) {
// This is a blocking function, that will restart (in the event of failure)
for {
select {
case <-svcCtx.Ctx.Done():
return
default:
leaseNamespace, serviceLease := lease.ServiceName(service)
id := lease.NewID(p.config.LeaderElectionType, leaseNamespace, serviceLease)
l := p.leaseMgr.Get(id)
l.Lock()
if !l.Elected.Load() {
l.Unlock()
err := serviceFunc(svcCtx, service, wg, true)
if err != nil {
log.Error(err.Error())
}
} else {
l.Unlock()
time.Sleep(time.Millisecond * 200)
// if the context isn't cancelled restart
if ctx.Err() != context.Canceled {
*leaderElectionActive = true
err := serviceFunc(ctx, service)
if err != nil {
log.Error(err.Error())
}
*leaderElectionActive = false
} else {
*leaderElectionActive = false
break
}
}
}
func hasV6(endpoints []string) bool {
for _, e := range endpoints {
ip := net.ParseIP(e)
if ip != nil {
if ip.To4() == nil {
return true
}
}
}
return false
}
func getEndpoint(endpoints []string, family string) string {
for _, e := range endpoints {
ip := net.ParseIP(e)
if family == utils.IPv4Family && ip.To4() != nil {
return e
}
if family == utils.IPv6Family && ip.To4() == nil {
return e
}
}
return ""
}

View File

@@ -7,7 +7,6 @@ import (
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
)
@@ -24,19 +23,20 @@ func newBGP(generic generic, bgpServer *bgp.Server) endpointWorker {
}
}
func (b *BGP) processInstance(svcCtx *servicecontext.Context, service *v1.Service) error {
func (b *BGP) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
if !svcCtx.IsNetworkConfigured(cluster.Network[i].IP()) {
if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) {
log.Debug("attempting to advertise BGP service", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP())
err := b.bgpServer.AddHost(svcCtx.Ctx, cluster.Network[i].CIDR(), lease.ServiceNamespacedName(service))
err := b.bgpServer.AddHost(cluster.Network[i].CIDR())
if err != nil {
log.Error("error adding BGP host", "provider", b.provider.GetLabel(), "err", err)
} else {
log.Info("added BGP host", "provider",
b.provider.GetLabel(), "ip", cluster.Network[i].CIDR(), "service name", service.Name, "namespace", service.Namespace)
svcCtx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
*leaderElectionActive = true
}
}
}
@@ -45,19 +45,20 @@ func (b *BGP) processInstance(svcCtx *servicecontext.Context, service *v1.Servic
return nil
}
func (b *BGP) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service) {
func (b *BGP) clear(ctx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
// If BGP mode is enabled - routes should be deleted
if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
err := b.bgpServer.DelHost(svcCtx.Ctx, cluster.Network[i].CIDR(), lease.ServiceNamespacedName(service))
err := b.bgpServer.DelHost(cluster.Network[i].CIDR())
if err != nil {
log.Error("deleting BGP host", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "err", err)
} else {
log.Info("deleted BGP host", "provider",
b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace)
svcCtx.ConfiguredNetworks.Delete(cluster.Network[i].IP())
ctx.ConfiguredNetworks.Delete(cluster.Network[i])
*leaderElectionActive = false
}
}
}
@@ -65,18 +66,14 @@ func (b *BGP) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *strin
}
}
b.clearEgress(lastKnownGoodEndpoint, service)
if svcCtx.LeaderCancel != nil {
svcCtx.LeaderCancel()
}
b.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (b *BGP) getEndpoints(service *v1.Service, id string) ([]string, error) {
return b.getAllEndpoints(service, id)
}
func (b *BGP) delete(ctx context.Context, service *v1.Service, id string) error {
func (b *BGP) delete(service *v1.Service, id string) error {
// When no-leader-elecition mode
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
// find all existing local endpoints
@@ -87,40 +84,36 @@ func (b *BGP) delete(ctx context.Context, service *v1.Service, id string) error
// If there were local endpoints deleted
if len(endpoints) > 0 {
b.deleteAction(ctx, service)
b.deleteAction(service)
}
}
return nil
}
func (b *BGP) deleteAction(ctx context.Context, service *v1.Service) {
b.clearBGPHosts(ctx, service)
func (b *BGP) deleteAction(service *v1.Service) {
b.clearBGPHosts(service)
}
func (b *BGP) clearBGPHosts(ctx context.Context, service *v1.Service) {
ClearBGPHosts(ctx, service, b.instances, b.bgpServer)
func (b *BGP) clearBGPHosts(service *v1.Service) {
ClearBGPHosts(service, b.instances, b.bgpServer)
}
func (b *BGP) setInstanceEndpointsStatus(_ context.Context, _ *v1.Service, _ []string) error {
func (b *BGP) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
return nil
}
func ClearBGPHosts(ctx context.Context, service *v1.Service, instances *[]*instance.Instance, bgpServer *bgp.Server) {
func ClearBGPHosts(service *v1.Service, instances *[]*instance.Instance, bgpServer *bgp.Server) {
if instance := instance.FindServiceInstance(service, *instances); instance != nil {
ClearBGPHostsByInstance(ctx, instance, bgpServer)
ClearBGPHostsByInstance(instance, bgpServer)
}
}
func ClearBGPHostsByInstance(ctx context.Context, instance *instance.Instance, bgpServer *bgp.Server) {
if instance == nil {
log.Error("failed to clear BGP host for nil instance")
return
}
func ClearBGPHostsByInstance(instance *instance.Instance, bgpServer *bgp.Server) {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
network := cluster.Network[i]
err := bgpServer.DelHost(ctx, network.CIDR(), lease.ServiceNamespacedName(instance.ServiceSnapshot))
err := bgpServer.DelHost(network.CIDR())
if err != nil {
log.Error("[endpoint] error deleting BGP host", "err", err)
} else {

View File

@@ -10,31 +10,24 @@ import (
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/wireguard"
v1 "k8s.io/api/core/v1"
)
type endpointWorker interface {
processInstance(svcCtx *servicecontext.Context, service *v1.Service) error
clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service)
processInstance(svcCtx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error
clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool)
getEndpoints(service *v1.Service, id string) ([]string, error)
removeEgress(service *v1.Service, lastKnownGoodEndpoint *string)
delete(ctx context.Context, service *v1.Service, id string) error
setInstanceEndpointsStatus(ctx context.Context, service *v1.Service, endpoints []string) error
delete(service *v1.Service, id string) error
setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error
}
func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server, instances *[]*instance.Instance,
leaseMgr *lease.Manager, tunnelMgr *wireguard.TunnelManager, routeMgr *route.Manager) endpointWorker {
generic := newGeneric(config, provider, instances, leaseMgr)
func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server, instances *[]*instance.Instance) endpointWorker {
generic := newGeneric(config, provider, instances)
if config.EnableWireguard {
return newWireguardWorker(config, provider, bgpServer, instances, leaseMgr, tunnelMgr)
}
if config.EnableRoutingTable {
return newRoutingTable(generic, routeMgr)
return newRoutingTable(generic)
}
if config.EnableBGP {
return newBGP(generic, bgpServer)
@@ -47,49 +40,60 @@ type generic struct {
config *kubevip.Config
provider providers.Provider
instances *[]*instance.Instance
leaseMgr *lease.Manager
}
func newGeneric(config *kubevip.Config, provider providers.Provider, instances *[]*instance.Instance, leaseMgr *lease.Manager) generic {
func newGeneric(config *kubevip.Config, provider providers.Provider, instances *[]*instance.Instance) generic {
return generic{
config: config,
provider: provider,
instances: instances,
leaseMgr: leaseMgr,
}
}
func (g *generic) processInstance(_ *servicecontext.Context, _ *v1.Service) error {
func (g *generic) processInstance(_ *servicecontext.Context, _ *v1.Service, _ *bool) error {
return nil
}
func (g *generic) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service) {
g.clearEgress(lastKnownGoodEndpoint, service)
if svcCtx.LeaderCancel != nil {
svcCtx.LeaderCancel()
}
func (g *generic) clear(_ *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
g.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service) {
func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if *lastKnownGoodEndpoint != "" {
log.Warn("existing endpoint has been removed, no remaining endpoints for leaderElection", "provider", g.provider.GetLabel(), "endpoint", lastKnownGoodEndpoint)
log.Warn("existing endpoint has been removed, no remaining endpoints for leaderElection", "provider", g.provider.GetLabel(), "endpoint", lastKnownGoodEndpoint)
if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP, service.Namespace, string(service.UID), service.Annotations, g.config.EgressWithNftables); err != nil {
log.Error("error removing redundant egress rules", "err", err)
}
*lastKnownGoodEndpoint = "" // reset endpoint
if g.config.EnableServicesElection || g.config.EnableLeaderElection {
cancel() // stop services watcher
}
*leaderElectionActive = false
}
}
func (g *generic) getEndpoints(service *v1.Service, id string) ([]string, error) {
return g.getAllEndpoints(service, id)
func (g *generic) getEndpoints(_ *v1.Service, id string) ([]string, error) {
return g.getLocalEndpoints(id)
}
func (g *generic) getLocalEndpoints(id string) ([]string, error) {
// Build endpoints
var endpoints []string
var err error
if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
}
return endpoints, nil
}
func (g *generic) getAllEndpoints(service *v1.Service, id string) ([]string, error) {
// Build endpoints
var err error
var endpoints []string
if service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
if !g.config.EnableLeaderElection && !g.config.EnableServicesElection &&
service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
if endpoints, err = g.provider.GetAllEndpoints(); err != nil {
return nil, fmt.Errorf("[%s] error getting all endpoints: %w", g.provider.GetLabel(), err)
}
@@ -105,10 +109,10 @@ func (g *generic) getAllEndpoints(service *v1.Service, id string) ([]string, err
func (g *generic) removeEgress(_ *v1.Service, _ *string) {
}
func (g *generic) delete(_ context.Context, _ *v1.Service, _ string) error {
func (g *generic) delete(_ *v1.Service, _ string) error {
return nil
}
func (g *generic) setInstanceEndpointsStatus(_ context.Context, _ *v1.Service, _ []string) error {
func (g *generic) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
return nil
}

View File

@@ -2,46 +2,64 @@ package endpoints
import (
"context"
"errors"
"fmt"
"net"
"sync"
"syscall"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/egress"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
)
type RoutingTable struct {
generic
mtx sync.Mutex
routeMgr *route.Manager
}
func newRoutingTable(generic generic, routeMgr *route.Manager) endpointWorker {
func newRoutingTable(generic generic) endpointWorker {
return &RoutingTable{
generic: generic,
routeMgr: routeMgr,
generic: generic,
}
}
func (rt *RoutingTable) processInstance(svcCtx *servicecontext.Context, service *v1.Service) error {
inst := instance.FindServiceInstance(service, *rt.instances)
if inst != nil {
for _, cluster := range inst.Clusters {
func (rt *RoutingTable) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
instance := instance.FindServiceInstance(service, *rt.instances)
if instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
if !svcCtx.IsNetworkConfigured(cluster.Network[i].IP()) && cluster.Network[i].HasEndpoints() {
if err := rt.routeMgr.Add(lease.ServiceNamespacedName(service), cluster.Network[i], false, true); err != nil {
return fmt.Errorf("[%s] error adding route: %s", rt.provider.GetLabel(), err.Error())
if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) && cluster.Network[i].HasEndpoints() {
err := cluster.Network[i].AddRoute(false)
if err != nil {
if errors.Is(err, syscall.EEXIST) {
// If route exists, but protocol is not set (e.g. the route was created by the older version
// of kube-vip) try to update it if necessary
isUpdated, err := cluster.Network[i].UpdateRoutes()
if err != nil {
return fmt.Errorf("[%s] error updating existing routes: %w", rt.provider.GetLabel(), err)
}
if isUpdated {
log.Info("updated route", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
} else {
log.Info("route already present", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
}
} else {
// If other error occurs, return error
return fmt.Errorf("[%s] error adding route: %s", rt.provider.GetLabel(), err.Error())
}
} else {
log.Info("added route", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
svcCtx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
*leaderElectionActive = true
}
}
}
@@ -51,11 +69,9 @@ func (rt *RoutingTable) processInstance(svcCtx *servicecontext.Context, service
return nil
}
func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service) {
rt.mtx.Lock()
defer rt.mtx.Unlock()
if !rt.config.EnableServicesElection {
if errs := ClearRoutes(service, rt.instances, rt.routeMgr); len(errs) == 0 {
func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
if errs := ClearRoutes(service, rt.instances); len(errs) == 0 {
svcCtx.ConfiguredNetworks.Clear()
} else {
for _, err := range errs {
@@ -64,11 +80,7 @@ func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpo
}
}
rt.clearEgress(lastKnownGoodEndpoint, service)
if svcCtx.LeaderCancel != nil {
svcCtx.LeaderCancel()
}
rt.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (rt *RoutingTable) getEndpoints(service *v1.Service, id string) ([]string, error) {
@@ -82,7 +94,7 @@ func (rt *RoutingTable) removeEgress(service *v1.Service, lastKnownGoodEndpoint
}
}
func (rt *RoutingTable) delete(_ context.Context, service *v1.Service, id string) error {
func (rt *RoutingTable) delete(service *v1.Service, id string) error {
// When no-leader-elecition mode
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
// find all existing local endpoints
@@ -101,27 +113,23 @@ func (rt *RoutingTable) delete(_ context.Context, service *v1.Service, id string
}
func (rt *RoutingTable) deleteAction(service *v1.Service) {
ClearRoutes(service, rt.instances, rt.routeMgr)
ClearRoutes(service, rt.instances)
}
func (rt *RoutingTable) setInstanceEndpointsStatus(ctx context.Context, service *v1.Service, endpoints []string) error {
inst := instance.FindServiceInstance(service, *rt.instances)
if inst == nil {
log.Error("failed to find the instance", "namespace", service.Namespace, "name", service.Name, "uid", service.UID, "provider", rt.provider.GetLabel())
func (rt *RoutingTable) setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error {
instance := instance.FindServiceInstance(service, *rt.instances)
if instance == nil {
log.Error("failed to find the instance", "service", service.UID, "provider", rt.provider.GetLabel())
} else {
for _, c := range inst.Clusters {
for _, c := range instance.Clusters {
for n := range c.Network {
// if there are no endpoints set HasEndpoints false just in case
if len(endpoints) < 1 {
c.Network[n].SetHasEndpoints(false)
} else {
// check if endpoint are available and are of same IP family as service
for _, ep := range endpoints {
if (net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(ep).To4() == nil) {
c.Network[n].SetHasEndpoints(true)
break
}
}
}
// check if endpoint are available and are of same IP family as service
if len(endpoints) > 0 && ((net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(endpoints[0]).To4() == nil)) {
c.Network[n].SetHasEndpoints(true)
}
}
}
@@ -130,32 +138,41 @@ func (rt *RoutingTable) setInstanceEndpointsStatus(ctx context.Context, service
return nil
}
func ClearRoutes(service *v1.Service, instances *[]*instance.Instance, routeMgr *route.Manager) []error {
func ClearRoutes(service *v1.Service, instances *[]*instance.Instance) []error {
errs := []error{}
if svcInst := instance.FindServiceInstance(service, *instances); svcInst != nil {
clearErrs := ClearRoutesByInstance(service, svcInst, instances, routeMgr)
errs = append(errs, clearErrs...)
}
return errs
}
func ClearRoutesByInstance(service *v1.Service, svcInst *instance.Instance, instances *[]*instance.Instance, routeMgr *route.Manager) []error {
if svcInst == nil {
return []error{fmt.Errorf("failed to remove routes for nil instance of service %s/%s, uid: %s", service.Namespace, service.Name, service.UID)}
}
errs := []error{}
for _, cluster := range svcInst.Clusters {
for i := range cluster.Network {
err := routeMgr.Delete(lease.ServiceNamespacedName(service), cluster.Network[i])
if err != nil {
log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
errs = append(errs, err)
if instance := instance.FindServiceInstance(service, *instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
route := cluster.Network[i].PrepareRoute()
// check if route we are about to delete is not referenced by more than one service
if CountRouteReferences(route, instances) <= 1 {
err := cluster.Network[i].DeleteRoute()
if err != nil && !errors.Is(err, syscall.ESRCH) {
log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
errs = append(errs, err)
}
log.Debug("deleted route", "ip",
cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface())
}
}
log.Debug("deleted route", "ip",
cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface())
}
}
return errs
}
func CountRouteReferences(route *netlink.Route, instances *[]*instance.Instance) int {
cnt := 0
for _, instance := range *instances {
for _, cluster := range instance.Clusters {
for n := range cluster.Network {
if cluster.Network[n].HasEndpoints() {
r := cluster.Network[n].PrepareRoute()
if r.Dst.String() == route.Dst.String() {
cnt++
}
}
}
}
}
return cnt
}

View File

@@ -1,289 +0,0 @@
package endpoints
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/wireguard"
v1 "k8s.io/api/core/v1"
)
// wireguardWorker handles endpoint changes for WireGuard-based services
type wireguardWorker struct {
config *kubevip.Config
provider providers.Provider
bgpServer *bgp.Server
instances *[]*instance.Instance
leaseMgr *lease.Manager
tunnelMgr *wireguard.TunnelManager
}
func newWireguardWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
instances *[]*instance.Instance, leaseMgr *lease.Manager, tunnelMgr *wireguard.TunnelManager) *wireguardWorker {
return &wireguardWorker{
config: config,
provider: provider,
bgpServer: bgpServer,
instances: instances,
leaseMgr: leaseMgr,
tunnelMgr: tunnelMgr,
}
}
// processInstance updates nftables DNAT rules when endpoints change
// This is called by the endpoint watcher when endpoints are added/modified
func (w *wireguardWorker) processInstance(svcCtx *servicecontext.Context, service *v1.Service) error {
log.Debug("[wireguard] processing instance for endpoint change", "service", service.Name, "namespace", service.Namespace)
// Get the target endpoint for this service
// For ExternalTrafficPolicy=Local, only use local endpoints
// For ExternalTrafficPolicy=Cluster, use all endpoints
var endpoints []string
var err error
if service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
endpoints, err = w.provider.GetLocalEndpoints(w.config.NodeName, w.config)
} else {
endpoints, err = w.provider.GetAllEndpoints()
}
if err != nil {
return fmt.Errorf("failed to get endpoints: %w", err)
}
if len(endpoints) == 0 {
log.Debug("[wireguard] no endpoints available", "service", service.Name)
w.clear(svcCtx, nil, service)
return nil
}
// Find the service processor to call updateServiceWireguardEndpoints
// Note: This requires access to the service processor which we don't have here
// So we'll recreate the DNAT rules directly
// First, clear existing rules
w.clear(svcCtx, nil, service)
// Get service VIPs
serviceIPs, err := utils.FetchServiceIPs(service)
if err != nil {
return fmt.Errorf("failed to get service IPs: %w", err)
}
// Create service identifier
serviceID := utils.SanitizeServiceID(fmt.Sprintf("%s_%s", service.Namespace, service.Name))
log.Info("[wireguard] updating DNAT rules for endpoint change",
"service", service.Name,
"namespace", service.Namespace,
"endpoints", endpoints,
"vips", serviceIPs)
// Update DNAT rules for each port
for _, port := range service.Spec.Ports {
// Determine target port (resolve named ports if necessary)
targetPort := w.provider.ResolvePort(port)
log.Info("[wireguard] resolved port", "service", service.Name, "servicePort", port.Port, "targetPort", targetPort, "targetPortName", port.TargetPort.StrVal)
// Build targets list from all endpoints
targets := make([]nftables.DNATTarget, len(endpoints))
for i, ep := range endpoints {
targets[i] = nftables.DNATTarget{
IP: ep,
Port: uint16(targetPort), //nolint:gosec // Port range validated by Kubernetes
}
}
for _, vip := range serviceIPs {
// Strip CIDR notation if present
vipAddr := utils.StripCIDR(vip)
// Get WireGuard interface name from TunnelManager for this VIP
if w.tunnelMgr == nil {
log.Error("[wireguard] TunnelManager not configured; cannot update DNAT rules",
"service", service.Name,
"namespace", service.Namespace)
return fmt.Errorf("TunnelManager not configured")
}
tunnelConfig := w.tunnelMgr.GetConfigForVIP(vipAddr)
if tunnelConfig == nil {
log.Error("[wireguard] WireGuard interface name not configured; cannot update DNAT rules",
"service", service.Name,
"namespace", service.Namespace,
"vip", vipAddr)
return fmt.Errorf("wireguard interface name not configured for VIP %s", vipAddr)
}
wgInterface := tunnelConfig.InterfaceName
portServiceID := fmt.Sprintf("%s_p%d", serviceID, port.Port)
log.Info("[wireguard] applying DNAT rule with load balancing",
"service", service.Name,
"vip", vipAddr,
"interface", wgInterface,
"sourcePort", port.Port,
"targets", targets,
"chainID", portServiceID)
// Apply the DNAT rule with load balancing across all endpoints
// localEndpoint=true when using ExternalTrafficPolicy=Local, which preserves client source IP
isLocalEndpoint := service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal
err := nftables.ApplyDNAT(
wgInterface,
vipAddr,
uint16(port.Port), //nolint:gosec // Port range validated by Kubernetes
targets,
portServiceID,
port.Protocol,
isLocalEndpoint,
tunnelConfig.ListenPort,
)
if err != nil {
log.Error("[wireguard] failed to update DNAT rule",
"service", service.Name,
"vip", vipAddr,
"port", port.Port,
"err", err)
continue
}
log.Debug("[wireguard] DNAT rule updated successfully",
"service", service.Name,
"vip", vipAddr,
"port", port.Port,
"targetCount", len(targets))
}
}
return nil
}
// clear removes DNAT rules when no endpoints are available
func (w *wireguardWorker) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service) {
log.Info("[wireguard] clearing DNAT rules (no endpoints)", "service", service.Name, "namespace", service.Namespace)
serviceID := utils.SanitizeServiceID(fmt.Sprintf("%s_%s", service.Namespace, service.Name))
// Get service IPs to determine IPv4 vs IPv6
serviceIPs, _ := utils.FetchServiceIPs(service)
// Delete DNAT chains for each port
for _, port := range service.Spec.Ports {
if port.Protocol != v1.ProtocolTCP && port.Protocol != v1.ProtocolUDP {
continue
}
portServiceID := fmt.Sprintf("%s_p%d", serviceID, port.Port)
// Determine if we have IPv4 or IPv6
hasIPv4, hasIPv6 := false, false
for _, vip := range serviceIPs {
if isIPv6Address(vip) {
hasIPv6 = true
} else {
hasIPv4 = true
}
}
if hasIPv4 {
if err := nftables.DeleteIngressChains(false, portServiceID); err != nil {
log.Warn("[wireguard] failed to delete IPv4 DNAT chains",
"service", service.Name,
"port", port.Port,
"err", err)
}
}
if hasIPv6 {
if err := nftables.DeleteIngressChains(true, portServiceID); err != nil {
log.Warn("[wireguard] failed to delete IPv6 DNAT chains",
"service", service.Name,
"port", port.Port,
"err", err)
}
}
}
if svcCtx.LeaderCancel != nil {
svcCtx.LeaderCancel()
}
}
// getEndpoints retrieves the list of endpoints for a service
// For ExternalTrafficPolicy=Local, only local endpoints are returned
// For ExternalTrafficPolicy=Cluster, all endpoints are returned
func (w *wireguardWorker) getEndpoints(service *v1.Service, id string) ([]string, error) {
var endpoints []string
var err error
if service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
endpoints, err = w.provider.GetLocalEndpoints(id, w.config)
} else {
endpoints, err = w.provider.GetAllEndpoints()
}
if err != nil {
return nil, fmt.Errorf("[wireguard] failed to get endpoints: %w", err)
}
log.Debug("[wireguard] retrieved endpoints", "service", service.Name, "count", len(endpoints), "endpoints", endpoints)
return endpoints, nil
}
// removeEgress is a no-op for WireGuard since egress is handled separately
func (w *wireguardWorker) removeEgress(service *v1.Service, lastKnownGoodEndpoint *string) {
// WireGuard doesn't use egress in the same way as other modes
log.Debug("[wireguard] removeEgress called (no-op)", "service", service.Name)
}
// delete removes all DNAT rules for a service
func (w *wireguardWorker) delete(ctx context.Context, service *v1.Service, id string) error {
log.Info("[wireguard] deleting DNAT rules for service", "service", service.Name, "namespace", service.Namespace)
w.clear(nil, nil, service)
return nil
}
// setInstanceEndpointsStatus updates the endpoint status on the service instance
func (w *wireguardWorker) setInstanceEndpointsStatus(_ context.Context, service *v1.Service, endpoints []string) error {
hasEndpoints := len(endpoints) > 0
log.Debug("[wireguard] setting instance endpoint status",
"service", service.Name,
"hasEndpoints", hasEndpoints,
"endpointCount", len(endpoints))
// Find the service instance
for _, inst := range *w.instances {
if inst.ServiceSnapshot == nil {
continue
}
if inst.ServiceSnapshot.UID == service.UID {
// Update the network status for all clusters
for _, cluster := range inst.Clusters {
for i := range cluster.Network {
cluster.Network[i].SetHasEndpoints(hasEndpoints)
}
}
log.Debug("[wireguard] updated instance endpoint status",
"service", service.Name,
"hasEndpoints", hasEndpoints)
return nil
}
}
log.Debug("[wireguard] instance not found for endpoint status update", "service", service.Name)
return nil
}
func isIPv6Address(ip string) bool {
// Strip CIDR notation if present before checking
addr := utils.StripCIDR(ip)
return utils.IsIPv6(addr)
}

View File

@@ -97,11 +97,11 @@ func (ep *Endpoints) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string,
return localEndpoints, nil
}
func (ep *Endpoints) UpdateServiceAnnotation(ctx context.Context, endpoint string, _ string, service *v1.Service, clientSet *kubernetes.Clientset) error {
func (ep *Endpoints) UpdateServiceAnnotation(endpoint string, _ string, service *v1.Service, clientSet *kubernetes.Clientset) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(ctx, service.Name, metav1.GetOptions{})
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
@@ -113,7 +113,7 @@ func (ep *Endpoints) UpdateServiceAnnotation(ctx context.Context, endpoint strin
currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(ctx, currentServiceCopy, metav1.UpdateOptions{})
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("error updating Service Spec", "label", ep.GetLabel(), "name", currentServiceCopy.Name, "err", err)
return err
@@ -132,15 +132,6 @@ func (ep *Endpoints) GetLabel() string {
return ep.label
}
func (ep *Endpoints) ResolvePort(servicePort v1.ServicePort) int32 {
return ResolvePortWithLookup(servicePort, func(name string) int32 {
for _, subset := range ep.endpoints.Subsets {
for _, p := range subset.Ports {
if p.Name == name {
return p.Port
}
}
}
return 0
})
func (ep *Endpoints) GetProtocol() string {
return ""
}

View File

@@ -20,10 +20,8 @@ import (
)
type Endpointslices struct {
label string
endpointsv4 []discoveryv1.Endpoint
endpointsv6 []discoveryv1.Endpoint
ports []discoveryv1.EndpointPort
label string
endpoints *discoveryv1.EndpointSlice
}
func NewEndpointslices() Provider {
@@ -58,42 +56,27 @@ func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.Ca
cancel()
return fmt.Errorf("[%s] error casting endpoints to v1.Endpoints struct", ep.label)
}
if eps.AddressType == discoveryv1.AddressTypeIPv6 {
ep.endpointsv6 = eps.Endpoints
} else {
ep.endpointsv4 = eps.Endpoints
}
// Store ports for resolving named ports
ep.ports = eps.Ports
ep.endpoints = eps
return nil
}
func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
result := []string{}
for _, e := range ep.endpointsv4 {
result = append(result, e.Addresses...)
}
for _, e := range ep.endpointsv6 {
result = append(result, e.Addresses...)
for _, ep := range ep.endpoints.Endpoints {
result = append(result, ep.Addresses...)
}
return result, nil
}
func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
var localEndpoints []string
tmpEps := []discoveryv1.Endpoint{}
tmpEps = append(tmpEps, ep.endpointsv4...)
tmpEps = append(tmpEps, ep.endpointsv6...)
for _, endpoint := range tmpEps {
if endpoint.Conditions.Serving == nil || !*endpoint.Conditions.Serving {
for _, endpoint := range ep.endpoints.Endpoints {
if !*endpoint.Conditions.Serving {
continue
}
for _, address := range endpoint.Addresses {
log.Debug("processing endpoint", "provider", ep.label, "ip", address)
// 1. Compare the Nodename
if endpoint.NodeName != nil && id == *endpoint.NodeName {
if endpoint.Hostname != nil {
@@ -115,11 +98,11 @@ func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]str
return localEndpoints, nil
}
func (ep *Endpointslices) UpdateServiceAnnotation(ctx context.Context, endpoint, endpointIPv6 string, service *v1.Service, clientSet *kubernetes.Clientset) error {
func (ep *Endpointslices) UpdateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, clientSet *kubernetes.Clientset) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(ctx, service.Name, metav1.GetOptions{})
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
@@ -132,7 +115,7 @@ func (ep *Endpointslices) UpdateServiceAnnotation(ctx context.Context, endpoint,
currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
currentServiceCopy.Annotations[kubevip.ActiveEndpointIPv6] = endpointIPv6
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(ctx, currentServiceCopy, metav1.UpdateOptions{})
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("error updating Service Spec", "provider", ep.label, "service name", currentServiceCopy.Name, "err", err)
return err
@@ -151,13 +134,6 @@ func (ep *Endpointslices) GetLabel() string {
return ep.label
}
func (ep *Endpointslices) ResolvePort(servicePort v1.ServicePort) int32 {
return ResolvePortWithLookup(servicePort, func(name string) int32 {
for _, p := range ep.ports {
if p.Name != nil && *p.Name == name && p.Port != nil {
return *p.Port
}
}
return 0
})
func (ep *Endpointslices) GetProtocol() string {
return string(ep.endpoints.AddressType)
}

View File

@@ -16,24 +16,7 @@ type Provider interface {
GetAllEndpoints() ([]string, error)
GetLocalEndpoints(string, *kubevip.Config) ([]string, error)
GetLabel() string
UpdateServiceAnnotation(context.Context, string, string, *v1.Service, *kubernetes.Clientset) error
UpdateServiceAnnotation(string, string, *v1.Service, *kubernetes.Clientset) error
LoadObject(runtime.Object, context.CancelFunc) error
// ResolvePort resolves a service port to the actual target port.
// For named ports, it looks up the port number from the endpoint.
// For numeric ports, it returns the port as-is.
ResolvePort(servicePort v1.ServicePort) int32
}
// ResolvePortWithLookup is a helper that resolves a service port using a lookup function
// for named ports. This consolidates the common resolution logic.
func ResolvePortWithLookup(servicePort v1.ServicePort, lookupNamedPort func(string) int32) int32 {
if servicePort.TargetPort.IntVal != 0 {
return servicePort.TargetPort.IntVal
}
if servicePort.TargetPort.StrVal != "" {
if port := lookupNamedPort(servicePort.TargetPort.StrVal); port != 0 {
return port
}
}
return servicePort.Port
GetProtocol() string
}

View File

@@ -2,9 +2,7 @@ package etcd
import (
"context"
"fmt"
"hash/fnv"
"sync"
"time"
log "log/slog"
@@ -66,11 +64,10 @@ type ClientConfig struct {
}
// RunElectionOrDie behaves the same way as RunElection but panics if there is an error.
func RunElectionOrDie(ctx context.Context, config *LeaderElectionConfig) error {
func RunElectionOrDie(ctx context.Context, config *LeaderElectionConfig) {
if err := RunElection(ctx, config); err != nil {
return fmt.Errorf("leaderelection error: %w", err)
panic(err)
}
return nil
}
// RunElection starts a client with the provided config or panics.
@@ -119,12 +116,7 @@ func RunElection(ctx context.Context, config *LeaderElectionConfig) error {
leaseTTL: lease.TTL,
}
wg := sync.WaitGroup{}
defer wg.Wait()
wg.Go(func() {
m.tryToBeLeader(ctx, &wg)
})
go m.tryToBeLeader(ctx)
m.watchLeaderChanges(ctx)
return nil
@@ -193,7 +185,7 @@ watcher:
log.Debug("Exiting watcher", "id", m.memberID)
}
func (m *member) tryToBeLeader(ctx context.Context, wg *sync.WaitGroup) {
func (m *member) tryToBeLeader(ctx context.Context) {
if err := m.election.Campaign(ctx, m.memberID); err != nil {
log.Error("Failed trying to become the leader", "err", err)
// Resign just in case we acquired leadership just before failing
@@ -212,9 +204,7 @@ func (m *member) tryToBeLeader(ctx context.Context, wg *sync.WaitGroup) {
m.weAreTheLeader <- struct{}{}
// Once we are the leader, start the routine to resign if context is canceled
wg.Go(func() {
m.resignOnCancel(ctx)
})
go m.resignOnCancel(ctx)
// After becoming the leader, we wait for at least a lease TTL to wait for
// the previous leader to detect the new leadership (if there was one) and
@@ -231,7 +221,7 @@ func (m *member) tryToBeLeader(ctx context.Context, wg *sync.WaitGroup) {
func (m *member) resignOnCancel(ctx context.Context) {
<-ctx.Done()
if err := m.election.Resign(m.client.Ctx()); err != nil && !errors.Is(err, context.Canceled) {
if err := m.election.Resign(m.client.Ctx()); err != nil {
log.Error("Failed to resign after the context was canceled", "err", err)
}
}

View File

@@ -20,26 +20,19 @@ import (
func TestRunElectionWithMemberIDCollision(t *testing.T) {
t.Parallel()
g := NewWithT(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ctx := context.Background()
cli := client(g)
defer cli.Close()
electionName := randomElectionNameForTest("memberIDConflict")
log.Printf("Election name %s\n", electionName)
memberCtx, cancelMember1 := context.WithCancel(ctx)
// Use a channel to signal when the first member has observed a new leader
// This ensures proper ordering without relying on sleep timing
firstMemberObservedLeader := make(chan struct{})
var firstMemberObservedOnce sync.Once
config := &etcd.LeaderElectionConfig{
EtcdConfig: etcd.ClientConfig{
Client: cli,
},
Name: electionName,
MemberID: randomElectionNameForTest("my-host"),
MemberID: "my-host",
LeaseDurationSeconds: 1,
Callbacks: etcd.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
@@ -49,11 +42,6 @@ func TestRunElectionWithMemberIDCollision(t *testing.T) {
},
OnNewLeader: func(identity string) {
log.Printf("New leader: %s\n", identity)
// Signal that the first member has observed a leader
// This means the lease has been created
firstMemberObservedOnce.Do(func() {
close(firstMemberObservedLeader)
})
},
OnStoppedLeading: func() {
log.Println("I'm not the leader anymore")
@@ -71,18 +59,8 @@ func TestRunElectionWithMemberIDCollision(t *testing.T) {
go func() {
defer wg.Done()
// Wait for the first member to observe a leader, which means the lease has been created
select {
case <-firstMemberObservedLeader:
// First member has created the lease, now try to create a conflicting one
case <-time.After(5 * time.Second):
t.Error("timeout waiting for first member to observe leader")
return
}
// Use a cancellable context to prevent hanging if this goroutine unexpectedly succeeds
member2Ctx, cancelMember2 := context.WithTimeout(ctx, 5*time.Second)
defer cancelMember2()
g.Expect(etcd.RunElection(member2Ctx, config)).Should(MatchError(ContainSubstring("creating lease")))
time.Sleep(time.Millisecond * 50) // make sure the first one becomes leader
g.Expect(etcd.RunElection(ctx, config)).Should(MatchError(ContainSubstring("creating lease")))
}()
wg.Wait()
@@ -91,8 +69,7 @@ func TestRunElectionWithMemberIDCollision(t *testing.T) {
func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
t.Parallel()
g := NewWithT(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ctx := context.Background()
cli := client(g)
defer cli.Close()
@@ -108,27 +85,24 @@ func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
LeaseDurationSeconds: 1,
}
member1Ctx, cancelMember1 := context.WithCancel(ctx)
member1Ctx, _ := context.WithCancel(ctx)
member2Ctx, cancelMember2 := context.WithCancel(ctx)
config1 := configBase
config1.EtcdConfig.Client = cliMember1
config1.MemberID = randomElectionNameForTest("my-host")
config1.MemberID = "my-host"
uniqueID := rand.Uint64()
config1.MemberUniqueID = &uniqueID
config1.Callbacks = baseCallbacksForName(config1.MemberID)
syncMembers := make(chan (any))
config1.Callbacks.OnStartedLeading = func(_ context.Context) {
log.Println("I'm my-host, the new leader!!!!")
close(syncMembers)
log.Println("Losing the leadership on purpose by stopping renewing the lease")
g.Expect(cliMember1.Lease.Close()).To(Succeed())
log.Println("Member1 leases closed")
cancelMember1()
}
config2 := configBase
config2.MemberID = randomElectionNameForTest("my-other-host")
config2.MemberID = "my-other-host"
config2.Callbacks = baseCallbacksForName(config2.MemberID)
config2.Callbacks.OnStartedLeading = func(_ context.Context) {
log.Println("I'm my-other-host, the new leader!!!!")
@@ -142,18 +116,17 @@ func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
go func() {
defer wg.Done()
g.Expect(etcd.RunElection(member1Ctx, &config1)).To(Succeed())
log.Printf("%s routine done\n", config1.MemberID)
log.Println("Member1 routine done")
}()
go func() {
defer wg.Done()
<-syncMembers
time.Sleep(time.Millisecond * 50) // Make sure member1 becomes leader
g.Expect(etcd.RunElection(member2Ctx, &config2)).To(Succeed())
log.Printf("%s routine done\n", config2.MemberID)
log.Println("Member2 routine done")
}()
wg.Wait()
}
func baseCallbacksForName(name string) etcd.LeaderCallbacks {

View File

@@ -29,8 +29,7 @@ const (
func TestMain(m *testing.M) {
logrus.SetLevel(logrus.DebugLevel)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ctx := context.Background()
expectSuccess(startEtcd(ctx), "starting etcd")
os.Exit(runTestsWithCleanup(m, func() {

View File

@@ -1,14 +1,12 @@
package instance
import (
"context"
"fmt"
"net"
"slices"
"strconv"
"strings"
"sync"
"log/slog"
log "log/slog"
"github.com/vishvananda/netlink"
@@ -18,10 +16,7 @@ import (
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/node"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/sysctl"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
@@ -34,35 +29,18 @@ type Instance struct {
Clusters []*cluster.Cluster
// Service uses DHCP
IsDHCPv4 bool
IsDHCPv6 bool
IsDHCP bool
DHCPInterface string
DHCPInterfaceHwaddr string
DHCPInterfaceIP string
DHCPInterfaceIPv4 string
DHCPInterfaceIPv6 string
DHCPHostname string
DHCPv4Client vip.DHCPClient
DHCPv6Client vip.DHCPClient
// Service use Vlan
IsVLAN bool
VLANInterface string
DHCPClient *vip.DHCPClient
// External Gateway IP the service is forwarded from
UPNPGatewayIPs []string
// Kubernetes service mapping
ServiceSnapshot *v1.Service
dnsAddresses []string
// AddCalled determined that ActionAdd was already performed for the instance
AddCalled bool
// LabelAdded determined that node was labeled with
// service-provided.kube-vip.io label
LabelAdded bool
}
type Port struct {
@@ -70,42 +48,18 @@ type Port struct {
Type string
}
func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
intfMgr *networkinterface.Manager, arpMgr *arp.Manager, routeMgr *route.Manager,
nodeLabelMgr node.Labeler, wg *sync.WaitGroup) (*Instance, error) {
instanceAddresses, instanceHostnames := FetchServiceAddresses(svc)
log.Info("new instance", "namespace", svc.Namespace, "service", svc.Name, "addresses", instanceAddresses, "hostnames", instanceHostnames)
func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Instance, error) {
instanceAddresses := FetchServiceAddresses(svc)
//instanceUID := string(svc.UID)
var newVips []*kubevip.Config
var link netlink.Link
var err error
var dnsAddresses []string
// Create new service
instance := &Instance{
ServiceSnapshot: svc,
dnsAddresses: dnsAddresses,
}
for _, address := range instanceAddresses {
// Detect if we're using a specific interface for services
var svcInterface string
svcInterface = svc.Annotations[kubevip.ServiceVlan]
if svcInterface != "" {
parent, tag, err := utils.ParseVLANInterface(svcInterface)
if err != nil {
log.Error("failed to validate VLAN", "err", err)
}
if err := instance.addVLAN(parent, tag); err != nil {
log.Error("failed to create VLAN", "err", err)
}
} else {
// If no vlan defined use specific interface from annotation
svcInterface = svc.Annotations[kubevip.ServiceInterface]
}
svcInterface = svc.Annotations[kubevip.ServiceInterface] // If the service has a specific interface defined, then use it
if svcInterface == kubevip.Auto {
link, err = autoFindInterface(address)
if err != nil {
@@ -166,7 +120,7 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
subnet := ""
var err error
if utils.IsIPv4(address) {
if vip.IsIPv4(address) {
if ipv4AutoSubnet {
subnet, err = autoFindSubnet(link, address)
if err != nil {
@@ -176,7 +130,7 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
if cidrs[0] != "" && cidrs[0] != kubevip.Auto {
subnet = cidrs[0]
} else {
subnet = strconv.Itoa(vip.DefaultMaskIPv4)
subnet = "32"
}
}
} else {
@@ -189,84 +143,19 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
if len(cidrs) > 1 && cidrs[1] != "" && cidrs[1] != kubevip.Auto {
subnet = cidrs[1]
} else {
subnet = strconv.Itoa(vip.DefaultMaskIPv6)
subnet = "128"
}
}
}
// Generate new Virtual IP configuration
newVips = append(newVips, &kubevip.Config{
VIP: address,
Interface: svcInterface,
SingleNode: true,
EnableARP: config.EnableARP,
EnableBGP: config.EnableBGP,
VIPSubnet: subnet,
EnableRoutingTable: config.EnableRoutingTable,
RoutingTableID: config.RoutingTableID,
RoutingTableType: config.RoutingTableType,
RoutingProtocol: config.RoutingProtocol,
ArpBroadcastRate: config.ArpBroadcastRate,
EnableServiceSecurity: config.EnableServiceSecurity,
DNSMode: config.DNSMode,
DHCPMode: config.DHCPMode,
DHCPBackoffAttempts: config.DHCPBackoffAttempts,
DisableServiceUpdates: config.DisableServiceUpdates,
EnableServicesElection: config.EnableServicesElection,
PreserveVIPOnLeadershipLoss: config.PreserveVIPOnLeadershipLoss,
KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
EnableLeaderElection: config.EnableLeaderElection,
},
})
}
for _, hostname := range instanceHostnames {
log.Info("hostname", "addr", hostname)
// Detect if we're using a specific interface for services
var svcInterface string
svcInterface = svc.Annotations[kubevip.ServiceVlan]
if svcInterface != "" {
parent, tag, err := utils.ParseVLANInterface(svcInterface)
if err != nil {
log.Error("failed to validate VLAN", "err", err)
}
if err := instance.addVLAN(parent, tag); err != nil {
log.Error("failed to create VLAN", "err", err)
}
} else {
// If no vlan defined use specific interface from annotation
svcInterface = svc.Annotations[kubevip.ServiceInterface]
}
// If it is still blank then use the
if svcInterface == "" {
switch config.ServicesInterface {
case "":
svcInterface = config.Interface
default:
svcInterface = config.ServicesInterface
}
}
if link == nil {
if link, err = netlink.LinkByName(svcInterface); err != nil {
return nil, fmt.Errorf("failed to get interface %s: %w", svcInterface, err)
}
if link == nil {
return nil, fmt.Errorf("failed to get interface %s", svcInterface)
}
}
// Generate new Virtual IP configuration
newVips = append(newVips, &kubevip.Config{
VIP: hostname,
VIP: address,
Interface: svcInterface,
SingleNode: true,
EnableARP: config.EnableARP,
EnableBGP: config.EnableBGP,
VIPSubnet: config.VIPSubnet,
VIPSubnet: subnet,
EnableRoutingTable: config.EnableRoutingTable,
RoutingTableID: config.RoutingTableID,
RoutingTableType: config.RoutingTableType,
@@ -274,7 +163,6 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
ArpBroadcastRate: config.ArpBroadcastRate,
EnableServiceSecurity: config.EnableServiceSecurity,
DNSMode: config.DNSMode,
DHCPMode: config.DHCPMode,
DisableServiceUpdates: config.DisableServiceUpdates,
EnableServicesElection: config.EnableServicesElection,
KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
@@ -283,23 +171,22 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
})
}
// Create new service
instance := &Instance{
//UID: instanceUID,
//VIPs: instanceAddresses,
ServiceSnapshot: svc,
}
// for _, port := range svc.Spec.Ports {
// instance.ExternalPorts = append(instance.ExternalPorts, Port{
// Port: uint16(port.Port), //nolint
// Type: string(port.Protocol),
// })
// }
if svc.Annotations != nil {
instance.DHCPInterfaceHwaddr = svc.Annotations[kubevip.HwAddrKey]
requestedIP := svc.Annotations[kubevip.RequestedIP]
if requestedIP != "" {
requestedIPs := strings.Split(requestedIP, ",")
if len(requestedIPs) > 2 {
return nil, fmt.Errorf("annotation %q cannot request more than one IPv4 and one Ipv6 address", kubevip.RequestedIP)
}
for _, ip := range requestedIPs {
netip := net.ParseIP(ip)
if netip.To4() != nil {
instance.DHCPInterfaceIPv4 = ip
} else {
instance.DHCPInterfaceIPv6 = ip
}
}
}
instance.DHCPInterfaceIP = svc.Annotations[kubevip.RequestedIP]
instance.DHCPHostname = svc.Annotations[kubevip.LoadbalancerHostname]
}
@@ -323,82 +210,29 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
// Create Add configuration to the new service
instance.VIPConfigs = newVips
// If this was purposely created with the address '0.0.0.0', or '::'
// If this was purposely created with the address 0.0.0.0,
// we will create a macvlan on the main interface and a DHCP client
if len(instanceAddresses) > 2 && (slices.Contains(instanceAddresses, "0.0.0.0") || slices.Contains(instanceAddresses, "::")) {
return nil, fmt.Errorf("DHCP cannot be used if more than 2 addresses (one IPv4 and one IPv6) were specified")
}
for i := range instance.VIPConfigs {
if instance.VIPConfigs[i].VIP == "0.0.0.0" {
err := instance.startDHCP(ctx, i, config.DHCPBackoffAttempts, wg)
if err != nil {
return nil, err
}
select {
case err := <-instance.DHCPv4Client.ErrorChannel():
return nil, fmt.Errorf("error starting DHCPv4 for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
case ip := <-instance.DHCPv4Client.IPChannel():
instance.VIPConfigs[i].Interface = instance.DHCPInterface
instance.VIPConfigs[i].VIP = ip
instance.DHCPInterfaceIPv4 = ip
}
}
if instance.VIPConfigs[i].VIP == "::" {
err := instance.startDHCP(ctx, i, config.DHCPBackoffAttempts, wg)
if err != nil {
return nil, err
}
select {
case err := <-instance.DHCPv6Client.ErrorChannel():
return nil, fmt.Errorf("error starting DHCPv6 for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
case ip := <-instance.DHCPv6Client.IPChannel():
instance.VIPConfigs[i].Interface = instance.DHCPInterface
instance.VIPConfigs[i].VIP = ip
instance.DHCPInterfaceIPv6 = ip
}
}
ddnsAnnotation, exists := svc.Annotations[kubevip.ServiceDDNS]
if exists {
instance.VIPConfigs[i].DDNS, err = strconv.ParseBool(ddnsAnnotation)
if err != nil {
log.Error("Failed to add service", "err", err)
return nil, err
}
}
if len(svc.Spec.IPFamilies) > 0 {
if len(svc.Spec.IPFamilies) > 1 {
instance.VIPConfigs[i].DHCPMode = utils.DualFamily
instance.VIPConfigs[i].DNSMode = utils.DualFamily
switch *svc.Spec.IPFamilyPolicy {
case v1.IPFamilyPolicyRequireDualStack:
instance.VIPConfigs[i].IsDualStack = true
instance.VIPConfigs[i].RequireDualStack = true
case v1.IPFamilyPolicyPreferDualStack:
instance.VIPConfigs[i].IsDualStack = true
instance.VIPConfigs[i].RequireDualStack = false
default:
instance.VIPConfigs[i].IsDualStack = false
instance.VIPConfigs[i].RequireDualStack = false
}
} else {
if strings.EqualFold(string(svc.Spec.IPFamilies[0]), utils.IPv4Family) {
instance.VIPConfigs[i].DHCPMode = strings.ToLower(utils.IPv4Family)
instance.VIPConfigs[i].DNSMode = strings.ToLower(utils.IPv4Family)
} else {
instance.VIPConfigs[i].DHCPMode = strings.ToLower(utils.IPv6Family)
instance.VIPConfigs[i].DNSMode = strings.ToLower(utils.IPv6Family)
}
}
}
c, err := cluster.InitCluster(instance.VIPConfigs[i], false, intfMgr, arpMgr, routeMgr, nodeLabelMgr)
// TODO: Consider how best to handle DHCP with multiple addresses
if len(instanceAddresses) == 1 && instanceAddresses[0] == "0.0.0.0" {
err := instance.startDHCP()
if err != nil {
log.Error("failed to add service", "err", err)
return nil, err
}
select {
case err := <-instance.DHCPClient.ErrorChannel():
return nil, fmt.Errorf("error starting DHCP for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
case ip := <-instance.DHCPClient.IPChannel():
instance.VIPConfigs[0].Interface = instance.DHCPInterface
instance.VIPConfigs[0].VIP = ip
instance.DHCPInterfaceIP = ip
}
}
for _, vipConfig := range instance.VIPConfigs {
c, err := cluster.InitCluster(vipConfig, false, intfMgr, arpMgr)
if err != nil {
log.Error("Failed to add Service %s/%s", svc.Namespace, svc.Name)
return nil, err
}
@@ -407,7 +241,7 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
}
instance.Clusters = append(instance.Clusters, c)
log.Info("(svcs) adding VIP", "ip", instance.VIPConfigs[i].VIP, "interface", instance.VIPConfigs[i].Interface, "namespace", svc.Namespace, "name", svc.Name)
log.Info("(svcs) adding VIP", "ip", vipConfig.VIP, "interface", vipConfig.Interface, "namespace", svc.Namespace, "name", svc.Name)
}
return instance, nil
@@ -470,62 +304,11 @@ func getAutoInterfaceName(link netlink.Link, defaultInterface string) string {
return link.Attrs().Name
}
func (i *Instance) addVLAN(parentInterface string, tag int) error {
var parent netlink.Link
interfaceName := fmt.Sprintf("%s.%d", parentInterface, tag)
iface, err := netlink.LinkByName(interfaceName)
if err != nil {
// check if parent interface doesnt exist
parent, err = netlink.LinkByName(parentInterface)
if err != nil {
return fmt.Errorf("error finding VLAN parent interface %s: %v", parentInterface, err)
}
log.Info("Creating new VLAN interface", "interface", interfaceName)
vlan := &netlink.Vlan{
LinkAttrs: netlink.LinkAttrs{
Name: interfaceName,
ParentIndex: parent.Attrs().Index,
},
VlanId: tag,
VlanProtocol: netlink.VLAN_PROTOCOL_8021Q,
}
err = netlink.LinkAdd(vlan)
if err != nil {
return fmt.Errorf("could not add VLAN %s: %v", interfaceName, err)
}
err = netlink.LinkSetUp(vlan)
if err != nil {
return fmt.Errorf("could not bring up VLAN interface [%s] : %v", interfaceName, err)
}
_, err = net.InterfaceByName(interfaceName)
if err != nil {
return fmt.Errorf("error finding new VLAN interface by name [%v]", err)
}
} else {
log.Info("Using existing VLAN interface", "interface", interfaceName)
if err := utils.ValidateVLANInterface(iface, parent, tag); err != nil {
return err
}
func (i *Instance) startDHCP() error {
if len(i.VIPConfigs) != 1 {
return fmt.Errorf("DHCP requires exactly 1 VIP config, got: %v", len(i.VIPConfigs))
}
i.VLANInterface = interfaceName
i.IsVLAN = true
return nil
}
func (i *Instance) startDHCP(ctx context.Context, index int, backoffAttempts uint, wg *sync.WaitGroup) error {
if len(i.VIPConfigs) > 2 {
return fmt.Errorf("DHCP can be used with 2 VIP config maximally, got: %v", len(i.VIPConfigs))
}
parent, err := netlink.LinkByName(i.VIPConfigs[index].Interface)
parent, err := netlink.LinkByName(i.VIPConfigs[0].Interface)
if err != nil {
return fmt.Errorf("error finding VIP Interface, for building DHCP Link : %v", err)
}
@@ -576,161 +359,124 @@ func (i *Instance) startDHCP(ctx context.Context, index int, backoffAttempts uin
log.Info("Using existing macvlan interface for DHCP", "interface", interfaceName)
}
var initRebootFlag bool
ip := net.ParseIP(i.VIPConfigs[index].VIP)
// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
rpfilterSetting := "0"
var client vip.DHCPClient
if ip.To4() != nil {
// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
rpfilterSetting := "0"
// Check if we need to set an override rp_filter value for the interface
if i.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
// Check the rp_filter value
rpFilter, err := strconv.Atoi(i.ServiceSnapshot.Annotations[kubevip.RPFilter])
if err != nil {
log.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
// Check if we need to set an override rp_filter value for the interface
if i.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
// Check the rp_filter value
rpFilter, err := strconv.Atoi(i.ServiceSnapshot.Annotations[kubevip.RPFilter])
if err != nil {
slog.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
} else {
if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
rpfilterSetting = i.ServiceSnapshot.Annotations[kubevip.RPFilter]
} else {
if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
rpfilterSetting = i.ServiceSnapshot.Annotations[kubevip.RPFilter]
} else {
log.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
}
slog.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
}
}
err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
if err != nil {
log.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
}
if i.DHCPInterfaceIPv4 != "" {
initRebootFlag = true
}
client = vip.NewDHCPv4Client(iface, initRebootFlag, i.DHCPInterfaceIPv4, backoffAttempts)
// Add the client so that we can call it to stop function
i.DHCPv4Client = client
// Set that DHCPv4 is enabled
i.IsDHCPv4 = true
} else {
if i.DHCPInterfaceIPv6 != "" {
initRebootFlag = true
}
client, err = vip.NewDHCPv6Client(iface, parent, initRebootFlag, i.DHCPInterfaceIPv6, backoffAttempts)
if err != nil {
return fmt.Errorf("unable to create client: %w", err)
}
// Add the client so that we can call it to stop function
i.DHCPv6Client = client
// Set that DHCPv6 is enabled
i.IsDHCPv6 = true
}
err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
if err != nil {
slog.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
}
var initRebootFlag bool
if i.DHCPInterfaceIP != "" {
initRebootFlag = true
}
client := vip.NewDHCPClient(iface, initRebootFlag, i.DHCPInterfaceIP)
// Add hostname to dhcp client if annotated
if i.DHCPHostname != "" {
log.Info("Hostname specified for dhcp lease", "interface", interfaceName, "hostname", i.DHCPHostname)
client.WithHostName(i.DHCPHostname)
}
wg.Go(func() {
if err := client.Start(ctx); err != nil {
log.Error("[instance] DHCP client error: %w")
}
})
go client.Start()
// Set that DHCP is enabled
i.IsDHCP = true
// Set the name of the interface so that it can be removed on Service deletion
i.DHCPInterface = interfaceName
i.DHCPInterfaceHwaddr = iface.HardwareAddr.String()
// Add the client so that we can call it to stop function
i.DHCPClient = client
return nil
}
// FetchLoadBalancerIngressAddresses tries to get the addresses from status.loadBalancerIP
func FetchLoadBalancerIngress(s *v1.Service) ([]string, []string) {
// FetchIngressAddresses tries to get the addresses from status.loadBalancerIP
func FetchLoadBalancerIngressAddresses(s *v1.Service) []string {
// If the service has no status, return empty
lbStatusAddresses := []string{}
lbStatusHostnames := []string{}
if len(s.Status.LoadBalancer.Ingress) == 0 {
return lbStatusAddresses, lbStatusHostnames
return lbStatusAddresses
}
for _, ingress := range s.Status.LoadBalancer.Ingress {
if ingress.IP != "" {
lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
}
if ingress.Hostname != "" {
lbStatusHostnames = append(lbStatusHostnames, ingress.Hostname)
}
// TODO: Handle hostname if needed
}
return lbStatusAddresses, lbStatusHostnames
return lbStatusAddresses
}
// FetchServiceAddresses tries to get the addresses from annotations
// kube-vip.io/loadbalancerIPs, then from spec.loadbalancerIP
func FetchServiceAddresses(s *v1.Service) ([]string, []string) {
func FetchServiceAddresses(s *v1.Service) []string {
annotationAvailable := false
if s.Annotations != nil {
if v, annotationAvailable := s.Annotations[kubevip.LoadbalancerIPAnnotation]; annotationAvailable {
ips := strings.Split(v, ",")
var trimmedIPs []string
var trimmedHostnames []string
for _, a := range ips {
a = strings.TrimSpace(a)
ip := net.ParseIP(a)
if ip == nil {
// this is probably a DNS name
trimmedHostnames = append(trimmedHostnames, a)
} else {
trimmedIPs = append(trimmedIPs, ip.String())
}
for _, ip := range ips {
trimmedIPs = append(trimmedIPs, strings.TrimSpace(ip))
}
return trimmedIPs, trimmedHostnames
return trimmedIPs
}
}
lbStatusAddresses := []string{}
lbStatusHostnames := []string{}
if !annotationAvailable {
lbStatusAddresses, lbStatusHostnames = FetchLoadBalancerIngress(s)
}
// Spec.LoadBalancerIP legacy handling
// if the loadBalancerIP is different from Status.LoadBalancer.Ingress IPs
// return the legacy LB as spec wins over status.
if lbIP := net.ParseIP(s.Spec.LoadBalancerIP); lbIP != nil && len(lbStatusAddresses) > 0 {
isLbIPv4 := utils.IsIPv4(s.Spec.LoadBalancerIP)
for _, a := range lbStatusAddresses {
if lbStatusIP := net.ParseIP(a); lbStatusIP != nil && utils.IsIPv4(a) == isLbIPv4 && !lbIP.Equal(lbStatusIP) {
return []string{s.Spec.LoadBalancerIP}, []string{}
if len(s.Status.LoadBalancer.Ingress) > 0 {
for _, ingress := range s.Status.LoadBalancer.Ingress {
lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
}
}
}
if len(lbStatusAddresses) > 0 || len(lbStatusHostnames) > 0 {
return lbStatusAddresses, lbStatusHostnames
lbIP := net.ParseIP(s.Spec.LoadBalancerIP)
isLbIPv4 := vip.IsIPv4(s.Spec.LoadBalancerIP)
if len(lbStatusAddresses) > 0 {
for _, a := range lbStatusAddresses {
if lbStatusIP := net.ParseIP(a); lbStatusIP != nil && lbIP != nil && vip.IsIPv4(a) == isLbIPv4 && !lbIP.Equal(lbStatusIP) {
return []string{s.Spec.LoadBalancerIP}
}
}
return lbStatusAddresses
}
if s.Spec.LoadBalancerIP != "" {
return []string{s.Spec.LoadBalancerIP}, []string{}
return []string{s.Spec.LoadBalancerIP}
}
return []string{}, []string{}
return []string{}
}
func FindServiceInstance(svc *v1.Service, instances []*Instance) *Instance {
log.Debug("finding service", "namespace", svc.Namespace, "name", svc.Name, "UID", svc.UID)
log.Debug("finding service", "UID", svc.UID)
for i := range instances {
log.Debug("saved service", "instance", i, "UID", instances[i].ServiceSnapshot.UID)
if instances[i].ServiceSnapshot.UID == svc.UID {
return instances[i]
}
}
log.Debug("instance not found", "namespace", svc.Namespace, "name", svc.Name, "UID", svc.UID)
return nil
}

View File

@@ -40,34 +40,10 @@ const (
// Flush the conntrack rules (remove existing sessions) once Egress is configured
FlushContrack = "kube-vip.io/flush-conntrack"
// Configure LoadBalancer IPs instead of relying on a controller
LoadbalancerIPAnnotation = "kube-vip.io/loadbalancerIPs"
// Ignore the LoadBalancer Service
LoadbalancerIgnore = "kube-vip.io/ignore"
// Used to configure DHCP with a Hostname
LoadbalancerHostname = "kube-vip.io/loadbalancerHostname"
// Define an interface name to bind the address of the LoadBalancer to
ServiceInterface = "kube-vip.io/serviceInterface"
// Specify VLAN subinterface for service (e.g. eth0.200)
ServiceVlan = "kube-vip.io/serviceVLAN"
ServiceSecurityIgnore = "kube-vip.io/ignore-service-security"
// Enable UPNP on a Service
UpnpEnabled = "kube-vip.io/forwardUPNP"
// Set the UPNP lease duration for a specific service using duration format (e.g., "30s", "1h")
UpnpLeaseDuration = "kube-vip.io/upnp-lease-duration"
LoadbalancerHostname = "kube-vip.io/loadbalancerHostname"
ServiceInterface = "kube-vip.io/serviceInterface"
UpnpEnabled = "kube-vip.io/forwardUPNP"
RPFilter = "kube-vip.io/rp_filter" // Set the return path filter for a specific service interface
// Name of the service lease object
ServiceLease = "kube-vip.io/leaseName"
// Enable DDNS for the service
ServiceDDNS = "kube-vip.io/ddns"
)

View File

@@ -1,250 +0,0 @@
package kubevip
import (
"fmt"
"net"
"strconv"
"strings"
"github.com/kube-vip/kube-vip/pkg/utils"
api "github.com/osrg/gobgp/v3/api"
"github.com/vishvananda/netlink"
)
// Peer defines a BGP Peer
type BGPPeer struct {
Address string
Port uint16
Interface string
AS uint32
Password string
MultiHop bool
MpbgpNexthop string
MpbgpIPv4 string
MpbgpIPv6 string
}
// Config defines the BGP server configuration
type BGPConfig struct {
AS uint32
RouterID string
SourceIP string
SourceIF string
MpbgpNexthop string
MpbgpIPv4 string
MpbgpIPv6 string
HoldTime uint64
KeepaliveInterval uint64
Peers []BGPPeer
Zebra ZebraConfig
}
// Defines Zebra connection configuration. More on the topic - https://github.com/osrg/gobgp/blob/master/docs/sources/zebra.md#configuration
type ZebraConfig struct {
Enabled bool
URL string
Version uint32
SoftwareName string
}
// ParseBGPPeerConfig - take a string and parses it into an array of peers
func ParseBGPPeerConfig(config string) (bgpPeers []BGPPeer, err error) {
peers := strings.Split(config, ",")
if len(peers) == 0 {
return nil, fmt.Errorf("no BGP Peer configurations found")
}
for x := range peers {
peerStr := peers[x]
config := strings.Split(peerStr, "/")
peerStr = config[0]
if peerStr == "" {
continue
}
isV6Peer := peerStr[0] == '['
isUnnumberedPeer := strings.HasPrefix(peerStr, "unnumbered:")
address := ""
if isV6Peer {
addressEndPos := strings.IndexByte(peerStr, ']')
if addressEndPos == -1 {
return nil, fmt.Errorf("no matching ] found for IPv6 BGP Peer")
}
address = peerStr[1:addressEndPos]
peerStr = peerStr[addressEndPos+1:]
} else if isUnnumberedPeer {
unnumberedEndPos := strings.IndexByte(peerStr, ':')
peerStr = peerStr[unnumberedEndPos+1:]
}
peer := strings.Split(peerStr, ":")
if len(peer) < 2 && !isUnnumberedPeer {
return nil, fmt.Errorf("mandatory peering params <host>:<AS> incomplete")
}
iface := ""
if isUnnumberedPeer {
iface = peer[0]
} else if !isV6Peer {
address = peer[0]
}
var ASNumber uint64
if len(peer) >= 2 {
ASNumber, err = strconv.ParseUint(peer[1], 10, 32)
if err != nil {
return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
}
}
password := ""
if len(peer) >= 3 {
password = peer[2]
}
multiHop := false
if len(peer) >= 4 && peer[3] != "" {
multiHop, err = strconv.ParseBool(peer[3])
if err != nil {
return nil, fmt.Errorf("BGP MultiHop format error (true/false) [%s]", peer[3])
}
}
var port uint64
if len(peer) >= 5 {
port, err = strconv.ParseUint(peer[4], 10, 16)
if err != nil {
return nil, fmt.Errorf("BGP Peer Port format error [%s]", peer[4])
}
} else if !isUnnumberedPeer {
port = 179
}
var mpbgpNexthop, mpbgpIPv4, mpbgpIPv6 string
if len(config) > 1 {
configData := strings.Split(config[1], ";")
for _, cfg := range configData {
c := strings.Split(cfg, "=")
if len(c) < 2 {
return nil, fmt.Errorf("peer configuration parameter '%s' is missing a value (expected key=value)", c[0])
}
switch c[0] {
case "mpbgp_nexthop":
mpbgpNexthop = c[1]
case "mpbgp_ipv4":
mpbgpIPv4 = c[1]
case "mpbgp_ipv6":
mpbgpIPv6 = c[1]
default:
return nil, fmt.Errorf("peer configuration parameter '%s' is not supported", c[0])
}
}
}
peerConfig := BGPPeer{
Address: address,
//nolint:gosec // previously parsed into uint32
AS: uint32(ASNumber),
Port: uint16(port),
Interface: iface,
Password: password,
MultiHop: multiHop,
MpbgpNexthop: mpbgpNexthop,
MpbgpIPv4: mpbgpIPv4,
MpbgpIPv6: mpbgpIPv6,
}
bgpPeers = append(bgpPeers, peerConfig)
}
return
}
func (p *BGPPeer) FindMpbgpAddresses(ap *api.Peer, server *BGPConfig) (string, string, error) {
var ipv4Address, ipv6Address string
switch p.MpbgpNexthop {
case "fixed":
ap.Transport.LocalAddress = server.SourceIP
if p.MpbgpIPv4 == "" && p.MpbgpIPv6 == "" {
return "", "", fmt.Errorf("to use MP-BGP with fixed address at least one IPv4 or IPv6 address has to be provided [current - IPv4: %s, IPv6: %s]",
p.MpbgpIPv4, p.MpbgpIPv6)
}
if p.MpbgpIPv4 != "" {
if net.ParseIP(p.MpbgpIPv4) == nil {
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv4 address", p.MpbgpIPv4)
}
}
if p.MpbgpIPv6 != "" {
if net.ParseIP(p.MpbgpIPv6) == nil {
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv6 address", p.MpbgpIPv6)
}
}
ipv4Address = p.MpbgpIPv4
ipv6Address = p.MpbgpIPv6
case "auto_sourceip":
ap.Transport.LocalAddress = server.SourceIP
// Resolve the local interface by SourceIP
iface, err := utils.GetInterfaceByIP(server.SourceIP)
if err != nil {
return "", "", fmt.Errorf("failed to get interface by IP: %v", err)
}
if utils.IsIPv4(server.SourceIP) {
// Get the non link-local IPv6 address on that interface
ipv6Address, err = utils.GetNonLinkLocalIP(iface, netlink.FAMILY_V6)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
}
} else {
// Get the non link-local IPv4 address on that interface
ipv4Address, err = utils.GetNonLinkLocalIP(iface, netlink.FAMILY_V4)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
}
}
case "auto_sourceif":
ap.Transport.BindInterface = server.SourceIF
iface, err := netlink.LinkByName(server.SourceIF)
if err != nil {
return "", "", fmt.Errorf("failed to get interface by name: %v", err)
}
// Get the non link-local IPv4 address on that interface
ipv4Address, err = utils.GetNonLinkLocalIP(&iface, netlink.FAMILY_V4)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
}
// Get the non link-local IPv6 address on that interface
ipv6Address, err = utils.GetNonLinkLocalIP(&iface, netlink.FAMILY_V6)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
}
default:
return "", "", fmt.Errorf("option %s for MP-BPG nexthop is not supported", server.MpbgpNexthop)
}
return ipv4Address, ipv6Address, nil
}
func (p *BGPPeer) SetMpbgpOptions(server *BGPConfig) {
if p.MpbgpNexthop == "" {
p.MpbgpNexthop = server.MpbgpNexthop
}
if p.MpbgpIPv4 == "" {
p.MpbgpIPv4 = server.MpbgpIPv4
}
if p.MpbgpIPv6 == "" {
p.MpbgpIPv6 = server.MpbgpIPv6
}
}

View File

@@ -6,13 +6,11 @@ import (
"math"
"math/bits"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/detector"
"github.com/kube-vip/kube-vip/pkg/utils"
"sigs.k8s.io/yaml"
)
// ParseEnvironment - will popultate the configuration from environment variables
@@ -26,7 +24,7 @@ func ParseEnvironment(c *Config) error {
if env != "" {
logLevel, err := strconv.ParseInt(env, 10, 32)
if err != nil {
return fmt.Errorf("unable to parse environment variable [vip_loglevel], should be int: %w", err)
panic("Unable to parse environment variable [vip_loglevel], should be int")
}
c.Logging = int32(logLevel)
}
@@ -292,18 +290,6 @@ func ParseEnvironment(c *Config) error {
c.ArpBroadcastRate = 3000
}
// Determine if VIP should be preserved on leadership loss
// true: VIP addresses remain on interface, only ARP/NDP broadcasting stops
// false (default): VIP addresses are deleted on leadership loss (legacy behavior)
env = os.Getenv(vipPreserveOnLeadershipLoss)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.PreserveVIPOnLeadershipLoss = b
}
// Wireguard Mode
env = os.Getenv(vipWireguard)
if env != "" {
@@ -377,30 +363,6 @@ func ParseEnvironment(c *Config) error {
c.DNSMode = env
}
// DHCP mode
env = os.Getenv(dhcpMode)
if env != "" {
c.DHCPMode = env
} else {
if c.DNSMode != "first" {
c.DHCPMode = c.DNSMode
} else {
c.DHCPMode = strings.ToLower(utils.IPv4Family)
}
}
// DHCP backoff attempts
env = os.Getenv(dhcpBackoffAttempts)
if env != "" {
tmp, err := strconv.ParseInt(env, 10, 32)
if err != nil {
return err
}
if tmp >= 0 {
c.DHCPBackoffAttempts = uint(tmp)
}
}
// Disable updates for services (status.LoadBalancer.Ingress will not be updated)
env = os.Getenv(disableServiceUpdates)
if env != "" {
@@ -460,7 +422,7 @@ func ParseEnvironment(c *Config) error {
// Peer AS
env = os.Getenv(bgpPeers)
if env != "" {
peers, err := ParseBGPPeerConfig(env)
peers, err := bgp.ParseBGPPeerConfig(env)
if err != nil {
return err
}
@@ -539,40 +501,6 @@ func ParseEnvironment(c *Config) error {
c.BGPConfig.KeepaliveInterval = u64
}
// BGP health check options
env = os.Getenv(controlPlaneHealthCheckAddress)
if env != "" {
c.ControlPlaneHealthCheck.Address = env
}
env = os.Getenv(controlPlaneHealthCheckPeriodSeconds)
if env != "" {
i, err := strconv.ParseInt(env, 10, 32)
if err != nil {
return fmt.Errorf("parsing env var %s (value: %s): %w", controlPlaneHealthCheckPeriodSeconds, env, err)
}
c.ControlPlaneHealthCheck.PeriodSeconds = int(i)
}
env = os.Getenv(controlPlaneHealthCheckTimeoutSeconds)
if env != "" {
i, err := strconv.ParseInt(env, 10, 32)
if err != nil {
return fmt.Errorf("parsing env var %s (value: %s): %w", controlPlaneHealthCheckTimeoutSeconds, env, err)
}
c.ControlPlaneHealthCheck.TimeoutSeconds = int(i)
}
env = os.Getenv(controlPlaneHealthCheckFailureThreshold)
if env != "" {
i, err := strconv.ParseInt(env, 10, 32)
if err != nil {
return fmt.Errorf("parsing env var %s (value: %s): %w", controlPlaneHealthCheckFailureThreshold, env, err)
}
c.ControlPlaneHealthCheck.FailureThreshold = int(i)
}
env = os.Getenv(controlPlaneHealthCheckCAPath)
if env != "" {
c.ControlPlaneHealthCheck.CAPath = env
}
env = os.Getenv(zebraEnable)
if env != "" {
result, err := strconv.ParseBool(env)
@@ -739,317 +667,5 @@ func ParseEnvironment(c *Config) error {
c.EgressClean = b
}
// check for configuration file path
env = os.Getenv(configFile)
if env != "" {
c.ConfigFile = env
}
return nil
}
// LoadConfigFromFile loads configuration from a JSON or YAML file
func LoadConfigFromFile(configFilePath string) (*Config, error) {
if configFilePath == "" {
return nil, fmt.Errorf("config file path is empty")
}
// Check if file exists
if _, err := os.Stat(configFilePath); os.IsNotExist(err) {
return nil, fmt.Errorf("config file does not exist: %s", configFilePath)
}
// Read file content
data, err := os.ReadFile(configFilePath)
if err != nil {
return nil, fmt.Errorf("failed to read config file %s: %v", configFilePath, err)
}
var config Config
ext := strings.ToLower(filepath.Ext(configFilePath))
switch ext {
case ".json":
err = json.Unmarshal(data, &config)
if err != nil {
return nil, fmt.Errorf("failed to parse JSON config file %s: %v", configFilePath, err)
}
case ".yaml", ".yml":
err = yaml.Unmarshal(data, &config)
if err != nil {
return nil, fmt.Errorf("failed to parse YAML config file %s: %v", configFilePath, err)
}
default:
return nil, fmt.Errorf("unsupported config file format %s. Supported formats: .json, .yaml, .yml", ext)
}
return &config, nil
}
// MergeConfigFromFile merges configuration loaded from file with existing config
// Priority: command line flags > environment variables > config file
func MergeConfigFromFile(c *Config, configFilePath string) error {
if configFilePath == "" {
return nil // No config file specified, nothing to merge
}
fileConfig, err := LoadConfigFromFile(configFilePath)
if err != nil {
return err
}
// Merge file config with existing config
// Only set values from file if they haven't been set by flags or env vars
mergeConfigValues(c, fileConfig)
return nil
}
// mergeConfigValues merges values from fileConfig into baseConfig
// Only overwrites zero values in baseConfig
func mergeConfigValues(baseConfig, fileConfig *Config) {
// Basic configuration
if baseConfig.Logging == 0 && fileConfig.Logging != 0 {
baseConfig.Logging = fileConfig.Logging
}
// Network configuration
if baseConfig.Interface == "" && fileConfig.Interface != "" {
baseConfig.Interface = fileConfig.Interface
}
if baseConfig.ServicesInterface == "" && fileConfig.ServicesInterface != "" {
baseConfig.ServicesInterface = fileConfig.ServicesInterface
}
if baseConfig.VIP == "" && fileConfig.VIP != "" {
baseConfig.VIP = fileConfig.VIP
}
if baseConfig.VIPSubnet == "" && fileConfig.VIPSubnet != "" {
baseConfig.VIPSubnet = fileConfig.VIPSubnet
}
if baseConfig.Address == "" && fileConfig.Address != "" {
baseConfig.Address = fileConfig.Address
}
if baseConfig.Port == 0 && fileConfig.Port != 0 {
baseConfig.Port = fileConfig.Port
}
if baseConfig.NodeName == "" && fileConfig.NodeName != "" {
baseConfig.NodeName = fileConfig.NodeName
}
// Boolean flags - only merge if not explicitly set
if !baseConfig.EnableARP && fileConfig.EnableARP {
baseConfig.EnableARP = fileConfig.EnableARP
}
if !baseConfig.EnableBGP && fileConfig.EnableBGP {
baseConfig.EnableBGP = fileConfig.EnableBGP
}
if !baseConfig.EnableWireguard && fileConfig.EnableWireguard {
baseConfig.EnableWireguard = fileConfig.EnableWireguard
}
if !baseConfig.EnableRoutingTable && fileConfig.EnableRoutingTable {
baseConfig.EnableRoutingTable = fileConfig.EnableRoutingTable
}
if !baseConfig.EnableControlPlane && fileConfig.EnableControlPlane {
baseConfig.EnableControlPlane = fileConfig.EnableControlPlane
}
if !baseConfig.DetectControlPlane && fileConfig.DetectControlPlane {
baseConfig.DetectControlPlane = fileConfig.DetectControlPlane
}
if !baseConfig.EnableServices && fileConfig.EnableServices {
baseConfig.EnableServices = fileConfig.EnableServices
}
if !baseConfig.EnableServicesElection && fileConfig.EnableServicesElection {
baseConfig.EnableServicesElection = fileConfig.EnableServicesElection
}
if !baseConfig.EnableNodeLabeling && fileConfig.EnableNodeLabeling {
baseConfig.EnableNodeLabeling = fileConfig.EnableNodeLabeling
}
if !baseConfig.EnableLoadBalancer && fileConfig.EnableLoadBalancer {
baseConfig.EnableLoadBalancer = fileConfig.EnableLoadBalancer
}
if !baseConfig.DDNS && fileConfig.DDNS {
baseConfig.DDNS = fileConfig.DDNS
}
if !baseConfig.SingleNode && fileConfig.SingleNode {
baseConfig.SingleNode = fileConfig.SingleNode
}
if !baseConfig.StartAsLeader && fileConfig.StartAsLeader {
baseConfig.StartAsLeader = fileConfig.StartAsLeader
}
if !baseConfig.PreserveVIPOnLeadershipLoss && fileConfig.PreserveVIPOnLeadershipLoss {
baseConfig.PreserveVIPOnLeadershipLoss = fileConfig.PreserveVIPOnLeadershipLoss
}
// Service configuration
if baseConfig.Namespace == "" && fileConfig.Namespace != "" {
baseConfig.Namespace = fileConfig.Namespace
}
if baseConfig.ServiceNamespace == "" && fileConfig.ServiceNamespace != "" {
baseConfig.ServiceNamespace = fileConfig.ServiceNamespace
}
if baseConfig.ServicesLeaseName == "" && fileConfig.ServicesLeaseName != "" {
baseConfig.ServicesLeaseName = fileConfig.ServicesLeaseName
}
// LoadBalancer configuration
if baseConfig.LoadBalancerPort == 0 && fileConfig.LoadBalancerPort != 0 {
baseConfig.LoadBalancerPort = fileConfig.LoadBalancerPort
}
if baseConfig.LoadBalancerForwardingMethod == "" && fileConfig.LoadBalancerForwardingMethod != "" {
baseConfig.LoadBalancerForwardingMethod = fileConfig.LoadBalancerForwardingMethod
}
if baseConfig.LoadBalancerClassName == "" && fileConfig.LoadBalancerClassName != "" {
baseConfig.LoadBalancerClassName = fileConfig.LoadBalancerClassName
}
// Routing Table configuration
if baseConfig.RoutingTableID == 0 && fileConfig.RoutingTableID != 0 {
baseConfig.RoutingTableID = fileConfig.RoutingTableID
}
if baseConfig.RoutingTableType == 0 && fileConfig.RoutingTableType != 0 {
baseConfig.RoutingTableType = fileConfig.RoutingTableType
}
if baseConfig.RoutingProtocol == 0 && fileConfig.RoutingProtocol != 0 {
baseConfig.RoutingProtocol = fileConfig.RoutingProtocol
}
// BGP configuration
mergeBGPConfig(&baseConfig.BGPConfig, &fileConfig.BGPConfig)
// Kubernetes configuration
if baseConfig.K8sConfigFile == "" && fileConfig.K8sConfigFile != "" {
baseConfig.K8sConfigFile = fileConfig.K8sConfigFile
}
// Leader Election configuration
mergeLeaderElectionConfig(&baseConfig.KubernetesLeaderElection, &fileConfig.KubernetesLeaderElection)
// BGP health check configuration
mergeHealthCheck(&baseConfig.ControlPlaneHealthCheck, &fileConfig.ControlPlaneHealthCheck)
// Prometheus configuration
if baseConfig.PrometheusHTTPServer == "" && fileConfig.PrometheusHTTPServer != "" {
baseConfig.PrometheusHTTPServer = fileConfig.PrometheusHTTPServer
}
// DNS configuration
if baseConfig.DNSMode == "" && fileConfig.DNSMode != "" {
baseConfig.DNSMode = fileConfig.DNSMode
}
// DHCP configuration - mode
if baseConfig.DHCPMode == "" && fileConfig.DHCPMode != "" {
baseConfig.DHCPMode = fileConfig.DHCPMode
}
// DHCP configuration - backoff attempts
if baseConfig.DHCPBackoffAttempts == DefaultDHCPBackoffAttempts && fileConfig.DHCPBackoffAttempts != DefaultDHCPBackoffAttempts {
baseConfig.DHCPBackoffAttempts = fileConfig.DHCPBackoffAttempts
}
// Health check configuration (HTTP listener for kube-vip readiness)
if baseConfig.HealthCheckPort == 0 && fileConfig.HealthCheckPort != 0 {
baseConfig.HealthCheckPort = fileConfig.HealthCheckPort
}
// Egress configuration
if baseConfig.EgressPodCidr == "" && fileConfig.EgressPodCidr != "" {
baseConfig.EgressPodCidr = fileConfig.EgressPodCidr
}
if baseConfig.EgressServiceCidr == "" && fileConfig.EgressServiceCidr != "" {
baseConfig.EgressServiceCidr = fileConfig.EgressServiceCidr
}
// Mirror configuration
if baseConfig.MirrorDestInterface == "" && fileConfig.MirrorDestInterface != "" {
baseConfig.MirrorDestInterface = fileConfig.MirrorDestInterface
}
// Iptables configuration
if baseConfig.IptablesBackend == "" && fileConfig.IptablesBackend != "" {
baseConfig.IptablesBackend = fileConfig.IptablesBackend
}
// Backend health check interval
if baseConfig.BackendHealthCheckInterval == 0 && fileConfig.BackendHealthCheckInterval != 0 {
baseConfig.BackendHealthCheckInterval = fileConfig.BackendHealthCheckInterval
}
// ARP broadcast rate
if baseConfig.ArpBroadcastRate == 0 && fileConfig.ArpBroadcastRate != 0 {
baseConfig.ArpBroadcastRate = fileConfig.ArpBroadcastRate
}
// Annotations
if baseConfig.Annotations == "" && fileConfig.Annotations != "" {
baseConfig.Annotations = fileConfig.Annotations
}
// Load balancers slice
if len(baseConfig.LoadBalancers) == 0 && len(fileConfig.LoadBalancers) > 0 {
baseConfig.LoadBalancers = fileConfig.LoadBalancers
}
}
// mergeBGPConfig merges BGP configuration
func mergeBGPConfig(base, file *BGPConfig) {
if base.RouterID == "" && file.RouterID != "" {
base.RouterID = file.RouterID
}
if base.AS == 0 && file.AS != 0 {
base.AS = file.AS
}
if base.SourceIF == "" && file.SourceIF != "" {
base.SourceIF = file.SourceIF
}
if base.SourceIP == "" && file.SourceIP != "" {
base.SourceIP = file.SourceIP
}
if base.HoldTime == 0 && file.HoldTime != 0 {
base.HoldTime = file.HoldTime
}
if base.KeepaliveInterval == 0 && file.KeepaliveInterval != 0 {
base.KeepaliveInterval = file.KeepaliveInterval
}
if len(base.Peers) == 0 && len(file.Peers) > 0 {
base.Peers = file.Peers
}
}
// mergeLeaderElectionConfig merges leader election configuration
func mergeLeaderElectionConfig(base, file *KubernetesLeaderElection) {
if base.LeaseName == "" && file.LeaseName != "" {
base.LeaseName = file.LeaseName
}
if base.LeaseDuration == 0 && file.LeaseDuration != 0 {
base.LeaseDuration = file.LeaseDuration
}
if base.RenewDeadline == 0 && file.RenewDeadline != 0 {
base.RenewDeadline = file.RenewDeadline
}
if base.RetryPeriod == 0 && file.RetryPeriod != 0 {
base.RetryPeriod = file.RetryPeriod
}
if len(base.LeaseAnnotations) == 0 && len(file.LeaseAnnotations) > 0 {
base.LeaseAnnotations = file.LeaseAnnotations
}
}
// mergeHealthCheck merges HTTP health check configuration for BGP route advertisement.
func mergeHealthCheck(base, file *HealthCheck) {
if base.Address == "" && file.Address != "" {
base.Address = file.Address
}
if base.PeriodSeconds == 0 && file.PeriodSeconds != 0 {
base.PeriodSeconds = file.PeriodSeconds
}
if base.TimeoutSeconds == 0 && file.TimeoutSeconds != 0 {
base.TimeoutSeconds = file.TimeoutSeconds
}
if base.FailureThreshold == 0 && file.FailureThreshold != 0 {
base.FailureThreshold = file.FailureThreshold
}
if base.CAPath == "" && file.CAPath != "" {
base.CAPath = file.CAPath
}
}

View File

@@ -9,25 +9,22 @@ const (
// vip_arpRate - defines the rate of gARP broadcasts
vipArpRate = "vip_arpRate"
// vipPreserveOnLeadershipLoss - if true, VIP addresses will remain on interface when leadership is lost
vipPreserveOnLeadershipLoss = "vip_preserve_on_leadership_loss"
// vipLeaderElection - defines if the kubernetes algorithm should be used
vipLeaderElection = "vip_leaderelection"
// vipLeaseName - defines the name of the lease lock
vipLeaseName = "vip_leasename"
// vipLeaseDuration - defines how long the current leader is considered valid
// vipLeaderElection - defines if the kubernetes algorithm should be used
vipLeaseDuration = "vip_leaseduration"
// vipRenewDeadline - defines how long the leader has to renew the lease before losing leadership
// vipLeaderElection - defines if the kubernetes algorithm should be used
vipRenewDeadline = "vip_renewdeadline"
// vipRetryPeriod - defines the time between attempts to acquire/renew the lease
// vipLeaderElection - defines if the kubernetes algorithm should be used
vipRetryPeriod = "vip_retryperiod"
// vipLeaseAnnotations - defines the annotations given to the lease lock
// vipLeaderElection - defines the annotations given to the lease lock
vipLeaseAnnotations = "vip_leaseannotations"
// vipLogLevel - defines the level of logging to produce (5 being the most verbose)
@@ -112,16 +109,6 @@ const (
bgpHoldTime = "bgp_hold_time"
// bgpKeepaliveInterval defines bgp timers keepalive interval
bgpKeepaliveInterval = "bgp_keepalive_interval"
// controlPlaneHealthCheckAddress defines the URL for control-plane health checks (BGP route withdrawal)
controlPlaneHealthCheckAddress = "control_plane_health_check_address"
// controlPlaneHealthCheckPeriodSeconds defines the period between control-plane health checks
controlPlaneHealthCheckPeriodSeconds = "control_plane_health_check_period_seconds"
// controlPlaneHealthCheckTimeoutSeconds defines the timeout for each control-plane health check request
controlPlaneHealthCheckTimeoutSeconds = "control_plane_health_check_timeout_seconds"
// controlPlaneHealthCheckFailureThreshold defines consecutive failures before BGP route withdrawal
controlPlaneHealthCheckFailureThreshold = "control_plane_health_check_failure_threshold"
// controlPlaneHealthCheckCAPath defines the path to a CA certificate for control-plane health check TLS verification
controlPlaneHealthCheckCAPath = "control_plane_health_check_ca_path"
// zebraEnable defines if Zebra integraton should be enabled
zebraEnable = "zebra_enable"
@@ -222,12 +209,6 @@ const (
// dnsMode defines mode that DNS lookup will be performed with (first, ipv4, ipv6, dual)
dnsMode = "dns_mode"
// dhcpMode defines mode that DHCP lookup will be performed with (ipv4, ipv6, dual)
dhcpMode = "dhcp_mode"
// dhcpBackoffAttempts defines how many times DHCP client will try to obtain an IP address
dhcpBackoffAttempts = "dhcp_backoff_attempts"
// disableServiceUpdates disables service updating
disableServiceUpdates = "disable_service_updates"
@@ -253,7 +234,4 @@ const (
// egressClean enables egress cleaning on kube-vip's start
egressClean = "egress_clean"
// configFile defines the path to a JSON/YAML configuration file
configFile = "config_file"
)

View File

@@ -1,559 +0,0 @@
package kubevip
import (
"os"
"path/filepath"
"testing"
)
func TestLoadConfigFromFile(t *testing.T) {
// Create temporary directory for test files
tmpDir, err := os.MkdirTemp("", "kube-vip-config-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
tests := []struct {
name string
filename string
content string
expectedConfig *Config
wantErr bool
errContains string
}{
{
name: "Valid YAML config",
filename: "config.yaml",
content: `
logging: 2
enableARP: true
enableControlPlane: true
enableServices: true
address: "192.168.1.100"
port: 6443
interface: "eth0"
namespace: "kube-system"
vipSubnet: "192.168.1.0/24"
leaseName: "test-lease"
leaseDuration: 15
renewDeadline: 10
retryPeriod: 2
prometheusHTTPServer: ":2112"
`,
expectedConfig: &Config{
Logging: 2,
EnableARP: true,
EnableControlPlane: true,
EnableServices: true,
Address: "192.168.1.100",
Port: 6443,
Interface: "eth0",
Namespace: "kube-system",
VIPSubnet: "192.168.1.0/24",
PrometheusHTTPServer: ":2112",
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "test-lease",
LeaseDuration: 15,
RenewDeadline: 10,
RetryPeriod: 2,
},
},
wantErr: false,
},
{
name: "Valid JSON config",
filename: "config.json",
content: `{
"logging": 3,
"enableBGP": true,
"enableServices": true,
"address": "10.0.0.100",
"port": 8443,
"interface": "ens192",
"namespace": "kube-system",
"loadBalancers": [
{
"name": "control-plane",
"ports": [
{
"type": "TCP",
"port": 6443
}
],
"bindToVip": true,
"forwardingMethod": "local"
}
]
}`,
expectedConfig: &Config{
Logging: 3,
EnableBGP: true,
EnableServices: true,
Address: "10.0.0.100",
Port: 8443,
Interface: "ens192",
Namespace: "kube-system",
LoadBalancers: []LoadBalancer{
{
Name: "control-plane",
Ports: []Port{
{
Type: "TCP",
Port: 6443,
},
},
BindToVip: true,
ForwardingMethod: "local",
},
},
},
wantErr: false,
},
{
name: "Complex BGP config",
filename: "bgp-config.yaml",
content: `
enableBGP: true
bgpConfig:
routerID: "192.168.1.1"
as: 65000
sourceIF: "eth0"
holdTime: 60
keepaliveInterval: 20
peers:
- address: "192.168.1.2"
as: 65001
port: 179
multiHop: false
- address: "192.168.1.3"
as: 65002
port: 179
multiHop: true
`,
expectedConfig: &Config{
EnableBGP: true,
BGPConfig: BGPConfig{
RouterID: "192.168.1.1",
AS: 65000,
SourceIF: "eth0",
HoldTime: 60,
KeepaliveInterval: 20,
Peers: []BGPPeer{
{
Address: "192.168.1.2",
AS: 65001,
Port: 179,
MultiHop: false,
},
{
Address: "192.168.1.3",
AS: 65002,
Port: 179,
MultiHop: true,
},
},
},
},
wantErr: false,
},
{
name: "Invalid JSON",
filename: "invalid.json",
content: `{"logging": 2, "invalid": }`,
wantErr: true,
errContains: "failed to parse JSON config file",
},
{
name: "Invalid YAML",
filename: "invalid.yaml",
content: "logging: 2\ninvalid: [unclosed",
wantErr: true,
errContains: "failed to parse YAML config file",
},
{
name: "Unsupported format",
filename: "config.txt",
content: "logging=2",
wantErr: true,
errContains: "unsupported config file format",
},
{
name: "Empty path",
filename: "",
content: "",
wantErr: true,
errContains: "config file path is empty",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var filePath string
if tt.filename != "" {
filePath = filepath.Join(tmpDir, tt.filename)
if err := os.WriteFile(filePath, []byte(tt.content), 0600); err != nil {
t.Fatalf("Failed to write test file: %v", err)
}
}
config, err := LoadConfigFromFile(filePath)
if tt.wantErr {
if err == nil {
t.Errorf("LoadConfigFromFile() expected error, got nil")
return
}
if tt.errContains != "" && !containsString(err.Error(), tt.errContains) {
t.Errorf("LoadConfigFromFile() error = %v, expected to contain %v", err, tt.errContains)
}
return
}
if err != nil {
t.Errorf("LoadConfigFromFile() unexpected error = %v", err)
return
}
if config == nil {
t.Errorf("LoadConfigFromFile() returned nil config")
return
}
// Compare key fields
if config.Logging != tt.expectedConfig.Logging {
t.Errorf("Logging = %v, expected %v", config.Logging, tt.expectedConfig.Logging)
}
if config.EnableARP != tt.expectedConfig.EnableARP {
t.Errorf("EnableARP = %v, expected %v", config.EnableARP, tt.expectedConfig.EnableARP)
}
if config.EnableBGP != tt.expectedConfig.EnableBGP {
t.Errorf("EnableBGP = %v, expected %v", config.EnableBGP, tt.expectedConfig.EnableBGP)
}
if config.Address != tt.expectedConfig.Address {
t.Errorf("Address = %v, expected %v", config.Address, tt.expectedConfig.Address)
}
if config.Port != tt.expectedConfig.Port {
t.Errorf("Port = %v, expected %v", config.Port, tt.expectedConfig.Port)
}
if config.Interface != tt.expectedConfig.Interface {
t.Errorf("Interface = %v, expected %v", config.Interface, tt.expectedConfig.Interface)
}
// Test BGP config if present
if tt.expectedConfig.EnableBGP {
if config.BGPConfig.RouterID != tt.expectedConfig.BGPConfig.RouterID {
t.Errorf("BGPConfig.RouterID = %v, expected %v", config.BGPConfig.RouterID, tt.expectedConfig.BGPConfig.RouterID)
}
if config.BGPConfig.AS != tt.expectedConfig.BGPConfig.AS {
t.Errorf("BGPConfig.AS = %v, expected %v", config.BGPConfig.AS, tt.expectedConfig.BGPConfig.AS)
}
if len(config.BGPConfig.Peers) != len(tt.expectedConfig.BGPConfig.Peers) {
t.Errorf("BGPConfig.Peers length = %v, expected %v", len(config.BGPConfig.Peers), len(tt.expectedConfig.BGPConfig.Peers))
}
}
// Test LoadBalancers if present
if len(tt.expectedConfig.LoadBalancers) > 0 {
if len(config.LoadBalancers) != len(tt.expectedConfig.LoadBalancers) {
t.Errorf("LoadBalancers length = %v, expected %v", len(config.LoadBalancers), len(tt.expectedConfig.LoadBalancers))
}
}
})
}
}
func TestMergeConfigFromFile(t *testing.T) {
// Create temporary directory for test files
tmpDir, err := os.MkdirTemp("", "kube-vip-merge-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
// Create test config file
configFile := filepath.Join(tmpDir, "test-config.yaml")
configContent := `
logging: 3
enableARP: true
enableServices: true
address: "192.168.1.200"
port: 6443
interface: "eth1"
namespace: "test-namespace"
leaseName: "file-lease"
leaseDuration: 20
prometheusHTTPServer: ":3000"
`
if err := os.WriteFile(configFile, []byte(configContent), 0600); err != nil {
t.Fatalf("Failed to write test config file: %v", err)
}
tests := []struct {
name string
baseConfig *Config
configFilePath string
expected *Config
wantErr bool
}{
{
name: "Merge with empty base config",
baseConfig: &Config{},
configFilePath: configFile,
expected: &Config{
Logging: 3,
EnableARP: true,
EnableServices: true,
Address: "192.168.1.200",
Port: 6443,
Interface: "eth1",
Namespace: "test-namespace",
PrometheusHTTPServer: ":3000",
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "file-lease",
LeaseDuration: 20,
},
},
wantErr: false,
},
{
name: "Merge respects existing values (priority test)",
baseConfig: &Config{
Logging: 5, // Should not be overridden
Port: 8443, // Should not be overridden
Interface: "eth0", // Should not be overridden
},
configFilePath: configFile,
expected: &Config{
Logging: 5, // From base (higher priority)
EnableARP: true, // From file
EnableServices: true, // From file
Address: "192.168.1.200", // From file
Port: 8443, // From base (higher priority)
Interface: "eth0", // From base (higher priority)
Namespace: "test-namespace", // From file
PrometheusHTTPServer: ":3000", // From file
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "file-lease", // From file
LeaseDuration: 20, // From file
},
},
wantErr: false,
},
{
name: "Empty config file path",
baseConfig: &Config{
Logging: 1,
},
configFilePath: "",
expected: &Config{
Logging: 1, // Unchanged
},
wantErr: false,
},
{
name: "Non-existent config file",
baseConfig: &Config{
Logging: 1,
},
configFilePath: "/non/existent/file.yaml",
expected: &Config{
Logging: 1,
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := MergeConfigFromFile(tt.baseConfig, tt.configFilePath)
if tt.wantErr {
if err == nil {
t.Errorf("MergeConfigFromFile() expected error, got nil")
}
return
}
if err != nil {
t.Errorf("MergeConfigFromFile() unexpected error = %v", err)
return
}
// Compare key fields
if tt.baseConfig.Logging != tt.expected.Logging {
t.Errorf("Logging = %v, expected %v", tt.baseConfig.Logging, tt.expected.Logging)
}
if tt.baseConfig.EnableARP != tt.expected.EnableARP {
t.Errorf("EnableARP = %v, expected %v", tt.baseConfig.EnableARP, tt.expected.EnableARP)
}
if tt.baseConfig.Address != tt.expected.Address {
t.Errorf("Address = %v, expected %v", tt.baseConfig.Address, tt.expected.Address)
}
if tt.baseConfig.Port != tt.expected.Port {
t.Errorf("Port = %v, expected %v", tt.baseConfig.Port, tt.expected.Port)
}
if tt.baseConfig.Interface != tt.expected.Interface {
t.Errorf("Interface = %v, expected %v", tt.baseConfig.Interface, tt.expected.Interface)
}
if tt.baseConfig.Namespace != tt.expected.Namespace {
t.Errorf("Namespace = %v, expected %v", tt.baseConfig.Namespace, tt.expected.Namespace)
}
if tt.baseConfig.PrometheusHTTPServer != tt.expected.PrometheusHTTPServer {
t.Errorf("PrometheusHTTPServer = %v, expected %v", tt.baseConfig.PrometheusHTTPServer, tt.expected.PrometheusHTTPServer)
}
})
}
}
func TestMergeConfigValues(t *testing.T) {
tests := []struct {
name string
baseConfig *Config
fileConfig *Config
expectedBase *Config
}{
{
name: "Merge basic configuration",
baseConfig: &Config{
Logging: 5, // Should not be overridden
Port: 0, // Should be overridden
},
fileConfig: &Config{
Logging: 2,
Port: 6443,
Interface: "eth0",
Address: "192.168.1.100",
},
expectedBase: &Config{
Logging: 5, // From base (non-zero)
Port: 6443, // From file (base was zero)
Interface: "eth0", // From file (base was empty)
Address: "192.168.1.100", // From file (base was empty)
},
},
{
name: "Merge boolean flags",
baseConfig: &Config{
EnableARP: true, // Should not be overridden
},
fileConfig: &Config{
EnableARP: false, // Should not override true
EnableBGP: true, // Should be set
EnableServices: true, // Should be set
EnableWireguard: false, // Should not be set (false doesn't override false)
},
expectedBase: &Config{
EnableARP: true, // From base (true has priority)
EnableBGP: true, // From file
EnableServices: true, // From file
EnableWireguard: false, // Remains false
},
},
{
name: "Merge BGP configuration",
baseConfig: &Config{
BGPConfig: BGPConfig{
RouterID: "1.1.1.1", // Should not be overridden
},
},
fileConfig: &Config{
BGPConfig: BGPConfig{
RouterID: "2.2.2.2", // Should not override
AS: 65000, // Should be set
SourceIF: "eth0", // Should be set
HoldTime: 30, // Should be set
KeepaliveInterval: 10, // Should be set
},
},
expectedBase: &Config{
BGPConfig: BGPConfig{
RouterID: "1.1.1.1", // From base (non-empty)
AS: 65000, // From file (base was zero)
SourceIF: "eth0", // From file (base was empty)
HoldTime: 30, // From file (base was zero)
KeepaliveInterval: 10, // From file (base was zero)
},
},
},
{
name: "Merge leader election configuration",
baseConfig: &Config{
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "base-lease", // Should not be overridden
},
},
fileConfig: &Config{
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "file-lease", // Should not override
LeaseDuration: 15, // Should be set
RenewDeadline: 10, // Should be set
RetryPeriod: 2, // Should be set
},
},
expectedBase: &Config{
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "base-lease", // From base (non-empty)
LeaseDuration: 15, // From file (base was zero)
RenewDeadline: 10, // From file (base was zero)
RetryPeriod: 2, // From file (base was zero)
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mergeConfigValues(tt.baseConfig, tt.fileConfig)
// Compare results
if tt.baseConfig.Logging != tt.expectedBase.Logging {
t.Errorf("Logging = %v, expected %v", tt.baseConfig.Logging, tt.expectedBase.Logging)
}
if tt.baseConfig.Port != tt.expectedBase.Port {
t.Errorf("Port = %v, expected %v", tt.baseConfig.Port, tt.expectedBase.Port)
}
if tt.baseConfig.Interface != tt.expectedBase.Interface {
t.Errorf("Interface = %v, expected %v", tt.baseConfig.Interface, tt.expectedBase.Interface)
}
if tt.baseConfig.EnableARP != tt.expectedBase.EnableARP {
t.Errorf("EnableARP = %v, expected %v", tt.baseConfig.EnableARP, tt.expectedBase.EnableARP)
}
if tt.baseConfig.EnableBGP != tt.expectedBase.EnableBGP {
t.Errorf("EnableBGP = %v, expected %v", tt.baseConfig.EnableBGP, tt.expectedBase.EnableBGP)
}
if tt.baseConfig.BGPConfig.RouterID != tt.expectedBase.BGPConfig.RouterID {
t.Errorf("BGPConfig.RouterID = %v, expected %v", tt.baseConfig.BGPConfig.RouterID, tt.expectedBase.BGPConfig.RouterID)
}
if tt.baseConfig.BGPConfig.AS != tt.expectedBase.BGPConfig.AS {
t.Errorf("BGPConfig.AS = %v, expected %v", tt.baseConfig.BGPConfig.AS, tt.expectedBase.BGPConfig.AS)
}
if tt.baseConfig.KubernetesLeaderElection.LeaseName != tt.expectedBase.KubernetesLeaderElection.LeaseName {
t.Errorf("KubernetesLeaderElection.LeaseName = %v, expected %v", tt.baseConfig.KubernetesLeaderElection.LeaseName, tt.expectedBase.KubernetesLeaderElection.LeaseName)
}
})
}
}
func TestLoadConfigFromFile_FileNotExists(t *testing.T) {
_, err := LoadConfigFromFile("/non/existent/path/config.yaml")
if err == nil {
t.Error("LoadConfigFromFile() expected error for non-existent file, got nil")
}
if !containsString(err.Error(), "config file does not exist") {
t.Errorf("LoadConfigFromFile() error = %v, expected to contain 'config file does not exist'", err)
}
}
// Helper function to check if a string contains a substring
func containsString(str, substr string) bool {
return len(str) >= len(substr) && (str == substr || len(substr) == 0 ||
(len(substr) > 0 && func() bool {
for i := 0; i <= len(str)-len(substr); i++ {
if str[i:i+len(substr)] == substr {
return true
}
}
return false
}()))
}

View File

@@ -5,7 +5,6 @@ import (
"log"
"strconv"
"github.com/google/go-containerregistry/pkg/name"
appv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -14,6 +13,7 @@ import (
applyCoreV1 "k8s.io/client-go/applyconfigurations/core/v1"
applyMetaV1 "k8s.io/client-go/applyconfigurations/meta/v1"
applyRbacV1 "k8s.io/client-go/applyconfigurations/rbac/v1"
"sigs.k8s.io/yaml"
)
@@ -167,11 +167,7 @@ func GenerateRoleBinding(rolebinding bool, saCfg *applyCoreV1.ServiceAccountAppl
}
// generatePodSpec will take a kube-vip config and generate a Pod spec
func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*corev1.Pod, error) {
imageRef, err := name.NewTag(image, name.WeakValidation, name.WithDefaultTag(imageVersion))
if err != nil {
return nil, fmt.Errorf("cannot parse %q: %w", image, err)
}
func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod {
command := "manager"
// Determine where the pods should be living (for multi-tenancy)
@@ -255,28 +251,6 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
newEnvironment = append(newEnvironment, dnsModeSelector...)
}
if c.DHCPMode != "" {
// build environment variables
dhcpModeSelector := []corev1.EnvVar{
{
Name: dhcpMode,
Value: c.DHCPMode,
},
}
newEnvironment = append(newEnvironment, dhcpModeSelector...)
}
if c.DHCPBackoffAttempts != DefaultDHCPBackoffAttempts {
// build environment variables
dhcpBackoff := []corev1.EnvVar{
{
Name: dhcpBackoffAttempts,
Value: strconv.FormatUint(uint64(c.DHCPBackoffAttempts), 10),
},
}
newEnvironment = append(newEnvironment, dhcpBackoff...)
}
// If we're doing the hybrid mode
if c.EnableControlPlane {
cp := []corev1.EnvVar{
@@ -507,40 +481,6 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
}
if c.ControlPlaneHealthCheck.Address != "" {
healthCheckVars := []corev1.EnvVar{
{
Name: controlPlaneHealthCheckAddress,
Value: c.ControlPlaneHealthCheck.Address,
},
}
if c.ControlPlaneHealthCheck.PeriodSeconds > 0 {
healthCheckVars = append(healthCheckVars, corev1.EnvVar{
Name: controlPlaneHealthCheckPeriodSeconds,
Value: fmt.Sprintf("%d", c.ControlPlaneHealthCheck.PeriodSeconds),
})
}
if c.ControlPlaneHealthCheck.TimeoutSeconds > 0 {
healthCheckVars = append(healthCheckVars, corev1.EnvVar{
Name: controlPlaneHealthCheckTimeoutSeconds,
Value: fmt.Sprintf("%d", c.ControlPlaneHealthCheck.TimeoutSeconds),
})
}
if c.ControlPlaneHealthCheck.FailureThreshold > 0 {
healthCheckVars = append(healthCheckVars, corev1.EnvVar{
Name: controlPlaneHealthCheckFailureThreshold,
Value: fmt.Sprintf("%d", c.ControlPlaneHealthCheck.FailureThreshold),
})
}
if c.ControlPlaneHealthCheck.CAPath != "" {
healthCheckVars = append(healthCheckVars, corev1.EnvVar{
Name: controlPlaneHealthCheckCAPath,
Value: c.ControlPlaneHealthCheck.CAPath,
})
}
newEnvironment = append(newEnvironment, healthCheckVars...)
}
// If the load-balancer is enabled then add the configuration to the manifest
if c.EnableLoadBalancer {
lb := []corev1.EnvVar{
@@ -621,7 +561,7 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
var securityContext *corev1.SecurityContext
if c.LoadBalancerForwardingMethod == "masquerade" {
privileged := true
var privileged = true
securityContext = &corev1.SecurityContext{
Privileged: &privileged,
}
@@ -639,16 +579,6 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
}
}
if c.PreserveVIPOnLeadershipLoss {
preserveVIPOnLeadershipLoss := []corev1.EnvVar{
{
Name: vipPreserveOnLeadershipLoss,
Value: strconv.FormatBool(c.PreserveVIPOnLeadershipLoss),
},
}
newEnvironment = append(newEnvironment, preserveVIPOnLeadershipLoss...)
}
newManifest := &corev1.Pod{
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
@@ -662,7 +592,7 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
Containers: []corev1.Container{
{
Name: "kube-vip",
Image: imageRef.Name(),
Image: fmt.Sprintf("ghcr.io/kube-vip/kube-vip:%s", imageVersion),
ImagePullPolicy: corev1.PullIfNotPresent,
SecurityContext: securityContext,
Args: []string{
@@ -703,24 +633,18 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
newManifest.Spec.HostAliases = append(newManifest.Spec.HostAliases, hostAlias)
}
return newManifest, nil
return newManifest
}
// GeneratePodManifestFromConfig will take a kube-vip config and generate a manifest
func GeneratePodManifestFromConfig(c *Config, image, imageVersion string, inCluster bool) (string, error) {
newManifest, err := generatePodSpec(c, image, imageVersion, inCluster)
if err != nil {
return "", err
}
b, err := yaml.Marshal(newManifest)
if err != nil {
return "", fmt.Errorf("failed to marshal manifest: %w", err)
}
return string(b), nil
func GeneratePodManifestFromConfig(c *Config, imageVersion string, inCluster bool) string {
newManifest := generatePodSpec(c, imageVersion, inCluster)
b, _ := yaml.Marshal(newManifest)
return string(b)
}
// GenerateDaemonsetManifestFromConfig will take a kube-vip config and generate a manifest
func GenerateDaemonsetManifestFromConfig(c *Config, image, imageVersion string, inCluster, taint bool) (string, error) {
func GenerateDaemonsetManifestFromConfig(c *Config, imageVersion string, inCluster, taint bool) string {
// Determine where the pod should be deployed
var namespace string
if c.ServiceNamespace != "" {
@@ -729,11 +653,7 @@ func GenerateDaemonsetManifestFromConfig(c *Config, image, imageVersion string,
namespace = metav1.NamespaceSystem
}
pod, err := generatePodSpec(c, image, imageVersion, inCluster)
if err != nil {
return "", err
}
podSpec := generatePodSpec(c, imageVersion, inCluster).Spec
newManifest := &appv1.DaemonSet{
TypeMeta: metav1.TypeMeta{
Kind: "DaemonSet",
@@ -760,7 +680,7 @@ func GenerateDaemonsetManifestFromConfig(c *Config, image, imageVersion string,
"app.kubernetes.io/version": imageVersion,
},
},
Spec: pod.Spec,
Spec: podSpec,
},
},
}
@@ -810,5 +730,5 @@ func GenerateDaemonsetManifestFromConfig(c *Config, image, imageVersion string,
delete(m, "status")
b, _ = yaml.Marshal(m)
return string(b), nil
return string(b)
}

View File

@@ -1,9 +1,6 @@
package kubevip
import (
"os"
"testing"
)
import "testing"
func TestParseEnvironment(t *testing.T) {
@@ -12,8 +9,8 @@ func TestParseEnvironment(t *testing.T) {
c *Config
wantErr bool
}{
{"nil config", nil, false},
{"basic config", &Config{Interface: "eth0", ServicesInterface: "eth1"}, false},
{"", nil, false},
{"", &Config{Interface: "eth0", ServicesInterface: "eth1"}, false},
}
for _, tt := range tests {
t.Logf("%v", tt.c)
@@ -24,53 +21,3 @@ func TestParseEnvironment(t *testing.T) {
})
}
}
func TestParseEnvironmentConfigFile(t *testing.T) {
// Save original environment
originalConfigFile := os.Getenv("config_file")
defer func() {
if originalConfigFile != "" {
os.Setenv("config_file", originalConfigFile)
} else {
os.Unsetenv("config_file")
}
}()
tests := []struct {
name string
envValue string
expectedConfig string
}{
{
name: "config_file environment variable set",
envValue: "/etc/kube-vip/config.yaml",
expectedConfig: "/etc/kube-vip/config.yaml",
},
{
name: "config_file environment variable empty",
envValue: "",
expectedConfig: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Set environment variable
if tt.envValue != "" {
os.Setenv("config_file", tt.envValue)
} else {
os.Unsetenv("config_file")
}
config := &Config{}
err := ParseEnvironment(config)
if err != nil {
t.Errorf("ParseEnvironment() unexpected error = %v", err)
}
if config.ConfigFile != tt.expectedConfig {
t.Errorf("ConfigFile = %v, expected %v", config.ConfigFile, tt.expectedConfig)
}
})
}
}

View File

@@ -12,14 +12,6 @@ const (
Auto = "auto"
)
func (c *Config) CheckSubnetExists() error {
if c.VIPSubnet == "" && c.VIP != "" && c.Address == "" {
return fmt.Errorf("vip_subnet must be set if using vip_address instead of address environment variable")
}
return nil
}
func (c *Config) CheckInterface() error {
if c.Interface != "" {
if err := isValidInterface(c.Interface); err != nil {

View File

@@ -1,56 +0,0 @@
package kubevip
import "testing"
func TestCheckSubnetExists(t *testing.T) {
tests := []struct {
name string
config Config
wantErr bool
}{
{
name: "vip only without subnet returns error",
config: Config{
VIP: "172.18.0.20",
},
wantErr: true,
},
{
name: "vip with subnet does not return error",
config: Config{
VIP: "172.18.0.20",
VIPSubnet: "32",
},
wantErr: false,
},
{
name: "address only without subnet does not return error",
config: Config{
Address: "172.18.0.20",
},
wantErr: false,
},
{
name: "address overrides vip without subnet",
config: Config{
VIP: "172.18.0.20",
Address: "172.18.0.30",
},
wantErr: false,
},
{
name: "empty config does not return error",
config: Config{},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.config.CheckSubnetExists()
if (err != nil) != tt.wantErr {
t.Errorf("CheckSubnetExists() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

View File

@@ -1,5 +1,9 @@
package kubevip
import (
"github.com/kube-vip/kube-vip/pkg/bgp"
)
// Config defines all of the settings for the Kube-Vip Pod
type Config struct {
// Logging, settings
@@ -50,10 +54,6 @@ type Config struct {
// ArpBroadcastRate, defines how often kube-vip will update the network about updates to the network
ArpBroadcastRate int64 `yaml:"arpBroadcastRate"`
// PreserveVIPOnLeadershipLoss, if true, VIP addresses will remain on interface when leadership is lost (only ARP/NDP broadcasting stops)
// If false, VIP addresses are deleted on leadership loss (legacy behavior)
PreserveVIPOnLeadershipLoss bool `yaml:"preserveVipOnLeadershipLoss"`
// Annotations will define if we're going to wait and lookup configuration from Kubernetes node annotations
Annotations string
@@ -128,14 +128,10 @@ type Config struct {
CleanRoutingTable bool `yaml:"cleanRoutingTable"`
// BGP Configuration
BGPConfig BGPConfig
BGPPeerConfig BGPPeer
BGPConfig bgp.Config
BGPPeerConfig bgp.Peer
BGPPeers []string
// ControlPlaneHealthCheck configures HTTP polling of the control plane when using BGP without
// leader election. If the health check fails, the BGP route will be withdrawn.
ControlPlaneHealthCheck HealthCheck `yaml:"controlPlaneHealthCheck,omitempty"`
// LoadBalancers are the various services we can load balance over
LoadBalancers []LoadBalancer `yaml:"loadBalancers,omitempty"`
@@ -162,16 +158,6 @@ type Config struct {
// DNSMode, this will set the mode DSN lookup will be performed (first, ipv4, ipv6, dual)
DNSMode string `yaml:"dnsDualStackMode"`
// IsDualStack reports if service is DualStack.
IsDualStack bool
// RequireDualStack defines if DualStack is required for the service. Based on service's Spec.ipFamilyPolicy field.
RequireDualStack bool
// DNSMode, this will set the mode DHCP lookup will be performed for DDNS (ipv4, ipv6, dual). By default will be the same as DNSMode.
// If DNSMode is 'first', IPv4 will be used.
DHCPMode string `yaml:"dhcpDualStackMode"`
// DisableServiceUpdates, if true, kube-vip will only advertise service, but it will not update service's Status.LoadBalancer.Ingress slice
DisableServiceUpdates bool `yaml:"disableServiceUpdates"`
@@ -200,12 +186,6 @@ type Config struct {
// EgressClean, enables egress cleaning on Kube-vip's start
EgressClean bool `yaml:"egressClean"`
// ConfigFile defines the path to a JSON/YAML configuration file
ConfigFile string `yaml:"configFile"`
// DHCPBackoffAttempts defaines how many times will DHCP client try to obtain address (unlimited when 0)
DHCPBackoffAttempts uint `yaml:"dhcpBackoffAttempts"`
}
// KubernetesLeaderElection defines all of the settings for Kubernetes KubernetesLeaderElection
@@ -237,29 +217,12 @@ type Etcd struct {
Endpoints []string
}
// HealthCheck defines HTTP health-check settings for control-plane polling when using BGP
// without leader election.
type HealthCheck struct {
// Address is the URL to poll to check the health of the control-plane. If the health
// check fails, the BGP route will be withdrawn.
Address string `yaml:"address"`
// PeriodSeconds is the interval in seconds between health checks.
PeriodSeconds int `yaml:"periodSeconds"`
// TimeoutSeconds is the timeout per health check request. If a request takes longer
// than this timeout, the health check is considered failed.
TimeoutSeconds int `yaml:"timeoutSeconds"`
// FailureThreshold is the number of consecutive failures before route withdrawal.
FailureThreshold int `yaml:"failureThreshold"`
// CAPath is the CA certificate path used for TLS verification when Address is an HTTPS URL.
CAPath string `yaml:"caPath"`
}
// LoadBalancer contains the configuration of a load balancing instance
type LoadBalancer struct {
// Name of a LoadBalancer
Name string `yaml:"name"`
// Ports exposed by a LoadBalancer
//Ports exposed by a LoadBalancer
Ports []Port
// BindToVip will bind the load balancer port to the VIP itself

View File

@@ -1,35 +0,0 @@
package kubevip
import (
"fmt"
"net/url"
"strings"
)
// Validate runs configuration checks that are independent of host state.
// This should be called after all config sources (flags, file, env vars) are merged.
func (c *Config) Validate() error {
if err := validateHealthCheckAddress(c.ControlPlaneHealthCheck.Address); err != nil {
return err
}
return nil
}
func validateHealthCheckAddress(address string) error {
if address == "" {
return nil
}
parsedURL, err := url.ParseRequestURI(address)
if err != nil {
return fmt.Errorf("control_plane_health_check_address %q is not a valid URL: %w", address, err)
}
scheme := strings.ToLower(parsedURL.Scheme)
if scheme != "http" && scheme != "https" {
return fmt.Errorf("control_plane_health_check_address %q has unsupported scheme %q, expected http or https", address, parsedURL.Scheme)
}
return nil
}

View File

@@ -1,32 +0,0 @@
package kubevip
import (
"testing"
)
func TestValidate_HealthCheckAddress(t *testing.T) {
tests := []struct {
name string
address string
wantErr bool
}{
{"empty address (disabled)", "", false},
{"valid http URL", "http://localhost:6443/livez", false},
{"valid https URL", "https://localhost:6443/livez", false},
{"https with path", "https://127.0.0.1:6443/readyz?verbose", false},
{"invalid URL", "not-a-url", true},
{"ftp scheme", "ftp://localhost/file", true},
{"tcp scheme", "tcp://localhost:6443", true},
{"missing scheme", "localhost:6443/livez", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := &Config{ControlPlaneHealthCheck: HealthCheck{Address: tt.address}}
err := c.Validate()
if (err != nil) != tt.wantErr {
t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

View File

@@ -1,7 +0,0 @@
package kubevip
const (
LBClassName = "kube-vip.io/kube-vip-class"
DefaultDHCPBackoffAttempts = 3
)

View File

@@ -1,22 +0,0 @@
package kubevip
import (
"slices"
)
const (
// ServiceProvided is the name of the label that will be added to the node
ServiceProvided = "service-provided.kube-vip.io"
// label used on nodes, which announce the LoadBalancer IP
HasIP = "kube-vip.io/has-ip"
)
var kubevipLabelKeys = []string{
ServiceProvided,
HasIP,
}
func GetKeysForCleanup() []string {
return slices.Clone(kubevipLabelKeys)
}

View File

@@ -1,234 +0,0 @@
package lease
import (
"context"
"fmt"
log "log/slog"
"os"
"strings"
"sync"
"sync/atomic"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
)
// Manager is used to manage leases.
type Manager struct {
leases map[string]*Lease
lock sync.Mutex
}
// NewManager creates new lease manager.
func NewManager() *Manager {
return &Manager{
leases: make(map[string]*Lease),
}
}
// Add adds lease to the manager.
// It returns three values:
// - lease for the object
// - isNewObject, which reports if it is a new object that is being handled
// - isSharedLease, which is true if object shares the lease with another object
// If object is new but not shared, we should start leaderelection and sync it
// If object is new and shared, we should only sync it as the leaderelection should be already handled
// If object is not new we should do nothing
func (m *Manager) Add(ctx context.Context, id ID) *Lease {
m.lock.Lock()
defer m.lock.Unlock()
if _, exists := m.leases[id.NamespacedName()]; !exists {
leaseCtx, leaseCancel := context.WithCancel(ctx)
m.leases[id.NamespacedName()] = newLease(leaseCtx, leaseCancel)
}
return m.leases[id.NamespacedName()]
}
// Delete removes the lease and cancels it if the lease counter equals 0.
func (m *Manager) Delete(id ID, objectName string) {
m.lock.Lock()
defer m.lock.Unlock()
if _, exist := m.leases[id.NamespacedName()]; exist {
m.leases[id.NamespacedName()].delete(objectName)
if m.leases[id.NamespacedName()].cnt.Load() < 1 {
m.leases[id.NamespacedName()].Cancel()
delete(m.leases, id.NamespacedName())
}
}
}
// Get returns lease for the service.
func (m *Manager) Get(id ID) *Lease {
m.lock.Lock()
defer m.lock.Unlock()
if lease, exist := m.leases[id.NamespacedName()]; exist {
return lease
}
return nil
}
// Lease holds lease data.
type Lease struct {
Ctx context.Context
Cancel context.CancelFunc
Started chan any
services sync.Map
cnt atomic.Int64
Elected atomic.Bool
Mtx sync.Mutex
locked bool
}
func newLease(ctx context.Context, cancel context.CancelFunc) *Lease {
return &Lease{
Ctx: ctx,
Cancel: cancel,
Started: make(chan any),
}
}
// Add adds the object to the lease and increments counter
// it will return true if object was added
func (l *Lease) Add(name string) bool {
if _, exists := l.services.Load(name); !exists {
l.services.Store(name, true)
l.cnt.Add(1)
return true
}
return false
}
// delete removes the service from the lease and decrements the counter
func (l *Lease) delete(service string) {
if _, exists := l.services.Load(service); exists {
l.services.Delete(service)
l.cnt.Add(-1)
}
}
func (l *Lease) Lock() {
l.Mtx.Lock()
l.locked = true
}
func (l *Lease) Unlock() {
if l.locked {
l.locked = false
l.Mtx.Unlock()
}
}
// ServiceName gets lease name and id for the service.
func ServiceName(service *v1.Service) (string, string) {
name, exists := service.Annotations[kubevip.ServiceLease]
if !exists || name == "" {
name = fmt.Sprintf("kubevip-%s", service.Name)
}
serviceLeaseParts := strings.Split(name, "/")
namespace := service.Namespace
if len(serviceLeaseParts) > 1 {
namespace = serviceLeaseParts[0]
name = serviceLeaseParts[1]
}
return namespace, name
}
func ServiceNamespacedName(service *v1.Service) string {
return fmt.Sprintf("%s/%s", service.Namespace, service.Name)
}
func ObjectName(id ID, suffix string) string {
return fmt.Sprintf("%s-%s", id.NamespacedName(), suffix)
}
func NamespaceName(lease string, c *kubevip.Config) (string, string) {
leaseName := lease
leasnameParts := strings.Split(lease, "/")
var ns string
var err error
if len(leasnameParts) > 1 {
ns = leasnameParts[0]
leaseName = leasnameParts[1]
} else {
ns, err = returnNamespace()
if err != nil {
log.Warn("unable to auto-detect namespace, dropping to config", "namespace", c.Namespace)
ns = c.Namespace
}
}
return ns, leaseName
}
func returnNamespace() (string, error) {
if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
return ns, nil
}
return "", err
}
return "", fmt.Errorf("unable to find Namespace")
}
type ID interface {
Name() string
Namespace() string
NamespacedName() string
}
type CommonID struct {
namespace string
name string
}
func NewID(leaseType, namespace, name string) ID {
if leaseType == "etcd" {
return newEtcdID(namespace, name)
}
return newKubernetesID(namespace, name)
}
func newKubernetesID(namespace, name string) ID {
return &KubernetesID{
CommonID: CommonID{
namespace: namespace,
name: name,
},
}
}
func newEtcdID(namespace, name string) ID {
return &EtcdID{
CommonID: CommonID{
namespace: namespace,
name: name,
},
}
}
func (c *CommonID) Name() string {
return c.name
}
func (c *CommonID) Namespace() string {
return c.namespace
}
type KubernetesID struct {
CommonID
}
func (k *KubernetesID) NamespacedName() string {
return fmt.Sprintf("%s/%s", k.namespace, k.name)
}
type EtcdID struct {
CommonID
}
func (e *EtcdID) NamespacedName() string {
return fmt.Sprintf("%s-%s", e.namespace, e.name)
}

View File

@@ -1,939 +0,0 @@
package lease
import (
"context"
"sync"
"testing"
"time"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func createTestService(name, namespace string, annotations map[string]string) *v1.Service {
return &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Annotations: annotations,
},
}
}
func getSvcID(svc *v1.Service) ID {
namespace, name := ServiceName(svc)
id := NewID("kubernetes", namespace, name)
return id
}
func getSvcData(svc *v1.Service) (context.Context, ID) {
return context.TODO(), getSvcID(svc)
}
const serviceLeaseAnnotation = kubevip.ServiceLease
// TestManager_Add_NewLease tests adding a new service with a new lease
func TestManager_Add_NewLease(t *testing.T) {
mgr := NewManager()
svc := createTestService("test-svc", "default", nil)
leaseID := mgr.Add(getSvcData(svc))
isNew := leaseID.Add(ServiceNamespacedName(svc))
if !isNew {
t.Error("expected isNew to be true for first Add")
}
if leaseID == nil {
t.Fatal("expected lease to be non-nil")
}
if leaseID.Ctx == nil {
t.Error("expected lease context to be non-nil")
}
if leaseID.Cancel == nil {
t.Error("expected lease cancel func to be non-nil")
}
if leaseID.Started == nil {
t.Error("expected lease Started channel to be non-nil")
}
}
// TestManager_Add_ExistingLease tests adding a service with an existing lease
func TestManager_Add_ExistingLease(t *testing.T) {
mgr := NewManager()
svc := createTestService("test-svc", "default", nil)
leaseID1 := mgr.Add(getSvcData(svc))
isNew1 := leaseID1.Add(ServiceNamespacedName(svc))
leaseID2 := mgr.Add(getSvcData(svc))
isNew2 := leaseID2.Add(ServiceNamespacedName(svc))
if !isNew1 {
t.Error("expected first Add to return isNew=true")
}
if isNew2 {
t.Error("expected second Add to return isNew=false")
}
if leaseID1 != leaseID2 {
t.Error("expected same lease to be returned for same service")
}
}
// TestManager_Delete_DecrementCounter tests the decrement counter functionality
func TestManager_Delete_DecrementCounter(t *testing.T) {
mgr := NewManager()
svc := createTestService("test-svc", "default", nil)
// Add twice (simulating adding same service twice)
objectName := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
_ = lease1.Add(objectName)
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
_ = lease2.Add(objectName)
// Delete once - should remove the lease
mgr.Delete(leaseID1, objectName)
lease := mgr.Get(getSvcID(svc))
if lease != nil {
t.Error("expected lease to be removed after first delete if same service was processed twice")
}
}
// TestManager_Delete_CancelsContext tests the context cancellation on delete
func TestManager_Delete_CancelsContext(t *testing.T) {
mgr := NewManager()
svc := createTestService("test-svc", "default", nil)
objectName := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
_ = lease1.Add(objectName)
// Verify context is not cancelled
select {
case <-lease1.Ctx.Done():
t.Fatal("expected context to not be cancelled initially")
default:
// Expected
}
// Delete the lease
mgr.Delete(leaseID1, objectName)
// Verify context is cancelled
select {
case <-lease1.Ctx.Done():
// Expected
case <-time.After(100 * time.Millisecond):
t.Error("expected context to be cancelled after delete")
}
}
// TestManager_Add_AfterDelete_CreatesNewLease tests adding a service after deleting it
func TestManager_Add_AfterDelete_CreatesNewLease(t *testing.T) {
mgr := NewManager()
svc := createTestService("test-svc", "default", nil)
objectName := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
_ = lease1.Add(objectName)
mgr.Delete(leaseID1, objectName)
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
isNew := lease2.Add(objectName)
if !isNew {
t.Error("expected isNew to be true after delete and re-add")
}
if lease1 == lease2 {
t.Error("expected new lease to be different from old lease")
}
}
// TestManager_Add_DifferentServices tests adding services with different names
func TestManager_Add_DifferentServices(t *testing.T) {
mgr := NewManager()
svc1 := createTestService("svc1", "default", nil)
svc2 := createTestService("svc2", "default", nil)
objectName1 := ServiceNamespacedName(svc1)
ctx1, leaseID1 := getSvcData(svc1)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
objectName2 := ServiceNamespacedName(svc2)
ctx2, leaseID2 := getSvcData(svc2)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease1.Add(objectName2)
if !isNew1 || !isNew2 {
t.Error("expected both adds to return isNew=true")
}
if lease1 == lease2 {
t.Error("expected different leases for different services")
}
}
// TestManager_Add_SameNameDifferentNamespace tests adding services with the same name but different namespaces
func TestManager_Add_SameNameDifferentNamespace(t *testing.T) {
mgr := NewManager()
svc1 := createTestService("test-svc", "namespace1", nil)
svc2 := createTestService("test-svc", "namespace2", nil)
objectName1 := ServiceNamespacedName(svc1)
ctx1, leaseID1 := getSvcData(svc1)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
objectName2 := ServiceNamespacedName(svc2)
ctx2, leaseID2 := getSvcData(svc2)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease1.Add(objectName2)
if !isNew1 || !isNew2 {
t.Error("expected both adds to return isNew=true")
}
if lease1 == lease2 {
t.Error("expected different leases for services in different namespaces")
}
}
// TestManager_ConcurrentAccess tests concurrent access to the lease manager
func TestManager_ConcurrentAccess(t *testing.T) {
mgr := NewManager()
svc := createTestService("test-svc", "default", nil)
var wg sync.WaitGroup
const numGoroutines = 100
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
added := lease1.Add(objectName1)
if !added {
t.Error("expected lease to be added")
}
// Concurrent adds
for range numGoroutines {
wg.Go(func() {
added := lease1.Add(objectName1)
if added {
t.Error("expected lease to already exist")
}
})
}
wg.Wait()
mgr.Delete(leaseID1, objectName1)
// After a one delete, lease should be gone
lease := mgr.Get(getSvcID(svc))
if lease != nil {
t.Error("expected lease to be removed after all concurrent deletes")
}
}
// TestLease_StartedChannel tests the Started channel behavior
func TestLease_StartedChannel(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
lease := newLease(ctx, cancel)
// Started channel should be open initially
select {
case <-lease.Started:
t.Fatal("expected Started channel to be open initially")
default:
// Expected
}
// Close the channel
close(lease.Started)
// Now it should be closed
select {
case <-lease.Started:
// Expected
default:
t.Error("expected Started channel to be closed after close()")
}
}
// TestGetName_WithoutAnnotation tests with no annotation
func TestGetName_WithoutAnnotation(t *testing.T) {
svc := createTestService("my-service", "my-namespace", nil)
namespace, name := ServiceName(svc)
id := NewID("kubernetes", namespace, name)
expectedName := "kubevip-my-service"
expectedID := "my-namespace/kubevip-my-service"
if id.Name() != expectedName {
t.Errorf("expected name %q, got %q", expectedName, id.Name())
}
if id.NamespacedName() != expectedID {
t.Errorf("expected id %q, got %q", expectedID, id.NamespacedName())
}
}
// TestGetName_WithAnnotation tests with a shared lease annotation
func TestGetName_WithAnnotation(t *testing.T) {
svc := createTestService("my-service", "my-namespace", map[string]string{
serviceLeaseAnnotation: "shared-lease",
})
namespace, name := ServiceName(svc)
id := NewID("kubernetes", namespace, name)
expectedName := "shared-lease"
expectedID := "my-namespace/shared-lease"
if id.Name() != expectedName {
t.Errorf("expected name %q, got %q", expectedName, id.Name())
}
if id.NamespacedName() != expectedID {
t.Errorf("expected id %q, got %q", expectedID, id.NamespacedName())
}
}
// TestGetName_WithAnnotation tests with a shared lease annotation
func TestGetName_WithAnnotationAndOverriddenNamespace(t *testing.T) {
svc := createTestService("my-service", "my-namespace", map[string]string{
serviceLeaseAnnotation: "other-namespace/shared-lease",
})
namespace, name := ServiceName(svc)
id := NewID("kubernetes", namespace, name)
expectedName := "shared-lease"
expectedID := "other-namespace/shared-lease"
expectedNamespace := "other-namespace"
if id.Name() != expectedName {
t.Errorf("expected name %q, got %q", expectedName, id.Name())
}
if id.NamespacedName() != expectedID {
t.Errorf("expected id %q, got %q", expectedID, id.NamespacedName())
}
if id.Namespace() != expectedNamespace {
t.Errorf("expected namespace %q, got %q", expectedNamespace, id.Namespace())
}
}
// TestGetName_WithoutAnnotation_Etcd tests with no annotation
func TestGetName_WithoutAnnotation_Etcd(t *testing.T) {
svc := createTestService("my-service", "my-namespace", nil)
namespace, name := ServiceName(svc)
id := NewID("etcd", namespace, name)
expectedName := "kubevip-my-service"
expectedID := "my-namespace-kubevip-my-service"
if id.Name() != expectedName {
t.Errorf("expected name %q, got %q", expectedName, id.Name())
}
if id.NamespacedName() != expectedID {
t.Errorf("expected id %q, got %q", expectedID, id.NamespacedName())
}
}
// TestGetName_WithAnnotation_Etcd tests with a shared lease annotation
func TestGetName_WithAnnotation_Etcd(t *testing.T) {
svc := createTestService("my-service", "my-namespace", map[string]string{
serviceLeaseAnnotation: "shared-lease",
})
namespace, name := ServiceName(svc)
id := NewID("etcd", namespace, name)
expectedName := "shared-lease"
expectedID := "my-namespace-shared-lease"
if id.Name() != expectedName {
t.Errorf("expected name %q, got %q", expectedName, id.Name())
}
if id.NamespacedName() != expectedID {
t.Errorf("expected id %q, got %q", expectedID, id.NamespacedName())
}
}
// TestGetName_WithAnnotation_Etcd tests with a shared lease annotation
func TestGetName_WithAnnotationAndOverriddenNamespace_Etcd(t *testing.T) {
svc := createTestService("my-service", "my-namespace", map[string]string{
serviceLeaseAnnotation: "other-namespace/shared-lease",
})
namespace, name := ServiceName(svc)
id := NewID("etcd", namespace, name)
expectedName := "shared-lease"
expectedID := "other-namespace-shared-lease"
expectedNamespace := "other-namespace"
if id.Name() != expectedName {
t.Errorf("expected name %q, got %q", expectedName, id.Name())
}
if id.NamespacedName() != expectedID {
t.Errorf("expected id %q, got %q", expectedID, id.NamespacedName())
}
if id.Namespace() != expectedNamespace {
t.Errorf("expected namespace %q, got %q", expectedNamespace, id.Namespace())
}
}
// TestManager_LeaderElectionRestartScenario simulates the bug scenario where
// leadership is lost and the restartable service watcher tries to restart
// the leader election. This test verifies that after deleting the lease,
// a new lease can be created.
func TestManager_LeaderElectionRestartScenario_etcd(t *testing.T) {
mgr := NewManager()
svc := createTestService("traefik", "traefik", nil)
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
if !isNew1 {
t.Fatal("expected first add to return isNew=true")
}
// Simulate leadership acquired - close Started channel
close(lease1.Started)
// Simulate leadership lost - the leader election function should delete the lease
// This is the fix: delete the lease when RunOrDie returns
mgr.Delete(leaseID1, objectName1)
// Verify lease is removed
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to be removed after delete")
}
// Simulate restartable service watcher calling StartServicesLeaderElection again
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease1.Add(objectName1)
if !isNew2 {
t.Fatal("expected second add after delete to return isNew=true")
}
// Verify we got a new lease with a fresh Started channel
if lease1 == lease2 {
t.Error("expected new lease to be different from old lease")
}
// Verify the new Started channel is not closed
select {
case <-lease2.Started:
t.Error("expected new lease's Started channel to be open")
default:
// Expected
}
}
// TestManager_CommonLeaseScenario tests the common lease feature where
// multiple services share the same lease.
func TestManager_CommonLeaseScenario(t *testing.T) {
mgr := NewManager()
// Two services sharing the same lease via annotation
sharedLeaseAnnotations := map[string]string{
serviceLeaseAnnotation: "shared-lease",
}
svc1 := createTestService("svc1", "default", sharedLeaseAnnotations)
svc2 := createTestService("svc2", "default", sharedLeaseAnnotations)
// First service gets a new lease
objectName1 := ServiceNamespacedName(svc1)
ctx1, leaseID1 := getSvcData(svc1)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
if !isNew1 {
t.Error("expected first add to return isNew=true")
}
// Simulate first service starting leadership
close(lease1.Started)
objectName2 := ServiceNamespacedName(svc2)
ctx2, leaseID2 := getSvcData(svc2)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease2.Add(objectName2)
// Second service should get the same lease
if !isNew2 {
t.Error("expected second add with same lease name to return isNew=true")
}
if lease1 != lease2 {
t.Error("expected same lease for services with same lease annotation")
}
// Delete first service - lease should still exist
mgr.Delete(leaseID1, objectName1)
if mgr.Get(getSvcID(svc1)) == nil {
t.Error("expected lease to still exist after first delete")
}
// Delete second service - lease should be removed
mgr.Delete(leaseID2, objectName2)
if mgr.Get(getSvcID(svc2)) != nil {
t.Error("expected lease to be removed after all services deleted")
}
}
// TestManager_RaceCondition_LeaseExistsBeforeDelete tests the scenario where
// a second goroutine calls Add before the first goroutine's defer deletes the lease.
// This simulates the race condition that could cause the gaps in the logs where there is no leader.
func TestManager_RaceCondition_LeaseExistsBeforeDelete(t *testing.T) {
mgr := NewManager()
svc := createTestService("traefik", "traefik", nil)
// Simulate first leader election start
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
if !isNew1 {
t.Fatal("expected first add to return isNew=true")
}
// Simulate leadership acquired - close Started channel
close(lease1.Started)
// Simulate a second goroutine calling Add BEFORE the first goroutine's defer deletes the lease
// This is the race condition scenario
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease1.Add(objectName1)
if isNew2 {
t.Error("expected second add before delete to return isNew=false")
}
if lease1 != lease2 {
t.Error("expected same lease to be returned")
}
// The Started channel should be closed (from the first run)
select {
case <-lease2.Started:
// Expected - channel is closed
default:
t.Error("expected Started channel to be closed")
}
// Now the first goroutine's defer deletes the lease
mgr.Delete(leaseID1, objectName1)
// The lease should not still exist because same service was processed twice, so we do not increment the counter
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease tonot exist")
}
// Second delete does nothing
mgr.Delete(leaseID2, objectName1)
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to not exist")
}
}
// TestManager_NonCommonLease_MultipleAdds tests that multiple Adds for a non-common
// lease service increment the counter correctly.
func TestManager_NonCommonLease_MultipleAdds(t *testing.T) {
mgr := NewManager()
svc := createTestService("traefik", "traefik", nil) // No common lease annotation
// First Add
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
if !isNew1 {
t.Error("expected first add to return isNew=true")
}
// Close Started to simulate leadership acquired
close(lease1.Started)
// Second Add (simulating another goroutine or restart attempt)
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease1.Add(objectName1)
if isNew2 {
t.Error("expected second add to return isNew=false")
}
if lease1 != lease2 {
t.Error("expected same lease")
}
// Third Add
ctx3, leaseID3 := getSvcData(svc)
lease3 := mgr.Add(ctx3, leaseID3)
isNew3 := lease1.Add(objectName1)
if isNew3 {
t.Error("expected third add to return isNew=false")
}
if lease1 != lease3 {
t.Error("expected same lease")
}
// Need one delete to remove the lease, another delete runs do nothing
mgr.Delete(leaseID1, objectName1)
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to be deleted")
}
mgr.Delete(leaseID2, objectName1)
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to be deleted")
}
mgr.Delete(leaseID3, objectName1)
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to be deleted")
}
}
// TestManager_LeaseContextCancelledBeforeStarted tests the scenario where
// the lease context is cancelled before the Started channel is closed.
// This can happen if leadership is never acquired and the context times out.
func TestManager_LeaseContextCancelledBeforeStarted(t *testing.T) {
mgr := NewManager()
svc := createTestService("traefik", "traefik", nil)
// First Add
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
if !isNew1 {
t.Fatal("expected first add to return isNew=true")
}
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease1.Add(objectName1)
if isNew2 {
t.Error("expected second add to return isNew=false")
}
// Verify Started is not closed yet
select {
case <-lease2.Started:
t.Error("expected Started channel to be open")
default:
// Expected
}
// Cancel the lease context (simulating timeout or leadership loss before acquiring)
lease1.Cancel()
// Verify context is cancelled
select {
case <-lease2.Ctx.Done():
// Expected
case <-time.After(100 * time.Millisecond):
t.Error("expected context to be cancelled")
}
// Delete should still work
mgr.Delete(leaseID1, objectName1)
mgr.Delete(leaseID2, objectName1)
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to be removed")
}
}
// TestManager_RestartAfterLeaseContextCancelled tests that after the lease
// context is cancelled and the lease is deleted, a new lease can be created.
func TestManager_RestartAfterLeaseContextCancelled(t *testing.T) {
mgr := NewManager()
svc := createTestService("traefik", "traefik", nil)
// First Add
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
_ = lease1.Add(objectName1)
// Cancel context before Started is closed
lease1.Cancel()
// Delete the lease
mgr.Delete(leaseID1, objectName1)
// Verify lease is gone
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to be removed after delete")
}
// Add again - should create new lease
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease1.Add(objectName1)
if !isNew2 {
t.Error("expected new lease after delete")
}
// Verify new lease has fresh context and Started channel
select {
case <-lease2.Ctx.Done():
t.Error("expected new lease context to be active")
default:
// Expected
}
select {
case <-lease2.Started:
t.Error("expected new lease Started channel to be open")
default:
// Expected
}
}
// TestManager_NonCommonLease_WaitForLeaseContextDone tests the scenario where
// a non-common lease service calls Add while another leader election is running.
// The caller should wait for the lease context to be done before returning.
// This test verifies the fix for the tight spin loop issue.
func TestManager_NonCommonLease_WaitForLeaseContextDone(t *testing.T) {
mgr := NewManager()
svc := createTestService("egress-service", "default", nil) // Non-common lease
// First Add - simulates the first leader election starting
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
if !isNew1 {
t.Fatal("expected first add to return isNew=true")
}
// Simulate leadership acquired
close(lease1.Started)
// Second Add - simulates another goroutine trying to start leader election
// This should return isNew=false
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease1.Add(objectName1)
if isNew2 {
t.Error("expected second add to return isNew=false")
}
if lease1 != lease2 {
t.Error("expected same lease to be returned")
}
// Verify Started channel is closed (leadership was acquired by first)
select {
case <-lease2.Started:
// Expected - channel is closed
default:
t.Error("expected Started channel to be closed")
}
// In the actual code (leader.go), when isNew=false for non-common lease,
// the code waits on either svcCtx.Ctx.Done() or svcLease.Ctx.Done()
// Here we verify that the lease context gets cancelled when we delete the lease
// Start a goroutine that waits for the lease context to be done
// This simulates what the leader.go code does
waitDone := make(chan struct{})
go func() {
select {
case <-lease2.Ctx.Done():
close(waitDone)
case <-time.After(1 * time.Second):
// Timeout - test will fail
}
}()
// Verify the goroutine is still waiting (lease context not yet cancelled)
select {
case <-waitDone:
t.Fatal("goroutine should still be waiting")
case <-time.After(50 * time.Millisecond):
// Expected - still waiting
}
// Now simulate the first leader election ending (defer deletes the lease)
mgr.Delete(leaseID1, objectName1)
// The lease context should now be cancelled (because counter went to 0)
// But we added twice, so we need to delete twice
mgr.Delete(leaseID2, objectName1)
// Now the goroutine should have completed
select {
case <-waitDone:
// Expected - lease context was cancelled
case <-time.After(200 * time.Millisecond):
t.Error("expected goroutine to complete after lease context cancelled")
}
// Verify lease is removed
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to be removed")
}
}
// TestManager_NonCommonLease_SpinLoopPrevention tests that the fix prevents
// a tight spin loop when a non-common lease service repeatedly calls Add
// while leader election is running. The key behavior is that when isNew=false,
// the lease context should be used to block until the leader election ends.
func TestManager_NonCommonLease_SpinLoopPrevention(t *testing.T) {
mgr := NewManager()
svc := createTestService("egress-service", "default", nil) // Non-common lease
// First Add - leader election starts
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
if !isNew1 {
t.Fatal("expected first add to return isNew=true")
}
close(lease1.Started)
// Track how many times Add is called in a tight loop
// In the buggy code, this would spin forever
// In the fixed code, Add returns isNew=false and the caller blocks on lease.Ctx.Done()
addCount := 0
done := make(chan struct{})
go func() {
for i := 0; i < 100; i++ {
objectName1 := ServiceNamespacedName(svc)
ctxTmp, leaseIDTmp := getSvcData(svc)
leaseTmp := mgr.Add(ctxTmp, leaseIDTmp)
isNewTmp := leaseTmp.Add(objectName1)
addCount++
if isNewTmp {
// This shouldn't happen while the first lease exists
t.Error("unexpected isNew=true")
break
}
// In the fixed code, we would block here on lease.Ctx.Done()
// For this test, we just verify that isNew=false is returned
// and the same lease is returned each time
if leaseTmp != lease1 {
t.Error("expected same lease")
break
}
}
close(done)
}()
// Wait for the loop to complete
select {
case <-done:
// Expected
case <-time.After(1 * time.Second):
t.Fatal("loop timed out")
}
// All 100 adds should have completed (returning isNew=false)
if addCount != 100 {
t.Errorf("expected 100 adds, got %d", addCount)
}
mgr.Delete(leaseID1, objectName1)
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to be removed after first delete")
}
}
// TestManager_NonCommonLease_ServiceContextCancellation tests that when
// a service is deleted (svcCtx.Ctx cancelled), the waiting goroutine
// should also unblock. This is the other exit path from the wait.
func TestManager_NonCommonLease_ServiceContextCancellation(t *testing.T) {
mgr := NewManager()
svc := createTestService("egress-service", "default", nil)
// First Add - leader election starts
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
_ = lease1.Add(objectName1)
close(lease1.Started)
// Second Add - returns isNew=false
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease2.Add(objectName1)
if isNew2 {
t.Error("expected isNew=false")
}
// Create a simulated service context
svcCtx, svcCancel := context.WithCancel(context.Background())
// Start a goroutine that waits on either svcCtx or lease context
// This simulates the behavior in leader.go
waitDone := make(chan string)
go func() {
select {
case <-svcCtx.Done():
waitDone <- "svcCtx"
case <-lease2.Ctx.Done():
waitDone <- "leaseCtx"
case <-time.After(1 * time.Second):
waitDone <- "timeout"
}
}()
// Cancel the service context (simulates service deletion)
svcCancel()
// The goroutine should unblock via svcCtx.Done()
select {
case result := <-waitDone:
if result != "svcCtx" {
t.Errorf("expected to unblock via svcCtx, got %s", result)
}
case <-time.After(200 * time.Millisecond):
t.Error("goroutine should have unblocked")
}
}

View File

@@ -16,7 +16,6 @@ import (
"github.com/cloudflare/ipvs/netmask"
"github.com/kube-vip/kube-vip/pkg/backend"
"github.com/kube-vip/kube-vip/pkg/sysctl"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
)
@@ -51,53 +50,49 @@ type IPVSLoadBalancer struct {
backendMap backend.Map
interval int
lock sync.Mutex
killFunc func()
stop chan struct{}
networkInterface string
leaderCancel context.CancelFunc
signal chan os.Signal
address string
family ipvs.AddressFamily
}
func NewIPVSLB(ctx context.Context, address string, port uint16, forwardingMethod string, backendHealthCheckInterval int,
killFunc func(), wg *sync.WaitGroup) (*IPVSLoadBalancer, error) {
func NewIPVSLB(address string, port uint16, forwardingMethod string, backendHealthCheckInterval int, networkInterface string, leaderCancel context.CancelFunc, signal chan os.Signal) (*IPVSLoadBalancer, error) {
log.Info("Starting IPVS LoadBalancer", "address", address)
// Create IPVS client
c, err := ipvs.New()
if err != nil {
log.Error("ensure IPVS kernel modules are loaded")
log.Error("error starting IPVS", "err", err)
return nil, fmt.Errorf("starting IPVS: %w", err)
log.Error("Error starting IPVS", "err", err)
panic("")
}
i, err := c.Info()
if err != nil {
log.Error("ensure IPVS kernel modules are loaded")
log.Error("error retrieving IPVS info", "err", err)
log.Error("Error retrieving IPVS info", "err", err)
if errors.Is(err, os.ErrPermission) {
log.Error("no permission to get IPVS info - please ensure that kube-vip is running with proper capabilities/privileged mode")
}
return nil, fmt.Errorf("retrieving IPVS: %w", err)
panic("")
}
log.Info("IPVS Loadbalancer enabled", "version", fmt.Sprintf("%d.%d.%d", i.Version[0], i.Version[1], i.Version[2]))
ip, family := ipAndFamily(address)
if strings.ToLower(forwardingMethod) == "masquerade" {
if err := enableProcSys("/proc/sys/net/ipv4/vs/conntrack", "net.ipv4.vs.conntrack"); err != nil {
return nil, err
}
enableProcSys("/proc/sys/net/ipv4/vs/conntrack", "net.ipv4.vs.conntrack")
if family == ipvs.INET6 {
if err := enableProcSys("/proc/sys/net/ipv6/conf/all/forwarding", "net.ipv6.conf.all.forwarding"); err != nil {
return nil, err
}
enableProcSys("/proc/sys/net/ipv6/conf/all/forwarding", "net.ipv6.conf.all.forwarding")
} else {
if err := enableProcSys("/proc/sys/net/ipv4/ip_forward", "net.ipv4.ip_forward"); err != nil {
return nil, err
}
enableProcSys("/proc/sys/net/ipv4/ip_forward", "net.ipv4.ip_forward")
}
}
netMask := netmask.MaskFrom(31, vip.DefaultMaskIPv4) // For ipv4
netMask := netmask.MaskFrom(31, 32) // For ipv4
if family == ipvs.INET6 {
netMask = netmask.MaskFrom(128, vip.DefaultMaskIPv6) // For ipv6
netMask = netmask.MaskFrom(128, 128) // For ipv6
}
// Generate out API Server LoadBalancer instance
@@ -138,32 +133,34 @@ func NewIPVSLB(ctx context.Context, address string, port uint16, forwardingMetho
forwardingMethod: m,
interval: backendHealthCheckInterval,
backendMap: make(backend.Map),
killFunc: killFunc,
stop: make(chan struct{}),
networkInterface: networkInterface,
leaderCancel: leaderCancel,
signal: signal,
address: address,
family: family,
}
wg.Go(func() {
lb.healthCheck(ctx)
})
go lb.healthCheck()
// Return our created load-balancer
return lb, nil
}
func enableProcSys(path, name string) error {
func enableProcSys(path, name string) {
isSet, err := sysctl.EnableProcSys(path)
if err != nil {
return fmt.Errorf("ensuring %s enabled: %w", name, err)
log.Error(fmt.Sprintf("ensuring %s enabled", name), "err", err)
panic("")
}
if isSet {
log.Info(fmt.Sprintf("sysctl set %s to 1", name))
}
return nil
}
func (lb *IPVSLoadBalancer) RemoveIPVSLB() error {
log.Info("Stopping IPVS LoadBalancer", "address", lb.address)
close(lb.stop)
err := lb.client.RemoveService(lb.loadBalancerService)
if err != nil {
return fmt.Errorf("error removing existing IPVS service: %v", err)
@@ -186,7 +183,6 @@ func (lb *IPVSLoadBalancer) AddBackend(address string, port uint16) error {
if err != nil {
log.Error("checking if backend is local", "err", err)
}
log.Info("checked if backend is local", "addr", address, "local", isLocal)
}
backend := backend.Entry{Addr: address, Port: port, IsLocal: isLocal}
@@ -231,7 +227,7 @@ func (lb *IPVSLoadBalancer) addBackend(address string, port uint16) error {
// Fatal error at this point as IPVS is probably not working
log.Error("Unable to create an IPVS service, ensure IPVS kernel modules are loaded")
log.Error("IPVS service", "err", err)
return utils.NewPanicError(fmt.Sprintf("unable to create an IPVS service - %s", err))
panic("")
}
log.Info("load-Balancer services created", "address", lb.addrString(), "port", lb.Port)
@@ -316,8 +312,8 @@ func ipAndFamily(address string) (netip.Addr, ipvs.AddressFamily) {
return netip.AddrFrom4([4]byte(ipAddr.To4())), ipvs.INET
}
func (lb *IPVSLoadBalancer) healthCheck(ctx context.Context) {
backend.Watch(ctx, lb.interval, func() {
func (lb *IPVSLoadBalancer) healthCheck() {
backend.Watch(func() {
lb.lock.Lock()
defer lb.lock.Unlock()
for backend, oldStatus := range lb.backendMap {
@@ -342,50 +338,43 @@ func (lb *IPVSLoadBalancer) healthCheck(ctx context.Context) {
lb.backendMap[backend] = newStatus
}
if lb.forwardingMethod == ipvs.Local && !lb.localBackendExists() {
if lb.killFunc != nil {
log.Error("no local backends available, restarting kube-vip")
lb.killFunc()
if lb.signal != nil {
close(lb.signal)
}
if lb.leaderCancel != nil {
lb.leaderCancel()
}
}
}
}
})
}, lb.interval, lb.stop)
}
func (lb *IPVSLoadBalancer) isLocal(address string) (bool, error) {
link, err := netlink.LinkByName(lb.networkInterface)
if err != nil {
return false, fmt.Errorf("getting link '%s': %w", lb.networkInterface, err)
}
family := netlink.FAMILY_V6
if vip.IsIPv4(address) {
family = netlink.FAMILY_V4
}
target := net.ParseIP(address)
if target == nil {
return false, fmt.Errorf("unable to parse IP address %s", address)
return false, fmt.Errorf("address '%s' is not a valid IP address", address)
}
links, err := netlink.LinkList()
addrs, err := netlink.AddrList(link, family)
if err != nil {
return false, fmt.Errorf("listing links: %w", err)
return false, fmt.Errorf("listing addresses for link '%s': %w", lb.networkInterface, err)
}
family := netlink.FAMILY_V4
if utils.IsIPv6(address) {
family = netlink.FAMILY_V6
}
for _, link := range links {
if link.Type() == "veth" {
continue
}
addrs, err := netlink.AddrList(link, family)
if err != nil {
log.Error("listing addresses", "link", link.Attrs().Name, "error", err.Error())
continue
}
for _, addr := range addrs {
if addr.Scope != int(netlink.SCOPE_UNIVERSE) {
continue
}
if addr.IP.Equal(target) {
return true, nil
}
for _, addr := range addrs {
if addr.IP.Equal(target) {
return true, nil
}
}

View File

@@ -6,7 +6,6 @@ import (
"testing"
"github.com/cloudflare/ipvs"
"github.com/kube-vip/kube-vip/pkg/utils"
)
func Test_ipAndFamily(t *testing.T) {
@@ -20,7 +19,7 @@ func Test_ipAndFamily(t *testing.T) {
want1 ipvs.AddressFamily
}{
{
name: utils.IPv4Family,
name: "IPv4",
args: args{
address: "192.168.0.20",
},
@@ -28,7 +27,7 @@ func Test_ipAndFamily(t *testing.T) {
want1: ipvs.INET,
},
{
name: utils.IPv6Family,
name: "IPv6",
args: args{
address: "ff02::3",
},

30
pkg/manager/cluster.go Normal file
View File

@@ -0,0 +1,30 @@
package manager
import (
"github.com/pkg/errors"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/etcd"
)
func initClusterManager(sm *Manager) (*cluster.Manager, error) {
m := &cluster.Manager{
SignalChan: sm.signalChan,
}
switch sm.config.LeaderElectionType {
case "kubernetes", "":
m.KubernetesClient = sm.clientSet
m.RetryWatcherClient = sm.rwClientSet
case "etcd":
client, err := etcd.NewClient(sm.config)
if err != nil {
return nil, err
}
m.EtcdClient = client
default:
return nil, errors.Errorf("invalid LeaderElectionMode %s not supported", sm.config.LeaderElectionType)
}
return m, nil
}

View File

@@ -2,7 +2,6 @@ package manager
import (
"context"
"errors"
"fmt"
"net/http"
"os"
@@ -10,7 +9,6 @@ import (
"path/filepath"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
@@ -18,26 +16,19 @@ import (
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/manager/worker"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/node"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/services"
"github.com/kube-vip/kube-vip/pkg/upnp"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
const plunderLock = "plndr-svcs-lock"
// Manager degines the manager of the load-balancing services
type Manager struct {
clientSet *kubernetes.Clientset
@@ -54,7 +45,8 @@ type Manager struct {
// This channel is used to catch an OS signal and trigger a shutdown
signalChan chan os.Signal
sigint sync.Once
// This channel is used to signal a shutdown
shutdownChan chan struct{}
svcProcessor *services.Processor
@@ -62,6 +54,10 @@ type Manager struct {
// from the service watcher
countServiceWatchEvent *prometheus.CounterVec
// This is a prometheus gauge indicating the state of the sessions.
// 1 means "ESTABLISHED", 0 means "NOT ESTABLISHED"
bgpSessionInfoGauge *prometheus.GaugeVec
// This mutex is to protect calls from various goroutines
mutex sync.Mutex
@@ -70,27 +66,10 @@ type Manager struct {
// This tracks VIPs and performs ARP/NDP advertisement.
arpMgr *arp.Manager
// This tracks node labels and performs label management
// implementation will be decided in constructor
// based on config.EnableNodeLabeling
nodeLabelManager node.LabelManager
// This variable reports if manager is being closed
closing atomic.Bool
// Will be used for leaderelection when required
electionMgr *election.Manager
// Will handle leases
leaseMgr *lease.Manager
// Will handle routes
routeMgr *route.Manager
}
// New will create a new managing object
func New(ctx context.Context, configMap string, config *kubevip.Config) (*Manager, error) {
func New(configMap string, config *kubevip.Config) (*Manager, error) {
// Instance identity should be the same as k8s node name to ensure better compatibility.
// By default k8s sets node name to `hostname -s`,
@@ -105,7 +84,6 @@ func New(ctx context.Context, configMap string, config *kubevip.Config) (*Manage
}
config.NodeName = hostname
}
config.NodeName = normalizeNodeName(config.NodeName)
log.Info("using node name", "name", config.NodeName)
adminConfigPath := "/etc/kubernetes/admin.conf"
@@ -203,47 +181,21 @@ func New(ctx context.Context, configMap string, config *kubevip.Config) (*Manage
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(signalChan, syscall.SIGTERM)
// Add Notification for SIGUSR1 (for configuration dump)
signal.Notify(signalChan, syscall.SIGUSR1)
// All watchers and other goroutines should have an additional goroutine that blocks on this, to shut things down
shutdownChan := make(chan struct{})
intfMgr := networkinterface.NewManager()
arpMgr := arp.NewManager(config)
// create the node label manager
// constructor will decide if it should be a noop or not
nodeLabelManager := node.NewManager(config, clientset)
var bgpServer *bgp.Server
// If BGP is enabled then we start a server instance that will broadcast VIPs
if config.EnableBGP {
var err error
// If Annotations have been set then we will look them up
if config.Annotations != "" {
err = annotationsWatcher(ctx, clientset, rwClientSet, config)
} else {
log.Debug("No Node annotations to parse")
}
if err != nil {
return nil, err
}
bgpServer, err = bgp.NewBGPServer(config.BGPConfig)
bgpServer, err = bgp.NewBGPServer(&config.BGPConfig)
if err != nil {
return nil, fmt.Errorf("creating BGP server: %w", err)
}
}
electionMgr, err := election.NewManager(config, clientset, rwClientSet)
if err != nil {
return nil, fmt.Errorf("creating election manager: %w", err)
}
leaseMgr := lease.NewManager()
routeMgr := route.NewManager()
svcProcessor := services.NewServicesProcessor(config, bgpServer, clientset, rwClientSet,
intfMgr, arpMgr, nodeLabelManager, electionMgr, leaseMgr, routeMgr)
svcProcessor := services.NewServicesProcessor(config, bgpServer, clientset, rwClientSet, shutdownChan, intfMgr, arpMgr)
return &Manager{
clientSet: clientset,
@@ -256,22 +208,35 @@ func New(ctx context.Context, configMap string, config *kubevip.Config) (*Manage
Name: "all_services_events",
Help: "Count all events fired by the service watcher categorised by event type",
}, []string{"type"}),
signalChan: signalChan,
svcProcessor: svcProcessor,
intfMgr: intfMgr,
arpMgr: arpMgr,
bgpServer: bgpServer,
nodeLabelManager: nodeLabelManager,
electionMgr: electionMgr,
leaseMgr: leaseMgr,
routeMgr: routeMgr,
bgpSessionInfoGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "kube_vip",
Subsystem: "manager",
Name: "bgp_session_info",
Help: "Display state of session by setting metric for label value with current state to 1",
}, []string{"state", "peer"}),
signalChan: signalChan,
shutdownChan: shutdownChan,
svcProcessor: svcProcessor,
intfMgr: intfMgr,
arpMgr: arpMgr,
bgpServer: bgpServer,
}, nil
}
// Start will begin the Manager, which will start services and watch the configmap
func (sm *Manager) Start(ctx context.Context) error {
wg := sync.WaitGroup{}
defer wg.Wait()
func (sm *Manager) Start() error {
// listen for interrupts or the Linux SIGTERM signal and cancel
// our context, which the leader election code will observe and
// step down
sm.signalChan = make(chan os.Signal, 1)
// Add Notification for Userland interrupt
signal.Notify(sm.signalChan, syscall.SIGINT)
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(sm.signalChan, syscall.SIGTERM)
// All watchers and other goroutines should have an additional goroutine that blocks on this, to shut things down
sm.shutdownChan = make(chan struct{})
// HealthCheck
if sm.config.HealthCheckPort != 0 {
@@ -281,7 +246,7 @@ func (sm *Manager) Start(ctx context.Context) error {
http.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
fmt.Fprintf(w, "OK")
})
wg.Go(func() {
go func() {
server := &http.Server{
Addr: fmt.Sprintf(":%d", sm.config.HealthCheckPort),
ReadHeaderTimeout: 3 * time.Second,
@@ -290,19 +255,25 @@ func (sm *Manager) Start(ctx context.Context) error {
if err != nil {
log.Error("healthcheck", "unable to start", err)
}
})
}()
}
// on exit, clean up the node labels
defer func() {
if err := sm.nodeLabelManager.CleanUpLabels(10 * time.Second); err != nil {
log.Error("CleanUpNodeLabels", "unable to cleanup node labels", err)
// If BGP is enabled then we start a server instance that will broadcast VIPs
if sm.config.EnableBGP {
// If Annotations have been set then we will look them up
err := sm.parseAnnotations()
if err != nil {
return err
}
}()
log.Info("Starting Kube-vip Manager with the BGP engine")
return sm.startBGP()
}
if sm.config.EnableARP || sm.config.EnableWireguard {
if sm.config.EnableUPNP {
clients := upnp.GetConnectionClients(ctx)
clients := upnp.GetConnectionClients(context.TODO())
if len(clients) == 0 {
log.Error("Error Enabling UPNP. No Clients found")
// Set the struct to false so nothing should use it in future
@@ -316,131 +287,50 @@ func (sm *Manager) Start(ctx context.Context) error {
log.Info("Found UPNP IGD2 Gateway address", "ip", ip)
}
}
// TODO: It would be nice to run the UPNP refresh only on the leader.
wg.Go(func() {
sm.svcProcessor.RefreshUPNPForwards(ctx)
})
}
// TODO: It would be nice to run the UPNP refresh only on the leader.
go sm.svcProcessor.RefreshUPNPForwards()
}
return sm.startMode(ctx)
}
// Start will begin the Manager, which will start services and watch the configmap
func (sm *Manager) startMode(ctx context.Context) error {
var cpCluster *cluster.Cluster
var err error
w := worker.New(sm.arpMgr, sm.intfMgr, sm.config, &sm.closing, sm.Kill,
sm.svcProcessor, &sm.mutex, sm.clientSet, sm.bgpServer, sm.electionMgr,
sm.leaseMgr, sm.routeMgr, sm.nodeLabelManager)
// use a Go context so we can tell the leaderelection code when we
// want to step down
wg := sync.WaitGroup{}
modeCtx, cancel := context.WithCancel(ctx)
defer func() {
wg.Wait()
w.Cleanup()
cancel()
log.Info("Shutting down Kube-Vip")
}()
log.Info("starting Kube-vip Manager", "mode", w.Name())
if err := w.Configure(modeCtx, &wg); err != nil {
defer cancel()
return fmt.Errorf("failed to configure %s mode: %w", w.Name(), err)
// If ARP is enabled then we start a LeaderElection that will use ARP to advertise VIPs
if sm.config.EnableARP {
log.Info("Starting Kube-vip Manager with the ARP engine")
return sm.startARP(sm.config.NodeName)
}
if sm.config.EnableControlPlane {
err = w.InitControlPlane()
if err != nil {
defer cancel()
return err
}
if sm.config.EnableWireguard {
log.Info("Starting Kube-vip Manager with the Wireguard engine")
return sm.startWireguard(sm.config.NodeName)
}
// Shutdown function that will wait on this signal, unless we call it ourselves
wg.Go(func() {
sm.waitForShutdown(modeCtx, cancel, cpCluster)
})
if sm.config.EnableControlPlane {
wg.Go(func() {
w.StartControlPlane(modeCtx, sm.electionMgr)
})
}
if sm.config.EnableServices {
// This will tidy any dangling kube-vip iptables rules
if sm.config.EgressClean {
err := nftables.ClearTables()
if err != nil {
log.Warn("[egress]", "mode", "nftables-internal", "clearing error", err)
} else {
log.Info("[egress]", "mode", "nftables-internal", "tables", "cleared")
}
// TODO: Deprecate the iptables code v1.2.x
err = vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
if err != nil {
log.Info("[egress]", "legacy-iptables", sm.config.EgressWithNftables, "mode", "IPv4", "error", err)
}
err = vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv6)
if err != nil {
log.Info("[egress]", "legacy-iptables", sm.config.EgressWithNftables, "mode", "IPv6", "error", err)
}
}
w.ConfigureServices()
for {
select {
case <-modeCtx.Done():
return nil
default:
if err = w.StartServices(modeCtx); err != nil {
if errors.Is(err, &utils.PanicError{}) {
sm.Kill()
return fmt.Errorf("failed to reconcile services, non-recoverable error: %w", err)
} else {
log.Error("failed to reconcile services, restarting", "error", err)
}
}
}
}
if sm.config.EnableRoutingTable {
log.Info("Starting Kube-vip Manager with the Routing Table engine")
return sm.startTableMode(sm.config.NodeName)
}
log.Error("prematurely exiting Load-balancer as no modes [ARP/BGP/Wireguard] are enabled")
return nil
}
func (sm *Manager) waitForShutdown(ctx context.Context, cancel context.CancelFunc, cpCluster *cluster.Cluster) {
for {
sig := <-sm.signalChan
switch sig {
case syscall.SIGUSR1:
log.Info("Received SIGUSR1, dumping configuration")
sm.dumpConfiguration(ctx)
case syscall.SIGINT, syscall.SIGTERM:
sm.closing.Store(true)
log.Info("Received kube-vip termination, signaling shutdown")
if cpCluster != nil {
cpCluster.Stop()
}
// Cancel the context, which will in turn cancel the leadership and all goroutines
cancel()
return
func returnNameSpace() (string, error) {
if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
return ns, nil
}
return "", err
}
return "", fmt.Errorf("unable to find Namespace")
}
func (sm *Manager) Kill() {
sm.sigint.Do(func() {
sm.signalChan <- syscall.SIGINT
})
}
func (sm *Manager) parseAnnotations() error {
if sm.config.Annotations == "" {
log.Debug("No Node annotations to parse")
return nil
}
// normalizeNodeName ensures the local machine hostname conforms to
// Kubernetes RFC1123 node naming conventions (lowercase).
func normalizeNodeName(name string) string {
return strings.ToLower(name)
err := sm.annotationsWatcher()
if err != nil {
return err
}
return nil
}

159
pkg/manager/manager_arp.go Normal file
View File

@@ -0,0 +1,159 @@
package manager
import (
"context"
"syscall"
"time"
log "log/slog"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/vip"
)
// Start will begin the Manager, which will start services and watch the configmap
func (sm *Manager) startARP(id string) error {
var cpCluster *cluster.Cluster
var ns string
var err error
// use a Go context so we can tell the leaderelection code when we
// want to step down
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
log.Info("Start ARP/NDP advertisement")
go sm.arpMgr.StartAdvertisement(ctx)
// Shutdown function that will wait on this signal, unless we call it ourselves
go func() {
<-sm.signalChan
log.Info("Received kube-vip termination, signaling shutdown")
if sm.config.EnableControlPlane {
cpCluster.Stop()
}
// Close all go routines
close(sm.shutdownChan)
// Cancel the context, which will in turn cancel the leadership
cancel()
}()
if sm.config.EnableControlPlane {
cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
if err != nil {
return err
}
clusterManager, err := initClusterManager(sm)
if err != nil {
return err
}
go func() {
err := cpCluster.StartCluster(sm.config, clusterManager, nil)
if err != nil {
log.Error("starting control plane", "err", err)
// Trigger the shutdown of this manager instance
sm.signalChan <- syscall.SIGINT
}
}()
// Check if we're also starting the services, if not we can sit and wait on the closing channel and return here
if !sm.config.EnableServices {
<-sm.shutdownChan
log.Info("Shutting down Kube-Vip")
return nil
}
ns = sm.config.Namespace
} else {
ns, err = returnNameSpace()
if err != nil {
log.Warn("unable to auto-detect namespace, dropping to config", "namespace", sm.config.Namespace)
ns = sm.config.Namespace
}
}
// This will tidy any dangling kube-vip iptables rules
if sm.config.EgressClean {
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
}
// Start a services watcher (all kube-vip pods will watch services), upon a new service
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
} else {
log.Info("beginning services leadership", "namespace", ns, "lock name", sm.config.ServicesLeaseName, "id", id)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: sm.config.ServicesLeaseName,
Namespace: ns,
},
Client: sm.clientSet.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: id,
},
}
// start the leader election code loop
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(sm.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(sm.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
log.Error("service watcher", "err", err)
panic("") // TODO: - emulating log.fatal here
}
},
OnStoppedLeading: func() {
// we can do cleanup here
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "new leader", id)
sm.svcProcessor.Stop()
log.Error("lost leadership, restarting kube-vip")
panic("") // TODO: - emulating log.fatal here
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if sm.config.EnableNodeLabeling {
applyNodeLabel(sm.clientSet, sm.config.Address, id, identity)
}
if identity == id {
// I just got the lock
return
}
log.Info("new leader elected", "new leader", identity)
},
},
})
}
return nil
}

116
pkg/manager/manager_bgp.go Normal file
View File

@@ -0,0 +1,116 @@
package manager
import (
"context"
"fmt"
"syscall"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/cluster"
api "github.com/osrg/gobgp/v3/api"
"github.com/prometheus/client_golang/prometheus"
)
// Start will begin the Manager, which will start services and watch the configmap
func (sm *Manager) startBGP() error {
var cpCluster *cluster.Cluster
// var ns string
var err error
if sm.bgpServer == nil {
sm.bgpServer, err = bgp.NewBGPServer(&sm.config.BGPConfig)
if err != nil {
return fmt.Errorf("creating BGP server: %w", err)
}
}
log.Info("Starting the BGP server to advertise VIP routes to BGP peers")
if err := sm.bgpServer.Start(func(p *api.WatchEventResponse_PeerEvent) {
ipaddr := p.GetPeer().GetState().GetNeighborAddress()
port := uint64(179)
peerDescription := fmt.Sprintf("%s:%d", ipaddr, port)
for stateName, stateValue := range api.PeerState_SessionState_value {
metricValue := 0.0
if stateValue == int32(p.GetPeer().GetState().GetSessionState().Number()) {
metricValue = 1
}
sm.bgpSessionInfoGauge.With(prometheus.Labels{
"state": stateName,
"peer": peerDescription,
}).Set(metricValue)
}
}); err != nil {
return fmt.Errorf("starting BGP server: %w", err)
}
// use a Go context so we can tell the leaderelection code when we
// want to step down
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Defer a function to check if the bgpServer has been created and if so attempt to close it
defer func() {
if sm.bgpServer != nil {
sm.bgpServer.Close()
}
}()
// Shutdown function that will wait on this signal, unless we call it ourselves
go func() {
<-sm.signalChan
log.Info("Received termination, signaling shutdown")
if sm.config.EnableControlPlane {
if cpCluster != nil {
cpCluster.Stop()
}
}
// Cancel the context, which will in turn cancel the leadership
cancel()
}()
if sm.config.EnableControlPlane {
cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
if err != nil {
return err
}
clusterManager, err := initClusterManager(sm)
if err != nil {
return err
}
go func() {
if sm.config.EnableLeaderElection {
err = cpCluster.StartCluster(sm.config, clusterManager, sm.bgpServer)
} else {
err = cpCluster.StartVipService(sm.config, clusterManager, sm.bgpServer)
}
if err != nil {
log.Error("Control Plane", "err", err)
// Trigger the shutdown of this manager instance
sm.signalChan <- syscall.SIGINT
}
}()
// Check if we're also starting the services, if not we can sit and wait on the closing channel and return here
if !sm.config.EnableServices {
<-sm.signalChan
log.Info("Shutting down Kube-Vip")
return nil
}
}
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
return err
}
log.Info("Shutting down Kube-Vip")
return nil
}

View File

@@ -1,235 +0,0 @@
package manager
import (
"context"
"fmt"
"os"
"time"
"github.com/kube-vip/kube-vip/pkg/nftables"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// dumpConfiguration prints the current configuration to stdout when SIGUSR1 is received
func (sm *Manager) dumpConfiguration(ctx context.Context) {
sm.mutex.Lock()
defer sm.mutex.Unlock()
fmt.Printf("\n")
fmt.Printf("================================================================================\n")
fmt.Printf(" KUBE-VIP CONFIGURATION DUMP\n")
fmt.Printf("================================================================================\n")
fmt.Printf("Timestamp: %s\n", time.Now().Format(time.RFC3339))
fmt.Printf("Node Name: %s\n", sm.config.NodeName)
fmt.Printf("Process ID: %d\n", os.Getpid())
fmt.Printf("================================================================================\n")
fmt.Printf("\n")
sm.dumpConfigSection()
sm.dumpBGPSection()
sm.dumpARPSection()
sm.dumpServicesSection(ctx)
sm.dumpNetworkInterfacesSection()
sm.dumpLeaderElectionSection()
sm.dumpRuntimeSection()
sm.dumpNFTablesSection()
fmt.Printf("================================================================================\n")
fmt.Printf(" END OF CONFIGURATION DUMP\n")
fmt.Printf("================================================================================\n")
fmt.Printf("\n")
}
func (sm *Manager) dumpConfigSection() {
fmt.Printf("--- BASIC CONFIGURATION ---\n")
fmt.Printf("VIP: %s\n", sm.config.Address)
fmt.Printf("VIP Subnet: %s\n", sm.config.VIPSubnet)
fmt.Printf("Port: %d\n", sm.config.Port)
fmt.Printf("Namespace: %s\n", sm.config.Namespace)
fmt.Printf("Service Namespace: %s\n", sm.config.ServiceNamespace)
fmt.Printf("Interface: %s\n", sm.config.Interface)
fmt.Printf("Services Interface: %s\n", sm.config.ServicesInterface)
fmt.Printf("Single Node Mode: %t\n", sm.config.SingleNode)
fmt.Printf("Start As Leader: %t\n", sm.config.StartAsLeader)
fmt.Printf("\n")
}
func (sm *Manager) dumpBGPSection() {
fmt.Printf("--- BGP CONFIGURATION ---\n")
fmt.Printf("BGP Enabled: %t\n", sm.config.EnableBGP)
if sm.config.EnableBGP {
fmt.Printf("BGP AS: %d\n", sm.config.BGPConfig.AS)
fmt.Printf("BGP Router ID: %s\n", sm.config.BGPConfig.RouterID)
fmt.Printf("BGP Source IP: %s\n", sm.config.BGPConfig.SourceIP)
fmt.Printf("BGP Source Interface: %s\n", sm.config.BGPConfig.SourceIF)
fmt.Printf("BGP Hold Time: %d\n", sm.config.BGPConfig.HoldTime)
fmt.Printf("BGP Keepalive Interval: %d\n", sm.config.BGPConfig.KeepaliveInterval)
fmt.Printf("BGP Peers: %d\n", len(sm.config.BGPConfig.Peers))
for i, peer := range sm.config.BGPConfig.Peers {
fmt.Printf(" Peer %d: %s:%d (AS: %d, MultiHop: %t)\n",
i+1, peer.Address, peer.Port, peer.AS, peer.MultiHop)
}
}
fmt.Printf("\n")
}
func (sm *Manager) dumpARPSection() {
fmt.Printf("--- ARP/NDP CONFIGURATION ---\n")
fmt.Printf("ARP Enabled: %t\n", sm.config.EnableARP)
if sm.config.EnableARP {
fmt.Printf("ARP Broadcast Rate: %d\n", sm.config.ArpBroadcastRate)
}
fmt.Printf("Wireguard Enabled: %t\n", sm.config.EnableWireguard)
fmt.Printf("Routing Table Enabled: %t\n", sm.config.EnableRoutingTable)
if sm.config.EnableRoutingTable {
fmt.Printf("Routing Table ID: %d\n", sm.config.RoutingTableID)
fmt.Printf("Routing Protocol: %d\n", sm.config.RoutingProtocol)
fmt.Printf("Clean Routing Table: %t\n", sm.config.CleanRoutingTable)
}
fmt.Printf("\n")
}
func (sm *Manager) dumpServicesSection(ctx context.Context) {
fmt.Printf("--- SERVICES CONFIGURATION ---\n")
fmt.Printf("Services Enabled: %t\n", sm.config.EnableServices)
if sm.config.EnableServices {
fmt.Printf("Services Election: %t\n", sm.config.EnableServicesElection)
fmt.Printf("Load Balancer Class Only: %t\n", sm.config.LoadBalancerClassOnly)
fmt.Printf("Load Balancer Class Name: %s\n", sm.config.LoadBalancerClassName)
fmt.Printf("Disable Service Updates: %t\n", sm.config.DisableServiceUpdates)
fmt.Printf("Enable Endpoints: %t\n", sm.config.EnableEndpoints)
fmt.Printf("Service Security Enabled: %t\n", sm.config.EnableServiceSecurity)
if sm.svcProcessor != nil {
instances := sm.svcProcessor.ServiceInstances
fmt.Printf("Kube-vip Active Service Instances: %d\n", len(instances))
for i, inst := range instances {
if inst.ServiceSnapshot != nil {
svc := inst.ServiceSnapshot
vipConfigs := ""
for j, cfg := range svc.Status.LoadBalancer.Ingress {
if j > 0 {
vipConfigs += ", "
}
vipConfigs += cfg.IP
}
fmt.Printf(" Service %d: %s/%s (Type: %s, VIPs: %s)\n",
i+1, svc.Namespace, svc.Name, svc.Spec.Type, vipConfigs)
}
}
}
if sm.clientSet != nil {
fmt.Println()
// Kubernetes configuration
fmt.Println("--- KUBERNETES CONFIGURATION (SERVICES/ENDPOINTSLICES) ---")
fmt.Println("Service Configuration:")
svcList, err := sm.clientSet.CoreV1().Services(v1.NamespaceAll).List(ctx, metav1.ListOptions{})
if err != nil {
fmt.Println("Unable to retrieve all Services")
} else {
for x := range svcList.Items {
// Build all addresses
vipConfigs := ""
for j, cfg := range svcList.Items[x].Status.LoadBalancer.Ingress {
if j > 0 {
vipConfigs += ", "
}
vipConfigs += cfg.IP
}
fmt.Printf("Name=%s, UUID=%s, Addresses=%s\n", svcList.Items[x].Name, string(svcList.Items[x].UID), vipConfigs)
}
fmt.Println()
}
fmt.Println("EndpointSlice Configuration (note endpoint names have -XXXXX prefixed):")
epList, err := sm.clientSet.DiscoveryV1().EndpointSlices(v1.NamespaceAll).List(ctx, metav1.ListOptions{})
if err != nil {
fmt.Println("Unable to retrieve all EndpointSlices")
} else {
for x := range epList.Items {
// Build all addresses
fmt.Printf(" Endpoint Slice Name: %s\n", epList.Items[x].Name)
for _, ep := range epList.Items[x].Endpoints {
endpoints := ""
for i, addresses := range ep.Addresses {
if i > 0 {
endpoints += ", "
}
endpoints += addresses
}
nodeName := "Unknown"
targetPod := "Unknown"
if ep.NodeName != nil {
nodeName = *ep.NodeName
}
if ep.TargetRef != nil {
targetPod = ep.TargetRef.Name
}
fmt.Printf("\tNode: %s, Target Pod:%s, Addresses: %s\n", nodeName, targetPod, endpoints)
}
}
}
}
}
fmt.Printf("\n")
}
func (sm *Manager) dumpNetworkInterfacesSection() {
fmt.Printf("--- NETWORK INTERFACES ---\n")
fmt.Printf("Network Interface Manager: %t\n", sm.intfMgr != nil)
fmt.Printf("ARP Manager: %t\n", sm.arpMgr != nil)
fmt.Printf("\n")
}
func (sm *Manager) dumpLeaderElectionSection() {
fmt.Printf("--- LEADER ELECTION CONFIGURATION ---\n")
fmt.Printf("Control Plane Enabled: %t\n", sm.config.EnableControlPlane)
if sm.config.EnableControlPlane {
fmt.Printf("Detect Control Plane: %t\n", sm.config.DetectControlPlane)
}
fmt.Printf("Leader Election Type: %s\n", sm.config.LeaderElectionType)
fmt.Printf("Leader Election Enabled: %t\n", sm.config.EnableLeaderElection)
if sm.config.EnableLeaderElection {
fmt.Printf("Lease Name: %s\n", sm.config.LeaseName)
fmt.Printf("Lease Duration: %d seconds\n", sm.config.LeaseDuration)
fmt.Printf("Renew Deadline: %d seconds\n", sm.config.RenewDeadline)
fmt.Printf("Retry Period: %d seconds\n", sm.config.RetryPeriod)
}
fmt.Printf("Services Lease Name: %s\n", sm.config.ServicesLeaseName)
fmt.Printf("Node Labeling Enabled: %t\n", sm.config.EnableNodeLabeling)
fmt.Printf("\n")
}
func (sm *Manager) dumpRuntimeSection() {
fmt.Printf("--- RUNTIME STATISTICS ---\n")
fmt.Printf("Load Balancer Enabled: %t\n", sm.config.EnableLoadBalancer)
if sm.config.EnableLoadBalancer {
fmt.Printf("Load Balancer Port: %d\n", sm.config.LoadBalancerPort)
fmt.Printf("Load Balancer Forwarding Method: %s\n", sm.config.LoadBalancerForwardingMethod)
fmt.Printf("Load Balancers Configured: %d\n", len(sm.config.LoadBalancers))
}
fmt.Printf("Prometheus HTTP Server: %s\n", sm.config.PrometheusHTTPServer)
fmt.Printf("Health Check Port: %d\n", sm.config.HealthCheckPort)
fmt.Printf("UPNP Enabled: %t\n", sm.config.EnableUPNP)
fmt.Printf("Egress Clean Enabled: %t\n", sm.config.EgressClean)
if sm.config.EgressClean {
fmt.Printf("Egress with nftables: %t\n", sm.config.EgressWithNftables)
fmt.Printf("Egress Pod CIDR: %s\n", sm.config.EgressPodCidr)
fmt.Printf("Egress Service CIDR: %s\n", sm.config.EgressServiceCidr)
}
fmt.Printf("\n")
}
func (sm *Manager) dumpNFTablesSection() {
fmt.Printf("--- NFTABLES CONFIGURATION ---\n")
chains, err := nftables.ListChains()
if err != nil {
fmt.Printf("Unable to retrieve NFTables chains, error=%s", err)
}
for x := range chains {
fmt.Printf("Chain: %s\n", chains[x])
}
fmt.Println()
}

View File

@@ -1,239 +0,0 @@
package manager
import (
"bytes"
"context"
"io"
"os"
"testing"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/stretchr/testify/assert"
)
func TestDumpConfiguration(t *testing.T) {
config := &kubevip.Config{
Address: "192.168.1.100",
Interface: "eth0",
Port: 6443,
EnableARP: true,
EnableBGP: false,
EnableControlPlane: true,
EnableServices: true,
LeaderElectionType: "kubernetes",
Namespace: "kube-system",
NodeName: "test-node",
KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
EnableLeaderElection: true,
LeaseName: "test-lease",
},
}
mgr := &Manager{
config: config,
}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpConfiguration(context.TODO())
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "KUBE-VIP CONFIGURATION DUMP", "should contain header")
assert.Contains(t, output, "Node Name: test-node", "should contain node name")
assert.Contains(t, output, "VIP: 192.168.1.100", "should contain VIP address")
assert.Contains(t, output, "Interface: eth0", "should contain interface")
assert.Contains(t, output, "Port: 6443", "should contain port")
assert.Contains(t, output, "ARP Enabled: true", "should contain ARP status")
assert.Contains(t, output, "BGP Enabled: false", "should contain BGP status")
}
func TestDumpConfigSection(t *testing.T) {
config := &kubevip.Config{
Address: "192.168.1.100",
Interface: "eth0",
Port: 6443,
VIPSubnet: "/24",
EnableARP: true,
EnableBGP: false,
EnableControlPlane: true,
EnableServices: true,
LeaderElectionType: "kubernetes",
Namespace: "kube-system",
KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
EnableLeaderElection: true,
LeaseName: "test-lease",
},
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpConfigSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "--- BASIC CONFIGURATION ---")
assert.Contains(t, output, "VIP: 192.168.1.100")
assert.Contains(t, output, "VIP Subnet: /24")
assert.Contains(t, output, "Interface: eth0")
assert.Contains(t, output, "Port: 6443")
assert.Contains(t, output, "Namespace: kube-system")
assert.Contains(t, output, "Single Node Mode: false")
assert.Contains(t, output, "Start As Leader: false")
}
func TestDumpBGPSection(t *testing.T) {
t.Run("BGP disabled", func(t *testing.T) {
config := &kubevip.Config{
EnableBGP: false,
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpBGPSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "BGP Enabled: false")
})
t.Run("BGP enabled", func(t *testing.T) {
config := &kubevip.Config{
EnableBGP: true,
BGPConfig: kubevip.BGPConfig{
RouterID: "192.168.1.1",
AS: 65000,
Peers: []kubevip.BGPPeer{
{Address: "192.168.1.2", AS: 65001},
{Address: "192.168.1.3", AS: 65002},
},
},
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpBGPSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "BGP Enabled: true")
assert.Contains(t, output, "BGP Router ID: 192.168.1.1")
assert.Contains(t, output, "BGP AS: 65000")
assert.Contains(t, output, "BGP Peers: 2")
})
}
func TestDumpARPSection(t *testing.T) {
t.Run("ARP disabled", func(t *testing.T) {
config := &kubevip.Config{
EnableARP: false,
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpARPSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "ARP Enabled: false")
})
t.Run("ARP enabled", func(t *testing.T) {
config := &kubevip.Config{
EnableARP: true,
ArpBroadcastRate: 5,
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpARPSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "ARP Enabled: true")
assert.Contains(t, output, "ARP Broadcast Rate: 5")
})
}
func TestDumpRuntimeSection(t *testing.T) {
config := &kubevip.Config{
EnableLoadBalancer: false,
PrometheusHTTPServer: "",
HealthCheckPort: 0,
EnableUPNP: false,
EgressClean: false,
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpRuntimeSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "--- RUNTIME STATISTICS ---", "should contain runtime section header")
assert.Contains(t, output, "Load Balancer Enabled: false", "should contain load balancer status")
assert.Contains(t, output, "UPNP Enabled: false", "should contain UPNP status")
}

View File

@@ -0,0 +1,185 @@
package manager
import (
"context"
"fmt"
"syscall"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
)
// Start will begin the Manager, which will start services and watch the configmap
func (sm *Manager) startTableMode(id string) error {
var cpCluster *cluster.Cluster
var err error
// use a Go context so we can tell the leaderelection code when we
// want to step down
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
log.Info("destination for routes", "table", sm.config.RoutingTableID, "protocol", sm.config.RoutingProtocol)
if sm.config.CleanRoutingTable {
go func() {
// we assume that after 10s all services should be configured so we can delete redundant routes
time.Sleep(time.Second * 10)
if err := sm.cleanRoutes(); err != nil {
log.Error("error checking for old routes", "err", err)
}
}()
}
if sm.config.EgressClean {
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv6)
log.Debug("IPtables rules cleaned on startup")
}
// Shutdown function that will wait on this signal, unless we call it ourselves
go func() {
<-sm.signalChan
log.Info("Received kube-vip termination, signaling shutdown")
if sm.config.EnableControlPlane {
cpCluster.Stop()
}
// Cancel the context, which will in turn cancel the leadership
cancel()
}()
if sm.config.EnableControlPlane {
cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
if err != nil {
return fmt.Errorf("cluster initialization error: %w", err)
}
clusterManager, err := initClusterManager(sm)
if err != nil {
return fmt.Errorf("cluster manager initialization error: %w", err)
}
if err := cpCluster.StartVipService(sm.config, clusterManager, nil); err != nil {
log.Error("Control Plane", "err", err)
// Trigger the shutdown of this manager instance
sm.signalChan <- syscall.SIGINT
}
} else {
ns, err := returnNameSpace()
if err != nil {
log.Warn("unable to auto-detect namespace", "dropping to", sm.config.Namespace)
ns = sm.config.Namespace
}
// Start a services watcher (all kube-vip pods will watch services), upon a new service
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
} else if sm.config.EnableLeaderElection {
log.Info("beginning services leadership", "namespace", ns, "lock name", plunderLock, "id", id)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: plunderLock,
Namespace: ns,
},
Client: sm.clientSet.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: id,
},
}
// start the leader election code loop
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(sm.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(sm.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
log.Error(err.Error())
panic("")
}
},
OnStoppedLeading: func() {
// we can do cleanup here
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "id", id)
sm.svcProcessor.Stop()
log.Error("lost leadership, restarting kube-vip")
panic("")
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if identity == id {
// I just got the lock
return
}
log.Info("new leader elected", "id", identity)
},
},
})
} else {
log.Info("beginning watching services without leader election")
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
log.Error("Cannot watch services", "err", err)
}
}
}
return nil
}
func (sm *Manager) cleanRoutes() error {
sm.mutex.Lock()
defer sm.mutex.Unlock()
routes, err := vip.ListRoutes(sm.config.RoutingTableID, sm.config.RoutingProtocol)
if err != nil {
return fmt.Errorf("error getting routes: %w", err)
}
for i := range routes {
found := false
if sm.config.EnableControlPlane {
found = (routes[i].Dst.IP.String() == sm.config.Address)
} else {
found = endpoints.CountRouteReferences(&routes[i], &sm.svcProcessor.ServiceInstances) > 0
}
if !found {
err = netlink.RouteDel(&(routes[i]))
if err != nil {
log.Error("[route] deletion", "route", routes[i], "err", err)
}
log.Debug("[route] deletion", "route", routes[i])
}
}
return nil
}

View File

@@ -1,38 +0,0 @@
package manager
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestNormalizeNodeName(t *testing.T) {
tests := []struct {
name string
hostname string
expected string
}{
{
name: "All lowercase hostname remains unchanged",
hostname: "worker-node-1",
expected: "worker-node-1",
},
{
name: "Mixed case hostname is lowercased",
hostname: "Worker-Node-1",
expected: "worker-node-1",
},
{
name: "All uppercase hostname is lowercased",
hostname: "MASTER-NODE",
expected: "master-node",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := normalizeNodeName(tt.hostname)
assert.Equal(t, tt.expected, result, "The normalized node name did not match the expected RFC1123 compliant name")
})
}
}

View File

@@ -0,0 +1,122 @@
package manager
import (
"context"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/wireguard"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
)
// Start will begin the Manager, which will start services and watch the configmap
func (sm *Manager) startWireguard(id string) error {
var ns string
var err error
// use a Go context so we can tell the leaderelection code when we
// want to step down
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
log.Info("reading wireguard peer configuration from Kubernetes secret")
s, err := sm.clientSet.CoreV1().Secrets(sm.config.Namespace).Get(ctx, "wireguard", metav1.GetOptions{})
if err != nil {
return err
}
// parse all the details needed for Wireguard
peerPublicKey := s.Data["peerPublicKey"]
peerEndpoint := s.Data["peerEndpoint"]
privateKey := s.Data["privateKey"]
// Configure the interface to join the Wireguard VPN
err = wireguard.ConfigureInterface(string(privateKey), string(peerPublicKey), string(peerEndpoint))
if err != nil {
return err
}
// Shutdown function that will wait on this signal, unless we call it ourselves
go func() {
<-sm.signalChan
log.Info("Received termination, signaling shutdown")
// Cancel the context, which will in turn cancel the leadership
cancel()
}()
ns, err = returnNameSpace()
if err != nil {
log.Warn("unable to auto-detect namespace", "dropping to", sm.config.Namespace)
ns = sm.config.Namespace
}
// Start a services watcher (all kube-vip pods will watch services), upon a new service
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
} else {
log.Info("beginning services leadership", "namespace", ns, "lock name", plunderLock, "id", id)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: plunderLock,
Namespace: ns,
},
Client: sm.clientSet.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: id,
},
}
// start the leader election code loop
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(sm.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(sm.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
log.Error(err.Error())
panic("")
}
},
OnStoppedLeading: func() {
// we can do cleanup here
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "id", id)
sm.svcProcessor.Stop()
log.Error("lost leadership, restarting kube-vip")
panic("")
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if identity == id {
// I just got the lock
return
}
log.Info("new leader elected", "id", identity)
},
},
})
}
return nil
}

View File

@@ -0,0 +1,73 @@
package manager
import (
"context"
"encoding/json"
"fmt"
log "log/slog"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
)
const (
nodeLabelIndex = "kube-vip.io/has-ip"
nodeLabelJSONPath = `kube-vip.io~1has-ip`
)
type patchStringLabel struct {
Op string `json:"op"`
Path string `json:"path"`
Value string `json:"value"`
}
// applyNodeLabel add/remove node label `kube-vip.io/has-ip=<VIP-Address>` to/from
// the node where the virtual IP was added to/removed from.
func applyNodeLabel(clientSet *kubernetes.Clientset, address, id, identity string) {
ctx := context.Background()
node, err := clientSet.CoreV1().Nodes().Get(ctx, id, metav1.GetOptions{})
if err != nil {
log.Error("can't query node labels", "node", id, "err", err)
return
}
log.Debug(fmt.Sprintf("node %s labels: %+v", id, node.Labels))
value, ok := node.Labels[nodeLabelIndex]
path := fmt.Sprintf("/metadata/labels/%s", nodeLabelJSONPath)
log.Debug(fmt.Sprintf("Received identity: %s - id: %s", identity, id))
if ok && value == address {
log.Debug(fmt.Sprintf("removing node label `has-ip=%s` on %s", address, id))
// Remove label
applyPatchLabels(ctx, clientSet, id, "remove", path, address)
} else {
log.Debug(fmt.Sprintf("setting node label `has-ip=%s` on %s", address, id))
// Append label
applyPatchLabels(ctx, clientSet, id, "add", path, address)
}
}
// applyPatchLabels add/remove node labels
func applyPatchLabels(ctx context.Context, clientSet *kubernetes.Clientset,
name, operation, path, value string) {
patchLabels := []patchStringLabel{{
Op: operation,
Path: path,
Value: value,
}}
patchData, err := json.Marshal(patchLabels)
if err != nil {
log.Error("node patch marshaling failed", "err", err)
return
}
// patch node
node, err := clientSet.CoreV1().Nodes().Patch(ctx,
name, types.JSONPatchType, patchData, metav1.PatchOptions{})
if err != nil {
log.Error("node patching failed", "err", err)
return
}
log.Debug("updated", "node", name, "labels", node.Labels)
}

15
pkg/manager/prom.go Normal file
View File

@@ -0,0 +1,15 @@
package manager
import "github.com/prometheus/client_golang/prometheus"
// PrometheusCollector defines a service watch event counter.
func (sm *Manager) PrometheusCollector() []prometheus.Collector {
collectors := []prometheus.Collector{}
if sm.svcProcessor != nil {
collectors = append(collectors, sm.svcProcessor.CountServiceWatchEvent)
}
if sm.bgpServer != nil {
collectors = append(collectors, sm.bgpServer.BGPSessionInfoGauge)
}
return collectors
}

View File

@@ -10,14 +10,13 @@ import (
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/davecgh/go-spew/spew"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/apimachinery/pkg/watch"
@@ -26,19 +25,18 @@ import (
// This file handles the watching of node annotations for configuration, it will exit once the annotations are
// present
func annotationsWatcher(ctx context.Context, clientSet,
rwClientSet kubernetes.Interface, config *kubevip.Config) error {
func (sm *Manager) annotationsWatcher() error {
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
log.Info("Kube-Vip is waiting for annotation prefix to be present on this node", "prefix", config.Annotations)
log.Info("Kube-Vip is waiting for annotation prefix to be present on this node", "prefix", sm.config.Annotations)
labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/hostname": config.NodeName}}
labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/hostname": sm.config.NodeName}}
listOptions := metav1.ListOptions{
LabelSelector: labels.Set(labelSelector.MatchLabels).String(),
}
// First we'll check the annotations for the node and if
// they aren't what are expected, we'll drop into the watch until they are
nodeList, err := clientSet.CoreV1().Nodes().List(ctx, listOptions)
nodeList, err := sm.clientSet.CoreV1().Nodes().List(context.Background(), listOptions)
if err != nil {
return err
}
@@ -47,11 +45,11 @@ func annotationsWatcher(ctx context.Context, clientSet,
// there's probably bigger problems
node := nodeList.Items[0]
bgpConfig, bgpPeer, err := parseBgpAnnotations(config.BGPConfig, &node, config.Annotations)
bgpConfig, bgpPeer, err := parseBgpAnnotations(sm.config.BGPConfig, &node, sm.config.Annotations)
if err == nil {
// No error, the annotations already exist
config.BGPConfig = bgpConfig
config.BGPPeerConfig = bgpPeer
sm.config.BGPConfig = bgpConfig
sm.config.BGPPeerConfig = bgpPeer
return nil
}
@@ -59,19 +57,30 @@ func annotationsWatcher(ctx context.Context, clientSet,
// they're as needed
log.Warn(err.Error())
rw, err := watchtools.NewRetryWatcherWithContext(ctx, node.ResourceVersion, &cache.ListWatch{
// TODO, will need refactoring as part of rikatz work
rw, err := watchtools.NewRetryWatcherWithContext(context.TODO(), node.ResourceVersion, &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return rwClientSet.CoreV1().Nodes().Watch(ctx, listOptions)
return sm.rwClientSet.CoreV1().Nodes().Watch(context.Background(), listOptions)
},
})
if err != nil {
return fmt.Errorf("error creating annotations watcher: %s", err.Error())
}
defer func() {
rw.Stop()
log.Debug("[annotations] watcher stopped")
exitFunction := make(chan struct{})
go func() {
select {
case <-sm.shutdownChan:
log.Debug("[annotations] shutdown called")
// Stop the retry watcher
rw.Stop()
return
case <-exitFunction:
log.Debug("[annotations] function ending")
// Stop the retry watcher
rw.Stop()
return
}
}()
ch := rw.ResultChan()
@@ -85,17 +94,16 @@ func annotationsWatcher(ctx context.Context, clientSet,
return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
}
bgpConfig, bgpPeer, err := parseBgpAnnotations(config.BGPConfig, node, config.Annotations)
bgpConfig, bgpPeer, err := parseBgpAnnotations(sm.config.BGPConfig, node, sm.config.Annotations)
if err != nil {
log.Error(err.Error())
continue
}
config.BGPConfig = bgpConfig
config.BGPPeerConfig = bgpPeer
sm.config.BGPConfig = bgpConfig
sm.config.BGPPeerConfig = bgpPeer
log.Info("[annotations] exiting Annotations watcher - annotations found")
return nil
rw.Stop()
case watch.Deleted:
node, ok := event.Object.(*v1.Node)
if !ok {
@@ -122,8 +130,10 @@ func annotationsWatcher(ctx context.Context, clientSet,
default:
}
}
log.Info("[annotations] exiting annotations watcher")
close(exitFunction)
log.Info("Exiting Annotations watcher")
return nil
}
// parseNodeAnnotations parses the annotations on the node and updates the configuration
@@ -141,14 +151,12 @@ func annotationsWatcher(ctx context.Context, clientSet,
// * `<info>` is the relevant information, such as `node-asn` or `peer-ip`
// * `{{n}}` is the number of the peer, always starting with `0`
// * kube-vip is only designed to manage one peer, just look for {{n}} == 0
func parseBgpAnnotations(bgpConfig kubevip.BGPConfig, node *v1.Node, prefix string) (kubevip.BGPConfig, kubevip.BGPPeer, error) {
bgpPeer := kubevip.BGPPeer{}
prefix = regexp.QuoteMeta(prefix)
func parseBgpAnnotations(bgpConfig bgp.Config, node *v1.Node, prefix string) (bgp.Config, bgp.Peer, error) {
bgpPeer := bgp.Peer{}
nodeASN := ""
regex := regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?node-asn$", prefix))
for k, v := range node.Annotations {
regex := regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?node-asn", prefix))
if regex.Match([]byte(k)) {
nodeASN = v
}
@@ -165,8 +173,8 @@ func parseBgpAnnotations(bgpConfig kubevip.BGPConfig, node *v1.Node, prefix stri
bgpConfig.AS = uint32(u64)
srcIP := ""
regex = regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?src-ip$", prefix))
for k, v := range node.Annotations {
regex := regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?src-ip", prefix))
if regex.Match([]byte(k)) {
srcIP = v
}
@@ -180,8 +188,8 @@ func parseBgpAnnotations(bgpConfig kubevip.BGPConfig, node *v1.Node, prefix stri
bgpConfig.RouterID, bgpConfig.SourceIP = srcIP, srcIP
peerASN := ""
regex = regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?peer-asn$", prefix))
for k, v := range node.Annotations {
regex := regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?peer-asn", prefix))
if regex.Match([]byte(k)) {
peerASN = v
}
@@ -198,8 +206,8 @@ func parseBgpAnnotations(bgpConfig kubevip.BGPConfig, node *v1.Node, prefix stri
bgpPeer.AS = uint32(u64)
peerIPString := ""
regex = regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-[0-9]+-)?peer-ip$", prefix))
for k, v := range node.Annotations {
regex := regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-[0-9]+-)?peer-ip", prefix))
if regex.Match([]byte(k)) {
peerIPString += v + ","
}
@@ -207,13 +215,8 @@ func parseBgpAnnotations(bgpConfig kubevip.BGPConfig, node *v1.Node, prefix stri
peerIPString = strings.TrimRight(peerIPString, ",")
peerIPs := strings.Split(peerIPString, ",")
if len(peerIPs) >= 1 && peerIPs[0] == "" || len(peerIPs) == 0 {
return bgpConfig, bgpPeer, fmt.Errorf("peer-ip value missing or empty")
}
bgpConfig.Peers = make([]kubevip.BGPPeer, 0, len(peerIPs))
regexPass := regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?bgp-pass$", prefix))
regexMultiHop := regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?peer-multi-hop$", prefix))
bgpConfig.Peers = make([]bgp.Peer, 0, len(peerIPs))
for _, peerIP := range peerIPs {
ipAddr := strings.TrimSpace(peerIP)
@@ -222,7 +225,8 @@ func parseBgpAnnotations(bgpConfig kubevip.BGPConfig, node *v1.Node, prefix stri
// Check if we're also expecting a password for this peer
base64BGPPassword := ""
for k, v := range node.Annotations {
if regexPass.Match([]byte(k)) {
regex := regexp.MustCompile(fmt.Sprintf("^%s/(bgp-peers-0-)?bgp-pass", prefix))
if regex.Match([]byte(k)) {
base64BGPPassword = v
}
}
@@ -235,21 +239,6 @@ func parseBgpAnnotations(bgpConfig kubevip.BGPConfig, node *v1.Node, prefix stri
// Set the password for each peer
bgpPeer.Password = string(decodedPassword)
}
// Check if multi-hop is enabled.
for k, v := range node.Annotations {
if regexMultiHop.MatchString(k) {
switch v {
case "true":
bgpPeer.MultiHop = true
case "false":
bgpPeer.MultiHop = false
default:
return bgpConfig, bgpPeer,
fmt.Errorf("invalid %q annotation value: %q, must be \"true\" or \"false\"", k, v)
}
}
}
bgpConfig.Peers = append(bgpConfig.Peers, bgpPeer)
}
}

View File

@@ -4,7 +4,7 @@ import (
"reflect"
"testing"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -15,7 +15,7 @@ func TestParseBgpAnnotations(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{Name: "test", Annotations: map[string]string{}},
}
bgpConfigBase := kubevip.BGPConfig{
bgpConfigBase := bgp.Config{
HoldTime: 15,
KeepaliveInterval: 5,
}
@@ -25,11 +25,9 @@ func TestParseBgpAnnotations(t *testing.T) {
}
node.Annotations = map[string]string{
"bgp/node-asn": "65000",
"bgp/peer-asn": "64000",
"bgp/src-ip": "10.0.0.254",
"bgp/peer-ip": "10.0.0.1",
"bgp/peer-multi-hop": "true",
"bgp/node-asn": "65000",
"bgp/peer-asn": "64000",
"bgp/src-ip": "10.0.0.254",
}
bgpConfig, bgpPeer, err := parseBgpAnnotations(bgpConfigBase, node, "bgp")
@@ -40,17 +38,15 @@ func TestParseBgpAnnotations(t *testing.T) {
assert.Equal(t, uint32(65000), bgpConfig.AS, "bgpConfig.AS parsed incorrectly")
assert.Equal(t, uint32(64000), bgpPeer.AS, "bgpPeer.AS parsed incorrectly")
assert.Equal(t, "10.0.0.254", bgpConfig.RouterID, "bgpConfig.RouterID parsed incorrectly")
assert.Equal(t, true, bgpPeer.MultiHop, "bgpPeer.MultiHop parsed incorrectly")
assert.EqualValues(t, 15, bgpConfig.HoldTime, "base bgpConfig.HoldTime should not be overwritten")
assert.EqualValues(t, 5, bgpConfig.KeepaliveInterval, "base bgpConfig.KeepaliveInterval should not be overwritten")
node.Annotations = map[string]string{ //nolint:gosec
"bgp/node-asn": "65000",
"bgp/peer-asn": "64000",
"bgp/src-ip": "10.0.0.254",
"bgp/peer-ip": "10.0.0.1,10.0.0.2,10.0.0.3",
"bgp/bgp-pass": "cGFzc3dvcmQ=", // dummy password for the test, gosec linter disabled
"bgp/peer-multi-hop": "true",
node.Annotations = map[string]string{
"bgp/node-asn": "65000",
"bgp/peer-asn": "64000",
"bgp/src-ip": "10.0.0.254",
"bgp/peer-ip": "10.0.0.1,10.0.0.2,10.0.0.3",
"bgp/bgp-pass": "cGFzc3dvcmQ=", // password
}
bgpConfig, bgpPeer, err = parseBgpAnnotations(bgpConfigBase, node, "bgp")
@@ -58,10 +54,10 @@ func TestParseBgpAnnotations(t *testing.T) {
t.Fatal("Parsing BGP annotations should return nil when minimum config is met")
}
bgpPeers := []kubevip.BGPPeer{
{Address: "10.0.0.1", AS: uint32(64000), Password: "password", MultiHop: true},
{Address: "10.0.0.2", AS: uint32(64000), Password: "password", MultiHop: true},
{Address: "10.0.0.3", AS: uint32(64000), Password: "password", MultiHop: true},
bgpPeers := []bgp.Peer{
{Address: "10.0.0.1", AS: uint32(64000), Password: "password"},
{Address: "10.0.0.2", AS: uint32(64000), Password: "password"},
{Address: "10.0.0.3", AS: uint32(64000), Password: "password"},
}
assert.Equal(t, bgpPeers, bgpConfig.Peers, "bgpConfig.Peers parsed incorrectly")
assert.Equal(t, "10.0.0.3", bgpPeer.Address, "bgpPeer.Address parsed incorrectly")
@@ -81,7 +77,7 @@ func TestParseNewBgpAnnotations(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{Name: "test", Annotations: map[string]string{}},
}
bgpConfigBase := kubevip.BGPConfig{
bgpConfigBase := bgp.Config{
HoldTime: 15,
KeepaliveInterval: 5,
}
@@ -90,12 +86,12 @@ func TestParseNewBgpAnnotations(t *testing.T) {
t.Fatal("Parsing BGP annotations should return an error when no annotations exist")
}
node.Annotations = map[string]string{ //nolint:gosec
node.Annotations = map[string]string{
"bgp/bgp-peers-0-node-asn": "65000",
"bgp/bgp-peers-0-peer-asn": "64000",
"bgp/bgp-peers-0-peer-ip": "10.0.0.1,10.0.0.2,10.0.0.3",
"bgp/bgp-peers-0-src-ip": "10.0.0.254",
"bgp/bgp-peers-0-bgp-pass": "cGFzc3dvcmQ=", // dummy password for the test, gosec linter disabled
"bgp/bgp-peers-0-bgp-pass": "cGFzc3dvcmQ=", // password
}
bgpConfig, bgpPeer, err := parseBgpAnnotations(bgpConfigBase, node, "bgp")
@@ -103,7 +99,7 @@ func TestParseNewBgpAnnotations(t *testing.T) {
t.Fatalf("Parsing BGP annotations should return nil when minimum config is met [%v]", err)
}
bgpPeers := []kubevip.BGPPeer{
bgpPeers := []bgp.Peer{
{Address: "10.0.0.1", AS: uint32(64000), Password: "password"},
{Address: "10.0.0.2", AS: uint32(64000), Password: "password"},
{Address: "10.0.0.3", AS: uint32(64000), Password: "password"},
@@ -125,15 +121,15 @@ func Test_parseBgpAnnotations(t *testing.T) {
tests := []struct {
name string
args args
want kubevip.BGPConfig
want1 kubevip.BGPPeer
want bgp.Config
want1 bgp.Peer
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, got1, err := parseBgpAnnotations(kubevip.BGPConfig{}, tt.args.node, tt.args.prefix)
got, got1, err := parseBgpAnnotations(bgp.Config{}, tt.args.node, tt.args.prefix)
if (err != nil) != tt.wantErr {
t.Errorf("parseBgpAnnotations() error = %v, wantErr %v", err, tt.wantErr)
return

View File

@@ -1,73 +0,0 @@
package worker
import (
"context"
log "log/slog"
"sync"
"sync/atomic"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/node"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/services"
"k8s.io/client-go/kubernetes"
)
type ARP struct {
Common
}
func NewARP(arpMgr *arp.Manager, intfMgr *networkinterface.Manager,
config *kubevip.Config, closing *atomic.Bool, killFunc func(),
svcProcessor *services.Processor, mutex *sync.Mutex, clientSet *kubernetes.Clientset,
electionMgr *election.Manager, leaseMgr *lease.Manager, routeMgr *route.Manager,
nodeLabelMgr node.Labeler) *ARP {
return &ARP{
Common: *newCommon(arpMgr, intfMgr, config, closing, killFunc,
svcProcessor, mutex, clientSet, electionMgr, leaseMgr, routeMgr,
nodeLabelMgr),
}
}
func (a *ARP) Configure(ctx context.Context, wg *sync.WaitGroup) error {
log.Info("Start ARP/NDP advertisement Global")
wg.Go(func() {
a.arpMgr.StartAdvertisement(ctx)
})
return nil
}
func (a *ARP) StartControlPlane(ctx context.Context, electionManager *election.Manager) {
err := a.cpCluster.StartCluster(ctx, a.config, electionManager, nil, a.leaseMgr, a.killFunc)
if err != nil {
log.Error("starting control plane", "err", err)
}
// Trigger the shutdown of this manager instance
a.killFunc()
}
func (a *ARP) ConfigureServices() {
}
func (a *ARP) StartServices(ctx context.Context) error {
// Start a services watcher (all kube-vip pods will watch services), upon a new service
// a lock based upon that service is created that they will all leaderElection on
if a.config.EnableServicesElection {
if err := a.PerServiceLeader(ctx); err != nil {
return err
}
} else {
a.GlobalLeader(ctx, a.config.ServicesLeaseName)
}
return nil
}
func (a *ARP) Name() string {
return "ARP"
}

View File

@@ -1,121 +0,0 @@
package worker
import (
"context"
"fmt"
log "log/slog"
"sync"
"sync/atomic"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/metrics"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/node"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/services"
api "github.com/osrg/gobgp/v3/api"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/client-go/kubernetes"
)
type BGP struct {
Common
bgpServer *bgp.Server
}
func NewBGP(arpMgr *arp.Manager, intfMgr *networkinterface.Manager,
config *kubevip.Config, closing *atomic.Bool, killFunc func(),
svcProcessor *services.Processor, mutex *sync.Mutex, clientSet *kubernetes.Clientset,
bgpServer *bgp.Server, electionMgr *election.Manager, leaseMgr *lease.Manager,
routeMgr *route.Manager, nodeLabelMgr node.Labeler) *BGP {
return &BGP{
Common: *newCommon(arpMgr, intfMgr, config, closing, killFunc,
svcProcessor, mutex, clientSet, electionMgr, leaseMgr, routeMgr,
nodeLabelMgr),
bgpServer: bgpServer,
}
}
func (b *BGP) Configure(ctx context.Context, _ *sync.WaitGroup) error {
var err error
if b.bgpServer == nil {
b.bgpServer, err = bgp.NewBGPServer(b.config.BGPConfig)
if err != nil {
return fmt.Errorf("creating BGP server: %w", err)
}
}
log.Info("Starting the BGP server to advertise VIP routes to BGP peers")
if err := b.bgpServer.Start(ctx, func(p *api.WatchEventResponse_PeerEvent) {
ipaddr := p.GetPeer().GetState().GetNeighborAddress()
port := uint64(179)
peerDescription := fmt.Sprintf("%s:%d", ipaddr, port)
for stateName, stateValue := range api.PeerState_SessionState_value {
metricValue := 0.0
if stateValue == int32(p.GetPeer().GetState().GetSessionState().Number()) {
metricValue = 1
}
metrics.BGPSessionInfoGauge.With(prometheus.Labels{
"state": stateName,
"peer": peerDescription,
}).Set(metricValue)
}
}); err != nil {
return fmt.Errorf("starting BGP server: %w", err)
}
return nil
}
func (b *BGP) Cleanup() {
// Defer a function to check if the bgpServer has been created and if so attempt to close it
if b.bgpServer != nil {
b.bgpServer.Close()
}
}
func (b *BGP) StartControlPlane(ctx context.Context, electionManager *election.Manager) {
var err error
if b.config.EnableLeaderElection {
err = b.cpCluster.StartCluster(ctx, b.config, electionManager, b.bgpServer, b.leaseMgr, b.killFunc)
} else {
err = b.cpCluster.StartVipService(ctx, b.config, electionManager, b.bgpServer, b.killFunc)
}
if err != nil {
log.Error("Control Plane", "err", err)
// Trigger the shutdown of this manager instance
b.killFunc()
}
}
func (b *BGP) ConfigureServices() {
// No configuration required
}
func (b *BGP) StartServices(ctx context.Context) error {
if b.config.EnableServicesElection {
if err := b.PerServiceLeader(ctx); err != nil {
return err
}
} else {
if err := b.ServicesNoLeader(ctx); err != nil {
return err
}
}
return nil
}
func (b *BGP) ServicesGlobalLeader(ctx context.Context, id string) {
// NOT IMPLEMENTED
}
func (b *BGP) Name() string {
return "BGP"
}

View File

@@ -1,220 +0,0 @@
package worker
import (
"context"
"fmt"
log "log/slog"
"sync"
"sync/atomic"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/metrics"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/node"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/services"
"k8s.io/client-go/kubernetes"
)
type Common struct {
arpMgr *arp.Manager
cpCluster *cluster.Cluster
intfMgr *networkinterface.Manager
config *kubevip.Config
closing *atomic.Bool
killFunc func()
svcProcessor *services.Processor
mutex *sync.Mutex
clientSet *kubernetes.Clientset
electionMgr *election.Manager
leaseMgr *lease.Manager
routeMgr *route.Manager
nodeLabelMgr node.Labeler
}
func newCommon(arpMgr *arp.Manager, intfMgr *networkinterface.Manager,
config *kubevip.Config, closing *atomic.Bool, killFunc func(),
svcProcessor *services.Processor, mutex *sync.Mutex, clientSet *kubernetes.Clientset,
electionMgr *election.Manager, leaseMgr *lease.Manager, routeMgr *route.Manager,
nodeLabelMgr node.Labeler) *Common {
return &Common{
arpMgr: arpMgr,
intfMgr: intfMgr,
config: config,
closing: closing,
killFunc: killFunc,
svcProcessor: svcProcessor,
mutex: mutex,
clientSet: clientSet,
electionMgr: electionMgr,
leaseMgr: leaseMgr,
routeMgr: routeMgr,
nodeLabelMgr: nodeLabelMgr,
}
}
func (c *Common) InitControlPlane() error {
var err error
c.cpCluster, err = cluster.InitCluster(c.config, false, c.intfMgr, c.arpMgr, c.routeMgr, c.nodeLabelMgr)
if err != nil {
return fmt.Errorf("cluster initialization error: %w", err)
}
return nil
}
func (c *Common) PerServiceLeader(ctx context.Context) error {
log.Info("beginning watching services, leaderelection will happen for every service")
err := c.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
return nil
}
func (c *Common) GlobalLeader(ctx context.Context, leaseName string) {
c.runGlobalElection(ctx, c, leaseName, c.config, c.electionMgr)
}
func (c *Common) ServicesNoLeader(ctx context.Context) error {
log.Info("beginning watching services without leader election")
err := c.svcProcessor.ServicesWatcher(ctx, services.NewCallback(c.svcProcessor.SyncServices, false))
if err != nil {
return fmt.Errorf("error while watching services: %w", err)
}
return nil
}
func (c *Common) Cleanup() {
// NOT IMPLEMENTED
}
func (c *Common) OnStartedLeading(ctx context.Context) {
err := c.svcProcessor.ServicesWatcher(ctx, services.NewCallback(c.svcProcessor.SyncServices, false))
if err != nil {
log.Error("service watcher", "err", err)
c.killFunc()
}
}
func (c *Common) OnStoppedLeading() {
// we can do cleanup here
c.mutex.Lock()
defer c.mutex.Unlock()
log.Info("leader lost", "former leader", c.config.NodeName)
c.svcProcessor.Stop()
log.Error("lost services leadership, restarting kube-vip")
c.killFunc()
}
func (c *Common) OnNewLeader(identity string) {
if identity == c.config.NodeName {
// I just got the lock
return
}
log.Info("new leader elected", "new leader", identity)
}
func (c *Common) runGlobalElection(ctx context.Context, a election.Actions, leaseName string,
config *kubevip.Config, electionManager *election.Manager) {
ns, leaseName := lease.NamespaceName(leaseName, config)
leaseID := lease.NewID(config.LeaderElectionType, ns, leaseName)
objectName := lease.ObjectName(leaseID, "svcs0")
// objLease, isNew, isSharedLease := c.leaseMgr.Add(leaseID, objectName)
objLease := c.leaseMgr.Add(ctx, leaseID)
isNew := objLease.Add(objectName)
// this service was already processed so we do not need to do anything
if !isNew {
log.Debug("this election was already done, waiting for it to finish", "lease", c.config.ServicesLeaseName)
// Wait for either the service context or lease context to be done
select {
case <-ctx.Done():
// Service was deleted
c.leaseMgr.Delete(leaseID, objectName)
case <-objLease.Ctx.Done():
// Leader election ended (leadership lost or context cancelled)
}
return
}
objLease.Lock()
defer func() {
objLease.Unlock()
}()
if objLease.Elected.Load() {
objLease.Unlock()
log.Debug("this election was already done, shared lease", "lease", leaseID.Name())
// wait for leader election to start or context to be done
select {
case <-objLease.Started:
case <-objLease.Ctx.Done():
// Lease was cancelled (e.g., leader election ended), return immediately
// This allows the restart loop to create a fresh lease
log.Debug("lease context cancelled before leader election started", "lease", leaseID.Name())
return
}
a.OnStartedLeading(objLease.Ctx)
log.Debug("waiting for lease to finish", "lease", leaseID.Name())
// wait for leaderelection to be finished
<-objLease.Ctx.Done()
// we can do cleanup here
a.OnStoppedLeading()
log.Error("lost leadership, restarting kube-vip", "lease", leaseID.Name())
c.killFunc()
return
}
// For new leases (not shared), ensure cleanup when the leader election ends
// This is critical for the restartable service watcher to be able to restart
// the leader election after leadership loss
defer func() {
// Delete the lease from the manager so subsequent calls can create a fresh lease
// This handles the case where leader election ends due to:
// 1. Leadership loss (e.g., network timeout)
// 2. Context cancellation
// 3. Any other reason RunOrDie returns
c.leaseMgr.Delete(leaseID, objectName)
}()
run := &election.RunConfig{
Config: config,
LeaseID: leaseID,
LeaseAnnotations: map[string]string{},
Mgr: electionManager,
OnStartedLeading: func(ctx context.Context) {
objLease.Elected.Store(true)
objLease.Unlock()
close(objLease.Started)
a.OnStartedLeading(ctx)
metrics.LeaderTransitionsTotal.WithLabelValues(leaseID.Name()).Inc()
metrics.IsLeader.WithLabelValues(config.NodeName, leaseID.Name()).Set(1)
},
OnStoppedLeading: func() {
objLease.Elected.Store(false)
a.OnStoppedLeading()
metrics.IsLeader.WithLabelValues(config.NodeName, leaseID.Name()).Set(0)
},
OnNewLeader: a.OnNewLeader,
}
if err := election.RunOrDie(ctx, run, config); err != nil {
log.Error("leaderelection failed", "err", err, "id", config.NodeName, "name", leaseID.Name())
}
}

View File

@@ -1,116 +0,0 @@
package worker
import (
"context"
"fmt"
log "log/slog"
"sync"
"sync/atomic"
"time"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/node"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/services"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
"k8s.io/client-go/kubernetes"
)
type Table struct {
Common
}
func NewTable(arpMgr *arp.Manager, intfMgr *networkinterface.Manager,
config *kubevip.Config, closing *atomic.Bool, killFUnc func(),
svcProcessor *services.Processor, mutex *sync.Mutex, clientSet *kubernetes.Clientset,
electionMgr *election.Manager, leaseMgr *lease.Manager, routeMgr *route.Manager,
nodeLabelMgr node.Labeler) *Table {
return &Table{
Common: *newCommon(arpMgr, intfMgr, config, closing, killFUnc,
svcProcessor, mutex, clientSet, electionMgr, leaseMgr, routeMgr,
nodeLabelMgr),
}
}
func (t *Table) Configure(ctx context.Context, wg *sync.WaitGroup) error {
log.Info("destination for routes", "table", t.config.RoutingTableID, "protocol", t.config.RoutingProtocol)
if t.config.CleanRoutingTable {
wg.Go(func() {
// we assume that after 10s all services should be configured so we can delete redundant routes
time.Sleep(time.Second * 10)
if err := t.cleanRoutes(); err != nil {
log.Error("error checking for old routes", "err", err)
}
})
}
return nil
}
func (t *Table) StartControlPlane(ctx context.Context, electionManager *election.Manager) {
if err := t.cpCluster.StartVipService(ctx, t.config, electionManager, nil, t.killFunc); err != nil {
log.Error("Control Plane", "err", err)
// Trigger the shutdown of this manager instance
t.killFunc()
} else {
log.Debug("start VipServer for cluster manager successful")
}
}
func (t *Table) ConfigureServices() {
// No configuration required
}
func (t *Table) StartServices(ctx context.Context) error {
log.Debug("starting Services")
if t.config.EnableServicesElection {
if err := t.PerServiceLeader(ctx); err != nil {
return err
}
} else if t.config.EnableLeaderElection {
t.GlobalLeader(ctx, t.config.ServicesLeaseName)
} else {
if err := t.ServicesNoLeader(ctx); err != nil {
return err
}
}
return nil
}
func (t *Table) Name() string {
return "Routing Table"
}
func (t *Table) cleanRoutes() error {
t.mutex.Lock()
defer t.mutex.Unlock()
routes, err := vip.ListRoutes(t.config.RoutingTableID, t.config.RoutingProtocol)
if err != nil {
return fmt.Errorf("error getting routes: %w", err)
}
for i := range routes {
found := false
if t.config.EnableControlPlane {
found = (routes[i].Dst.IP.String() == t.config.Address)
} else {
found = t.routeMgr.Check(routes[i].String())
}
if !found {
err = netlink.RouteDel(&(routes[i]))
if err != nil {
log.Error("[route] deletion", "route", routes[i], "err", err)
}
log.Debug("[route] deletion", "route", routes[i])
}
}
return nil
}

View File

@@ -1,278 +0,0 @@
package worker
import (
"context"
"fmt"
log "log/slog"
"os"
"sync"
"sync/atomic"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/node"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/services"
"github.com/kube-vip/kube-vip/pkg/sysctl"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/wireguard"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
)
type WireGuard struct {
Common
tunnelMgr *wireguard.TunnelManager
kubeAPIHost string
kubeAPIPort string
endpointWatcherCtx context.Context
endpointWatcherStop context.CancelFunc
endpointWatcherWg sync.WaitGroup
}
func NewWireGuard(arpMgr *arp.Manager, intfMgr *networkinterface.Manager,
config *kubevip.Config, closing *atomic.Bool, killFUnc func(),
svcProcessor *services.Processor, mutex *sync.Mutex, clientSet *kubernetes.Clientset,
electionMgr *election.Manager, leaseMgr *lease.Manager, routeMgr *route.Manager,
nodeLabelMgr node.Labeler) *WireGuard {
return &WireGuard{
Common: *newCommon(arpMgr, intfMgr, config, closing, killFUnc,
svcProcessor, mutex, clientSet, electionMgr, leaseMgr, routeMgr,
nodeLabelMgr),
}
}
func (w *WireGuard) Configure(ctx context.Context, _ *sync.WaitGroup) error {
log.Info("reading wireguard tunnel configurations from Kubernetes secret")
tunnelMgr := wireguard.NewTunnelManager()
err := tunnelMgr.LoadConfigurationsFromSecret(ctx, w.clientSet, w.config.Namespace, "wireguard")
if err != nil {
return fmt.Errorf("failed to load WireGuard tunnel configurations: %w", err)
}
// Clean up any stale resources from previous runs (crash recovery for hostNetwork: true)
// Must be called AFTER loading configs so we know which interfaces/ports to clean
if err := tunnelMgr.CleanupStaleResources(); err != nil {
log.Warn("failed to cleanup stale resources", "err", err)
// Continue anyway - the cleanup is best-effort
}
if _, err := sysctl.EnableProcSys("/proc/sys/net/ipv4/conf/all/src_valid_mark"); err != nil {
return fmt.Errorf("net.ipv4.conf.all.src_valid_mark is disabled and could not be enabled %w", err)
}
if _, err := sysctl.EnableProcSys("/proc/sys/net/ipv4/conf/all/route_localnet"); err != nil {
return fmt.Errorf("net.ipv4.conf.all.route_localnet is disabled and could not be enabled %w", err)
}
w.tunnelMgr = tunnelMgr
configuredVIPs := tunnelMgr.ListConfiguredTunnels()
log.Info("loaded WireGuard tunnel configurations", "vips", configuredVIPs)
return nil
}
func (w *WireGuard) InitControlPlane() error {
// Get Kubernetes service IP and port from environment
w.kubeAPIHost = os.Getenv("KUBERNETES_SERVICE_HOST")
w.kubeAPIPort = os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")
if w.kubeAPIHost == "" || w.kubeAPIPort == "" {
return fmt.Errorf("KUBERNETES_SERVICE_HOST or KUBERNETES_SERVICE_PORT_HTTPS not set")
}
return nil
}
func (w *WireGuard) StartControlPlane(ctx context.Context, electionManager *election.Manager) {
if !w.tunnelMgr.HasConfigForVIP(w.config.VIP) {
log.Error("no WireGuard tunnel configuration found for control plane VIP", "vip", w.config.VIP)
return
}
w.runGlobalElection(ctx, w, w.config.LeaseName, w.config, electionManager)
}
func (w *WireGuard) ConfigureServices() {
w.svcProcessor.TunnelMgr = w.tunnelMgr
}
func (w *WireGuard) StartServices(ctx context.Context) error {
if w.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err := w.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
}
return nil
}
func (w *WireGuard) Name() string {
return "WireGuard"
}
func (w *WireGuard) OnStartedLeading(ctx context.Context) {
// Bring up the WireGuard tunnel for control plane VIP
err := w.tunnelMgr.BringUpTunnelForVIP(w.config.VIP)
if err != nil {
log.Error("could not start wireguard tunnel for control plane", "vip", w.config.VIP, "err", err)
_ = w.tunnelMgr.TearDownTunnelForVIP(w.config.VIP)
w.killFunc()
return
}
// Get the tunnel to access its configuration
wg := w.tunnelMgr.GetTunnelForVIP(w.config.VIP)
if wg == nil {
log.Error("failed to get wireguard tunnel after bringing up", "vip", w.config.VIP)
w.killFunc()
return
}
tunnelConfig := w.tunnelMgr.GetConfigForVIP(w.config.VIP)
if tunnelConfig == nil {
log.Error("failed to get tunnel configuration", "vip", w.config.VIP)
_ = w.tunnelMgr.TearDownTunnelForVIP(w.config.VIP)
w.killFunc()
return
}
// Start endpoint watcher - DNAT rules will be applied when endpoints arrive
w.endpointWatcherCtx, w.endpointWatcherStop = context.WithCancel(ctx)
w.endpointWatcherWg.Go(func() {
w.watchKubernetesEndpoints(w.endpointWatcherCtx, tunnelConfig)
})
if w.config.EnableServices && !w.config.EnableServicesElection {
if err := w.svcProcessor.ServicesWatcher(ctx, services.NewCallback(w.svcProcessor.SyncServices, false)); err != nil {
log.Error("failed to start services watcher", "err", err)
}
}
}
// watchKubernetesEndpoints watches the kubernetes service EndpointSlices for changes
// and updates the DNAT rules when API server endpoints change (e.g., when an API server goes down)
func (w *WireGuard) watchKubernetesEndpoints(ctx context.Context, tunnelConfig *wireguard.TunnelConfig) {
log.Info("starting kubernetes endpoint watcher for control plane")
kubeSvc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "kubernetes",
Namespace: "default",
},
}
provider := providers.NewEndpointslices()
rw, err := provider.CreateRetryWatcher(ctx, w.clientSet, kubeSvc)
if err != nil {
log.Error("failed to create kubernetes endpoint watcher", "err", err)
return
}
defer rw.Stop()
for event := range rw.ResultChan() {
select {
case <-ctx.Done():
log.Info("kubernetes endpoint watcher stopped")
return
default:
}
switch event.Type {
case watch.Added, watch.Modified, watch.Deleted:
if err := provider.LoadObject(event.Object, func() {}); err != nil {
log.Error("failed to load endpoint object", "err", err)
continue
}
endpoints, _ := provider.GetAllEndpoints()
log.Info("kubernetes endpoints changed, updating DNAT rules", "eventType", event.Type, "endpoints", endpoints)
if err := w.updateControlPlaneDNAT(tunnelConfig, endpoints); err != nil {
log.Error("failed to update control plane DNAT rules", "err", err)
}
case watch.Error:
log.Warn("kubernetes endpoint watch error", "event", event)
}
}
}
// updateControlPlaneDNAT updates the DNAT rules for the control plane with the given endpoints
func (w *WireGuard) updateControlPlaneDNAT(tunnelConfig *wireguard.TunnelConfig, endpoints []string) error {
if len(endpoints) == 0 {
log.Warn("no kubernetes API server endpoints available")
// Don't delete rules - keep routing to last known endpoints
return nil
}
// Build targets with default port 6443
targets := make([]nftables.DNATTarget, len(endpoints))
for i, ep := range endpoints {
targets[i] = nftables.DNATTarget{IP: ep, Port: 6443}
}
vipIP := utils.StripCIDR(w.config.VIP)
err := nftables.ApplyDNAT(
tunnelConfig.InterfaceName,
vipIP,
6443,
targets,
"controlplane",
v1.ProtocolTCP,
false,
tunnelConfig.ListenPort,
)
if err != nil {
return fmt.Errorf("failed to apply updated DNAT rule: %w", err)
}
log.Info("control plane DNAT rules updated", "targetCount", len(targets))
return nil
}
func (w *WireGuard) OnStoppedLeading() {
// we can do cleanup here
w.mutex.Lock()
defer w.mutex.Unlock()
log.Info("leader lost", "id", w.config.NodeName)
// Stop the kubernetes endpoint watcher and wait for it to finish
if w.endpointWatcherStop != nil {
w.endpointWatcherStop()
w.endpointWatcherWg.Wait()
}
log.Info("deleting nftables DNAT chains")
err := nftables.DeleteIngressChains(false, "controlplane")
if err != nil {
log.Error("could not delete DNAT ingress chains", "err", err)
} else {
log.Info("nftables DNAT chains deleted successfully")
}
// Tear down all tunnels (control plane + services)
err = w.tunnelMgr.TearDownAllTunnels()
if err != nil {
log.Error("failed to tear down tunnels", "err", err)
}
if w.config.EnableServices && !w.config.EnableServicesElection {
w.svcProcessor.Stop()
}
log.Error("lost control plane leadership, restarting kube-vip")
w.killFunc()
}
func (w *WireGuard) OnNewLeader(identity string) {
// we're notified when new leader elected
if identity == w.config.NodeName {
// I just got the lock
return
}
// safety check - tear down tunnel if we're not the leader
_ = w.tunnelMgr.TearDownTunnelForVIP(w.config.VIP)
log.Info("new leader elected", "id", identity)
}

View File

@@ -1,60 +0,0 @@
package worker
import (
"context"
"sync"
"sync/atomic"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/node"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/services"
"k8s.io/client-go/kubernetes"
)
type Worker interface {
Configure(context.Context, *sync.WaitGroup) error
InitControlPlane() error
StartControlPlane(context.Context, *election.Manager)
ConfigureServices()
StartServices(ctx context.Context) error
Name() string
Cleanup()
}
func New(arpMgr *arp.Manager, intfMgr *networkinterface.Manager,
config *kubevip.Config, closing *atomic.Bool, killFunc func(),
svcProcessor *services.Processor, mutex *sync.Mutex, clientSet *kubernetes.Clientset,
bgpServer *bgp.Server, electionMgr *election.Manager, leaseMgr *lease.Manager,
routeMgr *route.Manager, nodeLabelMgr node.Labeler) Worker {
if config.EnableARP {
return NewARP(arpMgr, intfMgr, config, closing, killFunc,
svcProcessor, mutex, clientSet, electionMgr, leaseMgr, routeMgr,
nodeLabelMgr)
}
if config.EnableBGP {
return NewBGP(arpMgr, intfMgr, config, closing, killFunc,
svcProcessor, mutex, clientSet, bgpServer,
electionMgr, leaseMgr, routeMgr, nodeLabelMgr)
}
if config.EnableRoutingTable {
return NewTable(arpMgr, intfMgr, config, closing, killFunc,
svcProcessor, mutex, clientSet, electionMgr, leaseMgr, routeMgr,
nodeLabelMgr)
}
if config.EnableWireguard {
return NewWireGuard(arpMgr, intfMgr, config, closing, killFunc,
svcProcessor, mutex, clientSet, electionMgr, leaseMgr, routeMgr,
nodeLabelMgr)
}
return nil
}

View File

@@ -1,68 +0,0 @@
package metrics
import "github.com/prometheus/client_golang/prometheus"
var (
// Service / VIP Lifecycle
ActiveServices = prometheus.NewGaugeVec(
prometheus.GaugeOpts{Name: "kube_vip_active_services", Help: "How many LB services are currently managed"},
[]string{"namespace"},
)
ServiceReconcileErrorsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{Name: "kube_vip_service_reconcile_errors_total", Help: "Count reconcile failures"},
[]string{"namespace", "name", "reason"},
)
ServiceReconcileDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{Name: "kube_vip_service_reconcile_duration_seconds", Help: "How long AddOrModify takes end-to-end"},
[]string{"namespace"},
)
// This is a prometheus counter used to count the number of events received
// from the service watcher
CountServiceWatchEvent = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "kube_vip",
Subsystem: "manager",
Name: "all_services_events",
Help: "Count all events fired by the service watcher categorised by event type",
}, []string{"type"})
// Leader Election
LeaderTransitionsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{Name: "kube_vip_leader_election_transitions_total", Help: "Frequent transitions mean instability"},
[]string{"lease_name"},
)
IsLeader = prometheus.NewGaugeVec(
prometheus.GaugeOpts{Name: "kube_vip_is_leader", Help: "1 if this node currently holds the lease"},
[]string{"node", "lease_name"},
)
// This is a prometheus gauge indicating the state of the sessions.
// 1 means "ESTABLISHED", 0 means "NOT ESTABLISHED"
BGPSessionInfoGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "kube_vip",
Subsystem: "manager",
Name: "bgp_session_info",
Help: "Display state of session by setting metric for label value with current state to 1",
}, []string{"state", "peer"},
)
// General Health
BuildInfo = prometheus.NewGaugeVec(
prometheus.GaugeOpts{Name: "kube_vip_build_info", Help: "Constant 1; track version skew across nodes"},
[]string{"version", "build", "node"},
)
)
func RegisterPrometheusMetrics() {
// Register all metrics with Prometheus
prometheus.MustRegister(
ActiveServices,
ServiceReconcileErrorsTotal,
ServiceReconcileDuration,
LeaderTransitionsTotal,
IsLeader,
BGPSessionInfoGauge,
BuildInfo,
CountServiceWatchEvent,
)
}

View File

@@ -1,7 +1,6 @@
package networkinterface
import (
log "log/slog"
"sync"
"github.com/vishvananda/netlink"
@@ -24,12 +23,6 @@ func NewManager() *Manager {
func (m *Manager) Get(intf netlink.Link) *Link {
if l, ok := m.interfaces[intf.Attrs().Name]; ok {
updated, err := netlink.LinkByName(l.Intf.Attrs().Name)
if err != nil {
log.Error("failed to get interface %q: %w", l.Intf.Attrs().Name, err)
return nil
}
l.Intf = updated
return l
}
result := &Link{

File diff suppressed because it is too large Load Diff

View File

@@ -1,136 +0,0 @@
package labeler
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/pkg/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
)
// labelOperation is the operation to perform on the node labels
type labelOperation string
// labelOperation constants
const (
labelOperationRemove labelOperation = "remove"
labelOperationAdd labelOperation = "add"
)
// NewManager creates a new Label Manager for the given node
func NewManager(nodeName string, clientSet *kubernetes.Clientset) *Manager {
return &Manager{
nodeName: nodeName,
clientSet: clientSet,
}
}
// Manager is the label Manager for the node
type Manager struct {
// nodeName is the name of the node to manage
nodeName string
// clientSet is the Kubernetes client set to use
clientSet *kubernetes.Clientset
}
// AddLabel a new label to the node
func (m *Manager) AddLabel(labels map[string]string) error {
log.Debug("add labels to node", "node", m.nodeName)
return m.patchNode(labelOperationAdd, labels)
}
// RemoveLabel a label from the node
func (m *Manager) RemoveLabel(labels map[string]string) error {
log.Debug("delete label from node")
return m.patchNode(labelOperationRemove, labels)
}
// CleanUpLabels purges the node labels
func (m *Manager) CleanUpLabels(timeout time.Duration) error {
log.Debug("cleaning up labels for node", "node", m.nodeName, "timeout", timeout)
// create new context for labels cleanup (independent)
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
// get the node
node, err := m.clientSet.CoreV1().Nodes().Get(ctx, m.nodeName, metav1.GetOptions{})
if err != nil {
return errors.Wrapf(err, "failed to get node %s", m.nodeName)
}
// collect all labels with the prefix to remove
labels := map[string]string{}
for _, k := range kubevip.GetKeysForCleanup() {
if _, ok := node.Labels[k]; ok {
labels[k] = ""
}
}
if len(labels) == 0 {
log.Debug("no labels to remove for node", "node", m.nodeName)
return nil
}
// patch the node with the labels to remove
return m.patchNode(labelOperationRemove, labels)
}
// patchNode patches the node with the given labels
func (m *Manager) patchNode(operation labelOperation, labels map[string]string) error {
type patchStringLabel struct {
Op string `json:"op"`
Path string `json:"path"`
Value string `json:"value"`
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
patchLabels := []patchStringLabel{}
// generate the patch
for k, v := range labels {
patchLabels = append(patchLabels, patchStringLabel{
Op: string(operation),
// replace all slashes with ~1
Path: fmt.Sprintf("/metadata/labels/%s", strings.ReplaceAll(k, "/", "~1")),
Value: v,
})
}
patchData, err := json.Marshal(patchLabels)
if err != nil {
log.Debug("node patch marshaling failed", "err", err, "labels", labels, "patch", patchLabels)
return errors.Wrapf(err, "node patch marshaling failed for labels %v", labels)
}
log.Debug("patching node",
"node", m.nodeName,
"patch", string(patchData),
"operation", operation,
"labels", labels,
"clientSetNil", m.clientSet == nil)
if m.clientSet == nil {
return errors.New("kubernetes client is not initialized")
}
// patch node
node, err := m.clientSet.CoreV1().Nodes().Patch(ctx, m.nodeName, types.JSONPatchType, patchData, metav1.PatchOptions{})
if err != nil {
log.Debug("node patching failed", "err", err, "patchData", patchData)
return errors.Wrapf(err, "node patching failed with patch %s", string(patchData))
}
log.Debug("updated", "node", m.nodeName, "labels", node.Labels)
return nil
}

View File

@@ -1,24 +0,0 @@
package noop
import (
"time"
)
// NewManager creates a new NoOp label manager
func NewManager() *Manager {
return &Manager{}
}
type Manager struct{}
func (m *Manager) AddLabel(labels map[string]string) error {
return nil
}
func (m *Manager) RemoveLabel(labels map[string]string) error {
return nil
}
func (m *Manager) CleanUpLabels(_ time.Duration) error {
return nil
}

View File

@@ -1,49 +0,0 @@
package node
import (
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/node/labeler"
"github.com/kube-vip/kube-vip/pkg/node/noop"
"k8s.io/client-go/kubernetes"
)
// LabelManager is the interface for the node label manager
type LabelManager interface {
Labeler
LabelCleaner
}
type Labeler interface {
// AddLabel adds a label to the node
AddLabel(labels map[string]string) error
// RemoveLabel removes the label from the node
RemoveLabel(labels map[string]string) error
}
type LabelCleaner interface {
// CleanUpLabels removes all labels from the node
CleanUpLabels(timeout time.Duration) error
}
// NewManager creates a new Label Manager for the given node
// NoOp implementation is returned if node labeling is disabled,
// or if the client is not ready
func NewManager(config *kubevip.Config, clientSet *kubernetes.Clientset) LabelManager {
if !config.EnableNodeLabeling {
return noop.NewManager()
}
if clientSet == nil {
log.Debug("Skip node labeling, client is not ready")
return noop.NewManager()
}
log.Debug("Node labeling enabled")
return labeler.NewManager(config.NodeName, clientSet)
}

View File

@@ -1,126 +0,0 @@
package route
import (
"errors"
"fmt"
log "log/slog"
"sync"
"syscall"
)
type Manager struct {
tracker map[string]*item
mtx sync.Mutex
}
func NewManager() *Manager {
return &Manager{
tracker: make(map[string]*item),
}
}
type item struct {
objects map[string]bool
route route
}
func newItem(r route) *item {
return &item{
objects: make(map[string]bool),
route: r,
}
}
type route interface {
AddRoute(bool) (bool, error)
UpdateRoutes() (bool, error)
DeleteRoute() error
RouteHash() string
Interface() string
}
// Add will add route
func (m *Manager) Add(object string, r route, precheck, update bool) error {
m.mtx.Lock()
defer m.mtx.Unlock()
key := r.RouteHash()
itm, exists := m.tracker[key]
if !exists {
m.tracker[key] = newItem(r)
itm = m.tracker[key]
added, err := r.AddRoute(precheck)
if err != nil {
if update && errors.Is(err, syscall.EEXIST) && update {
// If route exists, but protocol is not set (e.g. the route was created by the older version
// of kube-vip) try to update it if necessary
isUpdated, err := r.UpdateRoutes()
if err != nil {
return fmt.Errorf("error updating existing route: %w", err)
}
if isUpdated {
log.Debug("[RT] updated route", "path", key, "object", object, "interface", r.Interface())
}
} else {
// If other error occurs, return error
return fmt.Errorf("error adding route %q: %w", key, err)
}
}
if added {
log.Debug("[RT] added route", "path", key, "object", object)
}
}
itm.objects[object] = true
log.Debug("[RT] incremented route", "path", key, "object", object, "cnt", len(itm.objects))
return nil
}
// Delete will delete route
func (m *Manager) Delete(object string, r route) error {
m.mtx.Lock()
defer m.mtx.Unlock()
key := r.RouteHash()
itm, exists := m.tracker[key]
if !exists {
log.Debug("[RT] deleting route - nothing to delete", "path", key, "object", object)
return nil
}
delete(itm.objects, object)
log.Debug("[RT] decremented route", "path", key, "object", object, "cnt", len(itm.objects))
if len(itm.objects) == 0 {
if err := r.DeleteRoute(); err != nil {
itm.objects[object] = true
return fmt.Errorf("failed to delete route: %w", err)
}
delete(m.tracker, key)
log.Debug("[RT] deleted route", "path", key, "object", object)
}
return nil
}
func (m *Manager) Clear() {
for _, itm := range m.tracker {
if err := itm.route.DeleteRoute(); err != nil {
log.Warn("[RT] failed to delete route", "err", err.Error())
}
}
m.tracker = make(map[string]*item)
}
func (m *Manager) Check(key string) bool {
_, exists := m.tracker[key]
return exists
}

View File

@@ -1,215 +0,0 @@
package route
import (
"fmt"
"testing"
)
func Test_SingleAddDel(t *testing.T) {
t.Run("Add and delete single route", func(_ *testing.T) {
m := NewManager()
if m == nil { // nolint
t.Error("failed to create manager")
}
r := &mockRoute{hash: "123456"}
name := "test-service"
if err := m.Add(name, r, false, false); err != nil {
t.Errorf("add failed for route %q, object %q", r.RouteHash(), name)
}
expected := 1
if len(m.tracker) != expected { //nolint
t.Errorf("number of items in tracker should be %d, but is %d", expected, len(m.tracker))
}
item, exists := m.tracker[r.RouteHash()]
if !exists {
t.Errorf("object %q was not added to tracker for service %q", r.RouteHash(), name)
}
if len(item.objects) != 1 {
t.Errorf("number of objects for %q was %d instead of 1", name, len(item.objects))
}
if err := m.Delete(name, r); err != nil {
t.Errorf("deletion of object %q failed for service %q", r.RouteHash(), name)
}
expected = 0
if len(m.tracker) != expected {
t.Errorf("number of items in tracker should be %d, but is %d", expected, len(m.tracker))
}
})
}
func Test_MultipleAddDel(t *testing.T) {
t.Run("Add and delete same route multiple times", func(_ *testing.T) {
m := NewManager()
if m == nil {
t.Error("failed to create manager")
}
r := &mockRoute{hash: "123456"}
numOfServices := 50
expectedRoutes := 1
for i := range numOfServices {
name := fmt.Sprintf("test-service-%d", i)
if err := m.Add(name, r, false, false); err != nil {
t.Errorf("add failed for route %q, object %q", r.RouteHash(), name)
}
if len(m.tracker) != expectedRoutes {
t.Errorf("number of routes in tracker should be %d, but is %d", expectedRoutes, len(m.tracker))
}
item, exists := m.tracker[r.RouteHash()]
if !exists {
t.Errorf("route %q was not added to the tracker", r.RouteHash())
}
expected := i + 1
if len(item.objects) != expected {
t.Errorf("number of objects for %q was %d instead of 1", name, len(item.objects))
}
}
for i := range numOfServices {
name := fmt.Sprintf("test-service-%d", i)
if err := m.Delete(name, r); err != nil {
t.Errorf("delete failed for route %q, object %q", r.RouteHash(), name)
}
if i == numOfServices-1 {
expectedRoutes = 0
}
if len(m.tracker) != expectedRoutes {
t.Errorf("number of routes in tracker should be %d, but is %d", expectedRoutes, len(m.tracker))
}
exists := m.Check(r.RouteHash())
if i < numOfServices-1 {
if !exists {
t.Errorf("route %q was not added to the tracker", r.RouteHash())
}
} else {
if exists {
t.Errorf("route %q was not deleted from the tracker", r.RouteHash())
}
}
}
})
}
func Test_MultipleRoutesAddDel(t *testing.T) {
t.Run("Add and delete multiple routes multiple times", func(_ *testing.T) {
m := NewManager()
if m == nil {
t.Error("failed to create manager")
}
routes := []*mockRoute{}
numOfRoutes := 20
hashBase := 123456
for i := range numOfRoutes {
routes = append(routes, &mockRoute{hash: fmt.Sprintf("%d%d", hashBase, i)})
}
numOfServices := 50
for rtCnt, r := range routes {
for i := range numOfServices {
expectedRoutes := rtCnt + 1
name := fmt.Sprintf("test-service-%d", i)
if err := m.Add(name, r, false, false); err != nil {
t.Errorf("add failed for route %q, object %q", r.RouteHash(), name)
}
if len(m.tracker) != expectedRoutes {
t.Errorf("number of routes in tracker should be %d, but is %d", expectedRoutes, len(m.tracker))
}
item, exists := m.tracker[r.RouteHash()]
if !exists {
t.Errorf("route %q was not added to the tracker", r.RouteHash())
}
expected := i + 1
if len(item.objects) != expected {
t.Errorf("number of objects for %q was %d instead of 1", name, len(item.objects))
}
}
}
for rtCnt, r := range routes {
for i := range numOfServices {
name := fmt.Sprintf("test-service-%d", i)
if err := m.Delete(name, r); err != nil {
t.Errorf("delete failed for route %q, object %q", r.RouteHash(), name)
}
expectedRoutes := numOfRoutes - rtCnt
if i == numOfServices-1 {
expectedRoutes -= 1
}
if len(m.tracker) != expectedRoutes {
t.Errorf("number of routes in tracker should be %d, but is %d", expectedRoutes, len(m.tracker))
}
exists := m.Check(r.RouteHash())
if i < numOfServices-1 {
if !exists {
t.Errorf("route %q should not be deleted", r.RouteHash())
}
} else {
if exists {
t.Errorf("route %q was not deleted from the tracker", r.RouteHash())
}
}
}
}
})
}
type mockRoute struct {
added bool
addErr error
updated bool
updateErr error
delErr error
hash string
intf string
}
func (mr *mockRoute) AddRoute(_ bool) (bool, error) {
return mr.added, mr.addErr
}
func (mr *mockRoute) UpdateRoutes() (bool, error) {
return mr.updated, mr.updateErr
}
func (mr *mockRoute) DeleteRoute() error {
return mr.delErr
}
func (mr *mockRoute) RouteHash() string {
return mr.hash
}
func (mr *mockRoute) Interface() string {
return mr.intf
}

View File

@@ -3,27 +3,21 @@ package servicecontext
import (
"context"
"sync"
"sync/atomic"
)
type Context struct {
Ctx context.Context
Cancel context.CancelFunc
IsActive bool
IsWatched bool
ConfiguredNetworks sync.Map
EndpointsReady chan any
epReady sync.Once
Signalled atomic.Bool
LeaderCancel context.CancelFunc
}
func New(ctx context.Context) *Context {
// context and cancel stored for a future use, gosec linter disabled
svcCtx, svcCancel := context.WithCancel(ctx) //nolint:gosec
svcCtx, svcCancel := context.WithCancel(ctx)
return &Context{
Ctx: svcCtx,
Cancel: svcCancel,
EndpointsReady: make(chan any),
Ctx: svcCtx,
Cancel: svcCancel,
}
}
@@ -40,18 +34,3 @@ func (ctx *Context) IsNetworkConfigured(ip string) bool {
_, exists := ctx.ConfiguredNetworks.Load(ip)
return exists
}
func (ctx *Context) SignalReadiness() {
ctx.epReady.Do(func() {
close(ctx.EndpointsReady)
ctx.Signalled.Store(true)
})
}
func (ctx *Context) ResetReadiness() {
if ctx.Signalled.Load() {
ctx.EndpointsReady = make(chan any)
ctx.epReady = sync.Once{}
ctx.Signalled.Store(false)
}
}

View File

@@ -1,24 +0,0 @@
package services
import (
"sync"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
)
type Callback struct {
Function func(*servicecontext.Context, *v1.Service, *sync.WaitGroup, bool) error
UsesLeaderElection bool
}
func NewCallback(f func(*servicecontext.Context, *v1.Service, *sync.WaitGroup, bool) error, leaderElection bool) *Callback {
return &Callback{
Function: f,
UsesLeaderElection: leaderElection,
}
}
func (c *Callback) Run(svcCtx *servicecontext.Context, svc *v1.Service, wg *sync.WaitGroup) error {
return c.Function(svcCtx, svc, wg, c.UsesLeaderElection)
}

36
pkg/services/context.go Normal file
View File

@@ -0,0 +1,36 @@
package services
import (
"context"
"sync"
)
type Context struct {
Ctx context.Context
Cancel context.CancelFunc
IsActive bool
IsWatched bool
ConfiguredNetworks sync.Map
}
func NewContext(ctx context.Context) *Context {
svcCtx, svcCancel := context.WithCancel(ctx)
return &Context{
Ctx: svcCtx,
Cancel: svcCancel,
}
}
func (ctx *Context) HasConfiguredNetworks() bool {
cnt := 0
ctx.ConfiguredNetworks.Range(func(_ any, _ any) bool {
cnt++
return cnt < 1
})
return cnt > 0
}
func (ctx *Context) IsNetworkConfigured(ip string) bool {
_, exists := ctx.ConfiguredNetworks.Load(ip)
return exists
}

View File

@@ -13,7 +13,6 @@ import (
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -78,12 +77,12 @@ func (p *Processor) nftablesCheck() error {
func getSameFamilyCidr(sourceCidrs, ip string) string { //Todo: not sure how this ever worked
cidrs := strings.Split(sourceCidrs, ",")
isV6 := utils.IsIPv6(ip)
isV6 := vip.IsIPv6(ip)
matchingFamily := []string{}
for _, cidr := range cidrs {
// Is the ip an IPv6 address
if isV6 {
if utils.IsIPv6CIDR(cidr) {
if vip.IsIPv6CIDR(cidr) {
matchingFamily = append(matchingFamily, cidr)
selectedCIDR, err := checkCIDR(ip, cidr)
if err != nil {
@@ -95,7 +94,7 @@ func getSameFamilyCidr(sourceCidrs, ip string) string { //Todo: not sure how thi
}
}
} else {
if utils.IsIPv4CIDR(cidr) {
if vip.IsIPv4CIDR(cidr) {
matchingFamily = append(matchingFamily, cidr)
selectedCidr, err := checkCIDR(ip, cidr)
if err != nil {
@@ -135,7 +134,7 @@ func checkCIDR(ip, cidr string) (string, error) {
return "", nil
}
func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace, serviceUUID string, annotations map[string]string) error {
func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string, annotations map[string]string) error {
var podCidr, serviceCidr string
var autoServiceCIDR, autoPodCIDR string
var discoverErr error
@@ -147,7 +146,7 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
internalEgress := annotations[kubevip.EgressInternal]
if p.config.EgressPodCidr == "" || p.config.EgressServiceCidr == "" {
autoServiceCIDR, autoPodCIDR, discoverErr = p.AutoDiscoverCIDRs(ctx)
autoServiceCIDR, autoPodCIDR, discoverErr = p.AutoDiscoverCIDRs()
}
if discoverErr != nil {
@@ -164,7 +163,7 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
if podCidr == "" {
// There's no default IPv6 pod CIDR, therefore we silently back off if CIDR s not specified.
if !utils.IsIPv4(podIP) {
if !vip.IsIPv4(podIP) {
return fmt.Errorf("error with the CIDR [%s]", podIP)
}
podCidr = defaultPodCIDR
@@ -180,7 +179,7 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
if serviceCidr == "" {
// There's no default IPv6 service CIDR, therefore we silently back off if CIDR s not specified.
if !utils.IsIPv4(vipIP) {
if !vip.IsIPv4(vipIP) {
return nil
}
serviceCidr = defaultServiceCIDR
@@ -189,23 +188,23 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
log.Info("[Egress]", "podCIDR", podCidr, "serviceCIDR", serviceCidr, "vip", serviceCidr, "pod", podIP)
// checking if all addresses are of the same IP family
if utils.IsIPv4(podIP) != utils.IsIPv4CIDR(podCidr) {
if vip.IsIPv4(podIP) != vip.IsIPv4CIDR(podCidr) {
log.Error("[Egress] family is not matching. Backing off...", "pod", podIP, "podCIDR", podCidr)
return nil
}
if utils.IsIPv4(vipIP) != utils.IsIPv4CIDR(serviceCidr) {
if vip.IsIPv4(vipIP) != vip.IsIPv4CIDR(serviceCidr) {
log.Error("[Egress] family is not matching. Backing off...", "pod", podIP, "serviceCIDR", serviceCidr)
return nil
}
if utils.IsIPv4(vipIP) != utils.IsIPv4(podIP) {
if vip.IsIPv4(vipIP) != vip.IsIPv4(podIP) {
log.Error("[Egress] family is not matching. Backing off...", "pod", podIP, "vipIP", vipIP)
return nil
}
protocol := iptables.ProtocolIPv4
if utils.IsIPv6(vipIP) {
if vip.IsIPv6(vipIP) {
protocol = iptables.ProtocolIPv6
}
@@ -224,15 +223,8 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
}
// Add networks that we specifically should only SNAT for
allowCIDRs := []string{}
if allowedNetworks != "" {
networks := strings.Split(strings.TrimSpace(allowedNetworks), ",") //Remove whitespace characters and then create an array from the CIDRs
allowCIDRs = append(allowCIDRs, networks...)
}
// Apply the SNAT rules
err := nftables.ApplySNAT(podIP, vipIP, serviceUUID, destinationPorts, ignoreCIDRs, allowCIDRs, utils.IsIPv6(vipIP))
err := nftables.ApplySNAT(podIP, vipIP, serviceUUID, destinationPorts, ignoreCIDRs, vip.IsIPv6(vipIP))
if err != nil {
return fmt.Errorf("error performing netlink nftables [%s]", err)
}
@@ -276,9 +268,9 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
}
}
mask := fmt.Sprintf("/%d", vip.DefaultMaskIPv4)
if !utils.IsIPv4(podIP) {
mask = fmt.Sprintf("/%d", vip.DefaultMaskIPv6)
mask := "/32"
if !vip.IsIPv4(podIP) {
mask = "/128"
}
if allowedNetworks != "" {
@@ -338,12 +330,12 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
return nil
}
func (p *Processor) AutoDiscoverCIDRs(ctx context.Context) (serviceCIDR, podCIDR string, err error) {
func (p *Processor) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error) {
log.Debug("Trying to automatically discover Service and Pod CIDRs")
options := v1.ListOptions{
LabelSelector: "component=kube-controller-manager",
}
podList, err := p.clientSet.CoreV1().Pods("kube-system").List(ctx, options)
podList, err := p.clientSet.CoreV1().Pods("kube-system").List(context.TODO(), options)
if err != nil {
return "", "", fmt.Errorf("[Egress] Unable to get kube-controller-manager pod: %w", err)
}

View File

@@ -4,25 +4,38 @@ import (
"context"
"fmt"
"sync"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/metrics"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
)
var (
svcLocks map[string]*sync.Mutex
)
func init() {
svcLocks = make(map[string]*sync.Mutex)
}
// The StartServicesWatchForLeaderElection function will start a services watcher, the
func (p *Processor) StartServicesWatchForLeaderElection(ctx context.Context) error {
err := p.ServicesWatcher(ctx, NewCallback(p.StartServicesLeaderElection, true))
err := p.ServicesWatcher(ctx, p.StartServicesLeaderElection)
if err != nil {
return err
}
if p.config.EnableRoutingTable {
p.routeMgr.Clear()
for _, instance := range p.ServiceInstances {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
_ = cluster.Network[i].DeleteRoute()
}
cluster.Stop()
}
}
log.Info("Shutting down kube-Vip")
@@ -31,159 +44,84 @@ func (p *Processor) StartServicesWatchForLeaderElection(ctx context.Context) err
}
// The startServicesWatchForLeaderElection function will start a services watcher, the
func (p *Processor) StartServicesLeaderElection(svcCtx *servicecontext.Context, service *v1.Service, _ *sync.WaitGroup, _ bool) error {
if svcCtx == nil {
return fmt.Errorf("no context context for service %q with UID %q: nil context", service.Name, service.UID)
}
leaseNamespace, serviceLease := lease.ServiceName(service)
id := lease.NewID(p.config.LeaderElectionType, leaseNamespace, serviceLease)
objectName := lease.ServiceNamespacedName(service)
svcLease := p.leaseMgr.Get(id)
if svcLease == nil {
return fmt.Errorf("no existing lease found for service %q with UID %q", service.Name, service.UID)
}
isNew := svcLease.Add(objectName)
svcLease.Lock()
defer func() {
svcLease.Unlock()
}()
// this service was already processed so we do not need to do anything
if !isNew && svcLease.Elected.Load() {
svcLease.Unlock()
log.Debug("this service was already handled, waiting for it to finish", "service", service.Name, "uid", service.UID)
// Wait for either the service context or lease context to be done
select {
case <-svcCtx.Ctx.Done():
case <-svcLease.Ctx.Done():
}
return nil
}
wg := sync.WaitGroup{}
defer wg.Wait()
// Start a goroutine that will delete the lease when the service context is cancelled.
// This is important for proper cleanup when a service is deleted - it ensures that
// the lease context (svcLease.Ctx) gets cancelled, which causes RunOrDie to return.
// Without this, RunOrDie would continue running until leadership is naturally lost.
wg.Go(func() {
<-svcCtx.Ctx.Done()
p.leaseMgr.Delete(id, objectName)
})
select {
case <-svcCtx.Ctx.Done():
return fmt.Errorf("service context cancelled before election start: %w", svcCtx.Ctx.Err())
case <-svcLease.Ctx.Done():
return fmt.Errorf("lease context cancelled before election start: %w", svcLease.Ctx.Err())
case <-svcCtx.EndpointsReady:
}
// this service is sharing lease with another service
if svcLease.Elected.Load() {
svcLease.Unlock()
// wait for leader election to start or context to be done
select {
case <-svcLease.Started:
case <-svcLease.Ctx.Done():
// Lease was cancelled (e.g., leader election ended), return immediately
// This allows the restart loop to create a fresh lease
log.Debug("lease context cancelled before leader election started", "service", service.Name, "uid", service.UID)
return nil
}
if err := p.onStartedLeading(svcCtx, service, &wg); err != nil {
log.Error("error on started leading", "error", err)
}
// Block until service context is cancelled
<-svcCtx.Ctx.Done()
if err := p.onStoppedLeading(svcLease, service); err != nil {
log.Error("error on stopped leading", "error", err)
}
// wait for leaderelection to be finished
<-svcLease.Ctx.Done()
return nil
}
func (p *Processor) StartServicesLeaderElection(ctx context.Context, service *v1.Service) error {
serviceLease := fmt.Sprintf("kubevip-%s", service.Name)
log.Info("new leader election", "service", service.Name, "namespace", service.Namespace, "lock_name", serviceLease, "host_id", p.config.NodeName)
leaderCtx, leaderCancel := context.WithCancel(svcLease.Ctx)
svcCtx.LeaderCancel = leaderCancel
run := election.RunConfig{
Config: p.config,
LeaseID: id,
Mgr: p.electionMgr,
LeaseAnnotations: map[string]string{},
OnStartedLeading: func(_ context.Context) {
svcLease.Elected.Store(true)
svcLease.Unlock()
close(svcLease.Started)
// Mark this service as active (as we've started leading)
// we run this in background as it's blocking
if err := p.onStartedLeading(svcCtx, service, &wg); err != nil {
leaderCancel()
}
metrics.LeaderTransitionsTotal.WithLabelValues(id.Name()).Inc()
metrics.IsLeader.WithLabelValues(p.config.NodeName, id.Name()).Set(1)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: serviceLease,
Namespace: service.Namespace,
},
OnStoppedLeading: func() {
// we can do cleanup here
svcLease.Elected.Store(false)
log.Info("leadership lost", "service", service.Name, "uid", service.UID, "leader", p.config.NodeName)
if err := p.onStoppedLeading(svcLease, service); err != nil {
metrics.ServiceReconcileErrorsTotal.WithLabelValues(service.Namespace, service.Name, "delete_service").Inc()
leaderCancel()
}
metrics.IsLeader.WithLabelValues(p.config.NodeName, id.Name()).Set(0)
svcLease.Started = make(chan any)
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if identity == p.config.NodeName {
// I just got the lock
return
}
log.Info("new leader", "leader", identity, "service", service.Name, "uid", service.UID)
Client: p.clientSet.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: p.config.NodeName,
},
}
childCtx, childCancel := context.WithCancel(ctx)
defer childCancel()
if err := election.RunOrDie(leaderCtx, &run, p.config); err != nil {
return fmt.Errorf("services election failed: %w", err)
if _, ok := svcLocks[serviceLease]; !ok {
svcLocks[serviceLease] = new(sync.Mutex)
}
log.Info("stopping leader election", "service", service.Name, "uid", service.UID)
return nil
}
svcLocks[serviceLease].Lock()
defer svcLocks[serviceLease].Unlock()
func (p *Processor) onStartedLeading(svcCtx *servicecontext.Context, service *v1.Service, wg *sync.WaitGroup) error {
// Mark this service as active (as we've started leading)
// we run this in background as it's blocking
err := p.SyncServices(svcCtx, service, wg, true)
svcCtx, err := p.getServiceContext(service.UID)
if err != nil {
log.Error("service sync", "uid", service.UID, "err", err)
return err
return fmt.Errorf("failed to get context for service %q with UID %q: %w", service.Name, service.UID, err)
}
if svcCtx == nil {
return fmt.Errorf("failed to get context for service %q with UID %q: nil context", service.Name, service.UID)
}
return nil
}
func (p *Processor) onStoppedLeading(svcLease *lease.Lease, service *v1.Service) error {
log.Debug("deleting service due to lost leadership", "uid", service.UID)
err := p.deleteService(svcLease.Ctx, service.UID)
if err != nil {
log.Error("service deletion", "err", err)
return err
}
svcCtx.IsActive = true
// start the leader election code loop
leaderelection.RunOrDie(childCtx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(p.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(p.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(p.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
// Mark this service as active (as we've started leading)
// we run this in background as it's blocking
if err := p.SyncServices(ctx, service); err != nil {
log.Error("service sync", "err", err)
childCancel()
}
},
OnStoppedLeading: func() {
// we can do cleanup here
log.Info("leadership lost", "service", service.Name, "leader", p.config.NodeName)
if svcCtx.IsActive {
if err := p.deleteService(service.UID); err != nil {
log.Error("service deletion", "err", err)
}
}
// Mark this service is inactive
svcCtx.IsActive = false
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if identity == p.config.NodeName {
// I just got the lock
return
}
log.Info("new leader", "leader", identity)
},
},
})
log.Info("stopping leader election", "service", service.Name)
return nil
}

View File

@@ -2,32 +2,23 @@ package services
import (
"context"
"errors"
"fmt"
log "log/slog"
"reflect"
"sync"
"time"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/metrics"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/node"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/kube-vip/kube-vip/pkg/wireguard"
"github.com/prometheus/client_golang/prometheus"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
@@ -47,28 +38,19 @@ type Processor struct {
clientSet *kubernetes.Clientset
rwClientSet *kubernetes.Clientset
shutdownChan chan struct{}
// This is a prometheus counter used to count the number of events received
// from the service watcher
CountServiceWatchEvent *prometheus.CounterVec
intfMgr *networkinterface.Manager
arpMgr *arp.Manager
leaseMgr *lease.Manager
// nodeLabelManager is the manager for the node labels
nodeLabelManager node.Labeler
electionMgr *election.Manager
// TunnelMgr manages multiple WireGuard tunnels (one per service VIP)
TunnelMgr *wireguard.TunnelManager
routeMgr *route.Manager
}
// labelManager is the interface for the node label manager to add/remove labels
func NewServicesProcessor(config *kubevip.Config, bgpServer *bgp.Server,
clientSet *kubernetes.Clientset, rwClientSet *kubernetes.Clientset,
intfMgr *networkinterface.Manager, arpMgr *arp.Manager, nodeLabelManager node.Labeler,
electionMgr *election.Manager, leaseMgr *lease.Manager, routeMgr *route.Manager) *Processor {
clientSet *kubernetes.Clientset, rwClientSet *kubernetes.Clientset, shutdownChan chan struct{},
intfMgr *networkinterface.Manager, arpMgr *arp.Manager) *Processor {
lbClassFilterFunc := lbClassFilter
if config.LoadBalancerClassLegacyHandling {
lbClassFilterFunc = lbClassFilterLegacy
@@ -81,86 +63,66 @@ func NewServicesProcessor(config *kubevip.Config, bgpServer *bgp.Server,
bgpServer: bgpServer,
clientSet: clientSet,
rwClientSet: rwClientSet,
intfMgr: intfMgr,
arpMgr: arpMgr,
leaseMgr: leaseMgr,
nodeLabelManager: nodeLabelManager,
electionMgr: electionMgr,
TunnelMgr: wireguard.NewTunnelManager(),
routeMgr: routeMgr,
shutdownChan: shutdownChan,
CountServiceWatchEvent: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "kube_vip",
Subsystem: "manager",
Name: "all_services_events",
Help: "Count all events fired by the service watcher categorised by event type",
}, []string{"type"}),
intfMgr: intfMgr,
arpMgr: arpMgr,
}
}
func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceFunc *Callback, wg *sync.WaitGroup) error {
func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceFunc func(context.Context, *v1.Service) error) (bool, error) {
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
svc, ok := event.Object.(*v1.Service)
if !ok {
return fmt.Errorf("unable to parse Kubernetes services from API watcher")
return false, fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
timer := prometheus.NewTimer(metrics.ServiceReconcileDuration.WithLabelValues(svc.Namespace))
defer timer.ObserveDuration()
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
return nil
return true, nil
}
// Check if we ignore this service
if svc.Annotations[kubevip.LoadbalancerIgnore] == "true" {
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Info("ignore annotation for kube-vip", "service name", svc.Name)
return nil
return true, nil
}
// Check the loadBalancer class
if p.lbClassFilter(svc, p.config) {
return nil
return true, nil
}
svcAddresses, svcHostnames := instance.FetchServiceAddresses(svc)
svcAddresses := instance.FetchServiceAddresses(svc)
// We only care about LoadBalancer services that have been allocated an address
if len(svcAddresses) <= 0 && len(svcHostnames) <= 0 {
s, err := p.waitForAddress(ctx, svc)
if err != nil {
return fmt.Errorf("failed to get updated LB addresses for service %s/%s: %w", svc.Namespace, svc.Name, err)
}
svc = s
}
svcInstance := instance.FindServiceInstance(svc, p.ServiceInstances)
var err error
if svcInstance == nil {
svcInstance, err = instance.NewInstance(ctx, svc, p.config, p.intfMgr, p.arpMgr, p.routeMgr, p.nodeLabelManager, wg)
if err != nil {
metrics.ServiceReconcileErrorsTotal.WithLabelValues(svc.Namespace, svc.Name, "new_instance").Inc()
return fmt.Errorf("unable to create instance for service %s/%s", svc.Namespace, svc.Name)
}
p.ServiceInstances = append(p.ServiceInstances, svcInstance)
p.updateActiveServicesMetric()
}
_, usesCommonLease := svc.Annotations[kubevip.ServiceLease]
if usesCommonLease && svc.Spec.ExternalTrafficPolicy != v1.ServiceExternalTrafficPolicyTypeCluster {
metrics.ServiceReconcileErrorsTotal.WithLabelValues(svc.Namespace, svc.Name, "invalid_config").Inc()
return fmt.Errorf("annotation %q cannot be used with service traffic policy other than %q, service %s/%s",
kubevip.ServiceLease, v1.ServiceExternalTrafficPolicyTypeCluster, svc.Namespace, svc.Name)
if len(svcAddresses) <= 0 {
return true, nil
}
svcCtx, err := p.getServiceContext(svc.UID)
if err != nil {
metrics.ServiceReconcileErrorsTotal.WithLabelValues(svc.Namespace, svc.Name, "service_context").Inc()
return fmt.Errorf("failed to get service context: %w", err)
return false, fmt.Errorf("failed to get service context: %w", err)
}
// The modified event should only be triggered if the service has been modified (i.e. moved somewhere else)
if event.Type == watch.Modified {
shouldGarbageCollect := false
if svcInstance != nil {
shouldGarbageCollect = serviceChanged(svcInstance, svc)
i := instance.FindServiceInstance(svc, p.ServiceInstances)
var originalService []string
shouldGarbageCollect := true
if i != nil {
originalService = instance.FetchServiceAddresses(i.ServiceSnapshot)
shouldGarbageCollect = !reflect.DeepEqual(originalService, svcAddresses)
}
if shouldGarbageCollect {
for _, addr := range svcAddresses {
// log.Debugf("(svcs) Retrieving local addresses, to ensure that this modified address doesn't exist: %s", addr)
// log.Debugf("(svcs) Retreiving local addresses, to ensure that this modified address doesn't exist: %s", addr)
f, err := vip.GarbageCollect(p.config.Interface, addr, p.intfMgr)
if err != nil {
log.Error("(svcs) cleaning existing address error", "err", err)
@@ -169,161 +131,192 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
log.Warn("(svcs) already found existing config", "address", addr, "adapter", p.config.Interface)
}
}
// This service has been modified, but it was also active.
if svcCtx != nil {
log.Warn("(svcs) The load balancer has changed, cancelling original load balancer")
//Set it to inactive
svcCtx.Cancel()
}
// This service has been modified, but it was also active.
if svcCtx != nil && svcCtx.IsActive {
if i != nil {
originalService := instance.FetchServiceAddresses(i.ServiceSnapshot)
newService := instance.FetchServiceAddresses(svc)
if !reflect.DeepEqual(originalService, newService) {
if err := p.deleteService(ctx, svc.UID); err != nil {
metrics.ServiceReconcileErrorsTotal.WithLabelValues(svc.Namespace, svc.Name, "delete_service").Inc()
log.Error("(svc) unable to remove", "service", svc.UID)
// Calls the cancel function of the context
if svcCtx != nil {
log.Warn("(svcs) The load balancer has changed, cancelling original load balancer")
svcCtx.Cancel()
log.Warn("(svcs) waiting for load balancer to finish")
<-svcCtx.Ctx.Done()
}
err = p.deleteService(svc.UID)
if err != nil {
log.Error("(svc) unable to remove", "service", svc.UID)
}
p.svcMap.Delete(svc.UID)
}
// in theory this should never fail
p.svcMap.Delete(svc.UID)
// Reset the the svcCtx when it was garbage collected
// As the next function will create a new context when nil
svcCtx = nil
p.updateActiveServicesMetric()
}
}
}
ips, hostnames := instance.FetchServiceAddresses(svc)
log.Debug("(svcs) has been added/modified with addresses", "service name", svc.Name, "ips", ips, "hostnames", hostnames)
// Architecture walkthrough: (Had to do this as this code path is making my head hurt)
if svcCtx == nil {
ns, name := lease.ServiceName(svc)
leaseID := lease.NewID(p.config.LeaderElectionType, ns, name)
lease := p.leaseMgr.Add(ctx, leaseID)
svcCtx = servicecontext.New(lease.Ctx)
p.svcMap.Store(svc.UID, svcCtx)
}
// Is the service active (bool), if not then process this new service
// Does this service use an election per service?
//
// this goroutine starts service handling function (with or without leaderelection)
if !svcCtx.IsWatched {
wg.Go(func() {
watchWg := sync.WaitGroup{}
defer func() {
// wait for the sub-goroutines and tag service as not watched
watchWg.Wait()
svcCtx.IsWatched = false
}()
if svcCtx == nil || svcCtx != nil && !svcCtx.IsActive {
log.Debug("(svcs) has been added/modified with addresses", "service name", svc.Name, "ip", instance.FetchServiceAddresses(svc))
watchWg.Go(func() {
// start if service is not already watched/handled
// signal endpoints goroutine we are ready to start and run service handling function
log.Info("(svcs) service function starting", "uid", svc.UID)
err = serviceFunc.Run(svcCtx, svc, wg)
if svcCtx == nil {
svcCtx = servicecontext.New(ctx)
p.svcMap.Store(svc.UID, svcCtx)
}
if p.config.EnableServicesElection || // Service Election
((p.config.EnableRoutingTable || p.config.EnableBGP) && // Routing table mode or BGP
(!p.config.EnableLeaderElection && !p.config.EnableServicesElection)) { // No leaderelection or services election
// If this load balancer Traffic Policy is "local"
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Start an endpoint watcher if we're not watching it already
if !svcCtx.IsWatched {
// background the endpoint watcher
if (p.config.EnableRoutingTable || p.config.EnableBGP) && (!p.config.EnableLeaderElection && !p.config.EnableServicesElection) {
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Add Endpoint or EndpointSlices watcher
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
} else {
provider = providers.NewEndpointslices()
}
if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
}
}()
// We're now watching this service
svcCtx.IsWatched = true
}
} else if (p.config.EnableBGP || p.config.EnableRoutingTable) && (!p.config.EnableLeaderElection && !p.config.EnableServicesElection) {
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
if errors.Is(err, &utils.PanicError{}) {
// cancel service context on panic error
// TODO: should we quit kube-vip altogether here?
svcCtx.Cancel()
}
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
// Add Endpoint watcher
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
} else {
provider = providers.NewEndpointslices()
}
if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
}
}
log.Info("(svcs) service function done", "uid", svc.UID)
})
}()
// We're now watching this service
svcCtx.IsWatched = true
} else {
// this goroutine will watch endpoints for the service
watchWg.Go(func() {
// create provider and start watching the endpoints
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
} else {
provider = providers.NewEndpointslices()
}
if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
})
go func() {
for {
select {
case <-svcCtx.Ctx.Done():
log.Warn("(svcs) restartable service watcher ending", "uid", svc.UID)
return
default:
log.Info("(svcs) restartable service watcher starting", "uid", svc.UID)
err = serviceFunc(svcCtx.Ctx, svc)
})
if err != nil {
log.Error(err.Error())
}
}
}
// tag service as watched
svcCtx.IsWatched = true
}
if !p.config.EnableServicesElection {
log.Debug("Service now active", "name", svc.Name, "uid", svc.UID)
}
return nil
}
func (p *Processor) waitForAddress(ctx context.Context, svc *v1.Service) (*v1.Service, error) {
addressCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
ticker := time.NewTicker(time.Second)
for {
select {
case <-addressCtx.Done():
return nil, fmt.Errorf("failed to wait for the service LB address: %w", ctx.Err())
case <-ticker.C:
s, err := p.clientSet.CoreV1().Services(svc.Namespace).Get(addressCtx, svc.Name, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("failed to get updated service data: %w", err)
}()
}
addrs, hostnames := instance.FetchServiceAddresses(s)
if len(addrs) > 0 || len(hostnames) > 0 {
return s, nil
} else {
// Increment the waitGroup before the service Func is called (Done is completed in there)
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
svcCtx.IsActive = true
}
return false, nil
}
func (p *Processor) Delete(event watch.Event) error {
func (p *Processor) Delete(event watch.Event) (bool, error) {
svc, ok := event.Object.(*v1.Service)
if !ok {
return fmt.Errorf("(svcs) unable to parse Kubernetes services from API watcher")
return false, fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
svcCtx, err := p.getServiceContext(svc.UID)
if err != nil {
return fmt.Errorf("(svcs) unable to get context: %w", err)
return false, fmt.Errorf("(svcs) unable to get context: %w", err)
}
if svcCtx != nil {
if svcCtx != nil && svcCtx.IsActive {
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
return nil
return true, nil
}
// We can ignore this service
if svc.Annotations[kubevip.LoadbalancerIgnore] == "true" {
log.Info("(svcs) ignore annotation for kube-vip", "service name", svc.Name)
return nil
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Info("(svcs)ignore annotation for kube-vip", "service name", svc.Name)
return true, nil
}
// If no leader election is enabled, delete routes here
if !p.config.EnableLeaderElection && !p.config.EnableServicesElection &&
p.config.EnableRoutingTable && svcCtx.HasConfiguredNetworks() {
if errs := endpoints.ClearRoutes(svc, &p.ServiceInstances, p.routeMgr); len(errs) == 0 {
if errs := endpoints.ClearRoutes(svc, &p.ServiceInstances); len(errs) == 0 {
svcCtx.ConfiguredNetworks.Clear()
}
}
if !p.config.EnableServicesElection {
// If this is an active service then and additional leaderElection will handle stopping
err = p.deleteService(svcCtx.Ctx, svc.UID)
if err != nil {
log.Error(err.Error())
}
// If this is an active service then and additional leaderElection will handle stopping
err = p.deleteService(svc.UID)
if err != nil {
log.Error(err.Error())
}
// Calls the cancel function of the context
log.Warn("(svcs) The load balancer was deleted, cancelling context", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
log.Warn("(svcs) The load balancer was deleted, cancelling context")
svcCtx.Cancel()
log.Warn("(svcs) waiting for load balancer to finish")
<-svcCtx.Ctx.Done()
p.svcMap.Delete(svc.UID)
p.updateActiveServicesMetric()
}
if p.config.EnableLeaderElection && !p.config.EnableServicesElection {
if p.config.EnableBGP {
endpoints.ClearBGPHosts(svc, &p.ServiceInstances, p.bgpServer)
} else if p.config.EnableRoutingTable {
endpoints.ClearRoutes(svc, &p.ServiceInstances)
}
}
log.Info("(svcs) deleted", "service name", svc.Name, "namespace", svc.Namespace)
return nil
return true, nil
}
func (p *Processor) Stop() {
@@ -346,32 +339,6 @@ func (p *Processor) getServiceContext(uid types.UID) (*servicecontext.Context, e
return ctx, nil
}
func serviceChanged(i *instance.Instance, svc *v1.Service) bool {
svcAddresses, svcHostnames := instance.FetchServiceAddresses(svc)
originalServiceAddresses, originalServiceHostnames := instance.FetchServiceAddresses(i.ServiceSnapshot)
// Service addresses changed
return !reflect.DeepEqual(originalServiceAddresses, svcAddresses) ||
// Service hostnames changed
!reflect.DeepEqual(originalServiceHostnames, svcHostnames) ||
// ExternalTrafficPolicy changed
svc.Spec.ExternalTrafficPolicy != i.ServiceSnapshot.Spec.ExternalTrafficPolicy ||
// IP stack configuration changed
!reflect.DeepEqual(svc.Spec.IPFamilies, i.ServiceSnapshot.Spec.IPFamilies) ||
*svc.Spec.IPFamilyPolicy != *i.ServiceSnapshot.Spec.IPFamilyPolicy ||
// DDNS was disabled/enabled
svc.Annotations[kubevip.ServiceDDNS] != i.ServiceSnapshot.Annotations[kubevip.ServiceDDNS]
}
func (p *Processor) updateActiveServicesMetric() {
counts := map[string]int{}
for _, inst := range p.ServiceInstances {
if inst.ServiceSnapshot != nil {
counts[inst.ServiceSnapshot.Namespace]++
}
}
metrics.ActiveServices.Reset()
for ns, count := range counts {
metrics.ActiveServices.WithLabelValues(ns).Set(float64(count))
}
func (p *Processor) CountRouteReferences(route *netlink.Route) int {
return endpoints.CountRouteReferences(route, &p.ServiceInstances)
}

View File

@@ -2,12 +2,9 @@ package services
import (
"context"
"errors"
"fmt"
"net"
"slices"
"strings"
"sync"
"time"
log "log/slog"
@@ -15,7 +12,6 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
@@ -26,10 +22,7 @@ import (
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/upnp"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
@@ -39,106 +32,72 @@ const (
ActionDelete ServiceInstanceAction = "delete"
ActionAdd ServiceInstanceAction = "add"
ActionNone ServiceInstanceAction = "none"
// Default UPNP lease requested is 3600 seconds.
defaultUPNPLeaseDuration = 1 * time.Hour
)
func (p *Processor) SyncServices(ctx *servicecontext.Context, svc *v1.Service, wg *sync.WaitGroup, usesLeaderElection bool) error {
log.Debug("[STARTING] Service Sync", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
func (p *Processor) SyncServices(ctx context.Context, svc *v1.Service) error {
log.Debug("[STARTING] Service Sync", "namespace", svc.Namespace, "name", svc.Name)
// Iterate through the synchronising services
action, instance := p.getServiceInstanceAction(svc)
action := p.getServiceInstanceAction(svc)
switch action {
case ActionDelete:
log.Debug("[service] delete", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
if err := p.deleteService(ctx.Ctx, svc.UID); err != nil {
log.Debug("[service] delete", "namespace", svc.Namespace, "name", svc.Name)
if err := p.deleteService(svc.UID); err != nil {
return fmt.Errorf("error deleting service %s/%s: %w", svc.Namespace, svc.Name, err)
}
case ActionAdd:
log.Debug("[service] add", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
if instance != nil {
instance.AddCalled = true
}
if !usesLeaderElection {
select {
case <-ctx.Ctx.Done():
return nil
case <-ctx.EndpointsReady:
}
}
if err := p.addService(ctx.Ctx, instance, svc, wg); err != nil {
log.Debug("[service] add", "namespace", svc.Namespace, "name", svc.Name)
if err := p.addService(ctx, svc); err != nil {
return fmt.Errorf("error adding service %s/%s: %w", svc.Namespace, svc.Name, err)
}
case ActionNone:
log.Debug("[service] no action", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
log.Debug("[service] no action", "namespace", svc.Namespace, "name", svc.Name)
}
log.Debug("[FINISHED] Service Sync", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
log.Debug("[FINISHED] Service Sync", "namespace", svc.Namespace, "name", svc.Name)
return nil
}
func (p *Processor) getServiceInstanceAction(svc *v1.Service) (ServiceInstanceAction, *instance.Instance) {
func (p *Processor) getServiceInstanceAction(svc *v1.Service) ServiceInstanceAction {
// protect against multiple calls
// get the annotations or legacy values from manual configuration
addresses, hostnames := instance.FetchServiceAddresses(svc)
// get the status information of the LB Service
statusAddresses, _ := instance.FetchLoadBalancerIngress(svc)
addresses := instance.FetchServiceAddresses(svc)
ingressIPs := instance.FetchLoadBalancerIngressAddresses(svc)
p.mutex.Lock()
defer p.mutex.Unlock()
for _, instance := range p.ServiceInstances {
if instance != nil && instance.ServiceSnapshot.UID == svc.UID {
if !instance.AddCalled {
return ActionAdd, instance
}
for _, address := range addresses {
// handle the case where the service instance needs to be deleted
if instance.IsDHCPv4 {
if instance.IsDHCP {
if address != "0.0.0.0" {
return ActionDelete, instance
return ActionDelete
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(statusAddresses, instance.DHCPInterfaceIPv4) {
return ActionDelete, instance
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, instance.DHCPInterfaceIP) {
return ActionDelete
}
} else {
}
if !instance.IsDHCP {
if address == "0.0.0.0" {
return ActionDelete, instance
return ActionDelete
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(statusAddresses, address) {
return ActionDelete, instance
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, address) {
return ActionDelete
}
}
if instance.IsDHCPv6 {
if address != "::" {
return ActionDelete, instance
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(statusAddresses, instance.DHCPInterfaceIPv6) {
return ActionDelete, instance
}
} else {
if address == "::" {
return ActionDelete, instance
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(statusAddresses, address) {
return ActionDelete, instance
}
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !comparePortsAndPortStatuses(svc) {
return ActionDelete, instance
if !comparePortsAndPortStatuses(svc) {
return ActionDelete
}
}
// If we reach here, it means the service instance matches the service UID and is not a DHCP service, so we can return "no action"
return ActionNone, instance
return ActionNone
}
}
if len(addresses) > 0 || len(hostnames) > 0 {
log.Debug("no matching service instance found", "service", svc.Name, "namespace", svc.Namespace, "uid", svc.UID, "addresses", addresses, "hostnames", hostnames)
return ActionAdd, nil // If no matching instance is found, we need to add a new service instance
if len(addresses) > 0 {
log.Debug("No matching service instance found", "service", svc.Name, "namespace", svc.Namespace, "addresses", addresses)
return ActionAdd // If no matching instance is found, we need to add a new service instance
}
return ActionNone, nil
return ActionNone
}
func comparePortsAndPortStatuses(svc *v1.Service) bool {
@@ -157,125 +116,57 @@ func comparePortsAndPortStatuses(svc *v1.Service) bool {
return true
}
func (p *Processor) addService(ctx context.Context, inst *instance.Instance, svc *v1.Service, wg *sync.WaitGroup) error {
func (p *Processor) addService(ctx context.Context, svc *v1.Service) error {
// protect against addService while reading
p.mutex.Lock()
defer p.mutex.Unlock()
startTime := time.Now()
var err error
if inst == nil {
inst, err = instance.NewInstance(ctx, svc, p.config, p.intfMgr, p.arpMgr, p.routeMgr, p.nodeLabelManager, wg)
if err != nil {
return err
}
inst.AddCalled = true
p.ServiceInstances = append(p.ServiceInstances, inst)
newService, err := instance.NewInstance(svc, p.config, p.intfMgr, p.arpMgr)
if err != nil {
return err
}
if err := p.configureService(ctx, inst, svc, wg); err != nil {
return fmt.Errorf("failed to configure service: %w", err)
for x := range newService.VIPConfigs {
log.Debug("starting loadbalancer for service", "name", svc.Name, "namespace", svc.Namespace)
newService.Clusters[x].StartLoadBalancerService(ctx, newService.VIPConfigs[x], p.bgpServer, svc.Name, p.CountRouteReferences)
}
// add the label to the node after adding the service
labels := generateLabelsFromService(svc, kubevip.ServiceProvided)
if err := p.nodeLabelManager.AddLabel(labels); err != nil {
return fmt.Errorf("error adding label to node: %w", err)
}
inst.LabelAdded = true
p.upnpMap(ctx, newService)
finishTime := time.Since(startTime)
log.Info("[service]", "service", svc.Name, "namespace", svc.Namespace, "synchronised in", fmt.Sprintf("%dms", finishTime.Milliseconds()))
return nil
}
func (p *Processor) configureService(ctx context.Context, inst *instance.Instance, svc *v1.Service, wg *sync.WaitGroup) error {
// is not a global leader election mode
if p.config.EnableServicesElection || (!p.config.EnableARP && !p.config.EnableLeaderElection) || (!p.config.EnableARP && !p.config.EnableRoutingTable) {
for x := range inst.VIPConfigs {
log.Debug("starting loadbalancer for service", "name", svc.Name, "namespace", svc.Namespace, "uid", svc.UID)
if err := inst.Clusters[x].StartLoadBalancerService(ctx, inst.VIPConfigs[x], p.bgpServer, lease.ServiceNamespacedName(svc), wg); err != nil {
return fmt.Errorf("failed to start lb: %w", err)
}
}
}
p.upnpMap(ctx, inst)
if inst.IsDHCPv4 {
wg.Go(func() {
index := -1
for i := range inst.VIPConfigs {
ip := net.ParseIP(inst.VIPConfigs[i].VIP)
if ip.To4() != nil {
index = i
break
}
}
if index == -1 {
log.Error("unable to find proper VIPConfig for the DHCPv4")
} else {
for ip := range inst.DHCPv4Client.IPChannel() {
log.Debug("IP changed", "ip", ip)
inst.VIPConfigs[index].VIP = ip
inst.DHCPInterfaceIPv4 = ip
if !p.config.DisableServiceUpdates {
if err := p.updateStatus(ctx, inst); err != nil {
log.Warn("updating svc", "err", err)
}
if newService.IsDHCP && len(newService.VIPConfigs) == 1 {
go func() {
for ip := range newService.DHCPClient.IPChannel() {
log.Debug("IP changed", "ip", ip)
newService.VIPConfigs[0].VIP = ip
newService.DHCPInterfaceIP = ip
if !p.config.DisableServiceUpdates {
if err := p.updateStatus(newService); err != nil {
log.Warn("updating svc", "err", err)
}
}
log.Debug("IPv4 update channel closed, stopping")
}
})
log.Debug("IP update channel closed, stopping")
}()
}
if inst.IsDHCPv6 {
wg.Go(func() {
index := -1
for i := range inst.VIPConfigs {
ip := net.ParseIP(inst.VIPConfigs[i].VIP)
if ip.To4() == nil {
index = i
break
}
}
if index == -1 {
log.Error("unable to find proper VIPConfig for the DHCPv6")
} else {
for ip := range inst.DHCPv4Client.IPChannel() {
log.Debug("IP changed", "ip", ip)
inst.VIPConfigs[index].VIP = ip
inst.DHCPInterfaceIPv6 = ip
if !p.config.DisableServiceUpdates {
if err := p.updateStatus(ctx, inst); err != nil {
log.Warn("updating svc", "err", err)
}
}
}
log.Debug("IPv6 update channel closed, stopping")
}
})
}
p.ServiceInstances = append(p.ServiceInstances, newService)
if !p.config.DisableServiceUpdates {
log.Debug("[service] update", "namespace", inst.ServiceSnapshot.Namespace, "name", inst.ServiceSnapshot.Name)
if err := p.updateStatus(ctx, inst); err != nil {
log.Error("[service] updating status", "namespace", inst.ServiceSnapshot.Namespace, "name", inst.ServiceSnapshot.Name, "err", err)
log.Debug("[service] update", "namespace", newService.ServiceSnapshot.Namespace, "name", newService.ServiceSnapshot.Name)
if err := p.updateStatus(newService); err != nil {
log.Error("[service] updating status", "namespace", newService.ServiceSnapshot.Namespace, "name", newService.ServiceSnapshot.Name, "err", err)
}
}
serviceIPs, _ := instance.FetchServiceAddresses(svc)
serviceIPs := instance.FetchServiceAddresses(svc)
// Check if we need to flush any conntrack connections (due to some dangling conntrack connections)
if svc.Annotations[kubevip.FlushContrack] == "true" {
log.Debug("[service] Flushing conntrack rules", "service", svc.Name, "namespace", svc.Namespace)
for _, serviceIP := range serviceIPs {
err := vip.DeleteExistingSessions(serviceIP, false, svc.Annotations[kubevip.EgressDestinationPorts], svc.Annotations[kubevip.EgressSourcePorts])
err = vip.DeleteExistingSessions(serviceIP, false, svc.Annotations[kubevip.EgressDestinationPorts], svc.Annotations[kubevip.EgressSourcePorts])
if err != nil {
log.Error("[service] flushing any remaining egress connections", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
@@ -292,15 +183,15 @@ func (p *Processor) configureService(ctx context.Context, inst *instance.Instanc
// If we'er not using NFtables, then ensure that the correct iptables modules are loaded
if p.config.EgressWithNftables {
// Ensure that kernel modules are loaded and report back missing modules.
err := p.nftablesCheck()
err = p.nftablesCheck()
if err != nil {
log.Warn("[service] configuring nft egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
} else {
// Ensure that kernel modules are loaded and report back missing modules.
err := p.iptablesCheck()
err = p.iptablesCheck()
if err != nil {
log.Warn("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
var podIP string
@@ -311,14 +202,14 @@ func (p *Processor) configureService(ctx context.Context, inst *instance.Instanc
// Does the service have an active IPv6 endpoint
if svc.Annotations[kubevip.ActiveEndpointIPv6] != "" {
for _, serviceIP := range serviceIPs {
if !p.config.EnableEndpoints && utils.IsIPv6(serviceIP) {
if !p.config.EnableEndpoints && vip.IsIPv6(serviceIP) {
podIP = svc.Annotations[kubevip.ActiveEndpointIPv6]
err := p.configureEgress(ctx, serviceIP, podIP, svc.Namespace, string(svc.UID), svc.Annotations)
err = p.configureEgress(serviceIP, podIP, svc.Namespace, string(svc.UID), svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Warn("[service] configuring egress IPv6", "service", svc.Name, "namespace", svc.Namespace, "err", err)
log.Error("[service] configuring egress IPv6", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
@@ -326,13 +217,13 @@ func (p *Processor) configureService(ctx context.Context, inst *instance.Instanc
} else if svc.Annotations[kubevip.ActiveEndpoint] != "" { // Not expected to be IPv6, so should be an IPv4 address
for _, serviceIP := range serviceIPs {
podIPs := svc.Annotations[kubevip.ActiveEndpoint]
if !p.config.EnableEndpoints && utils.IsIPv6(serviceIP) {
if !p.config.EnableEndpoints && vip.IsIPv6(serviceIP) {
podIPs = svc.Annotations[kubevip.ActiveEndpointIPv6]
}
err := p.configureEgress(ctx, serviceIP, podIPs, svc.Namespace, string(svc.UID), svc.Annotations)
err = p.configureEgress(serviceIP, podIPs, svc.Namespace, string(svc.UID), svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Warn("[service] configuring egress IPv4", "service", svc.Name, "namespace", svc.Namespace, "err", err)
log.Error("[service] configuring egress IPv4", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
@@ -343,26 +234,20 @@ func (p *Processor) configureService(ctx context.Context, inst *instance.Instanc
} else {
provider = providers.NewEndpointslices()
}
err := provider.UpdateServiceAnnotation(ctx, svc.Annotations[kubevip.ActiveEndpoint], svc.Annotations[kubevip.ActiveEndpointIPv6], svc, p.clientSet)
err = provider.UpdateServiceAnnotation(svc.Annotations[kubevip.ActiveEndpoint], svc.Annotations[kubevip.ActiveEndpointIPv6], svc, p.clientSet)
if err != nil {
log.Warn("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
// Configure WireGuard DNAT rules if WireGuard is enabled
if p.config.EnableWireguard {
log.Debug("[service] configuring WireGuard DNAT rules", "service", svc.Name, "namespace", svc.Namespace)
if err := p.addServiceWireguard(ctx, svc); err != nil {
log.Warn("[service] failed to configure WireGuard DNAT", "service", svc.Name, "namespace", svc.Namespace, "err", err)
// Don't fail the entire service if WireGuard config fails
}
}
finishTime := time.Since(startTime)
log.Info("[service]", "service", svc.Name, "namespace", svc.Namespace, "synchronised in", fmt.Sprintf("%dms", finishTime.Milliseconds()))
return nil
}
func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
func (p *Processor) deleteService(uid types.UID) error {
// protect multiple calls
p.mutex.Lock()
defer p.mutex.Unlock()
@@ -371,7 +256,7 @@ func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
var serviceInstance *instance.Instance
found := false
for x := range p.ServiceInstances {
log.Debug("[service] lookup", "target UID", uid, "found UID", p.ServiceInstances[x].ServiceSnapshot.UID, "name", p.ServiceInstances[x].ServiceSnapshot.Name, "namespace", p.ServiceInstances[x].ServiceSnapshot.Namespace)
log.Debug("[service] lookup", "target UID", uid, "found UID ", p.ServiceInstances[x].ServiceSnapshot.UID, "name", p.ServiceInstances[x].ServiceSnapshot.Name, "namespace", p.ServiceInstances[x].ServiceSnapshot.Namespace)
// Add the running services to the new array
if p.ServiceInstances[x].ServiceSnapshot.UID != uid {
updatedInstances = append(updatedInstances, p.ServiceInstances[x])
@@ -381,19 +266,10 @@ func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
serviceInstance = p.ServiceInstances[x]
}
}
if serviceInstance.LabelAdded {
labels := generateLabelsFromService(serviceInstance.ServiceSnapshot, kubevip.ServiceProvided)
if err := p.nodeLabelManager.RemoveLabel(labels); err != nil {
return fmt.Errorf("error removing label from node: %w", err)
}
}
// If we've been through all services and not found the correct one then error
if !found {
// TODO: - fix UX
// return fmt.Errorf("unable to find/stop service [%s]", uid)
log.Warn("unable to find/stop service", "uid", uid)
return nil
}
@@ -403,60 +279,25 @@ func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
}
}
// Determine if this VIP is shared with other loadbalancers
// Determine if this this VIP is shared with other loadbalancers
shared := false
vipSet := make(map[string]interface{})
for x := range updatedInstances {
vips, _ := instance.FetchServiceAddresses(updatedInstances[x].ServiceSnapshot)
for _, vip := range vips { //updatedInstances[x].ServiceSnapshot.Spec.LoadBalancerIP {
for _, vip := range instance.FetchServiceAddresses(updatedInstances[x].ServiceSnapshot) { //updatedInstances[x].ServiceSnapshot.Spec.LoadBalancerIP {
vipSet[vip] = nil
}
}
vips, _ := instance.FetchServiceAddresses(serviceInstance.ServiceSnapshot)
for _, vip := range vips {
for _, vip := range instance.FetchServiceAddresses(serviceInstance.ServiceSnapshot) {
if _, found := vipSet[vip]; found {
shared = true
}
}
if p.config.EnableBGP {
endpoints.ClearBGPHostsByInstance(ctx, serviceInstance, p.bgpServer)
}
if p.config.EnableRoutingTable && (p.config.EnableLeaderElection || p.config.EnableServicesElection) {
if errs := endpoints.ClearRoutesByInstance(serviceInstance.ServiceSnapshot, serviceInstance, &p.ServiceInstances, p.routeMgr); len(errs) > 0 {
for _, err := range errs {
log.Error("unable to clear routes", "err", err)
}
}
}
if !shared {
for x := range serviceInstance.Clusters {
serviceInstance.Clusters[x].Stop()
}
if serviceInstance.IsVLAN {
vlan, err := netlink.LinkByName(serviceInstance.VLANInterface)
if err != nil {
return fmt.Errorf("[service] error finding VLAN Interface: %v", err)
}
err = netlink.LinkDel(vlan)
if err != nil {
return fmt.Errorf("[service] error deleting VLAN interface : %v", err)
}
}
if serviceInstance.IsDHCPv4 || serviceInstance.IsDHCPv6 {
if serviceInstance.IsDHCPv4 {
serviceInstance.DHCPv4Client.Stop()
}
if serviceInstance.IsDHCPv6 {
serviceInstance.DHCPv6Client.Stop()
}
if serviceInstance.IsDHCP {
serviceInstance.DHCPClient.Stop()
macvlan, err := netlink.LinkByName(serviceInstance.DHCPInterface)
if err != nil {
return fmt.Errorf("[service] error finding VIP Interface: %v", err)
@@ -467,6 +308,11 @@ func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
return fmt.Errorf("[service] error deleting DHCP Link : %v", err)
}
}
for i := range serviceInstance.VIPConfigs {
if serviceInstance.VIPConfigs[i].EnableBGP {
endpoints.ClearBGPHostsByInstance(serviceInstance, p.bgpServer)
}
}
// We will need to tear down the egress
if serviceInstance.ServiceSnapshot.Annotations[kubevip.Egress] == "true" {
@@ -483,183 +329,11 @@ func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
// Update the service array
p.ServiceInstances = updatedInstances
// Clean up WireGuard DNAT rules if WireGuard is enabled
if p.config.EnableWireguard {
log.Debug("[service] cleaning up WireGuard DNAT rules", "uid", uid, "name", serviceInstance.ServiceSnapshot.Name)
p.deleteServiceWireguard(ctx, serviceInstance.ServiceSnapshot)
}
log.Info("Removed instance from manager", "uid", uid, "name", serviceInstance.ServiceSnapshot.Name, "remaining advertised services", len(p.ServiceInstances))
log.Info("Removed instance from manager", "uid", uid, "remaining advertised services", len(p.ServiceInstances))
return nil
}
func (p *Processor) updateEgressConfiguration(ctx context.Context, svc *v1.Service) error {
p.mutex.Lock()
defer p.mutex.Unlock()
i := instance.FindServiceInstance(svc, p.ServiceInstances)
if i == nil {
return fmt.Errorf("service instance not found for %s/%s", svc.Namespace, svc.Name)
}
oldIPv4 := i.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint]
newIPv4 := svc.Annotations[kubevip.ActiveEndpoint]
oldIPv6 := i.ServiceSnapshot.Annotations[kubevip.ActiveEndpointIPv6]
newIPv6 := svc.Annotations[kubevip.ActiveEndpointIPv6]
// Skip update if endpoints haven't changed
if oldIPv4 == newIPv4 && oldIPv6 == newIPv6 {
return nil
}
log.Info("[service] updating egress configuration",
"service", svc.Name,
"namespace", svc.Namespace,
"old_ipv4", oldIPv4,
"new_ipv4", newIPv4,
"old_ipv6", oldIPv6,
"new_ipv6", newIPv6)
// Remove old egress rules if they exist
if oldIPv4 != "" || oldIPv6 != "" {
oldEndpoint := oldIPv4
if oldEndpoint == "" {
oldEndpoint = oldIPv6
}
serviceIPs, _ := instance.FetchServiceAddresses(i.ServiceSnapshot)
for _, serviceIP := range serviceIPs {
if err := egress.Teardown(
oldEndpoint,
serviceIP,
i.ServiceSnapshot.Namespace,
string(i.ServiceSnapshot.UID),
i.ServiceSnapshot.Annotations,
p.config.EgressWithNftables,
); err != nil {
log.Warn("[service] removing old egress rules", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
// Apply new egress rules with updated endpoint
serviceIPs, _ := instance.FetchServiceAddresses(svc)
errList := []error{}
// Check if egress should be IPv6
if svc.Annotations[kubevip.EgressIPv6] == "true" {
// Does the service have an active IPv6 endpoint
if newIPv6 != "" {
for _, serviceIP := range serviceIPs {
if !p.config.EnableEndpoints && utils.IsIPv6(serviceIP) {
podIP := newIPv6
err := p.configureEgress(ctx, serviceIP, podIP, svc.Namespace, string(svc.UID), svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Warn("[service] configuring egress IPv6", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
}
} else if newIPv4 != "" { // Not expected to be IPv6, so should be an IPv4 address
for _, serviceIP := range serviceIPs {
podIPs := newIPv4
if !p.config.EnableEndpoints && utils.IsIPv6(serviceIP) {
podIPs = newIPv6
}
err := p.configureEgress(ctx, serviceIP, podIPs, svc.Namespace, string(svc.UID), svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Warn("[service] configuring egress IPv4", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
if len(errList) > 0 {
return fmt.Errorf("errors configuring egress: %v", errList)
}
// Update the service snapshot to reflect the new state
// NOTE: Do NOT call UpdateServiceAnnotation here - the annotation was already updated
// by the endpoint processor, which is why we're being called in the first place.
// Calling it again would create an infinite loop of Modified events.
// svc is already a DeepCopy from the endpoint processor, so no need to copy again.
i.ServiceSnapshot = svc
log.Info("[service] egress configuration updated successfully", "service", svc.Name, "namespace", svc.Namespace)
return nil
}
// upnpLeaseDurationForService determines the UPNP lease duration for a given service, based on its annotations.
//
// The default lease duration is set to 1 hour, maintaining the default of 3600 seconds that was previously passed. If
// the service has an annotation of [kubevip.UpnpLeaseDuration], the function attempts to parse its value as a
// [time.Duration] using [time.ParseDuration].
//
// If parsing is successful, the lease duration is updated accordingly; otherwise, a warning is logged and the default
// duration is retained.
//
// Overriding the default lease duration can be useful for services that require longer or shorter UPNP port mappings,
// or for buggy UPNP implementations that may not handle renewals correctly. At least one router's implementation
// completely times out the mapping very shortly after creation if it is set to 3600 or 7200, but works fine if 0 is
// used.
//
// This function must therefore explicitly permit duration of 0, and callers and the underlying library must pass that
// value in XML correctly. A duration of 0 indicates to the UPNP gateway that the mapping should be permanent.
//
// It may be useful to update this function to read a global configuration option as well. This helper could also take
// in v1.Service instead of [instance.Instance], but the latter is more convenient for callers.
//
// Example where 0 was observed to stay on the problematic router: miniupnpc's test client, upnpc v2.2.4.
func upnpLeaseDurationForService(s *instance.Instance) time.Duration {
if s == nil || s.ServiceSnapshot == nil || s.ServiceSnapshot.Annotations == nil {
// No warning output. No annotation is unusual but perfectly ok.
return defaultUPNPLeaseDuration
}
// Constant is named `UpnpLeaseDuration` for consistency with `UpnpEnabled`. According to Go naming conventions
// regarding use of acronyms, `UPNPLeaseDuration` would be preferred. Cleanup of both of these is left for a future
// refactor as these are public symbols and might be used elsewhere in the ecosystem.
val, ok := s.ServiceSnapshot.Annotations[kubevip.UpnpLeaseDuration]
if !ok {
// No warning output. No annotation is common and perfectly ok.
return defaultUPNPLeaseDuration
}
if val == "" {
log.Warn("[UPNP] Lease duration annotation is empty, using default of 1 hour", "service", s.ServiceSnapshot.Name)
return defaultUPNPLeaseDuration
}
parsed, err := time.ParseDuration(val)
if err != nil {
log.Warn("[UPNP] Unable to parse lease duration from annotation, using default of 1 hour", "service", s.ServiceSnapshot.Name, "err", err)
return defaultUPNPLeaseDuration
}
if parsed < 0 {
log.Warn("[UPNP] Lease duration from annotation is negative, using default of 1 hour", "service", s.ServiceSnapshot.Name)
return defaultUPNPLeaseDuration
}
return parsed
}
// upnpLeaseDurationForServiceSec returns the UPNP lease duration for a service in uint32 seconds, as expected by the
// helper library. This is a convenience wrapper around [upnpLeaseDurationForService], and in case
// upnpLeaseDurationForService returns a duration that maps to a negative value of seconds or invalid float of seconds,
// it will return the default lease duration in seconds instead. (Technically, it will check for a reasonable range of
// seconds, e.g. ~10 years-ish.)
func upnpLeaseDurationForServiceSec(s *instance.Instance) uint32 {
duration := upnpLeaseDurationForService(s)
seconds := duration.Seconds()
// Check if within range.
if seconds >= 0 && seconds <= float64(10*365*24*60*60) {
return uint32(seconds)
}
return uint32(defaultUPNPLeaseDuration.Seconds())
}
// Set up UPNP forwards for a service
// We first try to use the more modern Pinhole API introduced in UPNPv2 and fall back to UPNPv2 Port Forwarding if no forward was successful
func (p *Processor) upnpMap(ctx context.Context, s *instance.Instance) {
@@ -676,23 +350,17 @@ func (p *Processor) upnpMap(ctx context.Context, s *instance.Instance) {
gateways := upnp.GetGatewayClients(ctx)
// Determine desired UPNP TTL / "lease duration". Passed into the library as integer seconds from now, as the
// underlying XML API wants integer seconds.
leaseDurationSec := upnpLeaseDurationForServiceSec(s)
// Reset Gateway IPs to remove stale addresses
s.UPNPGatewayIPs = make([]string, 0)
vips, _ := instance.FetchServiceAddresses(s.ServiceSnapshot)
for _, vip := range vips {
for _, vip := range instance.FetchServiceAddresses(s.ServiceSnapshot) {
for _, port := range s.ServiceSnapshot.Spec.Ports {
for _, gw := range gateways {
log.Info("[UPNP] Adding map", "vip", vip, "port", port.Port, "service", s.ServiceSnapshot.Name, "gateway", gw.WANIPv6FirewallControlClient.Location)
forwardSucessful := false
if gw.WANIPv6FirewallControlClient != nil {
log.Info("[UPNP] Adding map", "vip", vip, "port", port.Port, "service", s.ServiceSnapshot.Name, "gateway", gw.WANIPv6FirewallControlClient.Location, "leaseDurationSec", leaseDurationSec)
pinholeID, pinholeErr := gw.WANIPv6FirewallControlClient.AddPinholeCtx(ctx, "0.0.0.0", uint16(port.Port), vip, uint16(port.Port), upnp.MapProtocolToIANA(string(port.Protocol)), leaseDurationSec) //nolint TODO
pinholeID, pinholeErr := gw.WANIPv6FirewallControlClient.AddPinholeCtx(ctx, "0.0.0.0", uint16(port.Port), vip, uint16(port.Port), upnp.MapProtocolToIANA(string(port.Protocol)), 3600) //nolint TODO
if pinholeErr == nil {
forwardSucessful = true
log.Info("[UPNP] Service should be accessible externally", "port", port.Port, "pinhold ID", pinholeID)
@@ -703,17 +371,9 @@ func (p *Processor) upnpMap(ctx context.Context, s *instance.Instance) {
}
// Fallback to PortForward
if !forwardSucessful {
log.Info("[UPNP] Adding map", "vip", vip, "port", port.Port, "service", s.ServiceSnapshot.Name, "leaseDurationSec", leaseDurationSec)
portMappingErr := gw.ConnectionClient.AddPortMapping("0.0.0.0", uint16(port.Port), strings.ToUpper(string(port.Protocol)), uint16(port.Port), vip, true, s.ServiceSnapshot.Name, leaseDurationSec) //nolint TODO
portMappingErr := gw.ConnectionClient.AddPortMapping("0.0.0.0", uint16(port.Port), strings.ToUpper(string(port.Protocol)), uint16(port.Port), vip, true, s.ServiceSnapshot.Name, 3600) //nolint TODO
if portMappingErr == nil {
ip, err := gw.ConnectionClient.GetExternalIPAddress()
if err != nil {
// Log the error but continue on the off chance the mapping was successful
log.Error("[UPNP] Unable to get external IP address from gateway", "service", s.ServiceSnapshot.Name, "port", port.Port, "err", err)
} else {
log.Info("[UPNP] Service should be accessible externally", "service", s.ServiceSnapshot.Name, "port", port.Port, "externalip", ip)
}
log.Info("[UPNP] Service should be accessible externally", "port", port.Port)
forwardSucessful = true
} else {
//TODO: Cleanup
@@ -736,7 +396,7 @@ func (p *Processor) upnpMap(ctx context.Context, s *instance.Instance) {
s.UPNPGatewayIPs = slices.Compact(s.UPNPGatewayIPs)
}
func (p *Processor) updateStatus(ctx context.Context, i *instance.Instance) error {
func (p *Processor) updateStatus(i *instance.Instance) error {
// let's retry status update every 10ms for 30s
retryConfig := wait.Backoff{
Steps: 3000,
@@ -745,12 +405,10 @@ func (p *Processor) updateStatus(ctx context.Context, i *instance.Instance) erro
Jitter: 0.1,
}
// will retry for every error encountered, TODO: should a list of errors that will trigger retry be specified?
err := retry.OnError(retryConfig, func(err error) bool {
return !errors.Is(err, context.Canceled)
}, func() error {
err := retry.OnError(retryConfig, func(error) bool { return true }, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := p.clientSet.CoreV1().Services(i.ServiceSnapshot.Namespace).Get(ctx, i.ServiceSnapshot.Name, metav1.GetOptions{})
currentService, err := p.clientSet.CoreV1().Services(i.ServiceSnapshot.Namespace).Get(context.TODO(), i.ServiceSnapshot.Name, metav1.GetOptions{})
if err != nil {
return err
}
@@ -760,24 +418,14 @@ func (p *Processor) updateStatus(ctx context.Context, i *instance.Instance) erro
currentServiceCopy.Annotations = make(map[string]string)
}
// If we're using ARP then we can only broadcast the VIP from one place, also useful for other software when running BGP, add an annotation to the service
if p.config.EnableARP || p.config.EnableBGP {
// If we're using ARP then we can only broadcast the VIP from one place, add an annotation to the service
if p.config.EnableARP {
// Add the current host
currentServiceCopy.Annotations[kubevip.VipHost] = p.config.NodeName
}
if i.DHCPInterfaceHwaddr != "" || i.DHCPInterfaceIPv4 != "" || i.DHCPInterfaceIPv6 != "" {
if i.DHCPInterfaceHwaddr != "" || i.DHCPInterfaceIP != "" {
currentServiceCopy.Annotations[kubevip.HwAddrKey] = i.DHCPInterfaceHwaddr
dhcpInterfaceIP := ""
if i.DHCPInterfaceIPv4 != "" {
dhcpInterfaceIP = i.DHCPInterfaceIPv4
if i.DHCPInterfaceIPv6 != "" {
dhcpInterfaceIP += ","
}
}
if i.DHCPInterfaceIPv6 != "" {
dhcpInterfaceIP += i.DHCPInterfaceIPv6
}
currentServiceCopy.Annotations[kubevip.RequestedIP] = dhcpInterfaceIP
currentServiceCopy.Annotations[kubevip.RequestedIP] = i.DHCPInterfaceIP
}
if currentService.Annotations["development.kube-vip.io/synthetic-api-server-error-on-update"] == "true" {
@@ -786,7 +434,7 @@ func (p *Processor) updateStatus(ctx context.Context, i *instance.Instance) erro
}
if !cmp.Equal(currentService, currentServiceCopy) {
currentService, err = p.clientSet.CoreV1().Services(currentServiceCopy.Namespace).Update(ctx, currentServiceCopy, metav1.UpdateOptions{})
currentService, err = p.clientSet.CoreV1().Services(currentServiceCopy.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("updating Spec", "service", i.ServiceSnapshot.Name, "err", err)
return err
@@ -804,8 +452,8 @@ func (p *Processor) updateStatus(ctx context.Context, i *instance.Instance) erro
ingresses := []v1.LoadBalancerIngress{}
for _, c := range i.VIPConfigs {
if !utils.IsIP(c.VIP) {
ips, err := utils.LookupHost(c.VIP, c.DNSMode, *i.ServiceSnapshot.Spec.IPFamilyPolicy == v1.IPFamilyPolicyRequireDualStack)
if !vip.IsIP(c.VIP) {
ips, err := vip.LookupHost(c.VIP, p.config.DNSMode)
if err != nil {
return err
}
@@ -833,12 +481,10 @@ func (p *Processor) updateStatus(ctx context.Context, i *instance.Instance) erro
}
}
}
log.Debug("LB status", "current", currentService.Status.LoadBalancer.Ingress, "new", ingresses)
if !ingressEqual(currentService.Status.LoadBalancer.Ingress, ingresses) {
if !cmp.Equal(currentService.Status.LoadBalancer.Ingress, ingresses) {
currentService.Status.LoadBalancer.Ingress = ingresses
log.Debug("updating service status", "namespace", currentService.Namespace, "name", currentService.Name, "uid", currentService.UID)
_, err = p.clientSet.CoreV1().Services(currentService.Namespace).UpdateStatus(ctx, currentService, metav1.UpdateOptions{})
if err != nil && !apierrors.IsInvalid(err) {
_, err = p.clientSet.CoreV1().Services(currentService.Namespace).UpdateStatus(context.TODO(), currentService, metav1.UpdateOptions{})
if err != nil {
log.Error("updating Service", "namespace", i.ServiceSnapshot.Namespace, "name", i.ServiceSnapshot.Name, "err", err)
return err
}
@@ -849,61 +495,22 @@ func (p *Processor) updateStatus(ctx context.Context, i *instance.Instance) erro
return err
}
func ingressEqual(a, b []v1.LoadBalancerIngress) bool {
if len(a) != len(b) {
return false
}
for i := range len(a) {
if a[i].IP != b[i].IP || a[i].Hostname != b[i].Hostname ||
!cmp.Equal(a[i].Ports, b[i].Ports) {
return false
}
}
return true
}
func isUPNPEnabled(s *v1.Service) bool {
return metav1.HasAnnotation(s.ObjectMeta, kubevip.UpnpEnabled) && s.Annotations[kubevip.UpnpEnabled] == "true"
}
// Refresh UPNP Port Forwards for all Service Instances registered in the processor
func (p *Processor) RefreshUPNPForwards(ctx context.Context) {
func (p *Processor) RefreshUPNPForwards() {
log.Info("Starting UPNP Port Refresher")
ticker := time.NewTicker(300 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
// Skip logging if no service instances
if len(p.ServiceInstances) == 0 {
continue
}
time.Sleep(300 * time.Second)
log.Info("[UPNP] Refreshing Instances", "number of instances", len(p.ServiceInstances))
for i := range p.ServiceInstances {
p.upnpMap(ctx, p.ServiceInstances[i])
if err := p.updateStatus(ctx, p.ServiceInstances[i]); err != nil {
log.Warn("[UPNP] Error updating service", "ip", p.ServiceInstances[i].ServiceSnapshot.Name, "err", err)
}
log.Info("[UPNP] Refreshing Instances", "number of instances", len(p.ServiceInstances))
for i := range p.ServiceInstances {
p.upnpMap(context.TODO(), p.ServiceInstances[i])
if err := p.updateStatus(p.ServiceInstances[i]); err != nil {
log.Warn("[UPNP] Error updating service", "ip", p.ServiceInstances[i].ServiceSnapshot.Name, "err", err)
}
}
}
}
// GenerateLabelFromService generates a label key and value for the given service
func generateLabelsFromService(svc *v1.Service, labelKey string) map[string]string {
addresses, _ := instance.FetchServiceAddresses(svc)
sanitized := make([]string, len(addresses))
for i, addr := range addresses {
sanitized[i] = utils.SanitizeIPForLabel(addr)
}
return map[string]string{
fmt.Sprintf("%s/%s.%s", labelKey, svc.Name, svc.Namespace): strings.Join(sanitized, ","),
}
}

View File

@@ -1,68 +0,0 @@
package services
import (
"testing"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/kube-vip/kube-vip/pkg/instance"
)
// Test_upnpLeaseDurationForService tests whether the default lease duration is used, and whether the annotation
// overrides it correctly.
//
// For simplicity, this table driven test does not cover all edge cases (passing in nil instance, nil service snapshot,
// nil annotations, etc).
func Test_upnpLeaseDurationForService(t *testing.T) {
const annotation = "kube-vip.io/upnp-lease-duration" // Validating value of kubevip.UpnpLeaseDuration.
tcs := []struct {
name string
annotations map[string]string
want int // in seconds
}{
{
name: "No annotation uses default",
annotations: map[string]string{},
want: 3600,
},
{
name: "Valid annotation overrides default",
annotations: map[string]string{
annotation: "2h",
},
want: 7200,
},
{
name: "Valid short annotation overrides default",
annotations: map[string]string{
annotation: "30m",
},
want: 1800,
},
{
name: "Invalid annotation uses default",
annotations: map[string]string{
annotation: "invalid-duration",
},
want: 3600,
},
}
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
i := &instance.Instance{
ServiceSnapshot: &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Annotations: tc.annotations,
},
},
}
gotDuration := upnpLeaseDurationForService(i)
got := int(gotDuration.Seconds())
if got != tc.want {
t.Errorf("upnpLeaseDurationForService(%+v) = %v, want %v", tc.annotations, got, tc.want)
}
})
}
}

Some files were not shown because too many files have changed in this diff Show More