Compare commits

..

1 Commits

Author SHA1 Message Date
Dan Finneran
296c9c1580 Unintentionally created duplicate functions that can't be used
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-08-12 12:17:20 +00:00
140 changed files with 3642 additions and 14523 deletions

View File

@@ -22,10 +22,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@v4
with:
ref: ${{ github.ref_name }}
- name: Anchore SBOM Action
uses: anchore/sbom-action@v0.24.0
uses: anchore/sbom-action@v0.20.4
with:
format: cyclonedx-json

View File

@@ -9,63 +9,43 @@ jobs:
matrix:
mode: ["arp", "rt", "bgp"]
fail-fast: true
max-parallel: 1
steps:
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H-%M')"
- name: Ensure fs wont cause issues
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image locally
run: make dockerx86Local
- name: Run Manifest generation tests
run: make manifest-test
- name: Run ARP mode tests
run: E2E_KEEP_LOGS=true make e2e-tests-arp
- name: Run ARP mode tests v1.29.0 onwards
run: make e2e-tests129-arp
if: matrix.mode== 'arp'
- name: Run RT mode tests
run: E2E_KEEP_LOGS=true make e2e-tests-rt
- name: Run RT mode tests v1.29.0 onwards
run: make e2e-tests129-rt
if: matrix.mode== 'rt'
- name: Get GoBGP binaries
run: make get-gobgp
if: matrix.mode== 'bgp'
- name: Run BGP mode tests
run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true make e2e-tests-bgp
- name: Run BGP mode tests v1.29.0 onwards
run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 make e2e-tests129-bgp
if: matrix.mode== 'bgp'
- name: Change log directory permissions
run: sudo chmod -R 755 /tmp/kube-vip-test-${{ matrix.mode }}*
if: matrix.mode== 'bgp' && always()
- name: Save logs
uses: actions/upload-artifact@v7
with:
name: e2e-test-logs-${{ matrix.mode }}-${{ steps.date.outputs.date }}
path: /tmp/kube-vip-test-${{ matrix.mode }}*
if: always()
service-e2e-tests:
runs-on: ubuntu-latest
name: E2E service tests
steps:
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H-%M')"
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image with iptables
run: make dockerx86ActionIPTables
- name: Run tests
run: DOCKERTAG=action E2E_KEEP_LOGS=true make service-tests
- name: Save logs
uses: actions/upload-artifact@v7
with:
name: services-test-logs-${{ steps.date.outputs.date }}
path: /tmp/kube-vip-service-tests*
if: always()
run: DOCKERTAG=action make service-tests

View File

@@ -10,11 +10,11 @@ jobs:
- name: Init
run: sudo apt-get update && sudo apt-get install -y build-essential && sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Install golangci-lint
run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.7.2
run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.64.8
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: All checks
@@ -24,9 +24,9 @@ jobs:
name: Unit tests
steps:
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Run tests
@@ -36,9 +36,9 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Run tests
@@ -48,7 +48,7 @@ jobs:
name: Image vulnerability scan
steps:
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Build image with iptables
run: make dockerx86ActionIPTables
- name: Run Trivy vulnerability scanner

View File

@@ -38,16 +38,16 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version-file: go.mod
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v4
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@@ -58,7 +58,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v4
uses: github/codeql-action/autobuild@v3
# Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
@@ -72,4 +72,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v4
uses: github/codeql-action/analyze@v3

View File

@@ -11,25 +11,25 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to Github Packages
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build standard version
id: docker_build
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
@@ -39,7 +39,7 @@ jobs:
ghcr.io/kube-vip/kube-vip:${{ github.ref_name }}
- name: Build iptables version
id: docker_build_iptables
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: .
file: Dockerfile_iptables

View File

@@ -11,31 +11,31 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to Github Packages
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Generate Metadata
uses: docker/metadata-action@v6.0.0
uses: docker/metadata-action@v5.7.0
id: metadata
with:
labels: |
org.opencontainers.image.documentation=https://kube-vip.io/docs/
- name: Build and push main branch
id: docker_build
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
@@ -48,7 +48,7 @@ jobs:
ghcr.io/kube-vip/kube-vip:latest
- name: Build iptables version and push main branch
id: docker_build_iptables
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: .
file: Dockerfile_iptables

View File

@@ -1,38 +1,13 @@
version: "2"
run:
timeout: 10m
linters:
enable:
- bodyclose
- gosec
- misspell
- unconvert
- unparam
settings:
misspell:
ignore-rules:
- creater
staticcheck:
checks:
- all
# Disable QF1008 to retain embedded fields for better readability.
- "-QF1008"
exclusions:
generated: lax
presets:
- comments
- common-false-positives
- legacy
- std-error-handling
paths:
- third_party$
- builtin$
- examples$
formatters:
enable:
- gofmt
- goimports
exclusions:
generated: lax
paths:
- third_party$
- builtin$
- examples$
- bodyclose
- gofmt
- goimports
- revive
- gosec
- misspell
- unconvert
- unparam

View File

@@ -1,38 +0,0 @@
# Changelog
All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
### Fixed
- Retry on 403 Forbidden and 401 Unauthorized in `ServicesWatcher` at startup with exponential backoff. Fixes #1464.
### Added
- SIGUSR1 signal handler for runtime configuration dumps (#1301)
- Send SIGUSR1 to kube-vip process to dump current configuration to stdout
- Configuration dump includes:
- Basic configuration (VIP, interface, port, namespace settings)
- BGP configuration (enabled status, AS number, router ID, peers)
- ARP/NDP configuration (enabled status, broadcast rate)
- Services configuration (enabled status, load balancer settings)
- Network interfaces status
- Leader election configuration (type, lease details)
- Runtime statistics (load balancer, Prometheus, health check settings)
- Output format: Human-readable plaintext via fmt.Printf()
- Thread-safe implementation using mutex protection
- Non-disruptive: Process continues running after configuration dump
- Added comprehensive unit tests for all dump methods
- Added E2E tests for signal handling
### Changed
- Updated signal handlers in manager_arp.go, manager_bgp.go, manager_wireguard.go, and manager_table.go to use switch statement pattern for handling multiple signals (SIGUSR1, SIGINT, SIGTERM)
- wireguard.go now manages a complete wireguard interface on the current network namespace
- manager_wireguard.go uses the new wireguard.go implementation
## [v1.0.1] - Previous Release
### Previous changes
- See git history for changes prior to CHANGELOG.md introduction

View File

@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:experimental
FROM golang:1.26.1-alpine3.23 as dev
FROM golang:1.24.5-alpine3.22 as dev
RUN apk add --no-cache git ca-certificates make
RUN adduser -D appuser
COPY . /src/

View File

@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:experimental
FROM golang:1.26.1-alpine3.23 as dev
FROM golang:1.24.5-alpine3.22 as dev
RUN apk add --no-cache git make
RUN adduser -D appuser
COPY . /src/
@@ -11,7 +11,7 @@ RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
--mount=type=cache,sharing=locked,id=goroot,target=/root/.cache/go-build \
CGO_ENABLED=0 GOOS=linux make build
FROM alpine:3.23.3
FROM alpine:3.22.1
# Update pkgs and add iptables
RUN apk upgrade && \
apk add --no-cache iptables iptables-legacy

View File

@@ -5,7 +5,7 @@ TARGET := kube-vip
.DEFAULT_GOAL := $(TARGET)
# These will be provided to the target
VERSION := v1.1.1
VERSION := v1.0.0
BUILD := `git rev-parse HEAD`
@@ -15,11 +15,9 @@ TARGETOS=linux
# Use linker flags to provide version/build settings to the target
LDFLAGS=-ldflags "-s -w -X=main.Version=$(VERSION) -X=main.Build=$(BUILD) -extldflags -static"
DOCKERTAG ?= $(VERSION)
REPOSITORY ?= docker.io/plndr
GO_VERSION := 1.25.5
K8S_VERSION ?= v1.35.0
REPOSITORY ?= plndr
.PHONY: all build clean install uninstall simplify check run e2e-tests unit-tests integration-tests unit-tests-docker integration-tests-docker
.PHONY: all build clean install uninstall simplify check run e2e-tests
all: check install
@@ -111,45 +109,44 @@ run: install
manifests:
@make build
@mkdir -p ./docs/manifests/$(VERSION)/
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-arp.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-lb.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-bgp.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-arp.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-lb.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --bgp --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-bgp.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
@-rm ./kube-vip
manifest-test:
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --enableLoadBalancer
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --controlplane --services
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --enableLoadBalancer
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --bgp --controlplane --services
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster
unit-tests:
go test -race ./...
unit-tests-docker:
docker run --rm -w /kube-vip -v $$(pwd):/kube-vip golang:$(GO_VERSION) make unit-tests
go test ./...
integration-tests:
go test -tags=integration,e2e -v ./pkg/etcd
e2e-tests-arp: get-whoami
GOMAXPROCS=4 TEST_MODE=arp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests:
GOMAXPROCS=4 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd
e2e-tests-rt: get-whoami
GOMAXPROCS=4 TEST_MODE=rt K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129-arp:
GOMAXPROCS=4 TEST_MODE=arp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests-bgp: get-whoami
GOMAXPROCS=4 TEST_MODE=bgp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129-rt:
GOMAXPROCS=4 TEST_MODE=rt V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests: e2e-tests-arp e2e-tests-rt e2e-tests-bgp
e2e-tests129-bgp:
GOMAXPROCS=4 TEST_MODE=bgp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129: e2e-tests129-arp e2e-tests129-rt e2e-tests129-bgp
service-tests:
$(MAKE) -C testing/e2e/e2e dockerLocal
E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/services -Services -simple -deployments -leaderActive -leaderFailover -localDeploy -egress -egressIPv6 -dualStack
trivy: dockerx86ActionIPTables
@@ -169,10 +166,10 @@ kind-quick:
kubectl create configmap --namespace kube-system kubevip --from-literal range-global=172.18.100.10-172.18.100.30
kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
kind load docker-image --name kube-vip $(REPOSITORY)/$(TARGET):$(DOCKERTAG)
docker run --network host --rm $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --services --inCluster --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --servicesElection --interface eth0 | kubectl apply -f -
docker run --network host --rm $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --services --inCluster --arp --servicesElection --interface eth0 | kubectl apply -f -
kind-reload:
kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name services
kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name kube-vip
kubectl rollout restart -n kube-system daemonset/kube-vip-ds
get-gobgp:
@@ -180,5 +177,3 @@ get-gobgp:
wget -nc --directory-prefix=bin https://github.com/osrg/gobgp/releases/download/v3.37.0/gobgp_3.37.0_linux_amd64.tar.gz
tar -xvzf bin/gobgp_3.37.0_linux_amd64.tar.gz -C bin
get-whoami:
docker pull ghcr.io/traefik/whoami:v1.11

View File

@@ -66,17 +66,6 @@ Thanks for taking the time to join our community and start contributing! We welc
:warning: This project has issue compiling on MacOS, please compile it on linux distribution
Additionally it is now relatively easy and quick to develop with [skaffold](https://skaffold.dev/), and the `skaffold.yaml` exists within the root folder of the gir repository.
### Set up a kind development environment
1. `kind create cluster --config ./testing/kind.yaml`
2. `kubectl apply -f https://kube-vip.io/manifests/rbac.yaml`
3. Create a load balancer range `configMap` from the kind cluster
4. Apply the CCM manifest
5. Start skaffold with `skaffold dev`
6. Start developing and see changes applied in real-time.
## Star History
[![Star History Chart](https://api.star-history.com/svg?repos=kube-vip/kube-vip&type=Date)](https://star-history.com/#kube-vip/kube-vip&Date)

View File

@@ -58,18 +58,14 @@ var kubeKubeadmInit = &cobra.Command{
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating VIPSubnet", "err", err)
return
}
}
cfg, err := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
if err != nil {
log.Error("unable to create manifest", "err", err)
return
}
cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
fmt.Println(cfg) // output manifest to stdout
},
}
@@ -107,18 +103,14 @@ var kubeKubeadmJoin = &cobra.Command{
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating VIPSubnet", "err", err)
return
}
}
cfg, err := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
if err != nil {
log.Error("unable to create manifest", "err", err)
return
}
cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
fmt.Println(cfg) // output manifest to stdout
},
}

View File

@@ -19,7 +19,6 @@ var taint, role, rolebinding bool
func init() {
kubeManifest.PersistentFlags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
kubeManifest.PersistentFlags().StringVar(&image, "image", "ghcr.io/kube-vip/kube-vip", "Define a hardcoded image with or without tag for the manifest")
kubeManifestDaemon.PersistentFlags().BoolVar(&taint, "taint", false, "Taint the manifest for only running on control planes")
kubeManifestRbac.PersistentFlags().BoolVar(&role, "role", false, "Generate only a Role inside the serviceNamespace access")
kubeManifestRbac.PersistentFlags().BoolVar(&rolebinding, "rolebinding", false, "Generate only a RoleBinding for namespaced access")
@@ -54,24 +53,20 @@ var kubeManifestPod = &cobra.Command{
// The control plane has a requirement for a VIP being specified
if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
_ = cmd.Help()
log.Error("no address is specified for kube-vip to expose services on")
log.Error("No address is specified for kube-vip to expose services on")
return
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("config parse", "err", err)
return
}
}
cfg, err := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
if err != nil {
log.Error("unable to create manifest", "err", err)
return
}
cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
fmt.Println(cfg) // output manifest to stdout
},
}
@@ -91,24 +86,20 @@ var kubeManifestDaemon = &cobra.Command{
// The control plane has a requirement for a VIP being specified
if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
_ = cmd.Help()
log.Error("no address is specified for kube-vip to expose services on")
log.Error("No address is specified for kube-vip to expose services on")
return
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("config parse", "err", err)
return
}
}
cfg, err := kubevip.GenerateDaemonsetManifestFromConfig(&initConfig, image, Release.Version, inCluster, taint)
if err != nil {
log.Error("unable to create manifest", "err", err)
return
}
cfg := kubevip.GenerateDaemonsetManifestFromConfig(&initConfig, Release.Version, inCluster, taint)
fmt.Println(cfg) // output manifest to stdout
},
}
@@ -129,13 +120,13 @@ var kubeManifestRbac = &cobra.Command{
// The control plane has a requirement for a VIP being specified
if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
_ = cmd.Help()
log.Error("no address is specified for kube-vip to expose services on")
log.Error("No address is specified for kube-vip to expose services on")
return
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating VIPSubnet", "err", err)
return

View File

@@ -6,10 +6,7 @@ import (
"net"
"net/http"
"os"
"slices"
"strconv"
"strings"
"sync"
"time"
log "log/slog"
@@ -22,13 +19,9 @@ import (
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/manager"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
// Is an option to set the image
var image string
// Is kube-vip running within cluster
var inCluster bool
@@ -69,7 +62,6 @@ func init() {
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for VIP changes")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableWireguard, "wireguard", false, "Enable Wireguard for services VIPs")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableRoutingTable, "table", false, "Enable Routing Table for services VIPs")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.PreserveVIPOnLeadershipLoss, "preserveVipOnLeadershipLoss", false, "Preserve ARP VIP addresses on interface when leadership is lost (default: false for backward compatibility)")
// LoadBalancer flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "enableLoadBalancer", false, "enable loadbalancing on the VIP with IPVS")
@@ -82,9 +74,9 @@ func init() {
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLeaderElection, "leaderElection", false, "Use the Kubernetes leader election mechanism for clustering")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaderElectionType, "leaderElectionType", "kubernetes", "Defines the backend to run the leader election: kubernetes or etcd. Defaults to kubernetes.")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaseName, "leaseName", "plndr-cp-lock", "Name of the lease that is used for leader election")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 15, "Length of time (in seconds) a Kubernetes leader lease can be held for")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 10, "Length of time (in seconds) a Kubernetes leader can attempt to renew its lease")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 2, "Length of time (in seconds) the LeaderElector clients should wait between tries of actions")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 5, "Length of time (in seconds) a Kubernetes leader lease can be held for")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time (in seconds) a Kubernetes leader can attempt to renew its lease")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Length of time (in seconds) the LeaderElector clients should wait between tries of actions")
// BGP flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableBGP, "bgp", false, "This will enable BGP support within kube-vip")
@@ -128,16 +120,13 @@ func init() {
// Extended behaviour flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServicesElection, "servicesElection", false, "Enable leader election per kubernetes service")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassOnly, "lbClassOnly", false, fmt.Sprintf("Enable load balancing only for services with LoadBalancerClass %q", kubevip.LBClassName))
kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerClassName, "lbClassName", kubevip.LBClassName, fmt.Sprintf("Name of load balancer class for kube-VIP, defaults to %q", kubevip.LBClassName))
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassOnly, "lbClassOnly", false, "Enable load balancing only for services with LoadBalancerClass \"kube-vip.io/kube-vip-class\"")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerClassName, "lbClassName", "kube-vip.io/kube-vip-class", "Name of load balancer class for kube-VIP, defaults to \"kube-vip.io/kube-vip-class\"")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassLegacyHandling, "lbClassNameLegacyHandling", true, "Use legacy LoadBalancer class name handling (e.g. accepting services both with empty and non-empty class)")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServiceSecurity, "onlyAllowTrafficServicePorts", false, "Only allow traffic to service ports, others will be dropped, defaults to false")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableNodeLabeling, "enableNodeLabeling", false, fmt.Sprintf("Enable leader node labeling with %q, defaults to false", kubevip.HasIP))
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableNodeLabeling, "enableNodeLabeling", false, "Enable leader node labeling with \"kube-vip.io/has-ip=<VIP address>\", defaults to false")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesLeaseName, "servicesLeaseName", "plndr-svcs-lock", "Name of the lease that is used for leader election for services (in arp mode)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.DNSMode, "dnsMode", "first", "Name of the mode that DNS lookup will be performed (first, ipv4, ipv6, dual)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.DHCPMode, "dhcpMode", "", "Mode DHCP resolving will use to obtain IP addresses (ipv4, ipv6, dual)")
kubeVipCmd.PersistentFlags().UintVar(&initConfig.DHCPBackoffAttempts, "dhcpBackoffAttempts", kubevip.DefaultDHCPBackoffAttempts,
fmt.Sprintf("number of times DHCP client will try to obtain an IP address (defaults to: %d, 0 for unlimited retries)", kubevip.DefaultDHCPBackoffAttempts))
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DisableServiceUpdates, "disableServiceUpdates", false, "If true, kube-vip will process services as usual, but will not update service's Status.LoadBalancer.Ingress slice")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpoints, "enableEndpoints", false, "If enabled, kube-vip will only advertise services, but will use the (deprecated since v1.33) endpoints for IP addresses")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoInterfaceGlobalScope, "loInterfaceGlobalScope", false, "If true, kube-vip will set global scope when using the lo interface, otherwise a host scope will be used by default")
@@ -155,9 +144,6 @@ func init() {
// Kubernetes client specific flags
kubeVipCmd.PersistentFlags().StringVar(&initConfig.K8sConfigFile, "k8sConfigPath", "/etc/kubernetes/admin.conf", "Path to the configuration file used with the Kubernetes client")
// Configuration file flag
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ConfigFile, "config-file", "", "Path to a JSON/YAML configuration file to load settings from")
kubeVipCmd.AddCommand(kubeKubeadm)
kubeVipCmd.AddCommand(kubeManifest)
kubeVipCmd.AddCommand(kubeVipManager)
@@ -200,16 +186,7 @@ var kubeVipService = &cobra.Command{
Short: "Start the Virtual IP / Load balancer as a service within a Kubernetes cluster",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
// Load configuration from file if specified (lowest priority)
if initConfig.ConfigFile != "" {
err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
if err != nil {
log.Error("loading config file", "err", err)
return
}
}
// parse environment variables, these will overwrite anything loaded from config file
// parse environment variables, these will overwrite anything loaded or flags
err := kubevip.ParseEnvironment(&initConfig)
if err != nil {
log.Error("parsing env", "err", err)
@@ -237,7 +214,7 @@ var kubeVipService = &cobra.Command{
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating CIDR", "err", err)
return
@@ -251,11 +228,8 @@ var kubeVipService = &cobra.Command{
return
}
ctx, cancel := context.WithCancel(cmd.Context())
defer cancel()
// Start the service manager, this will watch the config Map and construct kube-vip services for it
err = mgr.Start(ctx)
err = mgr.Start()
if err != nil {
log.Error("manager start", "err", err)
return
@@ -267,16 +241,7 @@ var kubeVipManager = &cobra.Command{
Use: "manager",
Short: "Start the kube-vip manager",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
// Load configuration from file if specified (lowest priority)
if initConfig.ConfigFile != "" {
err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
if err != nil {
log.Error("loading config file", "err", err)
return
}
}
// parse environment variables, these will overwrite anything loaded from config file
// parse environment variables, these will overwrite anything loaded or flags
err := kubevip.ParseEnvironment(&initConfig)
if err != nil {
log.Error("parsing environment", "err", err)
@@ -293,7 +258,7 @@ var kubeVipManager = &cobra.Command{
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("No interface is specified for kube-vip to bind to")
return
@@ -303,19 +268,10 @@ var kubeVipManager = &cobra.Command{
// Welome messages
log.Info("kube-vip.io", "version", Release.Version, "build", Release.Build)
wg := sync.WaitGroup{}
defer wg.Wait()
// create main manager context
ctx, cancel := context.WithCancel(cmd.Context())
defer cancel()
// start prometheus server
if initConfig.PrometheusHTTPServer != "" {
wg.Go(func() {
servePrometheusHTTPServer(ctx, PrometheusHTTPServerConfig{
Addr: initConfig.PrometheusHTTPServer,
})
go servePrometheusHTTPServer(cmd.Context(), PrometheusHTTPServerConfig{
Addr: initConfig.PrometheusHTTPServer,
})
}
@@ -399,13 +355,13 @@ var kubeVipManager = &cobra.Command{
initConfig.Interface = defaultIF.Name
log.Info("kube-vip bind", "interface", initConfig.Interface)
wg.Go(func() {
if err := vip.MonitorDefaultInterface(ctx, defaultIF); err != nil {
go func() {
if err := vip.MonitorDefaultInterface(context.TODO(), defaultIF); err != nil {
log.Error("interface monitor", "err", err)
return
}
})
}()
}
}
// Perform a check on the state of the interface
@@ -430,7 +386,7 @@ var kubeVipManager = &cobra.Command{
prometheus.MustRegister(mgr.PrometheusCollector()...)
// Start the service manager, this will watch the config Map and construct kube-vip services for it
err = mgr.Start(ctx)
err = mgr.Start()
if err != nil {
log.Error("start manager", "err", err)
return
@@ -464,20 +420,19 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
ReadHeaderTimeout: 2 * time.Second,
}
wg := sync.WaitGroup{}
wg.Go(func() {
go func() {
if err = srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Error("prometheus HTTP server", "err", err)
return
}
})
}()
log.Info("prometheus HTTP server started")
<-ctx.Done()
// create prometheus shutdown context (independent of other contexts)
log.Info("prometheus HTTP server stopped")
ctxShutDown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer func() {
cancel()
@@ -491,43 +446,28 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
if err == http.ErrServerClosed {
err = nil
}
log.Info("prometheus HTTP server stopped")
wg.Wait()
}
func GenerateCidrRange(address string, dnsMode string) (string, error) {
func GenerateCidrRange(address string) (string, error) {
var cidrs []string
addresses := strings.Split(address, ",")
for _, a := range addresses {
ip := net.ParseIP(a)
if ip == nil {
// we probably are a DNS name
ips, err := utils.LookupHost(a, dnsMode, true)
ips, err := net.LookupIP(a)
if len(ips) == 0 || err != nil {
return "", fmt.Errorf("invalid IP address: %s from [%s], %v", a, address, err)
}
for _, addr := range ips {
ip = net.ParseIP(addr)
if ip.To4() != nil {
cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv4))
} else {
cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv6))
}
}
ip = ips[0]
}
if ip.To4() != nil {
cidrs = append(cidrs, "32")
} else {
if ip.To4() != nil {
cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv4))
} else {
cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv6))
}
cidrs = append(cidrs, "128")
}
}
// compact as DNS could have a lot of addresses
slices.Sort(cidrs)
cidrs = slices.Compact(cidrs)
slices.Reverse(cidrs)
return strings.Join(cidrs, ","), nil
}

View File

@@ -19,7 +19,7 @@ func main() {
var errorOccurred bool
for {
p := make([]byte, 2048)
conn, err := net.Dial("udp", net.JoinHostPort(*address, fmt.Sprint(port)))
conn, err := net.Dial("udp", fmt.Sprintf("%s:%d", *address, *port))
if err != nil {
if !errorOccurred {
errorTime = time.Now()

124
go.mod
View File

@@ -1,53 +1,50 @@
module github.com/kube-vip/kube-vip
go 1.25.7
go 1.24.4
require (
github.com/cloudflare/ipvs v0.11.0
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/docker/docker v28.5.2+incompatible
github.com/florianl/go-conntrack v0.7.0
github.com/docker/docker v28.3.3+incompatible
github.com/florianl/go-conntrack v0.4.0
github.com/google/go-cmp v0.7.0
github.com/google/go-containerregistry v0.21.3
github.com/google/nftables v0.3.0
github.com/gookit/slog v0.6.0
github.com/gookit/slog v0.5.8
github.com/huin/goupnp v1.3.0
github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d
github.com/jpillora/backoff v1.0.0
github.com/mdlayher/ndp v1.1.0
github.com/onsi/ginkgo/v2 v2.28.1
github.com/onsi/gomega v1.39.1
github.com/onsi/ginkgo/v2 v2.23.4
github.com/onsi/gomega v1.37.0
github.com/osrg/gobgp/v3 v3.37.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.23.2
github.com/sirupsen/logrus v1.9.4
github.com/spf13/cobra v1.10.2
github.com/stretchr/testify v1.11.1
github.com/prometheus/client_golang v1.22.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.9.1
github.com/stretchr/testify v1.10.0
github.com/vishvananda/netlink v1.3.1
go.etcd.io/etcd/api/v3 v3.6.9
go.etcd.io/etcd/client/pkg/v3 v3.6.9
go.etcd.io/etcd/client/v3 v3.6.9
go.uber.org/zap v1.27.1
go.etcd.io/etcd/api/v3 v3.6.1
go.etcd.io/etcd/client/pkg/v3 v3.6.1
go.etcd.io/etcd/client/v3 v3.6.1
go.uber.org/zap v1.27.0
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
golang.org/x/sync v0.20.0
golang.org/x/sys v0.42.0
golang.org/x/sync v0.15.0
golang.org/x/sys v0.33.0
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
google.golang.org/grpc v1.79.3
google.golang.org/protobuf v1.36.11
gopkg.in/yaml.v3 v3.0.1
k8s.io/api v0.35.3
k8s.io/apimachinery v0.35.3
k8s.io/client-go v0.35.2
k8s.io/klog/v2 v2.140.0
sigs.k8s.io/kind v0.31.0
sigs.k8s.io/yaml v1.6.0
google.golang.org/grpc v1.73.0
google.golang.org/protobuf v1.36.6
k8s.io/api v0.33.3
k8s.io/apimachinery v0.33.3
k8s.io/client-go v0.33.3
k8s.io/klog/v2 v2.130.1
sigs.k8s.io/kind v0.29.0
sigs.k8s.io/yaml v1.4.0
)
require (
al.essio.dev/pkg/shellescape v1.5.1 // indirect
github.com/BurntSushi/toml v1.4.0 // indirect
github.com/Masterminds/semver/v3 v3.4.0 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/Microsoft/go-winio v0.4.14 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/containerd/errdefs v1.0.0 // indirect
@@ -58,15 +55,15 @@ require (
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/dlclark/regexp2 v1.11.4 // indirect
github.com/docker/go-connections v0.6.0 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/eapache/channels v1.1.0 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/emicklei/go-restful/v3 v3.12.2 // indirect
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.8.0 // indirect
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
@@ -75,12 +72,12 @@ require (
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.7.0 // indirect
github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect
github.com/google/gnostic-models v0.6.9 // indirect
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gookit/color v1.6.0 // indirect
github.com/gookit/goutil v0.7.1 // indirect
github.com/gookit/gsr v0.1.1 // indirect
github.com/gookit/color v1.5.4 // indirect
github.com/gookit/goutil v0.6.18 // indirect
github.com/gookit/gsr v0.1.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
@@ -100,8 +97,8 @@ require (
github.com/moby/sys/atomicwriter v0.1.0 // indirect
github.com/moby/term v0.5.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
github.com/morikuni/aec v1.1.0 // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
@@ -109,16 +106,16 @@ require (
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/pierrec/lz4/v4 v4.1.22 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.62.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/sagikazarmark/locafero v0.6.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.7.1 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/spf13/pflag v1.0.6 // indirect
github.com/spf13/viper v1.19.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tj/go-spin v1.1.0 // indirect
@@ -129,39 +126,40 @@ require (
github.com/xlab/c-for-go v1.3.0 // indirect
github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
go.opentelemetry.io/otel v1.39.0 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.37.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 // indirect
go.opentelemetry.io/otel/metric v1.39.0 // indirect
go.opentelemetry.io/otel/trace v1.39.0 // indirect
go.opentelemetry.io/otel/metric v1.37.0 // indirect
go.opentelemetry.io/otel/sdk v1.37.0 // indirect
go.opentelemetry.io/otel/trace v1.37.0 // indirect
go.uber.org/automaxprocs v1.6.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.49.0 // indirect
golang.org/x/mod v0.34.0 // indirect
golang.org/x/net v0.52.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/term v0.41.0 // indirect
golang.org/x/text v0.35.0 // indirect
golang.org/x/crypto v0.39.0 // indirect
golang.org/x/mod v0.25.0 // indirect
golang.org/x/net v0.41.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/term v0.32.0 // indirect
golang.org/x/text v0.26.0 // indirect
golang.org/x/time v0.9.0 // indirect
golang.org/x/tools v0.43.0 // indirect
golang.org/x/tools v0.33.0 // indirect
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect
k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect
modernc.org/cc/v4 v4.24.4 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/opt v0.1.4 // indirect
modernc.org/sortutil v1.2.1 // indirect
modernc.org/strutil v1.2.1 // indirect
modernc.org/token v1.1.0 // indirect
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
sigs.k8s.io/randfill v1.0.0 // indirect
sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
)

395
go.sum
View File

@@ -2,18 +2,20 @@ al.essio.dev/pkg/shellescape v1.5.1 h1:86HrALUujYS/h+GtqoB26SBEdkWfmMI6FubjXlsXy
al.essio.dev/pkg/shellescape v1.5.1/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8=
github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
github.com/cloudflare/ipvs v0.11.0 h1:niLcbqfv8+RSYk+yI+jhiCIHobEdEhO++mcwCX0Znsw=
github.com/cloudflare/ipvs v0.11.0/go.mod h1:XsbuKcQpqb3rpjYPtsFsM8BNZlgLw+Z0iC6RhocqDd8=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
@@ -37,10 +39,10 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
@@ -49,26 +51,21 @@ github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4
github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0=
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU=
github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU=
github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/florianl/go-conntrack v0.7.0 h1:kWbRhLRUqBlmi2ncASmygN5yI281wlpnTUJs69tcs7g=
github.com/florianl/go-conntrack v0.7.0/go.mod h1:7kzfTjtQCRsAVkuornpI8KiwFfa79QmcZXZQex1IXhU=
github.com/florianl/go-conntrack v0.4.0 h1:TlYkxytdwgVayfU0cKwkHurQA0Rd1ZSEBRckRYDUu18=
github.com/florianl/go-conntrack v0.4.0/go.mod h1:iPDx4oIats2T7X7Jm3PFyRCJM1GfZhJaSHOWROYOrE8=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs=
github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo=
github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M=
github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk=
github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE=
github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc=
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -84,40 +81,40 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw=
github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-containerregistry v0.21.3 h1:Xr+yt3VvwOOn/5nJzd7UoOhwPGiPkYW0zWDLLUXqAi4=
github.com/google/go-containerregistry v0.21.3/go.mod h1:D5ZrJF1e6dMzvInpBPuMCX0FxURz7GLq2rV3Us9aPkc=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg=
github.com/google/nftables v0.3.0/go.mod h1:BCp9FsrbF1Fn/Yu6CLUc9GGZFw/+hsxfluNXXmxBfRM=
github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc=
github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gookit/assert v0.1.1 h1:lh3GcawXe/p+cU7ESTZ5Ui3Sm/x8JWpIis4/1aF0mY0=
github.com/gookit/assert v0.1.1/go.mod h1:jS5bmIVQZTIwk42uXl4lyj4iaaxx32tqH16CFj0VX2E=
github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA=
github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs=
github.com/gookit/goutil v0.7.1 h1:AaFJPN9mrdeYBv8HOybri26EHGCC34WJVT7jUStGJsI=
github.com/gookit/goutil v0.7.1/go.mod h1:vJS9HXctYTCLtCsZot5L5xF+O1oR17cDYO9R0HxBmnU=
github.com/gookit/gsr v0.1.1 h1:TaHD3M7qa6lcAf9D2J4mGNg+QjgDtD1bw7uctF8RXOM=
github.com/gookit/gsr v0.1.1/go.mod h1:7wv4Y4WCnil8+DlDYHBjidzrEzfHhXEoFjEA0pPPWpI=
github.com/gookit/slog v0.6.0 h1:KEQxOJxbTtk7oyqah6nJOEKjOdI0z5qoqkX7I6G65g4=
github.com/gookit/slog v0.6.0/go.mod h1:hPlpNi/WIcGmkEjHzQTS7s5JZkHmmnGy9sYo6csa08s=
github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
github.com/gookit/goutil v0.6.18 h1:MUVj0G16flubWT8zYVicIuisUiHdgirPAkmnfD2kKgw=
github.com/gookit/goutil v0.6.18/go.mod h1:AY/5sAwKe7Xck+mEbuxj0n/bc3qwrGNe3Oeulln7zBA=
github.com/gookit/gsr v0.1.0 h1:0gadWaYGU4phMs0bma38t+Do5OZowRMEVlHv31p0Zig=
github.com/gookit/gsr v0.1.0/go.mod h1:7wv4Y4WCnil8+DlDYHBjidzrEzfHhXEoFjEA0pPPWpI=
github.com/gookit/slog v0.5.8 h1:XZCeHLQvvOZWcSUDZcqxXITsL9+d1ESsKZoASBmK1lI=
github.com/gookit/slog v0.5.8/go.mod h1:s0ViFOY/IgUuT4MDPF0l9x5/npcciy8pL4xwWZadnoc=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
@@ -132,23 +129,34 @@ github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d h1:VkCNWh6tuQLgD
github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d/go.mod h1:VvGYjkZoJyKqlmT1yzakUs4mfKMNB0XdODP0+rdml6k=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE=
github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung=
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ=
github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok=
github.com/jsimonetti/rtnetlink v0.0.0-20201216134343-bde56ed16391/go.mod h1:cR77jAZG3Y3bsb8hF6fHJbFoyFukLFOkQ98S0pQz3xw=
github.com/jsimonetti/rtnetlink v0.0.0-20201220180245-69540ac93943/go.mod h1:z4c53zj6Eex712ROyh8WI0ihysb5j2ROyV42iNogmAs=
github.com/jsimonetti/rtnetlink v0.0.0-20210122163228-8d122574c736/go.mod h1:ZXpIyOK59ZnN7J0BV99cZUPmsqDRZ3eq5X+st7u/oSA=
github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b/go.mod h1:8w9Rh8m+aHZIG69YPGGem1i5VzoyRC8nw2kA8B+ik5U=
github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190/go.mod h1:NmKSdU4VGSiv1bMsdqNALI4RSvvjtz65tTMCnD05qLo=
github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/k-sone/critbitgo v1.4.0 h1:l71cTyBGeh6X5ATh6Fibgw3+rtNT80BA0uNNWgkPrbE=
github.com/k-sone/critbitgo v1.4.0/go.mod h1:7E6pyoyADnFxlUBEKcnfS49b7SUAQGMK+OAp/UQvo0s=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
@@ -157,24 +165,35 @@ github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a
github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo=
github.com/mdlayher/ethtool v0.0.0-20211028163843-288d040e9d60/go.mod h1:aYbhishWc4Ai3I2U4Gaa2n3kHWSwzme6EsG/46HRQbE=
github.com/mdlayher/genetlink v1.0.0/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc=
github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw=
github.com/mdlayher/genetlink v1.3.2/go.mod h1:tcC3pkCrPUGIKKsCsp0B3AdaaKuHtaxoJRz3cc+528o=
github.com/mdlayher/ndp v1.1.0 h1:QylGKGVtH60sKZUE88+IW5ila1Z/M9/OXhWdsVKuscs=
github.com/mdlayher/ndp v1.1.0/go.mod h1:FmgESgemgjl38vuOIyAHWUUL6vQKA/pQNkvXdWsdQFM=
github.com/mdlayher/netlink v1.6.2/go.mod h1:O1HXX2sIWSMJ3Qn1BYZk1yZM+7iMki/uYGGiwGyq/iU=
github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA=
github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M=
github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY=
github.com/mdlayher/netlink v1.1.1/go.mod h1:WTYpFb/WTvlRJAyKhZL5/uy69TDDpHHu2VZmb2XgV7o=
github.com/mdlayher/netlink v1.2.0/go.mod h1:kwVW1io0AZy9A1E2YYgaD4Cj+C+GPkU6klXCMzIJ9p8=
github.com/mdlayher/netlink v1.2.1/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU=
github.com/mdlayher/netlink v1.2.2-0.20210123213345-5cc92139ae3e/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU=
github.com/mdlayher/netlink v1.3.0/go.mod h1:xK/BssKuwcRXHrtN04UBkwQ6dY9VviGGuriDdoPSWys=
github.com/mdlayher/netlink v1.4.0/go.mod h1:dRJi5IABcZpBD2A3D0Mv/AiX8I9uDEu5oGkAVrekmf8=
github.com/mdlayher/netlink v1.4.1/go.mod h1:e4/KuJ+s8UhfUpO9z00/fDZZmhSrs+oxyqAS9cNgn6Q=
github.com/mdlayher/netlink v1.5.0/go.mod h1:1Kr8BBFxGyUyNmztC9WLOayqYVAd2wsgOZm18nqGuzQ=
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 h1:A1Cq6Ysb0GM0tpKMbdCXCIfBclan4oHk1Jb+Hrejirg=
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42/go.mod h1:BB4YCPDOzfy7FniQ/lxuYQ3dgmM2cZumHbK8RpTjN2o=
github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY=
github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4=
github.com/mdlayher/socket v0.2.3/go.mod h1:bz12/FozYNH/VbvC3q7TRIK/Y6dH1kCKsXaUeXi/FmY=
github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00/go.mod h1:GAFlyu4/XV68LkQKYzKhIo/WW7j3Zi0YRAz/BOoanUc=
github.com/mdlayher/socket v0.0.0-20211007213009-516dcbdf0267/go.mod h1:nFZ1EtZYK8Gi/k6QNu7z7CgO20i/4ExeQswwWuPmG/g=
github.com/mdlayher/socket v0.1.0/go.mod h1:mYV5YIZAfHh4dzDVzI8x8tWLWCliuX8Mon5Awbj+qDs=
github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos=
github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ=
github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE=
github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A=
github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws=
github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
@@ -190,17 +209,16 @@ github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFL
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/morikuni/aec v1.1.0 h1:vBBl0pUnvi/Je71dsRrhMBtreIqNMYErSAbEeb8jrXQ=
github.com/morikuni/aec v1.1.0/go.mod h1:xDRgiq/iw5l+zkao76YTKzKttOp2cwPEne25HDkJnBw=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI=
github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE=
github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28=
github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg=
github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus=
github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8=
github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y=
github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
@@ -215,59 +233,57 @@ github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNH
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk=
github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds=
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 h1:pyC9PaHYZFgEKFdlp3G8RaCKgVpHZnecvArXvPXcFkM=
@@ -288,97 +304,136 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.etcd.io/etcd/api/v3 v3.6.9 h1:UA7iKfEW1AzgihcBSGXci2kDGQiokSq41F9HMCI/RTI=
go.etcd.io/etcd/api/v3 v3.6.9/go.mod h1:csEk/qTfxKL36NqJdU15Tgtl65A8dyEY2BYo7PRsIwk=
go.etcd.io/etcd/client/pkg/v3 v3.6.9 h1:T8nuk8Lz64C+Hzb0coBFLMSlVSQZBpAtFk46swdM1DA=
go.etcd.io/etcd/client/pkg/v3 v3.6.9/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y=
go.etcd.io/etcd/client/v3 v3.6.9 h1:3X555hQXmhRr27O37wls53g68CpUiPOiHXrZfz2Al+o=
go.etcd.io/etcd/client/v3 v3.6.9/go.mod h1:KO7H1HLYh1qaljuVZJQwBFk1lRce6pJzt+C81GEnrlM=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=
go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.etcd.io/etcd/api/v3 v3.6.1 h1:yJ9WlDih9HT457QPuHt/TH/XtsdN2tubyxyQHSHPsEo=
go.etcd.io/etcd/api/v3 v3.6.1/go.mod h1:lnfuqoGsXMlZdTJlact3IB56o3bWp1DIlXPIGKRArto=
go.etcd.io/etcd/client/pkg/v3 v3.6.1 h1:CxDVv8ggphmamrXM4Of8aCC8QHzDM4tGcVr9p2BSoGk=
go.etcd.io/etcd/client/pkg/v3 v3.6.1/go.mod h1:aTkCp+6ixcVTZmrJGa7/Mc5nMNs59PEgBbq+HCmWyMc=
go.etcd.io/etcd/client/v3 v3.6.1 h1:KelkcizJGsskUXlsxjVrSmINvMMga0VWwFF0tSPGEP0=
go.etcd.io/etcd/client/v3 v3.6.1/go.mod h1:fCbPUdjWNLfx1A6ATo9syUmFVxqHH9bCnPLBZmnLmMY=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 h1:bDMKF3RUSxshZ5OjOTi8rsHGaPKsAt76FaqgvIUySLc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0/go.mod h1:dDT67G/IkA46Mr2l9Uj7HsQVwsjASyV9SjGofsiUZDA=
go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0=
go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs=
go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18=
go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE=
go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8=
go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew=
go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=
go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329 h1:9kj3STMvgqy3YA4VQXBrN7925ICMxD5wzMRcgA30588=
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20220923203811-8be639271d50/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210110051926-789bb1bd4061/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210123111255-9b0068b26619/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -387,43 +442,44 @@ golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uI
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA=
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10 h1:3GDAcqdIg1ozBNLgPy4SLT84nfcBjr6rhGtXYtrkWLU=
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10/go.mod h1:T97yPqesLiNrOYxkwmhMI0ZIlJDm+p0PMR8eRVeR5tQ=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls=
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY=
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok=
google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc=
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo=
gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4=
gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
k8s.io/api v0.35.3 h1:pA2fiBc6+N9PDf7SAiluKGEBuScsTzd2uYBkA5RzNWQ=
k8s.io/api v0.35.3/go.mod h1:9Y9tkBcFwKNq2sxwZTQh1Njh9qHl81D0As56tu42GA4=
k8s.io/apimachinery v0.35.3 h1:MeaUwQCV3tjKP4bcwWGgZ/cp/vpsRnQzqO6J6tJyoF8=
k8s.io/apimachinery v0.35.3/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns=
k8s.io/client-go v0.35.2 h1:YUfPefdGJA4aljDdayAXkc98DnPkIetMl4PrKX97W9o=
k8s.io/client-go v0.35.2/go.mod h1:4QqEwh4oQpeK8AaefZ0jwTFJw/9kIjdQi0jpKeYvz7g=
k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc=
k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0=
k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE=
k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ=
k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck=
k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8=
k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE=
k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA=
k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM=
k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA=
k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg=
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4=
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8=
k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0=
k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
modernc.org/cc/v4 v4.24.4 h1:TFkx1s6dCkQpd6dKurBNmpo+G8Zl4Sq/ztJ+2+DEsh0=
modernc.org/cc/v4 v4.24.4/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
modernc.org/ccorpus2 v1.5.2 h1:Ui+4tc58mf/W+2arcYCJR903y3zl3ecsI7Fpaaqozyw=
@@ -440,13 +496,14 @@ modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
sigs.k8s.io/kind v0.31.0 h1:UcT4nzm+YM7YEbqiAKECk+b6dsvc/HRZZu9U0FolL1g=
sigs.k8s.io/kind v0.31.0/go.mod h1:FSqriGaoTPruiXWfRnUXNykF8r2t+fHtK0P0m1AbGF8=
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
sigs.k8s.io/kind v0.29.0 h1:3TpCsyh908IkXXpcSnsMjWdwdWjIl7o9IMZImZCWFnI=
sigs.k8s.io/kind v0.29.0/go.mod h1:ldWQisw2NYyM6k64o/tkZng/1qQW7OlzcN5a8geJX3o=
sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco=
sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc=
sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps=
sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

View File

@@ -2,18 +2,16 @@ package arp
import (
"context"
"fmt"
log "log/slog"
"sync"
"time"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
type Manager struct {
instances sync.Map
instances map[string]*Instance
config *kubevip.Config
}
@@ -26,7 +24,8 @@ type Instance struct {
func NewManager(config *kubevip.Config) *Manager {
return &Manager{
config: config,
instances: make(map[string]*Instance),
config: config,
}
}
@@ -43,14 +42,10 @@ func (i *Instance) Name() string {
}
func (m *Manager) Insert(instance *Instance) {
i, err := m.get(instance.Name())
if err != nil {
log.Error("[ARP manager] unable to insert instance", "err", err)
return
}
if i == nil {
log.Info("[ARP manager] inserting ARP/NDP instance", "name", instance.Name())
m.instances.Store(instance.Name(), instance)
i, ok := m.instances[instance.Name()]
if !ok {
log.Info("inserting ARP/NDP instance", "name", instance.Name())
m.instances[instance.Name()] = instance
} else {
i.mu.Lock()
defer i.mu.Unlock()
@@ -59,50 +54,20 @@ func (m *Manager) Insert(instance *Instance) {
}
func (m *Manager) Remove(instance *Instance) {
m.RemoveWithIPDelete(instance, true)
}
// RemoveOnLeadershipLoss removes an ARP instance when leadership is lost
func (m *Manager) RemoveOnLeadershipLoss(instance *Instance) {
// Use the inverse of PreserveVIPOnLeadershipLoss to decide whether to delete the IP
// If preserve is true, don't delete IP (deleteIP = false)
// If preserve is false, delete IP (deleteIP = true), This is the legacy behavior
deleteIP := !m.config.PreserveVIPOnLeadershipLoss
m.RemoveWithIPDelete(instance, deleteIP)
}
func (m *Manager) RemoveWithIPDelete(instance *Instance, deleteIP bool) {
i, err := m.get(instance.Name())
if err != nil {
log.Error("[ARP manager] unable to remove the instance", "err", err)
return
}
if i != nil {
if i, ok := m.instances[instance.Name()]; ok {
i.mu.Lock()
defer i.mu.Unlock()
if i.counter > 1 {
i.counter--
} else {
log.Info("[ARP manager] removing ARP/NDP instance", "name", instance.Name())
if deleteIP {
if _, err := instance.network.DeleteIP(); err != nil {
log.Error("failed to delete IP", "address", instance.network.IP(), "err", err)
}
}
m.instances.Delete(instance.Name())
log.Info("removing ARP/NDP instance", "name", instance.Name())
delete(m.instances, instance.Name())
}
} else {
log.Warn("[ARP manager] unable to remove the instance - instance not found", "name", instance.Name())
}
}
func (m *Manager) Count(name string) int {
i, err := m.get(name)
if err != nil {
log.Error("[ARP manager] unable to count instance", "err", err)
return -1
}
if i != nil {
if i, ok := m.instances[name]; ok {
i.mu.Lock()
defer i.mu.Unlock()
return i.counter
@@ -111,46 +76,26 @@ func (m *Manager) Count(name string) int {
}
func (m *Manager) StartAdvertisement(ctx context.Context) {
log.Info("[ARP manager] starting ARP/NDP advertisement")
log.Info("Starting ARP/NDP advertisement")
for {
select {
case <-ctx.Done(): // if cancel() execute
return
default:
m.instances.Range(func(_ any, instance any) bool {
if i, ok := instance.(*Instance); ok {
if i.counter > 0 {
ensureIPAndSendGratuitous(i)
} else {
// this instance should not be advertised - delete the IP just in case...
if _, err := i.network.DeleteIP(); err != nil {
log.Error("[ARP manager] failed to delete IP", "address", i.network.IP(), "err", err)
}
}
for _, instance := range m.instances {
if instance.counter > 0 {
ensureIPAndSendGratuitous(instance)
}
return true
})
}
}
if m.config.ArpBroadcastRate < 500 {
log.Warn("[ARP manager] arp broadcast rate is too low", "rate (ms)", m.config.ArpBroadcastRate, "setting to (ms)", "3000")
log.Error("arp broadcast rate is too low", "rate (ms)", m.config.ArpBroadcastRate, "setting to (ms)", "3000")
m.config.ArpBroadcastRate = 3000
}
time.Sleep(time.Duration(m.config.ArpBroadcastRate) * time.Millisecond)
}
}
func (m *Manager) get(name string) (*Instance, error) {
i, exists := m.instances.Load(name)
if !exists {
return nil, nil
}
inst, ok := i.(*Instance)
if !ok {
return nil, fmt.Errorf("value for name %q is not of Instance pointer type", name)
}
return inst, nil
}
// ensureIPAndSendGratuitous - adds IP to the interface if missing, and send
// either a gratuitous ARP or gratuitous NDP. Re-adds the interface if it is IPv6
// and in a dadfailed state.
@@ -166,27 +111,23 @@ func ensureIPAndSendGratuitous(instance *Instance) {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted and recreating address with NODAD flag to skip DAD", "IP", ipString, "interface", iface)
// Re-add immediately without DAD check since we're recovering from DADFAILED
// The AddIP function will set IFA_F_NODAD flag for IPv6 addresses when skipDAD=true
if _, err := instance.network.AddIP(false, true); err != nil {
log.Error("failed to recreate address after DADFAILED", "IP", ipString, "interface", iface, "err", err)
} else {
log.Info("successfully recreated address after DADFAILED recovery", "IP", ipString, "interface", iface)
}
log.Info("deleted and recreating address", "IP", ipString, "interface", iface)
// if _, err := instance.network.AddIP(false); err != nil {
// log.Error("failed to recreate address", "IP", ipString, "interface", iface)
// }
}
// Return early after DADFAILED recovery to avoid double IP addition
return
}
// Normal case: add IP with precheck and normal DAD process
if added, err := instance.network.AddIP(true, false); err != nil {
// Ensure the address exists on the interface before attempting to ARP
// if instance.network.HasEndpoints() {
if added, err := instance.network.AddIP(true); err != nil {
log.Warn(err.Error())
} else if added {
log.Warn("Re-applied the VIP configuration", "ip", ipString, "interface", iface)
}
// }
if utils.IsIPv6(ipString) {
if vip.IsIPv6(ipString) {
// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
if instance.ndp == nil {
log.Error("NDP responder was not created")
@@ -204,4 +145,5 @@ func ensureIPAndSendGratuitous(instance *Instance) {
log.Warn(err.Error())
}
}
}

View File

@@ -1,7 +1,6 @@
package backend
import (
"context"
"fmt"
"time"
@@ -9,6 +8,7 @@ import (
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
@@ -31,7 +31,7 @@ func (e *Entry) Check() bool {
// homeConfigPath := filepath.Join(os.Getenv("HOME"), ".kube", "config")
var k8sAddr string
if utils.IsIPv4(e.Addr) {
if vip.IsIPv4(e.Addr) {
k8sAddr = fmt.Sprintf("%s:%v", e.Addr, e.Port)
} else {
k8sAddr = fmt.Sprintf("[%s]:%v", e.Addr, e.Port)
@@ -66,7 +66,7 @@ func (e *Entry) Check() bool {
return true
}
func Watch(ctx context.Context, tickAction func(), interval int) {
func Watch(tickAction func(), interval int, stop chan struct{}) {
if interval <= 0 {
interval = 5
}
@@ -76,7 +76,7 @@ func Watch(ctx context.Context, tickAction func(), interval int) {
for {
select {
case <-ctx.Done():
case <-stop:
ticker.Stop()
return
case <-ticker.C:

View File

@@ -9,7 +9,7 @@ import (
)
// AddHost will update peers of a host
func (b *Server) AddHost(ctx context.Context, addr string) (err error) {
func (b *Server) AddHost(addr string) (err error) {
ip, _, err := net.ParseCIDR(addr)
if err != nil {
return err
@@ -20,7 +20,7 @@ func (b *Server) AddHost(ctx context.Context, addr string) (err error) {
return fmt.Errorf("failed to get path for %v", ip)
}
_, err = b.s.AddPath(ctx, &api.AddPathRequest{
_, err = b.s.AddPath(context.Background(), &api.AddPathRequest{
Path: p,
})
@@ -32,7 +32,7 @@ func (b *Server) AddHost(ctx context.Context, addr string) (err error) {
}
// DelHost will inform peers to remove a host
func (b *Server) DelHost(ctx context.Context, addr string) (err error) {
func (b *Server) DelHost(addr string) (err error) {
ip, _, err := net.ParseCIDR(addr)
if err != nil {
return err
@@ -42,7 +42,7 @@ func (b *Server) DelHost(ctx context.Context, addr string) (err error) {
return
}
return b.s.DeletePath(ctx, &api.DeletePathRequest{
return b.s.DeletePath(context.Background(), &api.DeletePathRequest{
Path: p,
})
}

View File

@@ -5,20 +5,17 @@ import (
"fmt"
"net"
"strconv"
"strings"
//nolint
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/vip"
api "github.com/osrg/gobgp/v3/api"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/osrg/gobgp/v3/pkg/server"
"google.golang.org/protobuf/types/known/anypb"
)
// AddPeer will add peers to the BGP configuration
func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error) {
func (b *Server) AddPeer(peer Peer) (err error) {
p := &api.Peer{
Conf: &api.PeerConf{
NeighborAddress: peer.Address,
@@ -69,23 +66,23 @@ func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error)
},
}
peer.SetMpbgpOptions(b.c)
peer.setMpbgpOptions(b.c)
ipv4Address, ipv6Address, err := peer.FindMpbgpAddresses(p, b.c)
ipv4Address, ipv6Address, err := peer.findMpbgpAddresses(p, b.c)
if err != nil {
return fmt.Errorf("failed to get MP-BGP addresses: %w", err)
}
mask := strconv.Itoa(vip.DefaultMaskIPv6)
mask := "128"
address := ipv4Address
family := api.Family_AFI_IP
if utils.IsIPv4(p.Conf.NeighborAddress) {
mask = strconv.Itoa(vip.DefaultMaskIPv4)
if vip.IsIPv4(p.Conf.NeighborAddress) {
mask = "32"
address = ipv6Address
family = api.Family_AFI_IP6
}
err = b.s.AddDefinedSet(ctx, &api.AddDefinedSetRequest{
err = b.s.AddDefinedSet(context.Background(), &api.AddDefinedSetRequest{
DefinedSet: &api.DefinedSet{
DefinedType: api.DefinedType_NEIGHBOR,
Name: fmt.Sprintf("peer-%s", p.Conf.NeighborAddress),
@@ -97,7 +94,7 @@ func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error)
}
if address != "" {
if err := insertPolicy(ctx, b.s, address, p, family); err != nil {
if err := insertPolicy(b.s, address, p, family); err != nil {
return fmt.Errorf("failed to add policy: %w", err)
}
}
@@ -111,7 +108,7 @@ func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error)
}
}
if err := b.s.AddPeer(ctx, &api.AddPeerRequest{Peer: p}); err != nil {
if err := b.s.AddPeer(context.Background(), &api.AddPeerRequest{Peer: p}); err != nil {
return fmt.Errorf("failed to add peer: %v", err)
}
@@ -130,7 +127,7 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
//nolint
nlri, _ := anypb.New(&api.IPAddressPrefix{
Prefix: ip.String(),
PrefixLen: vip.DefaultMaskIPv4,
PrefixLen: 32,
})
//nolint
@@ -150,7 +147,7 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
//nolint
nlri, _ := anypb.New(&api.IPAddressPrefix{
Prefix: ip.String(),
PrefixLen: vip.DefaultMaskIPv6,
PrefixLen: 128,
})
v6Family := &api.Family{
@@ -174,7 +171,105 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
return
}
func insertPolicy(ctx context.Context, s *server.BgpServer, address string, p *api.Peer, family api.Family_Afi) error {
// ParseBGPPeerConfig - take a string and parses it into an array of peers
func ParseBGPPeerConfig(config string) (bgpPeers []Peer, err error) {
peers := strings.Split(config, ",")
if len(peers) == 0 {
return nil, fmt.Errorf("no BGP Peer configurations found")
}
for x := range peers {
peerStr := peers[x]
config := strings.Split(peerStr, "/")
peerStr = config[0]
if peerStr == "" {
continue
}
isV6Peer := peerStr[0] == '['
address := ""
if isV6Peer {
addressEndPos := strings.IndexByte(peerStr, ']')
if addressEndPos == -1 {
return nil, fmt.Errorf("no matching ] found for IPv6 BGP Peer")
}
address = peerStr[1:addressEndPos]
peerStr = peerStr[addressEndPos+1:]
}
peer := strings.Split(peerStr, ":")
if len(peer) < 2 {
return nil, fmt.Errorf("mandatory peering params <host>:<AS> incomplete")
}
if !isV6Peer {
address = peer[0]
}
ASNumber, err := strconv.ParseUint(peer[1], 10, 32)
if err != nil {
return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
}
password := ""
if len(peer) >= 3 {
password = peer[2]
}
multiHop := false
if len(peer) >= 4 {
multiHop, err = strconv.ParseBool(peer[3])
if err != nil {
return nil, fmt.Errorf("BGP MultiHop format error (true/false) [%s]", peer[1])
}
}
var port uint64
if len(peer) >= 5 {
port, err = strconv.ParseUint(peer[4], 10, 16)
if err != nil {
return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
}
} else {
port = 179
}
var mpbgpNexthop, mpbgpIPv4, mpbgpIPv6 string
if len(config) > 1 {
configData := strings.Split(config[1], ";")
for _, cfg := range configData {
c := strings.Split(cfg, "=")
switch c[0] {
case "mpbgp_nexthop":
mpbgpNexthop = c[1]
case "mpbgp_ipv4":
mpbgpIPv4 = c[1]
case "mpbgp_ipv6":
mpbgpIPv6 = c[1]
default:
return nil, fmt.Errorf("peer configuration parameter '%s' is not supported", c[0])
}
}
}
peerConfig := Peer{
Address: address,
AS: uint32(ASNumber),
Port: uint16(port),
Password: password,
MultiHop: multiHop,
MpbgpNexthop: mpbgpNexthop,
MpbgpIPv4: mpbgpIPv4,
MpbgpIPv6: mpbgpIPv6,
}
bgpPeers = append(bgpPeers, peerConfig)
}
return
}
func insertPolicy(s *server.BgpServer, address string, p *api.Peer, family api.Family_Afi) error {
familyType := "v4"
if family == api.Family_AFI_IP6 {
familyType = "v6"
@@ -220,14 +315,14 @@ func insertPolicy(ctx context.Context, s *server.BgpServer, address string, p *a
},
}
err := s.AddPolicy(ctx, &api.AddPolicyRequest{
err := s.AddPolicy(context.Background(), &api.AddPolicyRequest{
Policy: policy,
})
if err != nil {
return fmt.Errorf("failed to add policy: %w", err)
}
err = s.AddPolicyAssignment(ctx, &api.AddPolicyAssignmentRequest{
err = s.AddPolicyAssignment(context.Background(), &api.AddPolicyAssignmentRequest{
Assignment: &api.PolicyAssignment{
Name: "global",
Direction: api.PolicyDirection_EXPORT,

View File

@@ -3,8 +3,6 @@ package bgp
import (
"reflect"
"testing"
"github.com/kube-vip/kube-vip/pkg/kubevip"
)
func TestParseBGPPeerConfig(t *testing.T) {
@@ -14,13 +12,13 @@ func TestParseBGPPeerConfig(t *testing.T) {
tests := []struct {
name string
args args
wantBgpPeers []kubevip.BGPPeer
wantBgpPeers []Peer
wantErr bool
}{
{
name: "IPv4, default port",
args: args{config: "192.168.0.10:65000::false,192.168.0.11:65000::false"},
wantBgpPeers: []kubevip.BGPPeer{
wantBgpPeers: []Peer{
{Address: "192.168.0.10", Port: 179, AS: 65000, MultiHop: false},
{Address: "192.168.0.11", Port: 179, AS: 65000, MultiHop: false},
},
@@ -28,7 +26,7 @@ func TestParseBGPPeerConfig(t *testing.T) {
{
name: "IPv4, different port",
args: args{config: "192.168.0.10:65000::false:180,192.168.0.11:65000::false:190"},
wantBgpPeers: []kubevip.BGPPeer{
wantBgpPeers: []Peer{
{Address: "192.168.0.10", Port: 180, AS: 65000, MultiHop: false},
{Address: "192.168.0.11", Port: 190, AS: 65000, MultiHop: false},
},
@@ -36,14 +34,14 @@ func TestParseBGPPeerConfig(t *testing.T) {
{
name: "IPv6, multi-protocol",
args: args{config: "[fd00:1111:2222:3333:c7d9:7235:6bf7:5d52]:65501::false/mpbgp_nexthop=auto_sourceif"},
wantBgpPeers: []kubevip.BGPPeer{
wantBgpPeers: []Peer{
{Address: "fd00:1111:2222:3333:c7d9:7235:6bf7:5d52", Port: 179, AS: 65501, MultiHop: false, MpbgpNexthop: "auto_sourceif"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotBgpPeers, err := kubevip.ParseBGPPeerConfig(tt.args.config)
gotBgpPeers, err := ParseBGPPeerConfig(tt.args.config)
if (err != nil) != tt.wantErr {
t.Errorf("ParseBGPPeerConfig() error = %v, wantErr %v", err, tt.wantErr)
return

View File

@@ -7,24 +7,13 @@ import (
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
api "github.com/osrg/gobgp/v3/api"
gobgp "github.com/osrg/gobgp/v3/pkg/server"
"github.com/prometheus/client_golang/prometheus"
)
// Server manages a server object
type Server struct {
s *gobgp.BgpServer
c *kubevip.BGPConfig
// This is a prometheus gauge indicating the state of the sessions.
// 1 means "ESTABLISHED", 0 means "NOT ESTABLISHED"
BGPSessionInfoGauge *prometheus.GaugeVec
}
// NewBGPServer takes a configuration and returns a running BGP server instance
func NewBGPServer(c kubevip.BGPConfig) (b *Server, err error) {
func NewBGPServer(c *Config) (b *Server, err error) {
if c.AS == 0 {
return nil, fmt.Errorf("you need to provide AS")
}
@@ -39,7 +28,7 @@ func NewBGPServer(c kubevip.BGPConfig) (b *Server, err error) {
b = &Server{
s: gobgp.NewBgpServer(),
c: &c,
c: c,
BGPSessionInfoGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "kube_vip",
@@ -52,10 +41,10 @@ func NewBGPServer(c kubevip.BGPConfig) (b *Server, err error) {
}
// Start starts the BGP server
func (b *Server) Start(ctx context.Context, peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (err error) {
func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (err error) {
go b.s.Serve()
if err = b.s.StartBgp(ctx, &api.StartBgpRequest{
if err = b.s.StartBgp(context.Background(), &api.StartBgpRequest{
Global: &api.Global{
Asn: b.c.AS,
RouterId: b.c.RouterID,
@@ -65,7 +54,7 @@ func (b *Server) Start(ctx context.Context, peerStateChangeCallback func(*api.Wa
return
}
if err = b.s.WatchEvent(ctx, &api.WatchEventRequest{Peer: &api.WatchEventRequest_Peer{}}, func(r *api.WatchEventResponse) {
if err = b.s.WatchEvent(context.Background(), &api.WatchEventRequest{Peer: &api.WatchEventRequest_Peer{}}, func(r *api.WatchEventResponse) {
if p := r.GetPeer(); p != nil && p.Type == api.WatchEventResponse_PeerEvent_STATE {
log.Info("[BGP]", "peer", p.String())
if peerStateChangeCallback != nil {
@@ -77,13 +66,13 @@ func (b *Server) Start(ctx context.Context, peerStateChangeCallback func(*api.Wa
}
for _, p := range b.c.Peers {
if err = b.AddPeer(ctx, p); err != nil {
if err = b.AddPeer(p); err != nil {
return
}
}
if b.c.Zebra.Enabled {
if err = b.s.EnableZebra(ctx, &api.EnableZebraRequest{
if err = b.s.EnableZebra(context.Background(), &api.EnableZebraRequest{
Url: b.c.Zebra.URL,
Version: b.c.Zebra.Version,
SoftwareName: b.c.Zebra.SoftwareName,
@@ -98,8 +87,7 @@ func (b *Server) Start(ctx context.Context, peerStateChangeCallback func(*api.Wa
// Close will stop a running BGP Server
func (b *Server) Close() error {
// create new BGP stop context (independent)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
ctx, cf := context.WithTimeout(context.Background(), 5*time.Second)
defer cf()
return b.s.StopBgp(ctx, &api.StopBgpRequest{})
}

View File

@@ -1,19 +1,18 @@
package kubevip
package bgp
import (
"fmt"
"net"
"strconv"
"strings"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
api "github.com/osrg/gobgp/v3/api"
gobgp "github.com/osrg/gobgp/v3/pkg/server"
"github.com/prometheus/client_golang/prometheus"
"github.com/vishvananda/netlink"
)
// Peer defines a BGP Peer
type BGPPeer struct {
type Peer struct {
Address string
Port uint16
AS uint32
@@ -24,131 +23,21 @@ type BGPPeer struct {
MpbgpIPv6 string
}
// Config defines the BGP server configuration
type BGPConfig struct {
AS uint32
RouterID string
SourceIP string
SourceIF string
MpbgpNexthop string
MpbgpIPv4 string
MpbgpIPv6 string
HoldTime uint64
KeepaliveInterval uint64
Peers []BGPPeer
Zebra ZebraConfig
}
// Defines Zebra connection configuration. More on the topic - https://github.com/osrg/gobgp/blob/master/docs/sources/zebra.md#configuration
type ZebraConfig struct {
Enabled bool
URL string
Version uint32
SoftwareName string
}
// ParseBGPPeerConfig - take a string and parses it into an array of peers
func ParseBGPPeerConfig(config string) (bgpPeers []BGPPeer, err error) {
peers := strings.Split(config, ",")
if len(peers) == 0 {
return nil, fmt.Errorf("no BGP Peer configurations found")
func (p *Peer) setMpbgpOptions(server *Config) {
if p.MpbgpNexthop == "" {
p.MpbgpNexthop = server.MpbgpNexthop
}
for x := range peers {
peerStr := peers[x]
config := strings.Split(peerStr, "/")
peerStr = config[0]
if peerStr == "" {
continue
}
isV6Peer := peerStr[0] == '['
address := ""
if isV6Peer {
addressEndPos := strings.IndexByte(peerStr, ']')
if addressEndPos == -1 {
return nil, fmt.Errorf("no matching ] found for IPv6 BGP Peer")
}
address = peerStr[1:addressEndPos]
peerStr = peerStr[addressEndPos+1:]
}
peer := strings.Split(peerStr, ":")
if len(peer) < 2 {
return nil, fmt.Errorf("mandatory peering params <host>:<AS> incomplete")
}
if !isV6Peer {
address = peer[0]
}
ASNumber, err := strconv.ParseUint(peer[1], 10, 32)
if err != nil {
return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
}
password := ""
if len(peer) >= 3 {
password = peer[2]
}
multiHop := false
if len(peer) >= 4 {
multiHop, err = strconv.ParseBool(peer[3])
if err != nil {
return nil, fmt.Errorf("BGP MultiHop format error (true/false) [%s]", peer[1])
}
}
var port uint64
if len(peer) >= 5 {
port, err = strconv.ParseUint(peer[4], 10, 16)
if err != nil {
return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
}
} else {
port = 179
}
var mpbgpNexthop, mpbgpIPv4, mpbgpIPv6 string
if len(config) > 1 {
configData := strings.Split(config[1], ";")
for _, cfg := range configData {
c := strings.Split(cfg, "=")
switch c[0] {
case "mpbgp_nexthop":
mpbgpNexthop = c[1]
case "mpbgp_ipv4":
mpbgpIPv4 = c[1]
case "mpbgp_ipv6":
mpbgpIPv6 = c[1]
default:
return nil, fmt.Errorf("peer configuration parameter '%s' is not supported", c[0])
}
}
}
peerConfig := BGPPeer{
Address: address,
AS: uint32(ASNumber),
Port: uint16(port),
Password: password,
MultiHop: multiHop,
MpbgpNexthop: mpbgpNexthop,
MpbgpIPv4: mpbgpIPv4,
MpbgpIPv6: mpbgpIPv6,
}
bgpPeers = append(bgpPeers, peerConfig)
if p.MpbgpIPv4 == "" {
p.MpbgpIPv4 = server.MpbgpIPv4
}
if p.MpbgpIPv6 == "" {
p.MpbgpIPv6 = server.MpbgpIPv6
}
return
}
func (p *BGPPeer) FindMpbgpAddresses(ap *api.Peer, server *BGPConfig) (string, string, error) {
func (p *Peer) findMpbgpAddresses(ap *api.Peer, server *Config) (string, string, error) {
var ipv4Address, ipv6Address string
switch p.MpbgpNexthop {
case "fixed":
@@ -175,20 +64,20 @@ func (p *BGPPeer) FindMpbgpAddresses(ap *api.Peer, server *BGPConfig) (string, s
ap.Transport.LocalAddress = server.SourceIP
// Resolve the local interface by SourceIP
iface, err := utils.GetInterfaceByIP(server.SourceIP)
iface, err := vip.GetInterfaceByIP(server.SourceIP)
if err != nil {
return "", "", fmt.Errorf("failed to get interface by IP: %v", err)
}
if utils.IsIPv4(server.SourceIP) {
if vip.IsIPv4(server.SourceIP) {
// Get the non link-local IPv6 address on that interface
ipv6Address, err = utils.GetNonLinkLocalIP(iface, netlink.FAMILY_V6)
ipv6Address, err = vip.GetNonLinkLocalIP(iface, netlink.FAMILY_V6)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
}
} else {
// Get the non link-local IPv4 address on that interface
ipv4Address, err = utils.GetNonLinkLocalIP(iface, netlink.FAMILY_V4)
ipv4Address, err = vip.GetNonLinkLocalIP(iface, netlink.FAMILY_V4)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
}
@@ -202,13 +91,13 @@ func (p *BGPPeer) FindMpbgpAddresses(ap *api.Peer, server *BGPConfig) (string, s
}
// Get the non link-local IPv4 address on that interface
ipv4Address, err = utils.GetNonLinkLocalIP(&iface, netlink.FAMILY_V4)
ipv4Address, err = vip.GetNonLinkLocalIP(&iface, netlink.FAMILY_V4)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
}
// Get the non link-local IPv6 address on that interface
ipv6Address, err = utils.GetNonLinkLocalIP(&iface, netlink.FAMILY_V6)
ipv6Address, err = vip.GetNonLinkLocalIP(&iface, netlink.FAMILY_V6)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
}
@@ -219,16 +108,38 @@ func (p *BGPPeer) FindMpbgpAddresses(ap *api.Peer, server *BGPConfig) (string, s
return ipv4Address, ipv6Address, nil
}
func (p *BGPPeer) SetMpbgpOptions(server *BGPConfig) {
if p.MpbgpNexthop == "" {
p.MpbgpNexthop = server.MpbgpNexthop
}
// Config defines the BGP server configuration
type Config struct {
AS uint32
RouterID string
SourceIP string
SourceIF string
MpbgpNexthop string
MpbgpIPv4 string
MpbgpIPv6 string
if p.MpbgpIPv4 == "" {
p.MpbgpIPv4 = server.MpbgpIPv4
}
HoldTime uint64
KeepaliveInterval uint64
if p.MpbgpIPv6 == "" {
p.MpbgpIPv6 = server.MpbgpIPv6
}
Peers []Peer
Zebra ZebraConfig
}
// Defines Zebra connection configuration. More on the topic - https://github.com/osrg/gobgp/blob/master/docs/sources/zebra.md#configuration
type ZebraConfig struct {
Enabled bool
URL string
Version uint32
SoftwareName string
}
// Server manages a server object
type Server struct {
s *gobgp.BgpServer
c *Config
// This is a prometheus gauge indicating the state of the sessions.
// 1 means "ESTABLISHED", 0 means "NOT ESTABLISHED"
BGPSessionInfoGauge *prometheus.GaugeVec
}

View File

@@ -13,10 +13,11 @@ import (
// Cluster - The Cluster object manages the state of the cluster for a particular node
type Cluster struct {
stop chan bool
once sync.Once
Network []vip.Network
arpMgr *arp.Manager
stop chan bool
completed chan bool
once sync.Once
Network []vip.Network
arpMgr *arp.Manager
}
// InitCluster - Will attempt to initialise all of the required settings for the cluster
@@ -35,7 +36,6 @@ func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.M
newCluster := &Cluster{
Network: networks,
arpMgr: arpMgr,
stop: make(chan bool),
}
log.Debug("service security", "enabled", c.EnableServiceSecurity)
@@ -54,9 +54,9 @@ func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vi
networks := []vip.Network{}
for _, addr := range addresses {
network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.DHCPMode,
c.RequireDualStack, c.IsDualStack, c.RoutingTableID, c.RoutingTableType, c.RoutingProtocol, c.DNSMode,
c.LoadBalancerForwardingMethod, c.IptablesBackend, c.EnableLoadBalancer, c.EnableServiceSecurity, intfMgr)
network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.RoutingTableID,
c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend,
c.EnableLoadBalancer, c.EnableServiceSecurity, intfMgr)
if err != nil {
return nil, err
}
@@ -74,4 +74,7 @@ func (cluster *Cluster) Stop() {
close(cluster.stop)
})
}
// Wait until the completed channel is closed, signallign all shutdown tasks completed
<-cluster.completed
}

View File

@@ -2,7 +2,6 @@ package cluster
import (
"context"
"sync"
"github.com/kube-vip/kube-vip/pkg/vip"
)
@@ -13,14 +12,16 @@ import (
// during runtime if IP changes, startDDNS don't have to do reconfigure because
// dnsUpdater already have the functionality to keep trying resolve the IP
// and update the VIP configuration if it changes
func (cluster *Cluster) StartDDNS(ctx context.Context, network vip.Network, backoffAttempts uint, wg *sync.WaitGroup) error {
ddnsMgr := vip.NewDDNSManager(network, backoffAttempts)
ip, err := ddnsMgr.Start(ctx, wg)
if err != nil {
return err
}
if err = network.SetIP(ip); err != nil {
return err
func (cluster *Cluster) StartDDNS(ctx context.Context) error {
for i := range cluster.Network {
ddnsMgr := vip.NewDDNSManager(ctx, cluster.Network[i])
ip, err := ddnsMgr.Start()
if err != nil {
return err
}
if err = cluster.Network[i].SetIP(ip); err != nil {
return err
}
}
return nil

View File

@@ -3,59 +3,131 @@ package cluster
import (
"context"
"fmt"
"sync"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/etcd"
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/loadbalancer"
log "log/slog"
clientv3 "go.etcd.io/etcd/client/v3"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
watchtools "k8s.io/client-go/tools/watch"
)
// StartCluster - Begins a running instance of the Leader Election cluster
func (cluster *Cluster) StartCluster(ctx context.Context, c *kubevip.Config,
em *election.Manager, bgpServer *bgp.Server, leaseMgr *lease.Manager, killFunc func()) error {
// Manager degines the manager of the load-balancing services
type Manager struct {
KubernetesClient *kubernetes.Clientset
RetryWatcherClient *kubernetes.Clientset
// This channel is used to signal a shutdown
SignalChan chan os.Signal
ns, leaseName := lease.NamespaceName(c.LeaseName, c)
EtcdClient *clientv3.Client
}
leaseID := lease.NewID(c.LeaderElectionType, ns, leaseName)
// NewManager will create a new managing object
func NewManager(path string, inCluster bool, port int) (*Manager, error) {
var hostname string
log.Info("cluster membership", "namespace", leaseID.Namespace(), "lock", leaseID.Name(), "id", c.NodeName)
// If inCluster is set then it will likely have started as a static pod or won't have the
// VIP up before trying to connect to the API server, we set the API endpoint to this machine to
// ensure connectivity. Else if the path passed is empty and not running in the cluster,
// attempt to look for a kubeconfig in the default HOME dir.
objectName := lease.ObjectName(leaseID, "cp")
objLease := leaseMgr.Add(ctx, leaseID)
isNew := objLease.Add(objectName)
hostname = fmt.Sprintf("kubernetes:%v", port)
wg := sync.WaitGroup{}
defer wg.Wait()
if len(path) == 0 && !inCluster {
path = filepath.Join(os.Getenv("HOME"), ".kube", "config")
// Start a goroutine that will delete the lease when the service context is cancelled.
// This is important for proper cleanup when a service is deleted - it ensures that
// the lease context (svcLease.Ctx) gets cancelled, which causes RunOrDie to return.
// Without this, RunOrDie would continue running until leadership is naturally lost.
wg.Go(func() {
<-objLease.Ctx.Done()
leaseMgr.Delete(leaseID, objectName)
})
if !isNew {
log.Debug("this election was already done, waiting for it to finish", "lease", leaseName)
<-objLease.Ctx.Done()
return nil
}
wg.Go(func() {
select {
case <-cluster.stop:
case <-ctx.Done():
// We modify the config so that we can always speak to the correct host
id, err := os.Hostname()
if err != nil {
return nil, err
}
hostname = fmt.Sprintf("%s:%v", id, port)
}
config, err := k8s.NewRestConfig(path, inCluster, hostname)
if err != nil {
return nil, fmt.Errorf("failed to create k8s REST config: %w", err)
}
clientset, err := k8s.NewClientset(config)
if err != nil {
return nil, fmt.Errorf("error creating a new k8s clientset: %v", err)
}
rwConfig, err := k8s.NewRestConfig(path, inCluster, hostname)
if err != nil {
return nil, fmt.Errorf("failed to create k8s REST config for retryClientSet: %w", err)
}
rwConfig.Timeout = 0 // empty value to disable the timeout
rwClientSet, err := k8s.NewClientset(rwConfig)
if err != nil {
return nil, fmt.Errorf("failed to create k8s client for retry watcher: %w", err)
}
return &Manager{
KubernetesClient: clientset,
RetryWatcherClient: rwClientSet,
}, nil
}
// StartCluster - Begins a running instance of the Leader Election cluster
func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *bgp.Server) error {
var err error
log.Info("cluster membership", "namespace", c.Namespace, "lock", c.LeaseName, "id", c.NodeName)
// use a Go context so we can tell the leaderelection code when we
// want to step down
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// use a Go context so we can tell the arp loop code when we
// want to step down
ctxArp, cancelArp := context.WithCancel(context.Background())
defer cancelArp()
// use a Go context so we can tell the dns loop code when we
// want to step down
ctxDNS, cancelDNS := context.WithCancel(context.Background())
defer cancelDNS()
// listen for interrupts or the Linux SIGTERM signal and cancel
// our context, which the leader election code will observe and
// step down
signalChan := make(chan os.Signal, 1)
// Add Notification for Userland interrupt
signal.Notify(signalChan, syscall.SIGINT)
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(signalChan, syscall.SIGTERM)
go func() {
<-signalChan
log.Info("Received termination, signaling cluster shutdown")
// Cancel the leader context, which will in turn cancel the leadership
objLease.Cancel()
})
// Cancel the context, which will in turn cancel the leadership
cancel()
// Cancel the arp context, which will in turn stop any broadcasts
}()
// (attempt to) Remove the virtual IP, in case it already exists
@@ -69,110 +141,54 @@ func (cluster *Cluster) StartCluster(ctx context.Context, c *kubevip.Config,
}
}
objLease.Lock()
// Defer a function to check if the bgpServer has been created and if so attempt to close it
defer func() {
objLease.Unlock()
if bgpServer != nil {
bgpServer.Close()
}
}()
// this object is sharing lease with another object
if objLease.Elected.Load() {
log.Debug("this election was already done, shared lease", "lease", leaseName)
// wait for leader election to start or context to be done
select {
case <-objLease.Started:
case <-objLease.Ctx.Done():
// Lease was cancelled (e.g., leader election ended), return immediately
// This allows the restart loop to create a fresh lease
log.Debug("lease context cancelled before leader election started", "lease", leaseName)
return fmt.Errorf("lease %q context cancelled before leader election started", leaseName)
if c.EnableBGP && bgpServer == nil {
// Lets start BGP
log.Info("Starting the BGP server to advertise VIP routes to VGP peers")
bgpServer, err = bgp.NewBGPServer(&c.BGPConfig)
if err != nil {
log.Error("new BGP server", "err", err)
}
if err := bgpServer.Start(nil); err != nil {
log.Error("starting BGP server", "err", err)
}
cluster.OnStartedLeading(c, objLease, em, bgpServer, killFunc, true)
log.Debug("cluster waiting for leader context done", "lease", leaseName)
// wait for leaderelection to be finished
<-objLease.Ctx.Done()
cluster.OnStoppedLeading(c, objLease, bgpServer)
return nil
}
run := &election.RunConfig{
Config: c,
LeaseID: leaseID,
LeaseAnnotations: c.LeaseAnnotations,
Mgr: em,
OnStartedLeading: func(context.Context) { //nolint TODO: potential clean code
cluster.OnStartedLeading(c, objLease, em, bgpServer, killFunc, false)
},
OnStoppedLeading: func() {
objLease.Elected.Store(false)
cluster.OnStoppedLeading(c, objLease, bgpServer)
},
OnNewLeader: func(identity string) {
cluster.OnNewLeader(identity, c)
},
}
if err := election.RunOrDie(objLease.Ctx, run, c); err != nil {
objLease.Cancel()
return fmt.Errorf("leaderelection failed: %w", err)
}
return nil
}
func (cluster *Cluster) OnStartedLeading(c *kubevip.Config, objLease *lease.Lease,
em *election.Manager, bgpServer *bgp.Server, killFunc func(), isShared bool) {
objLease.Elected.Store(true)
objLease.Unlock()
// When we become leader, ensure we can take over VIPs even if they're preserved on other nodes
if !isShared {
close(objLease.Started)
}
if c.PreserveVIPOnLeadershipLoss {
log.Info("Becoming leader with VIP preservation enabled - ensuring VIP takeover")
// Force add the VIPs (this will work even if they exist due to the precheck logic)
for i := range cluster.Network {
added, err := cluster.Network[i].AddIP(true, false)
run := &runConfig{
config: c,
leaseID: c.NodeName,
sm: sm,
onStartedLeading: func(ctx context.Context) { //nolint TODO: potential clean code
// As we're leading lets start the vip service
err := cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, cancel)
if err != nil {
log.Error("failed to ensure VIP on leader takeover", "vip", cluster.Network[i].IP(), "err", err)
} else if added {
log.Info("took over VIP as new leader", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
} else {
log.Info("VIP already configured on interface", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
log.Error("starting VIP service on leader", "err", err)
}
}
}
},
onStoppedLeading: func() {
// we can do cleanup here
log.Info("This node is becoming a follower within the cluster")
// As we're leading lets start the vip service
err := cluster.StartVipService(objLease.Ctx, c, em, bgpServer, killFunc)
if err != nil {
log.Error("starting VIP service on leader", "err", err)
killFunc()
}
}
// Stop the dns context
cancelDNS()
// Stop the Arp context if it is running
cancelArp()
func (cluster *Cluster) OnStoppedLeading(c *kubevip.Config, objLease *lease.Lease,
bgpServer *bgp.Server) {
// we can do cleanup here
log.Info("This node is becoming a follower within the cluster")
// Stop the BGP server
if bgpServer != nil {
err := bgpServer.Close()
if err != nil {
log.Warn("close BGP server", "err", err)
}
}
// Stop the cluster context if it is running
objLease.Cancel()
// Handle VIP cleanup based on configuration
if c.PreserveVIPOnLeadershipLoss {
// For IPv6, we must remove VIPs immediately to avoid DAD failures on the new leader
// IPv6 Duplicate Address Detection will fail if the new leader tries to add an IP that is still present on this node's interface
// We need to check each VIP individually and only remove IPv6 VIPs
for i := range cluster.Network {
if utils.IsIPv6(cluster.Network[i].IP()) {
log.Info("Removing IPv6 VIP immediately (required to prevent DAD failures on new leader)", "ip", cluster.Network[i].IP())
for i := range cluster.Network {
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("delete VIP", "err", err)
@@ -180,47 +196,195 @@ func (cluster *Cluster) OnStoppedLeading(c *kubevip.Config, objLease *lease.Leas
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
} else {
log.Info("Preserving IPv4 VIP address on interface, only stopped ARP broadcasting", "ip", cluster.Network[i].IP())
}
}
} else {
// Legacy behavior: delete VIP addresses on leadership loss
log.Info("Deleting VIP addresses on leadership loss (legacy behavior)")
for i := range cluster.Network {
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("delete VIP", "err", err)
log.Error("lost leadership, restarting kube-vip")
panic("") // TODO - we could also return here
},
onNewLeader: func(identity string) {
// we're notified when new leader elected
log.Info("New leader", "leader", identity)
},
}
switch c.LeaderElectionType {
case "kubernetes", "":
cluster.runKubernetesLeaderElectionOrDie(ctx, run)
case "etcd":
cluster.runEtcdLeaderElectionOrDie(ctx, run)
default:
log.Info(fmt.Sprintf("LeaderElectionMode %s not supported, exiting", c.LeaderElectionType))
}
return nil
}
type runConfig struct {
config *kubevip.Config
leaseID string
sm *Manager
// onStartedLeading is called when this member starts leading.
onStartedLeading func(context.Context)
// onStoppedLeading is called when this member stops leading.
onStoppedLeading func()
// onNewLeader is called when the client observes a leader that is
// not the previously observed leader. This includes the first observed
// leader when the client starts.
onNewLeader func(identity string)
}
func (cluster *Cluster) runKubernetesLeaderElectionOrDie(ctx context.Context, run *runConfig) {
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: run.config.LeaseName,
Namespace: run.config.Namespace,
Annotations: run.config.LeaseAnnotations,
},
Client: run.sm.KubernetesClient.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: run.leaseID,
},
}
// start the leader election code loop
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(run.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(run.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(run.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: run.onStartedLeading,
OnStoppedLeading: run.onStoppedLeading,
OnNewLeader: run.onNewLeader,
},
})
}
func (cluster *Cluster) runEtcdLeaderElectionOrDie(ctx context.Context, run *runConfig) {
etcd.RunElectionOrDie(ctx, &etcd.LeaderElectionConfig{
EtcdConfig: etcd.ClientConfig{Client: run.sm.EtcdClient},
Name: run.config.LeaseName,
MemberID: run.leaseID,
LeaseDurationSeconds: int64(run.config.LeaseDuration),
Callbacks: etcd.LeaderCallbacks{
OnStartedLeading: run.onStartedLeading,
OnStoppedLeading: run.onStoppedLeading,
OnNewLeader: run.onNewLeader,
},
})
}
func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
log.Info("Kube-Vip is watching nodes for control-plane labels")
listOptions := metav1.ListOptions{
LabelSelector: "node-role.kubernetes.io/control-plane",
}
rw, err := watchtools.NewRetryWatcherWithContext(ctxArp, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.RetryWatcherClient.CoreV1().Nodes().Watch(context.Background(), listOptions)
},
})
if err != nil {
return fmt.Errorf("error creating label watcher: %s", err.Error())
}
go func() {
<-sm.SignalChan
log.Info("Received termination, signaling shutdown")
// Cancel the context
rw.Stop()
}()
ch := rw.ResultChan()
// defer rw.Stop()
for event := range ch {
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
node, ok := event.Object.(*v1.Node)
if !ok {
return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
// Find the node IP address (this isn't foolproof)
for x := range node.Status.Addresses {
if node.Status.Addresses[x].Type == v1.NodeInternalIP {
if checkIfNodeIsReady(node) {
err = lb.AddBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("add IPVS backend", "err", err)
}
} else {
err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("remove IPVS backend", "err", err)
}
}
}
}
case watch.Deleted:
node, ok := event.Object.(*v1.Node)
if !ok {
return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
}
// Find the node IP address (this isn't foolproof)
for x := range node.Status.Addresses {
if node.Status.Addresses[x].Type == v1.NodeInternalIP {
err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("Del IPVS backend", "err", err)
}
}
}
log.Info("Node deleted", "name", node.Name)
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes Nodes")
// This round trip allows us to handle unstructured status
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Error("watcher", "status", status)
default:
}
}
log.Error("lost leadership, restarting kube-vip")
log.Info("Exiting Node watcher")
return nil
}
func (cluster *Cluster) OnNewLeader(identity string, c *kubevip.Config) {
// we're notified when new leader elected
log.Info("New leader", "leader", identity)
// If we're not the new leader and we have VIPs preserved from previous leadership,
// we need to clean them up to avoid conflicts.
if identity != c.NodeName && c.PreserveVIPOnLeadershipLoss {
log.Info("Cleaning up preserved VIPs as another node became leader", "new_leader", identity)
for i := range cluster.Network {
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("failed to cleanup preserved VIP", "vip", cluster.Network[i].IP(), "err", err)
}
if deleted {
log.Info("cleaned up preserved VIP to avoid conflict", "IP", cluster.Network[i].IP(),
"interface", cluster.Network[i].Interface(), "new_leader", identity)
} else {
log.Debug("VIP was not present on this node", "IP", cluster.Network[i].IP(),
"interface", cluster.Network[i].Interface())
func checkIfNodeIsReady(node *v1.Node) bool {
if node == nil {
return false
}
for _, condition := range node.Status.Conditions {
if condition.Type == v1.NodeReady {
if condition.Status == v1.ConditionTrue {
return true
}
}
}
return false
}

View File

@@ -7,20 +7,17 @@ import (
"io/fs"
"net"
"os"
"strings"
"os/signal"
"sync"
"syscall"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/backend"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/loadbalancer"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
corev1 "k8s.io/api/core/v1"
@@ -29,88 +26,82 @@ import (
"k8s.io/client-go/kubernetes"
)
func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config, em *election.Manager,
bgpServer *bgp.Server, killFunc func()) error {
func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Config, sm *Manager, bgpServer *bgp.Server, cancelLeaderElection context.CancelFunc) error {
var err error
var wg sync.WaitGroup
defer wg.Wait()
// listen for interrupts or the Linux SIGTERM signal and cancel
// our context, which the leader election code will observe and
// step down
signalChan := make(chan os.Signal, 1)
// Add Notification for Userland interrupt
signal.Notify(signalChan, syscall.SIGINT)
wg.Go(func() {
<-ctx.Done()
killFunc()
})
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(signalChan, syscall.SIGTERM)
loadbalancers := []*loadbalancer.IPVSLoadBalancer{}
var arpWG sync.WaitGroup
for i := range cluster.Network {
network := cluster.Network[i]
if network.IsDDNS() {
if err := cluster.StartDDNS(ctx, cluster.Network[i], c.DHCPBackoffAttempts, &wg); err != nil {
log.Error("failed to start DDNS", "err", err)
}
}
if err := network.SetMask(c.VIPSubnet); err != nil {
return fmt.Errorf("failed to set mask for subnet %q: %w", c.VIPSubnet, err)
log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
panic("")
}
if network.IsDDNS() {
if err := cluster.StartDDNS(ctxDNS); err != nil {
log.Error(err.Error())
}
}
// start the dns updater if address is dns
if network.IsDNS() {
log.Info("starting the DNS updater", "address", network.DNSName())
ipUpdater := vip.NewIPUpdater(network)
wg.Go(func() {
ipUpdater.Run(ctx)
})
ipUpdater.Run(ctxDNS)
}
if !c.EnableRoutingTable {
// Normal VIP addition, use skipDAD=false for normal DAD process
if _, err = network.AddIP(false, false); err != nil {
return fmt.Errorf("failed to add IP address %s: %w", network.IP(), err)
if _, err = network.AddIP(false); err != nil {
log.Error(err.Error())
}
}
if c.EnableBGP {
// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
log.Debug("Attempting to advertise over BGP", "address", network.CIDR())
err = bgpServer.AddHost(ctx, network.CIDR())
err = bgpServer.AddHost(network.CIDR())
if err != nil {
log.Error(err.Error())
}
}
if c.EnableLoadBalancer {
lb, err := loadbalancer.NewIPVSLB(ctx, network.IP(), c.LoadBalancerPort, c.LoadBalancerForwardingMethod,
c.BackendHealthCheckInterval, c.Interface, killFunc, &wg)
lb, err := loadbalancer.NewIPVSLB(network.IP(), c.LoadBalancerPort, c.LoadBalancerForwardingMethod, c.BackendHealthCheckInterval, c.Interface, cancelLeaderElection, signalChan)
if err != nil {
return fmt.Errorf("creating IPVS LoadBalance: %w", err)
log.Error("Error creating IPVS LoadBalancer", "err", err)
}
wg.Go(func() {
err = em.NodeWatcher(ctx, lb, c.Port)
go func() {
err = sm.NodeWatcher(ctxArp, lb, c.Port) //TODO: We're using the ctxARP as the context this will change when rkatz finishes his change
if err != nil {
log.Error("Error watching node labels", "err", err)
if errors.Is(err, &utils.PanicError{}) {
killFunc()
}
}
})
}()
loadbalancers = append(loadbalancers, lb)
}
if c.EnableARP {
wg.Go(func() {
cluster.layer2Update(ctx, network, c)
})
arpWG.Add(1)
go cluster.layer2Update(ctxArp, network, c, &arpWG)
}
}
if c.EnableLoadBalancer {
// Shutdown function that will wait on this signal, unless we call it ourselves
<-ctx.Done()
<-signalChan
for _, lb := range loadbalancers {
err = lb.RemoveIPVSLB()
if err != nil {
@@ -133,8 +124,8 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
ips := []string{}
if nodename != "" {
if ips, err = getNodeIPs(ctx, nodename, em.KubernetesClient); err != nil && !apierrors.IsNotFound(err) {
log.Error("failed to get IP of control-plane node", "err", err)
if ips, err = getNodeIPs(ctxArp, nodename, sm.KubernetesClient); err != nil && !apierrors.IsNotFound(err) {
log.Error("failed to get IP of control-plane nod", "err", err)
}
}
@@ -165,7 +156,15 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
}
}
backend.Watch(ctx, func() {
stop := make(chan struct{})
// will wait for system interrupt and will send stop signal to backend watch
go func() {
<-signalChan
stop <- struct{}{}
}()
backend.Watch(func() {
for i := range cluster.Network {
network := cluster.Network[i]
networkIP := network.IP()
@@ -174,7 +173,6 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
log.Error("failed to check IP type", "IP", networkIP, "error", err)
continue
}
log.Debug("current ip to process", "ip", networkIP)
backendMap := &backendMapV4
if isNetworkV6 {
@@ -182,11 +180,8 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
}
for entry := range *backendMap {
log.Debug("entry.Check() for entry", "entry", entry)
if entry.Check() {
log.Debug("entry.Check() true")
// Normal VIP addition with precheck, use skipDAD=false for normal DAD process
_, err = network.AddIP(true, false)
_, err = network.AddIP(true)
if err != nil {
log.Error("error adding address", "err", err)
}
@@ -226,19 +221,14 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
deleted, err := network.DeleteIP()
if err != nil {
log.Error("error deleting IP", "err", err)
killFunc()
return
panic("")
}
if deleted {
log.Info("deleted address", "IP", network.IP(), "interface", network.Interface())
}
}
}
}, c.BackendHealthCheckInterval)
}
if c.EnableBGP {
<-ctx.Done()
}, c.BackendHealthCheckInterval, stop)
}
return nil
@@ -267,99 +257,60 @@ func getNodeIPs(ctx context.Context, nodename string, client *kubernetes.Clients
}
// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp *bgp.Server, name string, CountRouteReferences func(*netlink.Route) int, wg *sync.WaitGroup) error {
func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp *bgp.Server, name string, CountRouteReferences func(*netlink.Route) int) {
// use a Go context so we can tell the arp loop code when we
// want to step down
//nolint
lbCtx, lbCancel := context.WithCancel(ctx)
ctxArp, cancelArp := context.WithCancel(ctx)
var lbWg sync.WaitGroup
cluster.stop = make(chan bool, 1)
cluster.completed = make(chan bool, 1)
var arpWG sync.WaitGroup
for i := range cluster.Network {
network := cluster.Network[i]
if network.IsDDNS() {
ddnsReady := make(chan struct{})
lbWg.Go(func() {
// start the DDNS if requested
log.Debug("(svcs) start DDNS", "name", network.DNSName())
if err := cluster.StartDDNS(lbCtx, cluster.Network[i], c.DHCPBackoffAttempts, &lbWg); err != nil {
log.Error("failed to start DDNS", "err", err)
}
close(ddnsReady)
<-lbCtx.Done()
})
<-ddnsReady
}
log.Debug("current ip to process", "ip", network.IP(), "mask", c.VIPSubnet)
if err := network.SetMask(c.VIPSubnet); err != nil {
log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
lbCancel()
return utils.NewPanicError(fmt.Sprintf("failed to set mask for subnet %q: %s", c.VIPSubnet, err.Error()))
panic("")
}
_, err := network.DeleteIP()
if err != nil {
log.Warn("attempted to clean existing VIP", "err", err)
}
log.Debug("config flags", "enable_routing_table", c.EnableRoutingTable, "enable_leader_election", c.EnableLeaderElection, "enable_services_election", c.EnableServicesElection)
if c.EnableRoutingTable && (c.EnableLeaderElection || c.EnableServicesElection) {
err = network.AddRoute(false)
if err != nil {
log.Warn(err.Error())
} else {
log.Info("successful add Route")
}
}
if !c.EnableRoutingTable && !c.EnableBGP && !c.EnableWireguard {
// Normal VIP addition, use skipDAD=false for normal DAD process
// Note: When WireGuard is enabled, the VIP is added to the tunnel interface
// instead of lo, so we skip adding it here.
if _, err = network.AddIP(false, false); err != nil {
} else if !c.EnableRoutingTable {
if _, err = network.AddIP(false); err != nil {
log.Warn(err.Error())
} else {
log.Info("successful add IP")
}
}
if c.EnableARP {
lbWg.Go(func() {
cluster.layer2Update(lbCtx, network, c)
})
arpWG.Add(1)
go cluster.layer2Update(ctxArp, network, c, &arpWG)
}
if c.EnableBGP && (c.EnableLeaderElection || c.EnableServicesElection) {
// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
log.Debug("(svcs) attempting to advertise over BGP", "address", network.CIDR())
err = bgp.AddHost(lbCtx, network.CIDR())
err = bgp.AddHost(network.CIDR())
if err != nil {
log.Error(err.Error())
}
}
}
wg.Go(func() {
for i := range cluster.Network {
network := cluster.Network[i]
// start the dns updater if address is dns
if network.IsDNS() {
log.Info("(svcs) starting the DNS updater", "address", network.DNSName(), "ip", network.IP())
ipUpdater := vip.NewIPUpdater(network)
wg.Go(func() {
ipUpdater.Run(lbCtx)
})
}
}
go func() {
<-cluster.stop
// Stop the loadbalancer context if it is running
lbCancel()
// Stop the Arp context if it is running
cancelArp()
lbWg.Wait() // wait for all cluster ARP/NDP to be finished
arpWG.Wait() // wait for all cluster ARP/NDP to be finished
log.Info("[LOADBALANCER] Stopping load balancers", "name", name)
@@ -375,57 +326,40 @@ func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip
}
}
close(cluster.completed)
return
}
for i := range cluster.Network {
if c.EnableARP && cluster.arpMgr.Count(cluster.Network[i].ARPName()) > 1 {
if c.EnableARP && cluster.arpMgr.Count(cluster.Network[i].ARPName()) > 0 {
continue
}
// Handle VIP cleanup based on configuration
if c.PreserveVIPOnLeadershipLoss {
// For IPv6, we must remove VIPs immediately to avoid DAD failures on the new leader
// IPv6 Duplicate Address Detection will fail if the new leader tries to add an IP
// that is still present on this node's interface
if utils.IsIPv6(cluster.Network[i].IP()) {
log.Info("[VIP] Removing IPv6 VIP immediately (required to prevent DAD failures on new leader)", "ip", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
} else {
log.Info("[VIP] Preserving IPv4 VIP address on interface, only stopped ARP broadcasting", "ip", cluster.Network[i].IP())
}
} else {
// Legacy behavior: delete VIP addresses on leadership loss
log.Info("[VIP] Deleting VIP", "ip", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
log.Info("[VIP] Deleting VIP", "ip", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
}
})
return nil
close(cluster.completed)
}()
}
// Layer2Update, handles the creation of the
func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config) {
func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config, arpWG *sync.WaitGroup) {
defer arpWG.Done()
log.Info("layer 2 broadcaster starting")
var ndp *vip.NdpResponder
var err error
ipString := network.IP()
if utils.IsIPv6(ipString) {
if vip.IsIPv6(ipString) {
if network.IPisLinkLocal() {
log.Error("layer2 is link-local can't use NDP", "address", ipString)
} else {
ndp, err = waitNDPResponder(ctx, network.Interface())
ndp, err = vip.NewNDPResponder(network.Interface())
if err != nil {
log.Error("failed to create new NDP Responder", "error", err)
} else {
@@ -436,7 +370,6 @@ func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c
}
}
log.Info("layer 2 broadcaster starting", "IP", network.IP(), "device", network.Interface())
log.Debug("layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
arpInstance := arp.NewInstance(network, ndp)
@@ -444,32 +377,5 @@ func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c
<-ctx.Done() // if cancel() execute
log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
cluster.arpMgr.RemoveOnLeadershipLoss(arpInstance)
}
func waitNDPResponder(ctx context.Context, ifaceName string) (*vip.NdpResponder, error) {
ndp, err := vip.NewNDPResponder(ifaceName)
if err != nil && strings.Contains(err.Error(), "no such device") {
log.Warn("unable to create NDP responder at first try", "interface", ifaceName, "err", err)
ndpCreateCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
ticker := time.NewTicker(time.Second)
for {
select {
case <-ndpCreateCtx.Done():
return nil, fmt.Errorf("failed to create NDP responder for interface %q: %w", ifaceName, ndpCreateCtx.Err())
case <-ticker.C:
ndp, err = vip.NewNDPResponder(ifaceName)
if err != nil {
log.Warn("unable to create NDP responder on retry", "interface", ifaceName, "err", err)
} else {
return ndp, nil
}
}
}
} else if err != nil {
return nil, fmt.Errorf("unable to create NDP responder for interface %q: %w", ifaceName, err)
}
return ndp, nil
cluster.arpMgr.Remove(arpInstance)
}

84
pkg/cluster/singleNode.go Normal file
View File

@@ -0,0 +1,84 @@
package cluster
import (
"context"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/vip"
)
// StartSingleNode will start a single node cluster
func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) error {
// Start kube-vip as a single node server
// TODO - Split all this code out as a separate function
log.Info("Starting kube-vip as a single node cluster")
log.Info("This node is assuming leadership of the cluster")
cluster.stop = make(chan bool, 1)
cluster.completed = make(chan bool, 1)
for i := range cluster.Network {
if !disableVIP {
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("Attempted to clean existing VIP", "err", err)
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
_, err = cluster.Network[i].AddIP(false)
if err != nil {
log.Warn(err.Error())
}
}
if c.EnableARP {
// Gratuitous ARP, will broadcast to new MAC <-> IP
err := vip.ARPSendGratuitous(cluster.Network[i].IP(), c.Interface)
if err != nil {
log.Warn(err.Error())
}
}
}
go func() {
<-cluster.stop
if !disableVIP {
for i := range cluster.Network {
log.Info("[VIP] Releasing the VIP", "address", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
}
}
close(cluster.completed)
}()
log.Info("Started Load Balancer and Virtual IP")
return nil
}
func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp.Server) error {
// use a Go context so we can tell the arp loop code when we
// want to step down
ctxArp, cancelArp := context.WithCancel(context.Background())
defer cancelArp()
// use a Go context so we can tell the dns loop code when we
// want to step down
ctxDNS, cancelDNS := context.WithCancel(context.Background())
defer cancelDNS()
return cluster.vipService(ctxArp, ctxDNS, c, sm, bgp, nil)
}

View File

@@ -37,5 +37,5 @@ func FindIPAddress(addrName string) (string, string, error) {
}
}
return "", "", fmt.Errorf("unknown interface [%s]", addrName)
return "", "", fmt.Errorf("Unknown interface [%s]", addrName)
}

View File

@@ -7,7 +7,6 @@ import (
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
@@ -20,7 +19,7 @@ func Teardown(podIP, vipIP, namespace, serviceUUID string, annotations map[strin
protocol := iptables.ProtocolIPv4
IPv6 := false
if utils.IsIPv6(podIP) {
if vip.IsIPv6(podIP) {
protocol = iptables.ProtocolIPv6
IPv6 = true
}

View File

@@ -1,261 +0,0 @@
package election
import (
"context"
"errors"
"fmt"
"sync"
"time"
log "log/slog"
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/etcd"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/loadbalancer"
"github.com/kube-vip/kube-vip/pkg/utils"
clientv3 "go.etcd.io/etcd/client/v3"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
watchtools "k8s.io/client-go/tools/watch"
)
type Manager struct {
KubernetesClient *kubernetes.Clientset
RetryWatcherClient *kubernetes.Clientset
// This channel is used to signal a shutdown
EtcdClient *clientv3.Client
}
// NewManager will create a new managing object
func NewManager(config *kubevip.Config, k8sClientset, rwClientset *kubernetes.Clientset) (*Manager, error) {
m := &Manager{}
switch config.LeaderElectionType {
case "kubernetes", "":
if k8sClientset == nil || rwClientset == nil {
return nil, fmt.Errorf("provided nil clientset")
}
m.KubernetesClient = k8sClientset
m.RetryWatcherClient = rwClientset
case "etcd":
client, err := etcd.NewClient(config)
if err != nil {
return nil, err
}
m.EtcdClient = client
default:
return nil, fmt.Errorf("invalid LeaderElectionMode %s not supported", config.LeaderElectionType)
}
return m, nil
}
func RunOrDie(ctx context.Context, run *RunConfig, c *kubevip.Config) error {
switch c.LeaderElectionType {
case "kubernetes", "":
runKubernetesLeaderElectionOrDie(ctx, run)
case "etcd":
if err := runEtcdLeaderElectionOrDie(ctx, run); err != nil {
return err
}
default:
log.Info(fmt.Sprintf("LeaderElectionMode %s not supported, exiting", c.LeaderElectionType))
}
return nil
}
func runKubernetesLeaderElectionOrDie(ctx context.Context, run *RunConfig) {
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: run.LeaseID.Name(),
Namespace: run.LeaseID.Namespace(),
Annotations: run.LeaseAnnotations,
},
Client: run.Mgr.KubernetesClient.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: run.Config.NodeName,
},
}
// start the leader election code loop
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(run.Config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(run.Config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(run.Config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: run.OnStartedLeading,
OnStoppedLeading: run.OnStoppedLeading,
OnNewLeader: run.OnNewLeader,
},
})
}
func runEtcdLeaderElectionOrDie(ctx context.Context, run *RunConfig) error {
if err := etcd.RunElectionOrDie(ctx, &etcd.LeaderElectionConfig{
EtcdConfig: etcd.ClientConfig{Client: run.Mgr.EtcdClient},
Name: run.LeaseID.NamespacedName(),
MemberID: run.Config.NodeName,
LeaseDurationSeconds: int64(run.Config.LeaseDuration),
Callbacks: etcd.LeaderCallbacks{
OnStartedLeading: run.OnStartedLeading,
OnStoppedLeading: run.OnStoppedLeading,
OnNewLeader: run.OnNewLeader,
},
}); err != nil {
return fmt.Errorf("etcd leaderelection: %w", err)
}
return nil
}
type Actions interface {
OnStartedLeading(ctx context.Context)
OnStoppedLeading()
OnNewLeader(identity string)
}
type RunConfig struct {
Config *kubevip.Config
LeaseID lease.ID
Mgr *Manager
LeaseAnnotations map[string]string
// onStartedLeading is called when this member starts leading.
OnStartedLeading func(context.Context)
// onStoppedLeading is called when this member stops leading.
OnStoppedLeading func()
// onNewLeader is called when the client observes a leader that is
// not the previously observed leader. This includes the first observed
// leader when the client starts.
OnNewLeader func(identity string)
}
func (em *Manager) NodeWatcher(ctx context.Context, lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
log.Info("Kube-Vip is watching nodes for control-plane labels")
listOptions := metav1.ListOptions{
LabelSelector: "node-role.kubernetes.io/control-plane",
}
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return em.RetryWatcherClient.CoreV1().Nodes().Watch(ctx, listOptions)
},
})
if err != nil {
return fmt.Errorf("error creating label watcher: %s", err.Error())
}
wg := sync.WaitGroup{}
defer wg.Wait()
wg.Go(func() {
<-ctx.Done()
log.Info("Received termination, signaling shutdown")
// Cancel the context
rw.Stop()
})
ch := rw.ResultChan()
// defer rw.Stop()
for event := range ch {
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
node, ok := event.Object.(*v1.Node)
if !ok {
return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
}
// Find the node IP address (this isn't foolproof)
for x := range node.Status.Addresses {
if node.Status.Addresses[x].Type == v1.NodeInternalIP {
if checkIfNodeIsReady(node) {
err = lb.AddBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("add IPVS backend", "err", err)
if errors.Is(err, &utils.PanicError{}) {
return fmt.Errorf("add IPVS backend: %w", err)
}
}
} else {
err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("remove IPVS backend", "err", err)
}
}
}
}
case watch.Deleted:
node, ok := event.Object.(*v1.Node)
if !ok {
return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
}
// Find the node IP address (this isn't foolproof)
for x := range node.Status.Addresses {
if node.Status.Addresses[x].Type == v1.NodeInternalIP {
err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("Del IPVS backend", "err", err)
}
}
}
log.Info("Node deleted", "name", node.Name)
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes Nodes")
// This round trip allows us to handle unstructured status
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Error("watcher", "status", status)
default:
}
}
log.Info("Exiting Node watcher")
return nil
}
func checkIfNodeIsReady(node *v1.Node) bool {
if node == nil {
return false
}
for _, condition := range node.Status.Conditions {
if condition.Type == v1.NodeReady {
if condition.Status == v1.ConditionTrue {
return true
}
}
}
return false
}

View File

@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"net"
"sync"
log "log/slog"
@@ -12,13 +11,10 @@ import (
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/wireguard"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
)
type Processor struct {
@@ -30,24 +26,23 @@ type Processor struct {
}
func NewEndpointProcessor(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
instances *[]*instance.Instance, leaseMgr *lease.Manager, tunnelMgr *wireguard.TunnelManager) *Processor {
instances *[]*instance.Instance) *Processor {
return &Processor{
config: config,
provider: provider,
bgpServer: bgpServer,
instances: instances,
worker: newEndpointWorker(config, provider, bgpServer, instances, leaseMgr, tunnelMgr),
worker: newEndpointWorker(config, provider, bgpServer, instances),
}
}
func (p *Processor) AddOrModify(svcCtx *servicecontext.Context, event watch.Event,
lastKnownGoodEndpoint *string, service *v1.Service, id string,
serviceFunc func(*servicecontext.Context, *v1.Service, *sync.WaitGroup) error, wg *sync.WaitGroup,
clientSet *kubernetes.Clientset,
egressUpdateFunc func(context.Context, *v1.Service) error) (bool, error) {
func (p *Processor) AddOrModify(ctx *servicecontext.Context, event watch.Event,
lastKnownGoodEndpoint *string, service *v1.Service, id string, leaderElectionActive *bool,
serviceFunc func(context.Context, *v1.Service) error,
leaderCtx *context.Context, cancel *context.CancelFunc) (bool, error) {
var err error
if err = p.provider.LoadObject(event.Object, svcCtx.Cancel); err != nil {
if err = p.provider.LoadObject(event.Object, *cancel); err != nil {
return false, fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
}
@@ -69,61 +64,50 @@ func (p *Processor) AddOrModify(svcCtx *servicecontext.Context, event watch.Even
// Check that we have local endpoints
if len(endpoints) != 0 {
// Ignore IPv4
if service.Annotations[kubevip.EgressIPv6] == "true" && !hasV6(endpoints) {
if service.Annotations[kubevip.EgressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
return true, nil
}
p.updateLastKnownGoodEndpoint(svcCtx, lastKnownGoodEndpoint, endpoints, service)
svcCtx.HasEndpoints.Store(true)
p.updateLastKnownGoodEndpoint(lastKnownGoodEndpoint, endpoints, service, leaderElectionActive, *cancel)
// start leader election if it's enabled and not already started
if !svcCtx.IsActive && p.config.EnableServicesElection {
wg.Go(func() {
startLeaderElection(svcCtx, service, serviceFunc, wg)
})
if !*leaderElectionActive && p.config.EnableServicesElection {
go func() {
*leaderCtx, *cancel = context.WithCancel(ctx.Ctx)
startLeaderElection(*leaderCtx, leaderElectionActive, service, serviceFunc)
}()
}
// There are local endpoints available on the node
// Process immediately if:
// - No services/leader election is enabled, OR
// - WireGuard is enabled (it always needs immediate DNAT rule updates)
if (!p.config.EnableServicesElection && !p.config.EnableLeaderElection) || p.config.EnableWireguard {
if err := p.worker.processInstance(svcCtx, service); err != nil {
if !p.config.EnableServicesElection && !p.config.EnableLeaderElection {
if err := p.worker.processInstance(ctx, service, leaderElectionActive); err != nil {
return false, fmt.Errorf("failed to process non-empty instance: %w", err)
}
}
} else {
svcCtx.HasEndpoints.Store(false)
// There are no local endpoints
p.worker.clear(svcCtx, lastKnownGoodEndpoint, service)
p.worker.clear(ctx, lastKnownGoodEndpoint, service, *cancel, leaderElectionActive)
}
// Set the service accordingly
p.updateAnnotations(service, lastKnownGoodEndpoint, clientSet, egressUpdateFunc)
p.updateAnnotations(service, lastKnownGoodEndpoint)
log.Debug("watcher", "provider",
p.provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", *lastKnownGoodEndpoint, "active leader election", svcCtx.IsActive)
p.provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", *lastKnownGoodEndpoint, "active leader election", *leaderElectionActive)
return false, nil
}
func (p *Processor) Delete(ctx context.Context, service *v1.Service, id string) error {
if err := p.worker.delete(ctx, service, id); err != nil {
func (p *Processor) Delete(service *v1.Service, id string) error {
if err := p.worker.delete(service, id); err != nil {
return fmt.Errorf("[%s] error deleting service: %w", p.provider.GetLabel(), err)
}
return nil
}
func (p *Processor) updateLastKnownGoodEndpoint(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service) {
func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service, leaderElectionActive *bool, cancel context.CancelFunc) {
// if we haven't populated one, then do so
family := utils.IPv4Family
if service.Annotations[kubevip.EgressIPv6] == "true" {
family = utils.IPv6Family
}
ep := getEndpoint(endpoints, family)
if *lastKnownGoodEndpoint == "" {
*lastKnownGoodEndpoint = ep
*lastKnownGoodEndpoint = endpoints[0]
return
}
@@ -138,122 +122,44 @@ func (p *Processor) updateLastKnownGoodEndpoint(svcCtx *servicecontext.Context,
// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
if !stillExists {
p.worker.removeEgress(service, lastKnownGoodEndpoint)
if svcCtx.IsActive && (p.config.EnableServicesElection || p.config.EnableLeaderElection) {
if *leaderElectionActive && (p.config.EnableServicesElection || p.config.EnableLeaderElection) {
log.Warn("existing endpoint has been removed, restarting leaderElection", "provider", p.provider.GetLabel(), "endpoint", *lastKnownGoodEndpoint)
// Stop the existing leaderElection
if svcCtx.Lease != nil {
svcCtx.Lease.Cancel()
}
cancel()
// disable last leaderElection flag
*leaderElectionActive = false
}
// Set our active endpoint to an existing one
*lastKnownGoodEndpoint = ep
*lastKnownGoodEndpoint = endpoints[0]
}
}
func (p *Processor) updateAnnotations(service *v1.Service, lastKnownGoodEndpoint *string,
clientSet *kubernetes.Clientset,
egressUpdateFunc func(context.Context, *v1.Service) error) {
func (p *Processor) updateAnnotations(service *v1.Service, lastKnownGoodEndpoint *string) {
// Set the service accordingly
if service.Annotations[kubevip.Egress] == "true" {
ip := net.ParseIP(*lastKnownGoodEndpoint)
activeEndpointAnnotation := kubevip.ActiveEndpoint
// Store old values from ServiceSnapshot to detect if annotation actually changed
// We use the ServiceSnapshot instead of the service parameter because the service parameter
// may have stale annotations if the last update failed
var oldEndpoint, oldEndpointIPv6 string
if p.instances != nil {
serviceInstance := instance.FindServiceInstance(service, *p.instances)
if serviceInstance != nil {
oldEndpoint = serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint]
oldEndpointIPv6 = serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpointIPv6]
}
}
// Fall back to service annotations if we couldn't find the instance
if oldEndpoint == "" && oldEndpointIPv6 == "" {
oldEndpoint = service.Annotations[kubevip.ActiveEndpoint]
oldEndpointIPv6 = service.Annotations[kubevip.ActiveEndpointIPv6]
}
// Determine which annotation to update based on IP version
var endpoint, endpointIPv6 string
if ip.To4() == nil && !p.config.EnableEndpoints {
// IPv6
endpointIPv6 = *lastKnownGoodEndpoint
endpoint = oldEndpoint // Preserve existing IPv4 if any
} else {
// IPv4
endpoint = *lastKnownGoodEndpoint
endpointIPv6 = oldEndpointIPv6 // Preserve existing IPv6 if any
}
// Check if annotation actually changed
annotationChanged := (oldEndpoint != endpoint) || (oldEndpointIPv6 != endpointIPv6)
if !annotationChanged {
return // Nothing to do
}
// Persist to Kubernetes
ctx := context.Background()
if err := p.provider.UpdateServiceAnnotation(ctx, endpoint, endpointIPv6, service, clientSet); err != nil {
log.Warn("failed to update service annotation", "service", service.Name, "namespace", service.Namespace, "err", err)
return
}
log.Debug("updated active endpoint annotation", "service", service.Name, "namespace", service.Namespace, "endpoint", *lastKnownGoodEndpoint)
// Trigger egress reconfiguration
// For services with leader election, the service watcher doesn't process Modified events
// after initial setup, so we need to directly call the update function
if egressUpdateFunc != nil {
// Create a copy of service with updated annotations
svcCopy := service.DeepCopy()
svcCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
svcCopy.Annotations[kubevip.ActiveEndpointIPv6] = endpointIPv6
if err := egressUpdateFunc(ctx, svcCopy); err != nil {
log.Error("failed to reconfigure egress", "service", service.Name, "namespace", service.Namespace, "err", err)
}
if !p.config.EnableEndpoints && p.provider.GetProtocol() == string(discoveryv1.AddressTypeIPv6) {
activeEndpointAnnotation = kubevip.ActiveEndpointIPv6
}
service.Annotations[activeEndpointAnnotation] = *lastKnownGoodEndpoint
}
}
func startLeaderElection(svcCtx *servicecontext.Context, service *v1.Service, serviceFunc func(*servicecontext.Context, *v1.Service, *sync.WaitGroup) error, wg *sync.WaitGroup) {
func startLeaderElection(ctx context.Context, leaderElectionActive *bool, service *v1.Service, serviceFunc func(context.Context, *v1.Service) error) {
// This is a blocking function, that will restart (in the event of failure)
for {
select {
case <-svcCtx.Ctx.Done():
return
default:
err := serviceFunc(svcCtx, service, wg)
// if the context isn't cancelled restart
if ctx.Err() != context.Canceled {
*leaderElectionActive = true
err := serviceFunc(ctx, service)
if err != nil {
log.Error(err.Error())
}
*leaderElectionActive = false
} else {
*leaderElectionActive = false
break
}
}
}
func hasV6(endpoints []string) bool {
for _, e := range endpoints {
ip := net.ParseIP(e)
if ip != nil {
if ip.To4() == nil {
return true
}
}
}
return false
}
func getEndpoint(endpoints []string, family string) string {
for _, e := range endpoints {
ip := net.ParseIP(e)
if family == utils.IPv4Family && ip.To4() != nil {
return e
}
if family == utils.IPv6Family && ip.To4() == nil {
return e
}
}
return ""
}

View File

@@ -23,19 +23,20 @@ func newBGP(generic generic, bgpServer *bgp.Server) endpointWorker {
}
}
func (b *BGP) processInstance(svcCtx *servicecontext.Context, service *v1.Service) error {
func (b *BGP) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
if !svcCtx.IsNetworkConfigured(cluster.Network[i].IP()) {
if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) {
log.Debug("attempting to advertise BGP service", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP())
err := b.bgpServer.AddHost(svcCtx.Ctx, cluster.Network[i].CIDR())
err := b.bgpServer.AddHost(cluster.Network[i].CIDR())
if err != nil {
log.Error("error adding BGP host", "provider", b.provider.GetLabel(), "err", err)
} else {
log.Info("added BGP host", "provider",
b.provider.GetLabel(), "ip", cluster.Network[i].CIDR(), "service name", service.Name, "namespace", service.Namespace)
svcCtx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
*leaderElectionActive = true
}
}
}
@@ -44,19 +45,20 @@ func (b *BGP) processInstance(svcCtx *servicecontext.Context, service *v1.Servic
return nil
}
func (b *BGP) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service) {
func (b *BGP) clear(ctx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
// If BGP mode is enabled - routes should be deleted
if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
err := b.bgpServer.DelHost(svcCtx.Ctx, cluster.Network[i].CIDR())
err := b.bgpServer.DelHost(cluster.Network[i].CIDR())
if err != nil {
log.Error("deleting BGP host", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "err", err)
} else {
log.Info("deleted BGP host", "provider",
b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace)
svcCtx.ConfiguredNetworks.Delete(cluster.Network[i].IP())
ctx.ConfiguredNetworks.Delete(cluster.Network[i])
*leaderElectionActive = false
}
}
}
@@ -64,18 +66,14 @@ func (b *BGP) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *strin
}
}
b.clearEgress(lastKnownGoodEndpoint, service)
if svcCtx.LeaderCancel != nil {
svcCtx.LeaderCancel()
}
b.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (b *BGP) getEndpoints(service *v1.Service, id string) ([]string, error) {
return b.getAllEndpoints(service, id)
}
func (b *BGP) delete(ctx context.Context, service *v1.Service, id string) error {
func (b *BGP) delete(service *v1.Service, id string) error {
// When no-leader-elecition mode
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
// find all existing local endpoints
@@ -86,40 +84,36 @@ func (b *BGP) delete(ctx context.Context, service *v1.Service, id string) error
// If there were local endpoints deleted
if len(endpoints) > 0 {
b.deleteAction(ctx, service)
b.deleteAction(service)
}
}
return nil
}
func (b *BGP) deleteAction(ctx context.Context, service *v1.Service) {
b.clearBGPHosts(ctx, service)
func (b *BGP) deleteAction(service *v1.Service) {
b.clearBGPHosts(service)
}
func (b *BGP) clearBGPHosts(ctx context.Context, service *v1.Service) {
ClearBGPHosts(ctx, service, b.instances, b.bgpServer)
func (b *BGP) clearBGPHosts(service *v1.Service) {
ClearBGPHosts(service, b.instances, b.bgpServer)
}
func (b *BGP) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
return nil
}
func ClearBGPHosts(ctx context.Context, service *v1.Service, instances *[]*instance.Instance, bgpServer *bgp.Server) {
func ClearBGPHosts(service *v1.Service, instances *[]*instance.Instance, bgpServer *bgp.Server) {
if instance := instance.FindServiceInstance(service, *instances); instance != nil {
ClearBGPHostsByInstance(ctx, instance, bgpServer)
ClearBGPHostsByInstance(instance, bgpServer)
}
}
func ClearBGPHostsByInstance(ctx context.Context, instance *instance.Instance, bgpServer *bgp.Server) {
if instance == nil {
log.Error("failed to clear BGP host for nil instance")
return
}
func ClearBGPHostsByInstance(instance *instance.Instance, bgpServer *bgp.Server) {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
network := cluster.Network[i]
err := bgpServer.DelHost(ctx, network.CIDR())
err := bgpServer.DelHost(network.CIDR())
if err != nil {
log.Error("[endpoint] error deleting BGP host", "err", err)
} else {

View File

@@ -10,27 +10,22 @@ import (
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/wireguard"
v1 "k8s.io/api/core/v1"
)
type endpointWorker interface {
processInstance(svcCtx *servicecontext.Context, service *v1.Service) error
clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service)
processInstance(svcCtx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error
clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool)
getEndpoints(service *v1.Service, id string) ([]string, error)
removeEgress(service *v1.Service, lastKnownGoodEndpoint *string)
delete(ctx context.Context, service *v1.Service, id string) error
delete(service *v1.Service, id string) error
setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error
}
func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server, instances *[]*instance.Instance, leaseMgr *lease.Manager, tunnelMgr *wireguard.TunnelManager) endpointWorker {
generic := newGeneric(config, provider, instances, leaseMgr)
func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server, instances *[]*instance.Instance) endpointWorker {
generic := newGeneric(config, provider, instances)
if config.EnableWireguard {
return newWireguardWorker(config, provider, bgpServer, instances, leaseMgr, tunnelMgr)
}
if config.EnableRoutingTable {
return newRoutingTable(generic)
}
@@ -45,37 +40,36 @@ type generic struct {
config *kubevip.Config
provider providers.Provider
instances *[]*instance.Instance
leaseMgr *lease.Manager
}
func newGeneric(config *kubevip.Config, provider providers.Provider, instances *[]*instance.Instance, leaseMgr *lease.Manager) generic {
func newGeneric(config *kubevip.Config, provider providers.Provider, instances *[]*instance.Instance) generic {
return generic{
config: config,
provider: provider,
instances: instances,
leaseMgr: leaseMgr,
}
}
func (g *generic) processInstance(_ *servicecontext.Context, _ *v1.Service) error {
func (g *generic) processInstance(_ *servicecontext.Context, _ *v1.Service, _ *bool) error {
return nil
}
func (g *generic) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service) {
g.clearEgress(lastKnownGoodEndpoint, service)
if svcCtx.LeaderCancel != nil {
svcCtx.LeaderCancel()
}
func (g *generic) clear(_ *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
g.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service) {
func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if *lastKnownGoodEndpoint != "" {
log.Warn("existing endpoint has been removed, no remaining endpoints for leaderElection", "provider", g.provider.GetLabel(), "endpoint", lastKnownGoodEndpoint)
log.Warn("existing endpoint has been removed, no remaining endpoints for leaderElection", "provider", g.provider.GetLabel(), "endpoint", lastKnownGoodEndpoint)
if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP, service.Namespace, string(service.UID), service.Annotations, g.config.EgressWithNftables); err != nil {
log.Error("error removing redundant egress rules", "err", err)
}
*lastKnownGoodEndpoint = "" // reset endpoint
if g.config.EnableServicesElection || g.config.EnableLeaderElection {
cancel() // stop services watcher
}
*leaderElectionActive = false
}
}
@@ -115,7 +109,7 @@ func (g *generic) getAllEndpoints(service *v1.Service, id string) ([]string, err
func (g *generic) removeEgress(_ *v1.Service, _ *string) {
}
func (g *generic) delete(_ context.Context, _ *v1.Service, _ string) error {
func (g *generic) delete(_ *v1.Service, _ string) error {
return nil
}

View File

@@ -26,7 +26,7 @@ func newRoutingTable(generic generic) endpointWorker {
}
}
func (rt *RoutingTable) processInstance(ctx *servicecontext.Context, service *v1.Service) error {
func (rt *RoutingTable) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
instance := instance.FindServiceInstance(service, *rt.instances)
if instance != nil {
for _, cluster := range instance.Clusters {
@@ -59,6 +59,7 @@ func (rt *RoutingTable) processInstance(ctx *servicecontext.Context, service *v1
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
*leaderElectionActive = true
}
}
}
@@ -68,7 +69,7 @@ func (rt *RoutingTable) processInstance(ctx *servicecontext.Context, service *v1
return nil
}
func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service) {
func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
if errs := ClearRoutes(service, rt.instances); len(errs) == 0 {
svcCtx.ConfiguredNetworks.Clear()
@@ -79,11 +80,7 @@ func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpo
}
}
rt.clearEgress(lastKnownGoodEndpoint, service)
if svcCtx.LeaderCancel != nil {
svcCtx.LeaderCancel()
}
rt.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (rt *RoutingTable) getEndpoints(service *v1.Service, id string) ([]string, error) {
@@ -97,7 +94,7 @@ func (rt *RoutingTable) removeEgress(service *v1.Service, lastKnownGoodEndpoint
}
}
func (rt *RoutingTable) delete(_ context.Context, service *v1.Service, id string) error {
func (rt *RoutingTable) delete(service *v1.Service, id string) error {
// When no-leader-elecition mode
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
// find all existing local endpoints
@@ -122,21 +119,17 @@ func (rt *RoutingTable) deleteAction(service *v1.Service) {
func (rt *RoutingTable) setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error {
instance := instance.FindServiceInstance(service, *rt.instances)
if instance == nil {
log.Error("failed to find the instance", "namespace", service.Namespace, "name", service.Name, "uid", service.UID, "provider", rt.provider.GetLabel())
log.Error("failed to find the instance", "service", service.UID, "provider", rt.provider.GetLabel())
} else {
for _, c := range instance.Clusters {
for n := range c.Network {
// if there are no endpoints set HasEndpoints false just in case
if len(endpoints) < 1 {
c.Network[n].SetHasEndpoints(false)
} else {
// check if endpoint are available and are of same IP family as service
for _, ep := range endpoints {
if (net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(ep).To4() == nil) {
c.Network[n].SetHasEndpoints(true)
break
}
}
}
// check if endpoint are available and are of same IP family as service
if len(endpoints) > 0 && ((net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(endpoints[0]).To4() == nil)) {
c.Network[n].SetHasEndpoints(true)
}
}
}
@@ -147,34 +140,23 @@ func (rt *RoutingTable) setInstanceEndpointsStatus(service *v1.Service, endpoint
func ClearRoutes(service *v1.Service, instances *[]*instance.Instance) []error {
errs := []error{}
if svcInst := instance.FindServiceInstance(service, *instances); svcInst != nil {
clearErrs := ClearRoutesByInstance(service, svcInst, instances)
errs = append(errs, clearErrs...)
}
return errs
}
func ClearRoutesByInstance(service *v1.Service, svcInst *instance.Instance, instances *[]*instance.Instance) []error {
if svcInst == nil {
return []error{fmt.Errorf("failed to remove routes for nil instance of service %s/%s, uid: %s", service.Namespace, service.Name, service.UID)}
}
errs := []error{}
for _, cluster := range svcInst.Clusters {
for i := range cluster.Network {
route := cluster.Network[i].PrepareRoute()
// check if route we are about to delete is not referenced by more than one service
if CountRouteReferences(route, instances) <= 1 {
err := cluster.Network[i].DeleteRoute()
if err != nil && !errors.Is(err, syscall.ESRCH) {
log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
errs = append(errs, err)
if instance := instance.FindServiceInstance(service, *instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
route := cluster.Network[i].PrepareRoute()
// check if route we are about to delete is not referenced by more than one service
if CountRouteReferences(route, instances) <= 1 {
err := cluster.Network[i].DeleteRoute()
if err != nil && !errors.Is(err, syscall.ESRCH) {
log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
errs = append(errs, err)
}
log.Debug("deleted route", "ip",
cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface())
}
log.Debug("deleted route", "ip",
cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface())
}
}
}
return errs
}

View File

@@ -1,289 +0,0 @@
package endpoints
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/wireguard"
v1 "k8s.io/api/core/v1"
)
// wireguardWorker handles endpoint changes for WireGuard-based services
type wireguardWorker struct {
config *kubevip.Config
provider providers.Provider
bgpServer *bgp.Server
instances *[]*instance.Instance
leaseMgr *lease.Manager
tunnelMgr *wireguard.TunnelManager
}
func newWireguardWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
instances *[]*instance.Instance, leaseMgr *lease.Manager, tunnelMgr *wireguard.TunnelManager) *wireguardWorker {
return &wireguardWorker{
config: config,
provider: provider,
bgpServer: bgpServer,
instances: instances,
leaseMgr: leaseMgr,
tunnelMgr: tunnelMgr,
}
}
// processInstance updates nftables DNAT rules when endpoints change
// This is called by the endpoint watcher when endpoints are added/modified
func (w *wireguardWorker) processInstance(svcCtx *servicecontext.Context, service *v1.Service) error {
log.Debug("[wireguard] processing instance for endpoint change", "service", service.Name, "namespace", service.Namespace)
// Get the target endpoint for this service
// For ExternalTrafficPolicy=Local, only use local endpoints
// For ExternalTrafficPolicy=Cluster, use all endpoints
var endpoints []string
var err error
if service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
endpoints, err = w.provider.GetLocalEndpoints(w.config.NodeName, w.config)
} else {
endpoints, err = w.provider.GetAllEndpoints()
}
if err != nil {
return fmt.Errorf("failed to get endpoints: %w", err)
}
if len(endpoints) == 0 {
log.Debug("[wireguard] no endpoints available", "service", service.Name)
w.clear(svcCtx, nil, service)
return nil
}
// Find the service processor to call updateServiceWireguardEndpoints
// Note: This requires access to the service processor which we don't have here
// So we'll recreate the DNAT rules directly
// First, clear existing rules
w.clear(svcCtx, nil, service)
// Get service VIPs
serviceIPs, err := utils.FetchServiceIPs(service)
if err != nil {
return fmt.Errorf("failed to get service IPs: %w", err)
}
// Create service identifier
serviceID := utils.SanitizeServiceID(fmt.Sprintf("%s_%s", service.Namespace, service.Name))
log.Info("[wireguard] updating DNAT rules for endpoint change",
"service", service.Name,
"namespace", service.Namespace,
"endpoints", endpoints,
"vips", serviceIPs)
// Update DNAT rules for each port
for _, port := range service.Spec.Ports {
// Determine target port (resolve named ports if necessary)
targetPort := w.provider.ResolvePort(port)
log.Info("[wireguard] resolved port", "service", service.Name, "servicePort", port.Port, "targetPort", targetPort, "targetPortName", port.TargetPort.StrVal)
// Build targets list from all endpoints
targets := make([]nftables.DNATTarget, len(endpoints))
for i, ep := range endpoints {
targets[i] = nftables.DNATTarget{
IP: ep,
Port: uint16(targetPort), //nolint:gosec // Port range validated by Kubernetes
}
}
for _, vip := range serviceIPs {
// Strip CIDR notation if present
vipAddr := utils.StripCIDR(vip)
// Get WireGuard interface name from TunnelManager for this VIP
if w.tunnelMgr == nil {
log.Error("[wireguard] TunnelManager not configured; cannot update DNAT rules",
"service", service.Name,
"namespace", service.Namespace)
return fmt.Errorf("TunnelManager not configured")
}
tunnelConfig := w.tunnelMgr.GetConfigForVIP(vipAddr)
if tunnelConfig == nil {
log.Error("[wireguard] WireGuard interface name not configured; cannot update DNAT rules",
"service", service.Name,
"namespace", service.Namespace,
"vip", vipAddr)
return fmt.Errorf("wireguard interface name not configured for VIP %s", vipAddr)
}
wgInterface := tunnelConfig.InterfaceName
portServiceID := fmt.Sprintf("%s_p%d", serviceID, port.Port)
log.Info("[wireguard] applying DNAT rule with load balancing",
"service", service.Name,
"vip", vipAddr,
"interface", wgInterface,
"sourcePort", port.Port,
"targets", targets,
"chainID", portServiceID)
// Apply the DNAT rule with load balancing across all endpoints
// localEndpoint=true when using ExternalTrafficPolicy=Local, which preserves client source IP
isLocalEndpoint := service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal
err := nftables.ApplyDNAT(
wgInterface,
vipAddr,
uint16(port.Port), //nolint:gosec // Port range validated by Kubernetes
targets,
portServiceID,
port.Protocol,
isLocalEndpoint,
tunnelConfig.ListenPort,
)
if err != nil {
log.Error("[wireguard] failed to update DNAT rule",
"service", service.Name,
"vip", vipAddr,
"port", port.Port,
"err", err)
continue
}
log.Debug("[wireguard] DNAT rule updated successfully",
"service", service.Name,
"vip", vipAddr,
"port", port.Port,
"targetCount", len(targets))
}
}
return nil
}
// clear removes DNAT rules when no endpoints are available
func (w *wireguardWorker) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service) {
log.Info("[wireguard] clearing DNAT rules (no endpoints)", "service", service.Name, "namespace", service.Namespace)
serviceID := utils.SanitizeServiceID(fmt.Sprintf("%s_%s", service.Namespace, service.Name))
// Get service IPs to determine IPv4 vs IPv6
serviceIPs, _ := utils.FetchServiceIPs(service)
// Delete DNAT chains for each port
for _, port := range service.Spec.Ports {
if port.Protocol != v1.ProtocolTCP && port.Protocol != v1.ProtocolUDP {
continue
}
portServiceID := fmt.Sprintf("%s_p%d", serviceID, port.Port)
// Determine if we have IPv4 or IPv6
hasIPv4, hasIPv6 := false, false
for _, vip := range serviceIPs {
if isIPv6Address(vip) {
hasIPv6 = true
} else {
hasIPv4 = true
}
}
if hasIPv4 {
if err := nftables.DeleteIngressChains(false, portServiceID); err != nil {
log.Warn("[wireguard] failed to delete IPv4 DNAT chains",
"service", service.Name,
"port", port.Port,
"err", err)
}
}
if hasIPv6 {
if err := nftables.DeleteIngressChains(true, portServiceID); err != nil {
log.Warn("[wireguard] failed to delete IPv6 DNAT chains",
"service", service.Name,
"port", port.Port,
"err", err)
}
}
}
if svcCtx.LeaderCancel != nil {
svcCtx.LeaderCancel()
}
}
// getEndpoints retrieves the list of endpoints for a service
// For ExternalTrafficPolicy=Local, only local endpoints are returned
// For ExternalTrafficPolicy=Cluster, all endpoints are returned
func (w *wireguardWorker) getEndpoints(service *v1.Service, id string) ([]string, error) {
var endpoints []string
var err error
if service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
endpoints, err = w.provider.GetLocalEndpoints(id, w.config)
} else {
endpoints, err = w.provider.GetAllEndpoints()
}
if err != nil {
return nil, fmt.Errorf("[wireguard] failed to get endpoints: %w", err)
}
log.Debug("[wireguard] retrieved endpoints", "service", service.Name, "count", len(endpoints), "endpoints", endpoints)
return endpoints, nil
}
// removeEgress is a no-op for WireGuard since egress is handled separately
func (w *wireguardWorker) removeEgress(service *v1.Service, lastKnownGoodEndpoint *string) {
// WireGuard doesn't use egress in the same way as other modes
log.Debug("[wireguard] removeEgress called (no-op)", "service", service.Name)
}
// delete removes all DNAT rules for a service
func (w *wireguardWorker) delete(ctx context.Context, service *v1.Service, id string) error {
log.Info("[wireguard] deleting DNAT rules for service", "service", service.Name, "namespace", service.Namespace)
w.clear(nil, nil, service)
return nil
}
// setInstanceEndpointsStatus updates the endpoint status on the service instance
func (w *wireguardWorker) setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error {
hasEndpoints := len(endpoints) > 0
log.Debug("[wireguard] setting instance endpoint status",
"service", service.Name,
"hasEndpoints", hasEndpoints,
"endpointCount", len(endpoints))
// Find the service instance
for _, inst := range *w.instances {
if inst.ServiceSnapshot == nil {
continue
}
if inst.ServiceSnapshot.UID == service.UID {
// Update the network status for all clusters
for _, cluster := range inst.Clusters {
for i := range cluster.Network {
cluster.Network[i].SetHasEndpoints(hasEndpoints)
}
}
log.Debug("[wireguard] updated instance endpoint status",
"service", service.Name,
"hasEndpoints", hasEndpoints)
return nil
}
}
log.Debug("[wireguard] instance not found for endpoint status update", "service", service.Name)
return nil
}
func isIPv6Address(ip string) bool {
// Strip CIDR notation if present before checking
addr := utils.StripCIDR(ip)
return utils.IsIPv6(addr)
}

View File

@@ -97,11 +97,11 @@ func (ep *Endpoints) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string,
return localEndpoints, nil
}
func (ep *Endpoints) UpdateServiceAnnotation(ctx context.Context, endpoint string, _ string, service *v1.Service, clientSet *kubernetes.Clientset) error {
func (ep *Endpoints) UpdateServiceAnnotation(endpoint string, _ string, service *v1.Service, clientSet *kubernetes.Clientset) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(ctx, service.Name, metav1.GetOptions{})
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
@@ -113,7 +113,7 @@ func (ep *Endpoints) UpdateServiceAnnotation(ctx context.Context, endpoint strin
currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(ctx, currentServiceCopy, metav1.UpdateOptions{})
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("error updating Service Spec", "label", ep.GetLabel(), "name", currentServiceCopy.Name, "err", err)
return err
@@ -132,15 +132,6 @@ func (ep *Endpoints) GetLabel() string {
return ep.label
}
func (ep *Endpoints) ResolvePort(servicePort v1.ServicePort) int32 {
return ResolvePortWithLookup(servicePort, func(name string) int32 {
for _, subset := range ep.endpoints.Subsets {
for _, p := range subset.Ports {
if p.Name == name {
return p.Port
}
}
}
return 0
})
func (ep *Endpoints) GetProtocol() string {
return ""
}

View File

@@ -20,10 +20,8 @@ import (
)
type Endpointslices struct {
label string
endpointsv4 []discoveryv1.Endpoint
endpointsv6 []discoveryv1.Endpoint
ports []discoveryv1.EndpointPort
label string
endpoints *discoveryv1.EndpointSlice
}
func NewEndpointslices() Provider {
@@ -58,42 +56,27 @@ func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.Ca
cancel()
return fmt.Errorf("[%s] error casting endpoints to v1.Endpoints struct", ep.label)
}
if eps.AddressType == discoveryv1.AddressTypeIPv6 {
ep.endpointsv6 = eps.Endpoints
} else {
ep.endpointsv4 = eps.Endpoints
}
// Store ports for resolving named ports
ep.ports = eps.Ports
ep.endpoints = eps
return nil
}
func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
result := []string{}
for _, e := range ep.endpointsv4 {
result = append(result, e.Addresses...)
}
for _, e := range ep.endpointsv6 {
result = append(result, e.Addresses...)
for _, ep := range ep.endpoints.Endpoints {
result = append(result, ep.Addresses...)
}
return result, nil
}
func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
var localEndpoints []string
tmpEps := []discoveryv1.Endpoint{}
tmpEps = append(tmpEps, ep.endpointsv4...)
tmpEps = append(tmpEps, ep.endpointsv6...)
for _, endpoint := range tmpEps {
if endpoint.Conditions.Serving == nil || !*endpoint.Conditions.Serving {
for _, endpoint := range ep.endpoints.Endpoints {
if !*endpoint.Conditions.Serving {
continue
}
for _, address := range endpoint.Addresses {
log.Debug("processing endpoint", "provider", ep.label, "ip", address)
// 1. Compare the Nodename
if endpoint.NodeName != nil && id == *endpoint.NodeName {
if endpoint.Hostname != nil {
@@ -115,11 +98,11 @@ func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]str
return localEndpoints, nil
}
func (ep *Endpointslices) UpdateServiceAnnotation(ctx context.Context, endpoint, endpointIPv6 string, service *v1.Service, clientSet *kubernetes.Clientset) error {
func (ep *Endpointslices) UpdateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, clientSet *kubernetes.Clientset) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(ctx, service.Name, metav1.GetOptions{})
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
@@ -132,7 +115,7 @@ func (ep *Endpointslices) UpdateServiceAnnotation(ctx context.Context, endpoint,
currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
currentServiceCopy.Annotations[kubevip.ActiveEndpointIPv6] = endpointIPv6
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(ctx, currentServiceCopy, metav1.UpdateOptions{})
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("error updating Service Spec", "provider", ep.label, "service name", currentServiceCopy.Name, "err", err)
return err
@@ -151,13 +134,6 @@ func (ep *Endpointslices) GetLabel() string {
return ep.label
}
func (ep *Endpointslices) ResolvePort(servicePort v1.ServicePort) int32 {
return ResolvePortWithLookup(servicePort, func(name string) int32 {
for _, p := range ep.ports {
if p.Name != nil && *p.Name == name && p.Port != nil {
return *p.Port
}
}
return 0
})
func (ep *Endpointslices) GetProtocol() string {
return string(ep.endpoints.AddressType)
}

View File

@@ -16,24 +16,7 @@ type Provider interface {
GetAllEndpoints() ([]string, error)
GetLocalEndpoints(string, *kubevip.Config) ([]string, error)
GetLabel() string
UpdateServiceAnnotation(context.Context, string, string, *v1.Service, *kubernetes.Clientset) error
UpdateServiceAnnotation(string, string, *v1.Service, *kubernetes.Clientset) error
LoadObject(runtime.Object, context.CancelFunc) error
// ResolvePort resolves a service port to the actual target port.
// For named ports, it looks up the port number from the endpoint.
// For numeric ports, it returns the port as-is.
ResolvePort(servicePort v1.ServicePort) int32
}
// ResolvePortWithLookup is a helper that resolves a service port using a lookup function
// for named ports. This consolidates the common resolution logic.
func ResolvePortWithLookup(servicePort v1.ServicePort, lookupNamedPort func(string) int32) int32 {
if servicePort.TargetPort.IntVal != 0 {
return servicePort.TargetPort.IntVal
}
if servicePort.TargetPort.StrVal != "" {
if port := lookupNamedPort(servicePort.TargetPort.StrVal); port != 0 {
return port
}
}
return servicePort.Port
GetProtocol() string
}

View File

@@ -2,9 +2,7 @@ package etcd
import (
"context"
"fmt"
"hash/fnv"
"sync"
"time"
log "log/slog"
@@ -66,11 +64,10 @@ type ClientConfig struct {
}
// RunElectionOrDie behaves the same way as RunElection but panics if there is an error.
func RunElectionOrDie(ctx context.Context, config *LeaderElectionConfig) error {
func RunElectionOrDie(ctx context.Context, config *LeaderElectionConfig) {
if err := RunElection(ctx, config); err != nil {
return fmt.Errorf("leaderelection error: %w", err)
panic(err)
}
return nil
}
// RunElection starts a client with the provided config or panics.
@@ -119,12 +116,7 @@ func RunElection(ctx context.Context, config *LeaderElectionConfig) error {
leaseTTL: lease.TTL,
}
wg := sync.WaitGroup{}
defer wg.Wait()
wg.Go(func() {
m.tryToBeLeader(ctx, &wg)
})
go m.tryToBeLeader(ctx)
m.watchLeaderChanges(ctx)
return nil
@@ -193,7 +185,7 @@ watcher:
log.Debug("Exiting watcher", "id", m.memberID)
}
func (m *member) tryToBeLeader(ctx context.Context, wg *sync.WaitGroup) {
func (m *member) tryToBeLeader(ctx context.Context) {
if err := m.election.Campaign(ctx, m.memberID); err != nil {
log.Error("Failed trying to become the leader", "err", err)
// Resign just in case we acquired leadership just before failing
@@ -212,9 +204,7 @@ func (m *member) tryToBeLeader(ctx context.Context, wg *sync.WaitGroup) {
m.weAreTheLeader <- struct{}{}
// Once we are the leader, start the routine to resign if context is canceled
wg.Go(func() {
m.resignOnCancel(ctx)
})
go m.resignOnCancel(ctx)
// After becoming the leader, we wait for at least a lease TTL to wait for
// the previous leader to detect the new leadership (if there was one) and
@@ -231,7 +221,7 @@ func (m *member) tryToBeLeader(ctx context.Context, wg *sync.WaitGroup) {
func (m *member) resignOnCancel(ctx context.Context) {
<-ctx.Done()
if err := m.election.Resign(m.client.Ctx()); err != nil && !errors.Is(err, context.Canceled) {
if err := m.election.Resign(m.client.Ctx()); err != nil {
log.Error("Failed to resign after the context was canceled", "err", err)
}
}

View File

@@ -20,26 +20,19 @@ import (
func TestRunElectionWithMemberIDCollision(t *testing.T) {
t.Parallel()
g := NewWithT(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ctx := context.Background()
cli := client(g)
defer cli.Close()
electionName := randomElectionNameForTest("memberIDConflict")
log.Printf("Election name %s\n", electionName)
memberCtx, cancelMember1 := context.WithCancel(ctx)
// Use a channel to signal when the first member has observed a new leader
// This ensures proper ordering without relying on sleep timing
firstMemberObservedLeader := make(chan struct{})
var firstMemberObservedOnce sync.Once
config := &etcd.LeaderElectionConfig{
EtcdConfig: etcd.ClientConfig{
Client: cli,
},
Name: electionName,
MemberID: randomElectionNameForTest("my-host"),
MemberID: "my-host",
LeaseDurationSeconds: 1,
Callbacks: etcd.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
@@ -49,11 +42,6 @@ func TestRunElectionWithMemberIDCollision(t *testing.T) {
},
OnNewLeader: func(identity string) {
log.Printf("New leader: %s\n", identity)
// Signal that the first member has observed a leader
// This means the lease has been created
firstMemberObservedOnce.Do(func() {
close(firstMemberObservedLeader)
})
},
OnStoppedLeading: func() {
log.Println("I'm not the leader anymore")
@@ -71,18 +59,8 @@ func TestRunElectionWithMemberIDCollision(t *testing.T) {
go func() {
defer wg.Done()
// Wait for the first member to observe a leader, which means the lease has been created
select {
case <-firstMemberObservedLeader:
// First member has created the lease, now try to create a conflicting one
case <-time.After(5 * time.Second):
t.Error("timeout waiting for first member to observe leader")
return
}
// Use a cancellable context to prevent hanging if this goroutine unexpectedly succeeds
member2Ctx, cancelMember2 := context.WithTimeout(ctx, 5*time.Second)
defer cancelMember2()
g.Expect(etcd.RunElection(member2Ctx, config)).Should(MatchError(ContainSubstring("creating lease")))
time.Sleep(time.Millisecond * 50) // make sure the first one becomes leader
g.Expect(etcd.RunElection(ctx, config)).Should(MatchError(ContainSubstring("creating lease")))
}()
wg.Wait()
@@ -91,8 +69,7 @@ func TestRunElectionWithMemberIDCollision(t *testing.T) {
func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
t.Parallel()
g := NewWithT(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ctx := context.Background()
cli := client(g)
defer cli.Close()
@@ -108,27 +85,24 @@ func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
LeaseDurationSeconds: 1,
}
member1Ctx, cancelMember1 := context.WithCancel(ctx)
member1Ctx, _ := context.WithCancel(ctx)
member2Ctx, cancelMember2 := context.WithCancel(ctx)
config1 := configBase
config1.EtcdConfig.Client = cliMember1
config1.MemberID = randomElectionNameForTest("my-host")
config1.MemberID = "my-host"
uniqueID := rand.Uint64()
config1.MemberUniqueID = &uniqueID
config1.Callbacks = baseCallbacksForName(config1.MemberID)
syncMembers := make(chan (any))
config1.Callbacks.OnStartedLeading = func(_ context.Context) {
log.Println("I'm my-host, the new leader!!!!")
close(syncMembers)
log.Println("Losing the leadership on purpose by stopping renewing the lease")
g.Expect(cliMember1.Lease.Close()).To(Succeed())
log.Println("Member1 leases closed")
cancelMember1()
}
config2 := configBase
config2.MemberID = randomElectionNameForTest("my-other-host")
config2.MemberID = "my-other-host"
config2.Callbacks = baseCallbacksForName(config2.MemberID)
config2.Callbacks.OnStartedLeading = func(_ context.Context) {
log.Println("I'm my-other-host, the new leader!!!!")
@@ -142,18 +116,17 @@ func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
go func() {
defer wg.Done()
g.Expect(etcd.RunElection(member1Ctx, &config1)).To(Succeed())
log.Printf("%s routine done\n", config1.MemberID)
log.Println("Member1 routine done")
}()
go func() {
defer wg.Done()
<-syncMembers
time.Sleep(time.Millisecond * 50) // Make sure member1 becomes leader
g.Expect(etcd.RunElection(member2Ctx, &config2)).To(Succeed())
log.Printf("%s routine done\n", config2.MemberID)
log.Println("Member2 routine done")
}()
wg.Wait()
}
func baseCallbacksForName(name string) etcd.LeaderCallbacks {

View File

@@ -29,8 +29,7 @@ const (
func TestMain(m *testing.M) {
logrus.SetLevel(logrus.DebugLevel)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ctx := context.Background()
expectSuccess(startEtcd(ctx), "starting etcd")
os.Exit(runTestsWithCleanup(m, func() {

View File

@@ -1,14 +1,12 @@
package instance
import (
"context"
"fmt"
"net"
"slices"
"strconv"
"strings"
"sync"
"log/slog"
log "log/slog"
"github.com/vishvananda/netlink"
@@ -19,7 +17,6 @@ import (
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/sysctl"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
@@ -32,27 +29,18 @@ type Instance struct {
Clusters []*cluster.Cluster
// Service uses DHCP
IsDHCPv4 bool
IsDHCPv6 bool
IsDHCP bool
DHCPInterface string
DHCPInterfaceHwaddr string
DHCPInterfaceIP string
DHCPInterfaceIPv4 string
DHCPInterfaceIPv6 string
DHCPHostname string
DHCPv4Client vip.DHCPClient
DHCPv6Client vip.DHCPClient
DHCPClient *vip.DHCPClient
// External Gateway IP the service is forwarded from
UPNPGatewayIPs []string
// Kubernetes service mapping
ServiceSnapshot *v1.Service
dnsAddresses []string
// AddCalled determined that ActionAdd was already performed for the instance
AddCalled bool
}
type Port struct {
@@ -60,14 +48,13 @@ type Port struct {
Type string
}
func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config, intfMgr *networkinterface.Manager, arpMgr *arp.Manager, wg *sync.WaitGroup) (*Instance, error) {
instanceAddresses, instanceHostnames := FetchServiceAddresses(svc)
log.Info("new instance", "namespace", svc.Namespace, "service", svc.Name, "addresses", instanceAddresses, "hostnames", instanceHostnames)
func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Instance, error) {
instanceAddresses := FetchServiceAddresses(svc)
//instanceUID := string(svc.UID)
var newVips []*kubevip.Config
var link netlink.Link
var err error
var dnsAddresses []string
for _, address := range instanceAddresses {
// Detect if we're using a specific interface for services
@@ -133,7 +120,7 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config, i
subnet := ""
var err error
if utils.IsIPv4(address) {
if vip.IsIPv4(address) {
if ipv4AutoSubnet {
subnet, err = autoFindSubnet(link, address)
if err != nil {
@@ -143,7 +130,7 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config, i
if cidrs[0] != "" && cidrs[0] != kubevip.Auto {
subnet = cidrs[0]
} else {
subnet = strconv.Itoa(vip.DefaultMaskIPv4)
subnet = "32"
}
}
} else {
@@ -156,70 +143,19 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config, i
if len(cidrs) > 1 && cidrs[1] != "" && cidrs[1] != kubevip.Auto {
subnet = cidrs[1]
} else {
subnet = strconv.Itoa(vip.DefaultMaskIPv6)
subnet = "128"
}
}
}
// Generate new Virtual IP configuration
newVips = append(newVips, &kubevip.Config{
VIP: address,
Interface: svcInterface,
SingleNode: true,
EnableARP: config.EnableARP,
EnableBGP: config.EnableBGP,
VIPSubnet: subnet,
EnableRoutingTable: config.EnableRoutingTable,
RoutingTableID: config.RoutingTableID,
RoutingTableType: config.RoutingTableType,
RoutingProtocol: config.RoutingProtocol,
ArpBroadcastRate: config.ArpBroadcastRate,
EnableServiceSecurity: config.EnableServiceSecurity,
DNSMode: config.DNSMode,
DHCPMode: config.DHCPMode,
DHCPBackoffAttempts: config.DHCPBackoffAttempts,
DisableServiceUpdates: config.DisableServiceUpdates,
EnableServicesElection: config.EnableServicesElection,
PreserveVIPOnLeadershipLoss: config.PreserveVIPOnLeadershipLoss,
KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
EnableLeaderElection: config.EnableLeaderElection,
},
})
}
for _, hostname := range instanceHostnames {
log.Info("hostname", "addr", hostname)
// Detect if we're using a specific interface for services
var svcInterface string
svcInterface = svc.Annotations[kubevip.ServiceInterface] // If the service has a specific interface defined, then use it
// If it is still blank then use the
if svcInterface == "" {
switch config.ServicesInterface {
case "":
svcInterface = config.Interface
default:
svcInterface = config.ServicesInterface
}
}
if link == nil {
if link, err = netlink.LinkByName(svcInterface); err != nil {
return nil, fmt.Errorf("failed to get interface %s: %w", svcInterface, err)
}
if link == nil {
return nil, fmt.Errorf("failed to get interface %s", svcInterface)
}
}
// Generate new Virtual IP configuration
newVips = append(newVips, &kubevip.Config{
VIP: hostname,
VIP: address,
Interface: svcInterface,
SingleNode: true,
EnableARP: config.EnableARP,
EnableBGP: config.EnableBGP,
VIPSubnet: config.VIPSubnet,
VIPSubnet: subnet,
EnableRoutingTable: config.EnableRoutingTable,
RoutingTableID: config.RoutingTableID,
RoutingTableType: config.RoutingTableType,
@@ -227,7 +163,6 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config, i
ArpBroadcastRate: config.ArpBroadcastRate,
EnableServiceSecurity: config.EnableServiceSecurity,
DNSMode: config.DNSMode,
DHCPMode: config.DHCPMode,
DisableServiceUpdates: config.DisableServiceUpdates,
EnableServicesElection: config.EnableServicesElection,
KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
@@ -238,27 +173,20 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config, i
// Create new service
instance := &Instance{
//UID: instanceUID,
//VIPs: instanceAddresses,
ServiceSnapshot: svc,
dnsAddresses: dnsAddresses,
}
// for _, port := range svc.Spec.Ports {
// instance.ExternalPorts = append(instance.ExternalPorts, Port{
// Port: uint16(port.Port), //nolint
// Type: string(port.Protocol),
// })
// }
if svc.Annotations != nil {
instance.DHCPInterfaceHwaddr = svc.Annotations[kubevip.HwAddrKey]
requestedIP := svc.Annotations[kubevip.RequestedIP]
if requestedIP != "" {
requestedIPs := strings.Split(requestedIP, ",")
if len(requestedIPs) > 2 {
return nil, fmt.Errorf("annotation %q cannot request more than one IPv4 and one Ipv6 address", kubevip.RequestedIP)
}
for _, ip := range requestedIPs {
netip := net.ParseIP(ip)
if netip.To4() != nil {
instance.DHCPInterfaceIPv4 = ip
} else {
instance.DHCPInterfaceIPv6 = ip
}
}
}
instance.DHCPInterfaceIP = svc.Annotations[kubevip.RequestedIP]
instance.DHCPHostname = svc.Annotations[kubevip.LoadbalancerHostname]
}
@@ -282,82 +210,29 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config, i
// Create Add configuration to the new service
instance.VIPConfigs = newVips
// If this was purposely created with the address '0.0.0.0', or '::'
// If this was purposely created with the address 0.0.0.0,
// we will create a macvlan on the main interface and a DHCP client
if len(instanceAddresses) > 2 && (slices.Contains(instanceAddresses, "0.0.0.0") || slices.Contains(instanceAddresses, "::")) {
return nil, fmt.Errorf("DHCP cannot be used if more than 2 addresses (one IPv4 and one IPv6) were specified")
}
for i := range instance.VIPConfigs {
if instance.VIPConfigs[i].VIP == "0.0.0.0" {
err := instance.startDHCP(ctx, i, config.DHCPBackoffAttempts, wg)
if err != nil {
return nil, err
}
select {
case err := <-instance.DHCPv4Client.ErrorChannel():
return nil, fmt.Errorf("error starting DHCPv4 for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
case ip := <-instance.DHCPv4Client.IPChannel():
instance.VIPConfigs[i].Interface = instance.DHCPInterface
instance.VIPConfigs[i].VIP = ip
instance.DHCPInterfaceIPv4 = ip
}
}
if instance.VIPConfigs[i].VIP == "::" {
err := instance.startDHCP(ctx, i, config.DHCPBackoffAttempts, wg)
if err != nil {
return nil, err
}
select {
case err := <-instance.DHCPv6Client.ErrorChannel():
return nil, fmt.Errorf("error starting DHCPv6 for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
case ip := <-instance.DHCPv6Client.IPChannel():
instance.VIPConfigs[i].Interface = instance.DHCPInterface
instance.VIPConfigs[i].VIP = ip
instance.DHCPInterfaceIPv6 = ip
}
}
ddnsAnnotation, exists := svc.Annotations[kubevip.ServiceDDNS]
if exists {
instance.VIPConfigs[i].DDNS, err = strconv.ParseBool(ddnsAnnotation)
if err != nil {
log.Error("Failed to add service", "err", err)
return nil, err
}
}
if len(svc.Spec.IPFamilies) > 0 {
if len(svc.Spec.IPFamilies) > 1 {
instance.VIPConfigs[i].DHCPMode = utils.DualFamily
instance.VIPConfigs[i].DNSMode = utils.DualFamily
switch *svc.Spec.IPFamilyPolicy {
case v1.IPFamilyPolicyRequireDualStack:
instance.VIPConfigs[i].IsDualStack = true
instance.VIPConfigs[i].RequireDualStack = true
case v1.IPFamilyPolicyPreferDualStack:
instance.VIPConfigs[i].IsDualStack = true
instance.VIPConfigs[i].RequireDualStack = false
default:
instance.VIPConfigs[i].IsDualStack = false
instance.VIPConfigs[i].RequireDualStack = false
}
} else {
if strings.EqualFold(string(svc.Spec.IPFamilies[0]), utils.IPv4Family) {
instance.VIPConfigs[i].DHCPMode = strings.ToLower(utils.IPv4Family)
instance.VIPConfigs[i].DNSMode = strings.ToLower(utils.IPv4Family)
} else {
instance.VIPConfigs[i].DHCPMode = strings.ToLower(utils.IPv6Family)
instance.VIPConfigs[i].DNSMode = strings.ToLower(utils.IPv6Family)
}
}
}
c, err := cluster.InitCluster(instance.VIPConfigs[i], false, intfMgr, arpMgr)
// TODO: Consider how best to handle DHCP with multiple addresses
if len(instanceAddresses) == 1 && instanceAddresses[0] == "0.0.0.0" {
err := instance.startDHCP()
if err != nil {
log.Error("failed to add service", "err", err)
return nil, err
}
select {
case err := <-instance.DHCPClient.ErrorChannel():
return nil, fmt.Errorf("error starting DHCP for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
case ip := <-instance.DHCPClient.IPChannel():
instance.VIPConfigs[0].Interface = instance.DHCPInterface
instance.VIPConfigs[0].VIP = ip
instance.DHCPInterfaceIP = ip
}
}
for _, vipConfig := range instance.VIPConfigs {
c, err := cluster.InitCluster(vipConfig, false, intfMgr, arpMgr)
if err != nil {
log.Error("Failed to add Service %s/%s", svc.Namespace, svc.Name)
return nil, err
}
@@ -366,7 +241,7 @@ func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config, i
}
instance.Clusters = append(instance.Clusters, c)
log.Info("(svcs) adding VIP", "ip", instance.VIPConfigs[i].VIP, "interface", instance.VIPConfigs[i].Interface, "namespace", svc.Namespace, "name", svc.Name)
log.Info("(svcs) adding VIP", "ip", vipConfig.VIP, "interface", vipConfig.Interface, "namespace", svc.Namespace, "name", svc.Name)
}
return instance, nil
@@ -429,11 +304,11 @@ func getAutoInterfaceName(link netlink.Link, defaultInterface string) string {
return link.Attrs().Name
}
func (i *Instance) startDHCP(ctx context.Context, index int, backoffAttempts uint, wg *sync.WaitGroup) error {
if len(i.VIPConfigs) > 2 {
return fmt.Errorf("DHCP can be used with 2 VIP config maximally, got: %v", len(i.VIPConfigs))
func (i *Instance) startDHCP() error {
if len(i.VIPConfigs) != 1 {
return fmt.Errorf("DHCP requires exactly 1 VIP config, got: %v", len(i.VIPConfigs))
}
parent, err := netlink.LinkByName(i.VIPConfigs[index].Interface)
parent, err := netlink.LinkByName(i.VIPConfigs[0].Interface)
if err != nil {
return fmt.Errorf("error finding VIP Interface, for building DHCP Link : %v", err)
}
@@ -484,161 +359,124 @@ func (i *Instance) startDHCP(ctx context.Context, index int, backoffAttempts uin
log.Info("Using existing macvlan interface for DHCP", "interface", interfaceName)
}
var initRebootFlag bool
ip := net.ParseIP(i.VIPConfigs[index].VIP)
// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
rpfilterSetting := "0"
var client vip.DHCPClient
if ip.To4() != nil {
// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
rpfilterSetting := "0"
// Check if we need to set an override rp_filter value for the interface
if i.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
// Check the rp_filter value
rpFilter, err := strconv.Atoi(i.ServiceSnapshot.Annotations[kubevip.RPFilter])
if err != nil {
log.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
// Check if we need to set an override rp_filter value for the interface
if i.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
// Check the rp_filter value
rpFilter, err := strconv.Atoi(i.ServiceSnapshot.Annotations[kubevip.RPFilter])
if err != nil {
slog.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
} else {
if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
rpfilterSetting = i.ServiceSnapshot.Annotations[kubevip.RPFilter]
} else {
if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
rpfilterSetting = i.ServiceSnapshot.Annotations[kubevip.RPFilter]
} else {
log.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
}
slog.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
}
}
err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
if err != nil {
log.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
}
if i.DHCPInterfaceIPv4 != "" {
initRebootFlag = true
}
client = vip.NewDHCPv4Client(iface, initRebootFlag, i.DHCPInterfaceIPv4, backoffAttempts)
// Add the client so that we can call it to stop function
i.DHCPv4Client = client
// Set that DHCPv4 is enabled
i.IsDHCPv4 = true
} else {
if i.DHCPInterfaceIPv6 != "" {
initRebootFlag = true
}
client, err = vip.NewDHCPv6Client(iface, parent, initRebootFlag, i.DHCPInterfaceIPv6, backoffAttempts)
if err != nil {
return fmt.Errorf("unable to create client: %w", err)
}
// Add the client so that we can call it to stop function
i.DHCPv6Client = client
// Set that DHCPv6 is enabled
i.IsDHCPv6 = true
}
err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
if err != nil {
slog.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
}
var initRebootFlag bool
if i.DHCPInterfaceIP != "" {
initRebootFlag = true
}
client := vip.NewDHCPClient(iface, initRebootFlag, i.DHCPInterfaceIP)
// Add hostname to dhcp client if annotated
if i.DHCPHostname != "" {
log.Info("Hostname specified for dhcp lease", "interface", interfaceName, "hostname", i.DHCPHostname)
client.WithHostName(i.DHCPHostname)
}
wg.Go(func() {
if err := client.Start(ctx); err != nil {
log.Error("[instance] DHCP client error: %w")
}
})
go client.Start()
// Set that DHCP is enabled
i.IsDHCP = true
// Set the name of the interface so that it can be removed on Service deletion
i.DHCPInterface = interfaceName
i.DHCPInterfaceHwaddr = iface.HardwareAddr.String()
// Add the client so that we can call it to stop function
i.DHCPClient = client
return nil
}
// FetchLoadBalancerIngressAddresses tries to get the addresses from status.loadBalancerIP
func FetchLoadBalancerIngress(s *v1.Service) ([]string, []string) {
// FetchIngressAddresses tries to get the addresses from status.loadBalancerIP
func FetchLoadBalancerIngressAddresses(s *v1.Service) []string {
// If the service has no status, return empty
lbStatusAddresses := []string{}
lbStatusHostnames := []string{}
if len(s.Status.LoadBalancer.Ingress) == 0 {
return lbStatusAddresses, lbStatusHostnames
return lbStatusAddresses
}
for _, ingress := range s.Status.LoadBalancer.Ingress {
if ingress.IP != "" {
lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
}
if ingress.Hostname != "" {
lbStatusHostnames = append(lbStatusHostnames, ingress.Hostname)
}
// TODO: Handle hostname if needed
}
return lbStatusAddresses, lbStatusHostnames
return lbStatusAddresses
}
// FetchServiceAddresses tries to get the addresses from annotations
// kube-vip.io/loadbalancerIPs, then from spec.loadbalancerIP
func FetchServiceAddresses(s *v1.Service) ([]string, []string) {
func FetchServiceAddresses(s *v1.Service) []string {
annotationAvailable := false
if s.Annotations != nil {
if v, annotationAvailable := s.Annotations[kubevip.LoadbalancerIPAnnotation]; annotationAvailable {
ips := strings.Split(v, ",")
var trimmedIPs []string
var trimmedHostnames []string
for _, a := range ips {
a = strings.TrimSpace(a)
ip := net.ParseIP(a)
if ip == nil {
// this is probably a DNS name
trimmedHostnames = append(trimmedHostnames, a)
} else {
trimmedIPs = append(trimmedIPs, ip.String())
}
for _, ip := range ips {
trimmedIPs = append(trimmedIPs, strings.TrimSpace(ip))
}
return trimmedIPs, trimmedHostnames
return trimmedIPs
}
}
lbStatusAddresses := []string{}
lbStatusHostnames := []string{}
if !annotationAvailable {
lbStatusAddresses, lbStatusHostnames = FetchLoadBalancerIngress(s)
}
// Spec.LoadBalancerIP legacy handling
// if the loadBalancerIP is different from Status.LoadBalancer.Ingress IPs
// return the legacy LB as spec wins over status.
if lbIP := net.ParseIP(s.Spec.LoadBalancerIP); lbIP != nil && len(lbStatusAddresses) > 0 {
isLbIPv4 := utils.IsIPv4(s.Spec.LoadBalancerIP)
for _, a := range lbStatusAddresses {
if lbStatusIP := net.ParseIP(a); lbStatusIP != nil && utils.IsIPv4(a) == isLbIPv4 && !lbIP.Equal(lbStatusIP) {
return []string{s.Spec.LoadBalancerIP}, []string{}
if len(s.Status.LoadBalancer.Ingress) > 0 {
for _, ingress := range s.Status.LoadBalancer.Ingress {
lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
}
}
}
if len(lbStatusAddresses) > 0 || len(lbStatusHostnames) > 0 {
return lbStatusAddresses, lbStatusHostnames
lbIP := net.ParseIP(s.Spec.LoadBalancerIP)
isLbIPv4 := vip.IsIPv4(s.Spec.LoadBalancerIP)
if len(lbStatusAddresses) > 0 {
for _, a := range lbStatusAddresses {
if lbStatusIP := net.ParseIP(a); lbStatusIP != nil && lbIP != nil && vip.IsIPv4(a) == isLbIPv4 && !lbIP.Equal(lbStatusIP) {
return []string{s.Spec.LoadBalancerIP}
}
}
return lbStatusAddresses
}
if s.Spec.LoadBalancerIP != "" {
return []string{s.Spec.LoadBalancerIP}, []string{}
return []string{s.Spec.LoadBalancerIP}
}
return []string{}, []string{}
return []string{}
}
func FindServiceInstance(svc *v1.Service, instances []*Instance) *Instance {
log.Debug("finding service", "namespace", svc.Namespace, "name", svc.Name, "UID", svc.UID)
log.Debug("finding service", "UID", svc.UID)
for i := range instances {
log.Debug("saved service", "instance", i, "UID", instances[i].ServiceSnapshot.UID)
if instances[i].ServiceSnapshot.UID == svc.UID {
return instances[i]
}
}
log.Debug("instance not found", "namespace", svc.Namespace, "name", svc.Name, "UID", svc.UID)
return nil
}

View File

@@ -40,31 +40,10 @@ const (
// Flush the conntrack rules (remove existing sessions) once Egress is configured
FlushContrack = "kube-vip.io/flush-conntrack"
// Configure LoadBalancer IPs instead of relying on a controller
LoadbalancerIPAnnotation = "kube-vip.io/loadbalancerIPs"
// Ignore the LoadBalancer Service
LoadbalancerIgnore = "kube-vip.io/ignore"
// Used to configure DHCP with a Hostname
LoadbalancerHostname = "kube-vip.io/loadbalancerHostname"
// Define an interface name to bind the address of the LoadBalancer to
ServiceInterface = "kube-vip.io/serviceInterface"
ServiceSecurityIgnore = "kube-vip.io/ignore-service-security"
// Enable UPNP on a Service
UpnpEnabled = "kube-vip.io/forwardUPNP"
// Set the UPNP lease duration for a specific service using duration format (e.g., "30s", "1h")
UpnpLeaseDuration = "kube-vip.io/upnp-lease-duration"
LoadbalancerHostname = "kube-vip.io/loadbalancerHostname"
ServiceInterface = "kube-vip.io/serviceInterface"
UpnpEnabled = "kube-vip.io/forwardUPNP"
RPFilter = "kube-vip.io/rp_filter" // Set the return path filter for a specific service interface
// Name of the service lease object
ServiceLease = "kube-vip.io/leaseName"
// Enable DDNS for the service
ServiceDDNS = "kube-vip.io/ddns"
)

View File

@@ -6,13 +6,11 @@ import (
"math"
"math/bits"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/detector"
"github.com/kube-vip/kube-vip/pkg/utils"
"sigs.k8s.io/yaml"
)
// ParseEnvironment - will popultate the configuration from environment variables
@@ -26,7 +24,7 @@ func ParseEnvironment(c *Config) error {
if env != "" {
logLevel, err := strconv.ParseInt(env, 10, 32)
if err != nil {
return fmt.Errorf("unable to parse environment variable [vip_loglevel], should be int: %w", err)
panic("Unable to parse environment variable [vip_loglevel], should be int")
}
c.Logging = int32(logLevel)
}
@@ -292,18 +290,6 @@ func ParseEnvironment(c *Config) error {
c.ArpBroadcastRate = 3000
}
// Determine if VIP should be preserved on leadership loss
// true: VIP addresses remain on interface, only ARP/NDP broadcasting stops
// false (default): VIP addresses are deleted on leadership loss (legacy behavior)
env = os.Getenv(vipPreserveOnLeadershipLoss)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.PreserveVIPOnLeadershipLoss = b
}
// Wireguard Mode
env = os.Getenv(vipWireguard)
if env != "" {
@@ -377,30 +363,6 @@ func ParseEnvironment(c *Config) error {
c.DNSMode = env
}
// DHCP mode
env = os.Getenv(dhcpMode)
if env != "" {
c.DHCPMode = env
} else {
if c.DNSMode != "first" {
c.DHCPMode = c.DNSMode
} else {
c.DHCPMode = strings.ToLower(utils.IPv4Family)
}
}
// DHCP backoff attempts
env = os.Getenv(dhcpBackoffAttempts)
if env != "" {
tmp, err := strconv.ParseInt(env, 10, 32)
if err != nil {
return err
}
if tmp >= 0 {
c.DHCPBackoffAttempts = uint(tmp)
}
}
// Disable updates for services (status.LoadBalancer.Ingress will not be updated)
env = os.Getenv(disableServiceUpdates)
if env != "" {
@@ -460,7 +422,7 @@ func ParseEnvironment(c *Config) error {
// Peer AS
env = os.Getenv(bgpPeers)
if env != "" {
peers, err := ParseBGPPeerConfig(env)
peers, err := bgp.ParseBGPPeerConfig(env)
if err != nil {
return err
}
@@ -705,295 +667,5 @@ func ParseEnvironment(c *Config) error {
c.EgressClean = b
}
// check for configuration file path
env = os.Getenv(configFile)
if env != "" {
c.ConfigFile = env
}
return nil
}
// LoadConfigFromFile loads configuration from a JSON or YAML file
func LoadConfigFromFile(configFilePath string) (*Config, error) {
if configFilePath == "" {
return nil, fmt.Errorf("config file path is empty")
}
// Check if file exists
if _, err := os.Stat(configFilePath); os.IsNotExist(err) {
return nil, fmt.Errorf("config file does not exist: %s", configFilePath)
}
// Read file content
data, err := os.ReadFile(configFilePath)
if err != nil {
return nil, fmt.Errorf("failed to read config file %s: %v", configFilePath, err)
}
var config Config
ext := strings.ToLower(filepath.Ext(configFilePath))
switch ext {
case ".json":
err = json.Unmarshal(data, &config)
if err != nil {
return nil, fmt.Errorf("failed to parse JSON config file %s: %v", configFilePath, err)
}
case ".yaml", ".yml":
err = yaml.Unmarshal(data, &config)
if err != nil {
return nil, fmt.Errorf("failed to parse YAML config file %s: %v", configFilePath, err)
}
default:
return nil, fmt.Errorf("unsupported config file format %s. Supported formats: .json, .yaml, .yml", ext)
}
return &config, nil
}
// MergeConfigFromFile merges configuration loaded from file with existing config
// Priority: command line flags > environment variables > config file
func MergeConfigFromFile(c *Config, configFilePath string) error {
if configFilePath == "" {
return nil // No config file specified, nothing to merge
}
fileConfig, err := LoadConfigFromFile(configFilePath)
if err != nil {
return err
}
// Merge file config with existing config
// Only set values from file if they haven't been set by flags or env vars
mergeConfigValues(c, fileConfig)
return nil
}
// mergeConfigValues merges values from fileConfig into baseConfig
// Only overwrites zero values in baseConfig
func mergeConfigValues(baseConfig, fileConfig *Config) {
// Basic configuration
if baseConfig.Logging == 0 && fileConfig.Logging != 0 {
baseConfig.Logging = fileConfig.Logging
}
// Network configuration
if baseConfig.Interface == "" && fileConfig.Interface != "" {
baseConfig.Interface = fileConfig.Interface
}
if baseConfig.ServicesInterface == "" && fileConfig.ServicesInterface != "" {
baseConfig.ServicesInterface = fileConfig.ServicesInterface
}
if baseConfig.VIP == "" && fileConfig.VIP != "" {
baseConfig.VIP = fileConfig.VIP
}
if baseConfig.VIPSubnet == "" && fileConfig.VIPSubnet != "" {
baseConfig.VIPSubnet = fileConfig.VIPSubnet
}
if baseConfig.Address == "" && fileConfig.Address != "" {
baseConfig.Address = fileConfig.Address
}
if baseConfig.Port == 0 && fileConfig.Port != 0 {
baseConfig.Port = fileConfig.Port
}
if baseConfig.NodeName == "" && fileConfig.NodeName != "" {
baseConfig.NodeName = fileConfig.NodeName
}
// Boolean flags - only merge if not explicitly set
if !baseConfig.EnableARP && fileConfig.EnableARP {
baseConfig.EnableARP = fileConfig.EnableARP
}
if !baseConfig.EnableBGP && fileConfig.EnableBGP {
baseConfig.EnableBGP = fileConfig.EnableBGP
}
if !baseConfig.EnableWireguard && fileConfig.EnableWireguard {
baseConfig.EnableWireguard = fileConfig.EnableWireguard
}
if !baseConfig.EnableRoutingTable && fileConfig.EnableRoutingTable {
baseConfig.EnableRoutingTable = fileConfig.EnableRoutingTable
}
if !baseConfig.EnableControlPlane && fileConfig.EnableControlPlane {
baseConfig.EnableControlPlane = fileConfig.EnableControlPlane
}
if !baseConfig.DetectControlPlane && fileConfig.DetectControlPlane {
baseConfig.DetectControlPlane = fileConfig.DetectControlPlane
}
if !baseConfig.EnableServices && fileConfig.EnableServices {
baseConfig.EnableServices = fileConfig.EnableServices
}
if !baseConfig.EnableServicesElection && fileConfig.EnableServicesElection {
baseConfig.EnableServicesElection = fileConfig.EnableServicesElection
}
if !baseConfig.EnableNodeLabeling && fileConfig.EnableNodeLabeling {
baseConfig.EnableNodeLabeling = fileConfig.EnableNodeLabeling
}
if !baseConfig.EnableLoadBalancer && fileConfig.EnableLoadBalancer {
baseConfig.EnableLoadBalancer = fileConfig.EnableLoadBalancer
}
if !baseConfig.DDNS && fileConfig.DDNS {
baseConfig.DDNS = fileConfig.DDNS
}
if !baseConfig.SingleNode && fileConfig.SingleNode {
baseConfig.SingleNode = fileConfig.SingleNode
}
if !baseConfig.StartAsLeader && fileConfig.StartAsLeader {
baseConfig.StartAsLeader = fileConfig.StartAsLeader
}
if !baseConfig.PreserveVIPOnLeadershipLoss && fileConfig.PreserveVIPOnLeadershipLoss {
baseConfig.PreserveVIPOnLeadershipLoss = fileConfig.PreserveVIPOnLeadershipLoss
}
// Service configuration
if baseConfig.Namespace == "" && fileConfig.Namespace != "" {
baseConfig.Namespace = fileConfig.Namespace
}
if baseConfig.ServiceNamespace == "" && fileConfig.ServiceNamespace != "" {
baseConfig.ServiceNamespace = fileConfig.ServiceNamespace
}
if baseConfig.ServicesLeaseName == "" && fileConfig.ServicesLeaseName != "" {
baseConfig.ServicesLeaseName = fileConfig.ServicesLeaseName
}
// LoadBalancer configuration
if baseConfig.LoadBalancerPort == 0 && fileConfig.LoadBalancerPort != 0 {
baseConfig.LoadBalancerPort = fileConfig.LoadBalancerPort
}
if baseConfig.LoadBalancerForwardingMethod == "" && fileConfig.LoadBalancerForwardingMethod != "" {
baseConfig.LoadBalancerForwardingMethod = fileConfig.LoadBalancerForwardingMethod
}
if baseConfig.LoadBalancerClassName == "" && fileConfig.LoadBalancerClassName != "" {
baseConfig.LoadBalancerClassName = fileConfig.LoadBalancerClassName
}
// Routing Table configuration
if baseConfig.RoutingTableID == 0 && fileConfig.RoutingTableID != 0 {
baseConfig.RoutingTableID = fileConfig.RoutingTableID
}
if baseConfig.RoutingTableType == 0 && fileConfig.RoutingTableType != 0 {
baseConfig.RoutingTableType = fileConfig.RoutingTableType
}
if baseConfig.RoutingProtocol == 0 && fileConfig.RoutingProtocol != 0 {
baseConfig.RoutingProtocol = fileConfig.RoutingProtocol
}
// BGP configuration
mergeBGPConfig(&baseConfig.BGPConfig, &fileConfig.BGPConfig)
// Kubernetes configuration
if baseConfig.K8sConfigFile == "" && fileConfig.K8sConfigFile != "" {
baseConfig.K8sConfigFile = fileConfig.K8sConfigFile
}
// Leader Election configuration
mergeLeaderElectionConfig(&baseConfig.KubernetesLeaderElection, &fileConfig.KubernetesLeaderElection)
// Prometheus configuration
if baseConfig.PrometheusHTTPServer == "" && fileConfig.PrometheusHTTPServer != "" {
baseConfig.PrometheusHTTPServer = fileConfig.PrometheusHTTPServer
}
// DNS configuration
if baseConfig.DNSMode == "" && fileConfig.DNSMode != "" {
baseConfig.DNSMode = fileConfig.DNSMode
}
// DHCP configuration - mode
if baseConfig.DHCPMode == "" && fileConfig.DHCPMode != "" {
baseConfig.DHCPMode = fileConfig.DHCPMode
}
// DHCP configuration - backoff attempts
if baseConfig.DHCPBackoffAttempts == DefaultDHCPBackoffAttempts && fileConfig.DHCPBackoffAttempts != DefaultDHCPBackoffAttempts {
baseConfig.DHCPBackoffAttempts = fileConfig.DHCPBackoffAttempts
}
// Health check configuration
if baseConfig.HealthCheckPort == 0 && fileConfig.HealthCheckPort != 0 {
baseConfig.HealthCheckPort = fileConfig.HealthCheckPort
}
// Egress configuration
if baseConfig.EgressPodCidr == "" && fileConfig.EgressPodCidr != "" {
baseConfig.EgressPodCidr = fileConfig.EgressPodCidr
}
if baseConfig.EgressServiceCidr == "" && fileConfig.EgressServiceCidr != "" {
baseConfig.EgressServiceCidr = fileConfig.EgressServiceCidr
}
// Mirror configuration
if baseConfig.MirrorDestInterface == "" && fileConfig.MirrorDestInterface != "" {
baseConfig.MirrorDestInterface = fileConfig.MirrorDestInterface
}
// Iptables configuration
if baseConfig.IptablesBackend == "" && fileConfig.IptablesBackend != "" {
baseConfig.IptablesBackend = fileConfig.IptablesBackend
}
// Backend health check interval
if baseConfig.BackendHealthCheckInterval == 0 && fileConfig.BackendHealthCheckInterval != 0 {
baseConfig.BackendHealthCheckInterval = fileConfig.BackendHealthCheckInterval
}
// ARP broadcast rate
if baseConfig.ArpBroadcastRate == 0 && fileConfig.ArpBroadcastRate != 0 {
baseConfig.ArpBroadcastRate = fileConfig.ArpBroadcastRate
}
// Annotations
if baseConfig.Annotations == "" && fileConfig.Annotations != "" {
baseConfig.Annotations = fileConfig.Annotations
}
// Load balancers slice
if len(baseConfig.LoadBalancers) == 0 && len(fileConfig.LoadBalancers) > 0 {
baseConfig.LoadBalancers = fileConfig.LoadBalancers
}
}
// mergeBGPConfig merges BGP configuration
func mergeBGPConfig(base, file *BGPConfig) {
if base.RouterID == "" && file.RouterID != "" {
base.RouterID = file.RouterID
}
if base.AS == 0 && file.AS != 0 {
base.AS = file.AS
}
if base.SourceIF == "" && file.SourceIF != "" {
base.SourceIF = file.SourceIF
}
if base.SourceIP == "" && file.SourceIP != "" {
base.SourceIP = file.SourceIP
}
if base.HoldTime == 0 && file.HoldTime != 0 {
base.HoldTime = file.HoldTime
}
if base.KeepaliveInterval == 0 && file.KeepaliveInterval != 0 {
base.KeepaliveInterval = file.KeepaliveInterval
}
if len(base.Peers) == 0 && len(file.Peers) > 0 {
base.Peers = file.Peers
}
}
// mergeLeaderElectionConfig merges leader election configuration
func mergeLeaderElectionConfig(base, file *KubernetesLeaderElection) {
if base.LeaseName == "" && file.LeaseName != "" {
base.LeaseName = file.LeaseName
}
if base.LeaseDuration == 0 && file.LeaseDuration != 0 {
base.LeaseDuration = file.LeaseDuration
}
if base.RenewDeadline == 0 && file.RenewDeadline != 0 {
base.RenewDeadline = file.RenewDeadline
}
if base.RetryPeriod == 0 && file.RetryPeriod != 0 {
base.RetryPeriod = file.RetryPeriod
}
if len(base.LeaseAnnotations) == 0 && len(file.LeaseAnnotations) > 0 {
base.LeaseAnnotations = file.LeaseAnnotations
}
}

View File

@@ -9,9 +9,6 @@ const (
// vip_arpRate - defines the rate of gARP broadcasts
vipArpRate = "vip_arpRate"
// vipPreserveOnLeadershipLoss - if true, VIP addresses will remain on interface when leadership is lost
vipPreserveOnLeadershipLoss = "vip_preserve_on_leadership_loss"
// vipLeaderElection - defines if the kubernetes algorithm should be used
vipLeaderElection = "vip_leaderelection"
@@ -212,12 +209,6 @@ const (
// dnsMode defines mode that DNS lookup will be performed with (first, ipv4, ipv6, dual)
dnsMode = "dns_mode"
// dhcpMode defines mode that DHCP lookup will be performed with (ipv4, ipv6, dual)
dhcpMode = "dhcp_mode"
// dhcpBackoffAttempts defines how many times DHCP client will try to obtain an IP address
dhcpBackoffAttempts = "dhcp_backoff_attempts"
// disableServiceUpdates disables service updating
disableServiceUpdates = "disable_service_updates"
@@ -243,7 +234,4 @@ const (
// egressClean enables egress cleaning on kube-vip's start
egressClean = "egress_clean"
// configFile defines the path to a JSON/YAML configuration file
configFile = "config_file"
)

View File

@@ -1,559 +0,0 @@
package kubevip
import (
"os"
"path/filepath"
"testing"
)
func TestLoadConfigFromFile(t *testing.T) {
// Create temporary directory for test files
tmpDir, err := os.MkdirTemp("", "kube-vip-config-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
tests := []struct {
name string
filename string
content string
expectedConfig *Config
wantErr bool
errContains string
}{
{
name: "Valid YAML config",
filename: "config.yaml",
content: `
logging: 2
enableARP: true
enableControlPlane: true
enableServices: true
address: "192.168.1.100"
port: 6443
interface: "eth0"
namespace: "kube-system"
vipSubnet: "192.168.1.0/24"
leaseName: "test-lease"
leaseDuration: 15
renewDeadline: 10
retryPeriod: 2
prometheusHTTPServer: ":2112"
`,
expectedConfig: &Config{
Logging: 2,
EnableARP: true,
EnableControlPlane: true,
EnableServices: true,
Address: "192.168.1.100",
Port: 6443,
Interface: "eth0",
Namespace: "kube-system",
VIPSubnet: "192.168.1.0/24",
PrometheusHTTPServer: ":2112",
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "test-lease",
LeaseDuration: 15,
RenewDeadline: 10,
RetryPeriod: 2,
},
},
wantErr: false,
},
{
name: "Valid JSON config",
filename: "config.json",
content: `{
"logging": 3,
"enableBGP": true,
"enableServices": true,
"address": "10.0.0.100",
"port": 8443,
"interface": "ens192",
"namespace": "kube-system",
"loadBalancers": [
{
"name": "control-plane",
"ports": [
{
"type": "TCP",
"port": 6443
}
],
"bindToVip": true,
"forwardingMethod": "local"
}
]
}`,
expectedConfig: &Config{
Logging: 3,
EnableBGP: true,
EnableServices: true,
Address: "10.0.0.100",
Port: 8443,
Interface: "ens192",
Namespace: "kube-system",
LoadBalancers: []LoadBalancer{
{
Name: "control-plane",
Ports: []Port{
{
Type: "TCP",
Port: 6443,
},
},
BindToVip: true,
ForwardingMethod: "local",
},
},
},
wantErr: false,
},
{
name: "Complex BGP config",
filename: "bgp-config.yaml",
content: `
enableBGP: true
bgpConfig:
routerID: "192.168.1.1"
as: 65000
sourceIF: "eth0"
holdTime: 60
keepaliveInterval: 20
peers:
- address: "192.168.1.2"
as: 65001
port: 179
multiHop: false
- address: "192.168.1.3"
as: 65002
port: 179
multiHop: true
`,
expectedConfig: &Config{
EnableBGP: true,
BGPConfig: BGPConfig{
RouterID: "192.168.1.1",
AS: 65000,
SourceIF: "eth0",
HoldTime: 60,
KeepaliveInterval: 20,
Peers: []BGPPeer{
{
Address: "192.168.1.2",
AS: 65001,
Port: 179,
MultiHop: false,
},
{
Address: "192.168.1.3",
AS: 65002,
Port: 179,
MultiHop: true,
},
},
},
},
wantErr: false,
},
{
name: "Invalid JSON",
filename: "invalid.json",
content: `{"logging": 2, "invalid": }`,
wantErr: true,
errContains: "failed to parse JSON config file",
},
{
name: "Invalid YAML",
filename: "invalid.yaml",
content: "logging: 2\ninvalid: [unclosed",
wantErr: true,
errContains: "failed to parse YAML config file",
},
{
name: "Unsupported format",
filename: "config.txt",
content: "logging=2",
wantErr: true,
errContains: "unsupported config file format",
},
{
name: "Empty path",
filename: "",
content: "",
wantErr: true,
errContains: "config file path is empty",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var filePath string
if tt.filename != "" {
filePath = filepath.Join(tmpDir, tt.filename)
if err := os.WriteFile(filePath, []byte(tt.content), 0600); err != nil {
t.Fatalf("Failed to write test file: %v", err)
}
}
config, err := LoadConfigFromFile(filePath)
if tt.wantErr {
if err == nil {
t.Errorf("LoadConfigFromFile() expected error, got nil")
return
}
if tt.errContains != "" && !containsString(err.Error(), tt.errContains) {
t.Errorf("LoadConfigFromFile() error = %v, expected to contain %v", err, tt.errContains)
}
return
}
if err != nil {
t.Errorf("LoadConfigFromFile() unexpected error = %v", err)
return
}
if config == nil {
t.Errorf("LoadConfigFromFile() returned nil config")
return
}
// Compare key fields
if config.Logging != tt.expectedConfig.Logging {
t.Errorf("Logging = %v, expected %v", config.Logging, tt.expectedConfig.Logging)
}
if config.EnableARP != tt.expectedConfig.EnableARP {
t.Errorf("EnableARP = %v, expected %v", config.EnableARP, tt.expectedConfig.EnableARP)
}
if config.EnableBGP != tt.expectedConfig.EnableBGP {
t.Errorf("EnableBGP = %v, expected %v", config.EnableBGP, tt.expectedConfig.EnableBGP)
}
if config.Address != tt.expectedConfig.Address {
t.Errorf("Address = %v, expected %v", config.Address, tt.expectedConfig.Address)
}
if config.Port != tt.expectedConfig.Port {
t.Errorf("Port = %v, expected %v", config.Port, tt.expectedConfig.Port)
}
if config.Interface != tt.expectedConfig.Interface {
t.Errorf("Interface = %v, expected %v", config.Interface, tt.expectedConfig.Interface)
}
// Test BGP config if present
if tt.expectedConfig.EnableBGP {
if config.BGPConfig.RouterID != tt.expectedConfig.BGPConfig.RouterID {
t.Errorf("BGPConfig.RouterID = %v, expected %v", config.BGPConfig.RouterID, tt.expectedConfig.BGPConfig.RouterID)
}
if config.BGPConfig.AS != tt.expectedConfig.BGPConfig.AS {
t.Errorf("BGPConfig.AS = %v, expected %v", config.BGPConfig.AS, tt.expectedConfig.BGPConfig.AS)
}
if len(config.BGPConfig.Peers) != len(tt.expectedConfig.BGPConfig.Peers) {
t.Errorf("BGPConfig.Peers length = %v, expected %v", len(config.BGPConfig.Peers), len(tt.expectedConfig.BGPConfig.Peers))
}
}
// Test LoadBalancers if present
if len(tt.expectedConfig.LoadBalancers) > 0 {
if len(config.LoadBalancers) != len(tt.expectedConfig.LoadBalancers) {
t.Errorf("LoadBalancers length = %v, expected %v", len(config.LoadBalancers), len(tt.expectedConfig.LoadBalancers))
}
}
})
}
}
func TestMergeConfigFromFile(t *testing.T) {
// Create temporary directory for test files
tmpDir, err := os.MkdirTemp("", "kube-vip-merge-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
// Create test config file
configFile := filepath.Join(tmpDir, "test-config.yaml")
configContent := `
logging: 3
enableARP: true
enableServices: true
address: "192.168.1.200"
port: 6443
interface: "eth1"
namespace: "test-namespace"
leaseName: "file-lease"
leaseDuration: 20
prometheusHTTPServer: ":3000"
`
if err := os.WriteFile(configFile, []byte(configContent), 0600); err != nil {
t.Fatalf("Failed to write test config file: %v", err)
}
tests := []struct {
name string
baseConfig *Config
configFilePath string
expected *Config
wantErr bool
}{
{
name: "Merge with empty base config",
baseConfig: &Config{},
configFilePath: configFile,
expected: &Config{
Logging: 3,
EnableARP: true,
EnableServices: true,
Address: "192.168.1.200",
Port: 6443,
Interface: "eth1",
Namespace: "test-namespace",
PrometheusHTTPServer: ":3000",
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "file-lease",
LeaseDuration: 20,
},
},
wantErr: false,
},
{
name: "Merge respects existing values (priority test)",
baseConfig: &Config{
Logging: 5, // Should not be overridden
Port: 8443, // Should not be overridden
Interface: "eth0", // Should not be overridden
},
configFilePath: configFile,
expected: &Config{
Logging: 5, // From base (higher priority)
EnableARP: true, // From file
EnableServices: true, // From file
Address: "192.168.1.200", // From file
Port: 8443, // From base (higher priority)
Interface: "eth0", // From base (higher priority)
Namespace: "test-namespace", // From file
PrometheusHTTPServer: ":3000", // From file
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "file-lease", // From file
LeaseDuration: 20, // From file
},
},
wantErr: false,
},
{
name: "Empty config file path",
baseConfig: &Config{
Logging: 1,
},
configFilePath: "",
expected: &Config{
Logging: 1, // Unchanged
},
wantErr: false,
},
{
name: "Non-existent config file",
baseConfig: &Config{
Logging: 1,
},
configFilePath: "/non/existent/file.yaml",
expected: &Config{
Logging: 1,
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := MergeConfigFromFile(tt.baseConfig, tt.configFilePath)
if tt.wantErr {
if err == nil {
t.Errorf("MergeConfigFromFile() expected error, got nil")
}
return
}
if err != nil {
t.Errorf("MergeConfigFromFile() unexpected error = %v", err)
return
}
// Compare key fields
if tt.baseConfig.Logging != tt.expected.Logging {
t.Errorf("Logging = %v, expected %v", tt.baseConfig.Logging, tt.expected.Logging)
}
if tt.baseConfig.EnableARP != tt.expected.EnableARP {
t.Errorf("EnableARP = %v, expected %v", tt.baseConfig.EnableARP, tt.expected.EnableARP)
}
if tt.baseConfig.Address != tt.expected.Address {
t.Errorf("Address = %v, expected %v", tt.baseConfig.Address, tt.expected.Address)
}
if tt.baseConfig.Port != tt.expected.Port {
t.Errorf("Port = %v, expected %v", tt.baseConfig.Port, tt.expected.Port)
}
if tt.baseConfig.Interface != tt.expected.Interface {
t.Errorf("Interface = %v, expected %v", tt.baseConfig.Interface, tt.expected.Interface)
}
if tt.baseConfig.Namespace != tt.expected.Namespace {
t.Errorf("Namespace = %v, expected %v", tt.baseConfig.Namespace, tt.expected.Namespace)
}
if tt.baseConfig.PrometheusHTTPServer != tt.expected.PrometheusHTTPServer {
t.Errorf("PrometheusHTTPServer = %v, expected %v", tt.baseConfig.PrometheusHTTPServer, tt.expected.PrometheusHTTPServer)
}
})
}
}
func TestMergeConfigValues(t *testing.T) {
tests := []struct {
name string
baseConfig *Config
fileConfig *Config
expectedBase *Config
}{
{
name: "Merge basic configuration",
baseConfig: &Config{
Logging: 5, // Should not be overridden
Port: 0, // Should be overridden
},
fileConfig: &Config{
Logging: 2,
Port: 6443,
Interface: "eth0",
Address: "192.168.1.100",
},
expectedBase: &Config{
Logging: 5, // From base (non-zero)
Port: 6443, // From file (base was zero)
Interface: "eth0", // From file (base was empty)
Address: "192.168.1.100", // From file (base was empty)
},
},
{
name: "Merge boolean flags",
baseConfig: &Config{
EnableARP: true, // Should not be overridden
},
fileConfig: &Config{
EnableARP: false, // Should not override true
EnableBGP: true, // Should be set
EnableServices: true, // Should be set
EnableWireguard: false, // Should not be set (false doesn't override false)
},
expectedBase: &Config{
EnableARP: true, // From base (true has priority)
EnableBGP: true, // From file
EnableServices: true, // From file
EnableWireguard: false, // Remains false
},
},
{
name: "Merge BGP configuration",
baseConfig: &Config{
BGPConfig: BGPConfig{
RouterID: "1.1.1.1", // Should not be overridden
},
},
fileConfig: &Config{
BGPConfig: BGPConfig{
RouterID: "2.2.2.2", // Should not override
AS: 65000, // Should be set
SourceIF: "eth0", // Should be set
HoldTime: 30, // Should be set
KeepaliveInterval: 10, // Should be set
},
},
expectedBase: &Config{
BGPConfig: BGPConfig{
RouterID: "1.1.1.1", // From base (non-empty)
AS: 65000, // From file (base was zero)
SourceIF: "eth0", // From file (base was empty)
HoldTime: 30, // From file (base was zero)
KeepaliveInterval: 10, // From file (base was zero)
},
},
},
{
name: "Merge leader election configuration",
baseConfig: &Config{
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "base-lease", // Should not be overridden
},
},
fileConfig: &Config{
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "file-lease", // Should not override
LeaseDuration: 15, // Should be set
RenewDeadline: 10, // Should be set
RetryPeriod: 2, // Should be set
},
},
expectedBase: &Config{
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "base-lease", // From base (non-empty)
LeaseDuration: 15, // From file (base was zero)
RenewDeadline: 10, // From file (base was zero)
RetryPeriod: 2, // From file (base was zero)
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mergeConfigValues(tt.baseConfig, tt.fileConfig)
// Compare results
if tt.baseConfig.Logging != tt.expectedBase.Logging {
t.Errorf("Logging = %v, expected %v", tt.baseConfig.Logging, tt.expectedBase.Logging)
}
if tt.baseConfig.Port != tt.expectedBase.Port {
t.Errorf("Port = %v, expected %v", tt.baseConfig.Port, tt.expectedBase.Port)
}
if tt.baseConfig.Interface != tt.expectedBase.Interface {
t.Errorf("Interface = %v, expected %v", tt.baseConfig.Interface, tt.expectedBase.Interface)
}
if tt.baseConfig.EnableARP != tt.expectedBase.EnableARP {
t.Errorf("EnableARP = %v, expected %v", tt.baseConfig.EnableARP, tt.expectedBase.EnableARP)
}
if tt.baseConfig.EnableBGP != tt.expectedBase.EnableBGP {
t.Errorf("EnableBGP = %v, expected %v", tt.baseConfig.EnableBGP, tt.expectedBase.EnableBGP)
}
if tt.baseConfig.BGPConfig.RouterID != tt.expectedBase.BGPConfig.RouterID {
t.Errorf("BGPConfig.RouterID = %v, expected %v", tt.baseConfig.BGPConfig.RouterID, tt.expectedBase.BGPConfig.RouterID)
}
if tt.baseConfig.BGPConfig.AS != tt.expectedBase.BGPConfig.AS {
t.Errorf("BGPConfig.AS = %v, expected %v", tt.baseConfig.BGPConfig.AS, tt.expectedBase.BGPConfig.AS)
}
if tt.baseConfig.KubernetesLeaderElection.LeaseName != tt.expectedBase.KubernetesLeaderElection.LeaseName {
t.Errorf("KubernetesLeaderElection.LeaseName = %v, expected %v", tt.baseConfig.KubernetesLeaderElection.LeaseName, tt.expectedBase.KubernetesLeaderElection.LeaseName)
}
})
}
}
func TestLoadConfigFromFile_FileNotExists(t *testing.T) {
_, err := LoadConfigFromFile("/non/existent/path/config.yaml")
if err == nil {
t.Error("LoadConfigFromFile() expected error for non-existent file, got nil")
}
if !containsString(err.Error(), "config file does not exist") {
t.Errorf("LoadConfigFromFile() error = %v, expected to contain 'config file does not exist'", err)
}
}
// Helper function to check if a string contains a substring
func containsString(str, substr string) bool {
return len(str) >= len(substr) && (str == substr || len(substr) == 0 ||
(len(substr) > 0 && func() bool {
for i := 0; i <= len(str)-len(substr); i++ {
if str[i:i+len(substr)] == substr {
return true
}
}
return false
}()))
}

View File

@@ -5,7 +5,6 @@ import (
"log"
"strconv"
"github.com/google/go-containerregistry/pkg/name"
appv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -14,6 +13,7 @@ import (
applyCoreV1 "k8s.io/client-go/applyconfigurations/core/v1"
applyMetaV1 "k8s.io/client-go/applyconfigurations/meta/v1"
applyRbacV1 "k8s.io/client-go/applyconfigurations/rbac/v1"
"sigs.k8s.io/yaml"
)
@@ -167,11 +167,7 @@ func GenerateRoleBinding(rolebinding bool, saCfg *applyCoreV1.ServiceAccountAppl
}
// generatePodSpec will take a kube-vip config and generate a Pod spec
func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*corev1.Pod, error) {
imageRef, err := name.NewTag(image, name.WeakValidation, name.WithDefaultTag(imageVersion))
if err != nil {
return nil, fmt.Errorf("cannot parse %q: %w", image, err)
}
func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod {
command := "manager"
// Determine where the pods should be living (for multi-tenancy)
@@ -255,28 +251,6 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
newEnvironment = append(newEnvironment, dnsModeSelector...)
}
if c.DHCPMode != "" {
// build environment variables
dhcpModeSelector := []corev1.EnvVar{
{
Name: dhcpMode,
Value: c.DHCPMode,
},
}
newEnvironment = append(newEnvironment, dhcpModeSelector...)
}
if c.DHCPBackoffAttempts != DefaultDHCPBackoffAttempts {
// build environment variables
dhcpBackoff := []corev1.EnvVar{
{
Name: dhcpBackoffAttempts,
Value: strconv.FormatUint(uint64(c.DHCPBackoffAttempts), 10),
},
}
newEnvironment = append(newEnvironment, dhcpBackoff...)
}
// If we're doing the hybrid mode
if c.EnableControlPlane {
cp := []corev1.EnvVar{
@@ -605,16 +579,6 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
}
}
if c.PreserveVIPOnLeadershipLoss {
preserveVIPOnLeadershipLoss := []corev1.EnvVar{
{
Name: vipPreserveOnLeadershipLoss,
Value: strconv.FormatBool(c.PreserveVIPOnLeadershipLoss),
},
}
newEnvironment = append(newEnvironment, preserveVIPOnLeadershipLoss...)
}
newManifest := &corev1.Pod{
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
@@ -628,7 +592,7 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
Containers: []corev1.Container{
{
Name: "kube-vip",
Image: imageRef.Name(),
Image: fmt.Sprintf("ghcr.io/kube-vip/kube-vip:%s", imageVersion),
ImagePullPolicy: corev1.PullIfNotPresent,
SecurityContext: securityContext,
Args: []string{
@@ -669,24 +633,18 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
newManifest.Spec.HostAliases = append(newManifest.Spec.HostAliases, hostAlias)
}
return newManifest, nil
return newManifest
}
// GeneratePodManifestFromConfig will take a kube-vip config and generate a manifest
func GeneratePodManifestFromConfig(c *Config, image, imageVersion string, inCluster bool) (string, error) {
newManifest, err := generatePodSpec(c, image, imageVersion, inCluster)
if err != nil {
return "", err
}
b, err := yaml.Marshal(newManifest)
if err != nil {
return "", fmt.Errorf("failed to marshal manifest: %w", err)
}
return string(b), nil
func GeneratePodManifestFromConfig(c *Config, imageVersion string, inCluster bool) string {
newManifest := generatePodSpec(c, imageVersion, inCluster)
b, _ := yaml.Marshal(newManifest)
return string(b)
}
// GenerateDaemonsetManifestFromConfig will take a kube-vip config and generate a manifest
func GenerateDaemonsetManifestFromConfig(c *Config, image, imageVersion string, inCluster, taint bool) (string, error) {
func GenerateDaemonsetManifestFromConfig(c *Config, imageVersion string, inCluster, taint bool) string {
// Determine where the pod should be deployed
var namespace string
if c.ServiceNamespace != "" {
@@ -695,11 +653,7 @@ func GenerateDaemonsetManifestFromConfig(c *Config, image, imageVersion string,
namespace = metav1.NamespaceSystem
}
pod, err := generatePodSpec(c, image, imageVersion, inCluster)
if err != nil {
return "", err
}
podSpec := generatePodSpec(c, imageVersion, inCluster).Spec
newManifest := &appv1.DaemonSet{
TypeMeta: metav1.TypeMeta{
Kind: "DaemonSet",
@@ -726,7 +680,7 @@ func GenerateDaemonsetManifestFromConfig(c *Config, image, imageVersion string,
"app.kubernetes.io/version": imageVersion,
},
},
Spec: pod.Spec,
Spec: podSpec,
},
},
}
@@ -776,5 +730,5 @@ func GenerateDaemonsetManifestFromConfig(c *Config, image, imageVersion string,
delete(m, "status")
b, _ = yaml.Marshal(m)
return string(b), nil
return string(b)
}

View File

@@ -1,9 +1,6 @@
package kubevip
import (
"os"
"testing"
)
import "testing"
func TestParseEnvironment(t *testing.T) {
@@ -12,8 +9,8 @@ func TestParseEnvironment(t *testing.T) {
c *Config
wantErr bool
}{
{"nil config", nil, false},
{"basic config", &Config{Interface: "eth0", ServicesInterface: "eth1"}, false},
{"", nil, false},
{"", &Config{Interface: "eth0", ServicesInterface: "eth1"}, false},
}
for _, tt := range tests {
t.Logf("%v", tt.c)
@@ -24,53 +21,3 @@ func TestParseEnvironment(t *testing.T) {
})
}
}
func TestParseEnvironmentConfigFile(t *testing.T) {
// Save original environment
originalConfigFile := os.Getenv("config_file")
defer func() {
if originalConfigFile != "" {
os.Setenv("config_file", originalConfigFile)
} else {
os.Unsetenv("config_file")
}
}()
tests := []struct {
name string
envValue string
expectedConfig string
}{
{
name: "config_file environment variable set",
envValue: "/etc/kube-vip/config.yaml",
expectedConfig: "/etc/kube-vip/config.yaml",
},
{
name: "config_file environment variable empty",
envValue: "",
expectedConfig: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Set environment variable
if tt.envValue != "" {
os.Setenv("config_file", tt.envValue)
} else {
os.Unsetenv("config_file")
}
config := &Config{}
err := ParseEnvironment(config)
if err != nil {
t.Errorf("ParseEnvironment() unexpected error = %v", err)
}
if config.ConfigFile != tt.expectedConfig {
t.Errorf("ConfigFile = %v, expected %v", config.ConfigFile, tt.expectedConfig)
}
})
}
}

View File

@@ -1,5 +1,9 @@
package kubevip
import (
"github.com/kube-vip/kube-vip/pkg/bgp"
)
// Config defines all of the settings for the Kube-Vip Pod
type Config struct {
// Logging, settings
@@ -50,10 +54,6 @@ type Config struct {
// ArpBroadcastRate, defines how often kube-vip will update the network about updates to the network
ArpBroadcastRate int64 `yaml:"arpBroadcastRate"`
// PreserveVIPOnLeadershipLoss, if true, VIP addresses will remain on interface when leadership is lost (only ARP/NDP broadcasting stops)
// If false, VIP addresses are deleted on leadership loss (legacy behavior)
PreserveVIPOnLeadershipLoss bool `yaml:"preserveVipOnLeadershipLoss"`
// Annotations will define if we're going to wait and lookup configuration from Kubernetes node annotations
Annotations string
@@ -128,8 +128,8 @@ type Config struct {
CleanRoutingTable bool `yaml:"cleanRoutingTable"`
// BGP Configuration
BGPConfig BGPConfig
BGPPeerConfig BGPPeer
BGPConfig bgp.Config
BGPPeerConfig bgp.Peer
BGPPeers []string
// LoadBalancers are the various services we can load balance over
@@ -158,16 +158,6 @@ type Config struct {
// DNSMode, this will set the mode DSN lookup will be performed (first, ipv4, ipv6, dual)
DNSMode string `yaml:"dnsDualStackMode"`
// IsDualStack reports if service is DualStack.
IsDualStack bool
// RequireDualStack defines if DualStack is required for the service. Based on service's Spec.ipFamilyPolicy field.
RequireDualStack bool
// DNSMode, this will set the mode DHCP lookup will be performed for DDNS (ipv4, ipv6, dual). By default will be the same as DNSMode.
// If DNSMode is 'first', IPv4 will be used.
DHCPMode string `yaml:"dhcpDualStackMode"`
// DisableServiceUpdates, if true, kube-vip will only advertise service, but it will not update service's Status.LoadBalancer.Ingress slice
DisableServiceUpdates bool `yaml:"disableServiceUpdates"`
@@ -196,12 +186,6 @@ type Config struct {
// EgressClean, enables egress cleaning on Kube-vip's start
EgressClean bool `yaml:"egressClean"`
// ConfigFile defines the path to a JSON/YAML configuration file
ConfigFile string `yaml:"configFile"`
// DHCPBackoffAttempts defaines how many times will DHCP client try to obtain address (unlimited when 0)
DHCPBackoffAttempts uint `yaml:"dhcpBackoffAttempts"`
}
// KubernetesLeaderElection defines all of the settings for Kubernetes KubernetesLeaderElection

View File

@@ -1,7 +0,0 @@
package kubevip
const (
LBClassName = "kube-vip.io/kube-vip-class"
DefaultDHCPBackoffAttempts = 3
)

View File

@@ -1,7 +0,0 @@
package kubevip
const (
// label used on nodes, which announce the LoadBalancer IP
HasIP = "kube-vip.io/has-ip"
HasIPJSONPath = `kube-vip.io~1has-ip`
)

View File

@@ -1,234 +0,0 @@
package lease
import (
"context"
"fmt"
log "log/slog"
"os"
"strings"
"sync"
"sync/atomic"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
)
// Manager is used to manage leases.
type Manager struct {
leases map[string]*Lease
lock sync.Mutex
}
// NewManager creates new lease manager.
func NewManager() *Manager {
return &Manager{
leases: make(map[string]*Lease),
}
}
// Add adds lease to the manager.
// It returns three values:
// - lease for the object
// - isNewObject, which reports if it is a new object that is being handled
// - isSharedLease, which is true if object shares the lease with another object
// If object is new but not shared, we should start leaderelection and sync it
// If object is new and shared, we should only sync it as the leaderelection should be already handled
// If object is not new we should do nothing
func (m *Manager) Add(ctx context.Context, id ID) *Lease {
m.lock.Lock()
defer m.lock.Unlock()
if _, exists := m.leases[id.NamespacedName()]; !exists {
leaseCtx, leaseCancel := context.WithCancel(ctx)
m.leases[id.NamespacedName()] = newLease(leaseCtx, leaseCancel)
}
return m.leases[id.NamespacedName()]
}
// Delete removes the lease and cancels it if the lease counter equals 0.
func (m *Manager) Delete(id ID, objectName string) {
m.lock.Lock()
defer m.lock.Unlock()
if _, exist := m.leases[id.NamespacedName()]; exist {
m.leases[id.NamespacedName()].delete(objectName)
if m.leases[id.NamespacedName()].cnt.Load() < 1 {
m.leases[id.NamespacedName()].Cancel()
delete(m.leases, id.NamespacedName())
}
}
}
// Get returns lease for the service.
func (m *Manager) Get(id ID) *Lease {
m.lock.Lock()
defer m.lock.Unlock()
if lease, exist := m.leases[id.NamespacedName()]; exist {
return lease
}
return nil
}
// Lease holds lease data.
type Lease struct {
Ctx context.Context
Cancel context.CancelFunc
Started chan any
services sync.Map
cnt atomic.Int64
Elected atomic.Bool
Mtx sync.Mutex
locked bool
}
func newLease(ctx context.Context, cancel context.CancelFunc) *Lease {
return &Lease{
Ctx: ctx,
Cancel: cancel,
Started: make(chan any),
}
}
// Add adds the object to the lease and increments counter
// it will return true if object was added
func (l *Lease) Add(name string) bool {
if _, exists := l.services.Load(name); !exists {
l.services.Store(name, true)
l.cnt.Add(1)
return true
}
return false
}
// delete removes the service from the lease and decrements the counter
func (l *Lease) delete(service string) {
if _, exists := l.services.Load(service); exists {
l.services.Delete(service)
l.cnt.Add(-1)
}
}
func (l *Lease) Lock() {
l.Mtx.Lock()
l.locked = true
}
func (l *Lease) Unlock() {
if l.locked {
l.locked = false
l.Mtx.Unlock()
}
}
// ServiceName gets lease name and id for the service.
func ServiceName(service *v1.Service) (string, string) {
name, exists := service.Annotations[kubevip.ServiceLease]
if !exists || name == "" {
name = fmt.Sprintf("kubevip-%s", service.Name)
}
serviceLeaseParts := strings.Split(name, "/")
namespace := service.Namespace
if len(serviceLeaseParts) > 1 {
namespace = serviceLeaseParts[0]
name = serviceLeaseParts[1]
}
return namespace, name
}
func ServiceNamespacedName(service *v1.Service) string {
return fmt.Sprintf("%s/%s", service.Namespace, service.Name)
}
func ObjectName(id ID, suffix string) string {
return fmt.Sprintf("%s-%s", id.NamespacedName(), suffix)
}
func NamespaceName(lease string, c *kubevip.Config) (string, string) {
leaseName := lease
leasnameParts := strings.Split(lease, "/")
var ns string
var err error
if len(leasnameParts) > 1 {
ns = leasnameParts[0]
leaseName = leasnameParts[1]
} else {
ns, err = returnNamespace()
if err != nil {
log.Warn("unable to auto-detect namespace, dropping to config", "namespace", c.Namespace)
ns = c.Namespace
}
}
return ns, leaseName
}
func returnNamespace() (string, error) {
if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
return ns, nil
}
return "", err
}
return "", fmt.Errorf("unable to find Namespace")
}
type ID interface {
Name() string
Namespace() string
NamespacedName() string
}
type CommonID struct {
namespace string
name string
}
func NewID(leaseType, namespace, name string) ID {
if leaseType == "etcd" {
return newEtcdID(namespace, name)
}
return newKubernetesID(namespace, name)
}
func newKubernetesID(namespace, name string) ID {
return &KubernetesID{
CommonID: CommonID{
namespace: namespace,
name: name,
},
}
}
func newEtcdID(namespace, name string) ID {
return &EtcdID{
CommonID: CommonID{
namespace: namespace,
name: name,
},
}
}
func (c *CommonID) Name() string {
return c.name
}
func (c *CommonID) Namespace() string {
return c.namespace
}
type KubernetesID struct {
CommonID
}
func (k *KubernetesID) NamespacedName() string {
return fmt.Sprintf("%s/%s", k.namespace, k.name)
}
type EtcdID struct {
CommonID
}
func (e *EtcdID) NamespacedName() string {
return fmt.Sprintf("%s-%s", e.namespace, e.name)
}

View File

@@ -1,939 +0,0 @@
package lease
import (
"context"
"sync"
"testing"
"time"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func createTestService(name, namespace string, annotations map[string]string) *v1.Service {
return &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Annotations: annotations,
},
}
}
func getSvcID(svc *v1.Service) ID {
namespace, name := ServiceName(svc)
id := NewID("kubernetes", namespace, name)
return id
}
func getSvcData(svc *v1.Service) (context.Context, ID) {
return context.TODO(), getSvcID(svc)
}
const serviceLeaseAnnotation = kubevip.ServiceLease
// TestManager_Add_NewLease tests adding a new service with a new lease
func TestManager_Add_NewLease(t *testing.T) {
mgr := NewManager()
svc := createTestService("test-svc", "default", nil)
leaseID := mgr.Add(getSvcData(svc))
isNew := leaseID.Add(ServiceNamespacedName(svc))
if !isNew {
t.Error("expected isNew to be true for first Add")
}
if leaseID == nil {
t.Fatal("expected lease to be non-nil")
}
if leaseID.Ctx == nil {
t.Error("expected lease context to be non-nil")
}
if leaseID.Cancel == nil {
t.Error("expected lease cancel func to be non-nil")
}
if leaseID.Started == nil {
t.Error("expected lease Started channel to be non-nil")
}
}
// TestManager_Add_ExistingLease tests adding a service with an existing lease
func TestManager_Add_ExistingLease(t *testing.T) {
mgr := NewManager()
svc := createTestService("test-svc", "default", nil)
leaseID1 := mgr.Add(getSvcData(svc))
isNew1 := leaseID1.Add(ServiceNamespacedName(svc))
leaseID2 := mgr.Add(getSvcData(svc))
isNew2 := leaseID2.Add(ServiceNamespacedName(svc))
if !isNew1 {
t.Error("expected first Add to return isNew=true")
}
if isNew2 {
t.Error("expected second Add to return isNew=false")
}
if leaseID1 != leaseID2 {
t.Error("expected same lease to be returned for same service")
}
}
// TestManager_Delete_DecrementCounter tests the decrement counter functionality
func TestManager_Delete_DecrementCounter(t *testing.T) {
mgr := NewManager()
svc := createTestService("test-svc", "default", nil)
// Add twice (simulating adding same service twice)
objectName := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
_ = lease1.Add(objectName)
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
_ = lease2.Add(objectName)
// Delete once - should remove the lease
mgr.Delete(leaseID1, objectName)
lease := mgr.Get(getSvcID(svc))
if lease != nil {
t.Error("expected lease to be removed after first delete if same service was processed twice")
}
}
// TestManager_Delete_CancelsContext tests the context cancellation on delete
func TestManager_Delete_CancelsContext(t *testing.T) {
mgr := NewManager()
svc := createTestService("test-svc", "default", nil)
objectName := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
_ = lease1.Add(objectName)
// Verify context is not cancelled
select {
case <-lease1.Ctx.Done():
t.Fatal("expected context to not be cancelled initially")
default:
// Expected
}
// Delete the lease
mgr.Delete(leaseID1, objectName)
// Verify context is cancelled
select {
case <-lease1.Ctx.Done():
// Expected
case <-time.After(100 * time.Millisecond):
t.Error("expected context to be cancelled after delete")
}
}
// TestManager_Add_AfterDelete_CreatesNewLease tests adding a service after deleting it
func TestManager_Add_AfterDelete_CreatesNewLease(t *testing.T) {
mgr := NewManager()
svc := createTestService("test-svc", "default", nil)
objectName := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
_ = lease1.Add(objectName)
mgr.Delete(leaseID1, objectName)
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
isNew := lease2.Add(objectName)
if !isNew {
t.Error("expected isNew to be true after delete and re-add")
}
if lease1 == lease2 {
t.Error("expected new lease to be different from old lease")
}
}
// TestManager_Add_DifferentServices tests adding services with different names
func TestManager_Add_DifferentServices(t *testing.T) {
mgr := NewManager()
svc1 := createTestService("svc1", "default", nil)
svc2 := createTestService("svc2", "default", nil)
objectName1 := ServiceNamespacedName(svc1)
ctx1, leaseID1 := getSvcData(svc1)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
objectName2 := ServiceNamespacedName(svc2)
ctx2, leaseID2 := getSvcData(svc2)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease1.Add(objectName2)
if !isNew1 || !isNew2 {
t.Error("expected both adds to return isNew=true")
}
if lease1 == lease2 {
t.Error("expected different leases for different services")
}
}
// TestManager_Add_SameNameDifferentNamespace tests adding services with the same name but different namespaces
func TestManager_Add_SameNameDifferentNamespace(t *testing.T) {
mgr := NewManager()
svc1 := createTestService("test-svc", "namespace1", nil)
svc2 := createTestService("test-svc", "namespace2", nil)
objectName1 := ServiceNamespacedName(svc1)
ctx1, leaseID1 := getSvcData(svc1)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
objectName2 := ServiceNamespacedName(svc2)
ctx2, leaseID2 := getSvcData(svc2)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease1.Add(objectName2)
if !isNew1 || !isNew2 {
t.Error("expected both adds to return isNew=true")
}
if lease1 == lease2 {
t.Error("expected different leases for services in different namespaces")
}
}
// TestManager_ConcurrentAccess tests concurrent access to the lease manager
func TestManager_ConcurrentAccess(t *testing.T) {
mgr := NewManager()
svc := createTestService("test-svc", "default", nil)
var wg sync.WaitGroup
const numGoroutines = 100
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
added := lease1.Add(objectName1)
if !added {
t.Error("expected lease to be added")
}
// Concurrent adds
for range numGoroutines {
wg.Go(func() {
added := lease1.Add(objectName1)
if added {
t.Error("expected lease to already exist")
}
})
}
wg.Wait()
mgr.Delete(leaseID1, objectName1)
// After a one delete, lease should be gone
lease := mgr.Get(getSvcID(svc))
if lease != nil {
t.Error("expected lease to be removed after all concurrent deletes")
}
}
// TestLease_StartedChannel tests the Started channel behavior
func TestLease_StartedChannel(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
lease := newLease(ctx, cancel)
// Started channel should be open initially
select {
case <-lease.Started:
t.Fatal("expected Started channel to be open initially")
default:
// Expected
}
// Close the channel
close(lease.Started)
// Now it should be closed
select {
case <-lease.Started:
// Expected
default:
t.Error("expected Started channel to be closed after close()")
}
}
// TestGetName_WithoutAnnotation tests with no annotation
func TestGetName_WithoutAnnotation(t *testing.T) {
svc := createTestService("my-service", "my-namespace", nil)
namespace, name := ServiceName(svc)
id := NewID("kubernetes", namespace, name)
expectedName := "kubevip-my-service"
expectedID := "my-namespace/kubevip-my-service"
if id.Name() != expectedName {
t.Errorf("expected name %q, got %q", expectedName, id.Name())
}
if id.NamespacedName() != expectedID {
t.Errorf("expected id %q, got %q", expectedID, id.NamespacedName())
}
}
// TestGetName_WithAnnotation tests with a shared lease annotation
func TestGetName_WithAnnotation(t *testing.T) {
svc := createTestService("my-service", "my-namespace", map[string]string{
serviceLeaseAnnotation: "shared-lease",
})
namespace, name := ServiceName(svc)
id := NewID("kubernetes", namespace, name)
expectedName := "shared-lease"
expectedID := "my-namespace/shared-lease"
if id.Name() != expectedName {
t.Errorf("expected name %q, got %q", expectedName, id.Name())
}
if id.NamespacedName() != expectedID {
t.Errorf("expected id %q, got %q", expectedID, id.NamespacedName())
}
}
// TestGetName_WithAnnotation tests with a shared lease annotation
func TestGetName_WithAnnotationAndOverriddenNamespace(t *testing.T) {
svc := createTestService("my-service", "my-namespace", map[string]string{
serviceLeaseAnnotation: "other-namespace/shared-lease",
})
namespace, name := ServiceName(svc)
id := NewID("kubernetes", namespace, name)
expectedName := "shared-lease"
expectedID := "other-namespace/shared-lease"
expectedNamespace := "other-namespace"
if id.Name() != expectedName {
t.Errorf("expected name %q, got %q", expectedName, id.Name())
}
if id.NamespacedName() != expectedID {
t.Errorf("expected id %q, got %q", expectedID, id.NamespacedName())
}
if id.Namespace() != expectedNamespace {
t.Errorf("expected namespace %q, got %q", expectedNamespace, id.Namespace())
}
}
// TestGetName_WithoutAnnotation_Etcd tests with no annotation
func TestGetName_WithoutAnnotation_Etcd(t *testing.T) {
svc := createTestService("my-service", "my-namespace", nil)
namespace, name := ServiceName(svc)
id := NewID("etcd", namespace, name)
expectedName := "kubevip-my-service"
expectedID := "my-namespace-kubevip-my-service"
if id.Name() != expectedName {
t.Errorf("expected name %q, got %q", expectedName, id.Name())
}
if id.NamespacedName() != expectedID {
t.Errorf("expected id %q, got %q", expectedID, id.NamespacedName())
}
}
// TestGetName_WithAnnotation_Etcd tests with a shared lease annotation
func TestGetName_WithAnnotation_Etcd(t *testing.T) {
svc := createTestService("my-service", "my-namespace", map[string]string{
serviceLeaseAnnotation: "shared-lease",
})
namespace, name := ServiceName(svc)
id := NewID("etcd", namespace, name)
expectedName := "shared-lease"
expectedID := "my-namespace-shared-lease"
if id.Name() != expectedName {
t.Errorf("expected name %q, got %q", expectedName, id.Name())
}
if id.NamespacedName() != expectedID {
t.Errorf("expected id %q, got %q", expectedID, id.NamespacedName())
}
}
// TestGetName_WithAnnotation_Etcd tests with a shared lease annotation
func TestGetName_WithAnnotationAndOverriddenNamespace_Etcd(t *testing.T) {
svc := createTestService("my-service", "my-namespace", map[string]string{
serviceLeaseAnnotation: "other-namespace/shared-lease",
})
namespace, name := ServiceName(svc)
id := NewID("etcd", namespace, name)
expectedName := "shared-lease"
expectedID := "other-namespace-shared-lease"
expectedNamespace := "other-namespace"
if id.Name() != expectedName {
t.Errorf("expected name %q, got %q", expectedName, id.Name())
}
if id.NamespacedName() != expectedID {
t.Errorf("expected id %q, got %q", expectedID, id.NamespacedName())
}
if id.Namespace() != expectedNamespace {
t.Errorf("expected namespace %q, got %q", expectedNamespace, id.Namespace())
}
}
// TestManager_LeaderElectionRestartScenario simulates the bug scenario where
// leadership is lost and the restartable service watcher tries to restart
// the leader election. This test verifies that after deleting the lease,
// a new lease can be created.
func TestManager_LeaderElectionRestartScenario_etcd(t *testing.T) {
mgr := NewManager()
svc := createTestService("traefik", "traefik", nil)
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
if !isNew1 {
t.Fatal("expected first add to return isNew=true")
}
// Simulate leadership acquired - close Started channel
close(lease1.Started)
// Simulate leadership lost - the leader election function should delete the lease
// This is the fix: delete the lease when RunOrDie returns
mgr.Delete(leaseID1, objectName1)
// Verify lease is removed
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to be removed after delete")
}
// Simulate restartable service watcher calling StartServicesLeaderElection again
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease1.Add(objectName1)
if !isNew2 {
t.Fatal("expected second add after delete to return isNew=true")
}
// Verify we got a new lease with a fresh Started channel
if lease1 == lease2 {
t.Error("expected new lease to be different from old lease")
}
// Verify the new Started channel is not closed
select {
case <-lease2.Started:
t.Error("expected new lease's Started channel to be open")
default:
// Expected
}
}
// TestManager_CommonLeaseScenario tests the common lease feature where
// multiple services share the same lease.
func TestManager_CommonLeaseScenario(t *testing.T) {
mgr := NewManager()
// Two services sharing the same lease via annotation
sharedLeaseAnnotations := map[string]string{
serviceLeaseAnnotation: "shared-lease",
}
svc1 := createTestService("svc1", "default", sharedLeaseAnnotations)
svc2 := createTestService("svc2", "default", sharedLeaseAnnotations)
// First service gets a new lease
objectName1 := ServiceNamespacedName(svc1)
ctx1, leaseID1 := getSvcData(svc1)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
if !isNew1 {
t.Error("expected first add to return isNew=true")
}
// Simulate first service starting leadership
close(lease1.Started)
objectName2 := ServiceNamespacedName(svc2)
ctx2, leaseID2 := getSvcData(svc2)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease2.Add(objectName2)
// Second service should get the same lease
if !isNew2 {
t.Error("expected second add with same lease name to return isNew=true")
}
if lease1 != lease2 {
t.Error("expected same lease for services with same lease annotation")
}
// Delete first service - lease should still exist
mgr.Delete(leaseID1, objectName1)
if mgr.Get(getSvcID(svc1)) == nil {
t.Error("expected lease to still exist after first delete")
}
// Delete second service - lease should be removed
mgr.Delete(leaseID2, objectName2)
if mgr.Get(getSvcID(svc2)) != nil {
t.Error("expected lease to be removed after all services deleted")
}
}
// TestManager_RaceCondition_LeaseExistsBeforeDelete tests the scenario where
// a second goroutine calls Add before the first goroutine's defer deletes the lease.
// This simulates the race condition that could cause the gaps in the logs where there is no leader.
func TestManager_RaceCondition_LeaseExistsBeforeDelete(t *testing.T) {
mgr := NewManager()
svc := createTestService("traefik", "traefik", nil)
// Simulate first leader election start
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
if !isNew1 {
t.Fatal("expected first add to return isNew=true")
}
// Simulate leadership acquired - close Started channel
close(lease1.Started)
// Simulate a second goroutine calling Add BEFORE the first goroutine's defer deletes the lease
// This is the race condition scenario
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease1.Add(objectName1)
if isNew2 {
t.Error("expected second add before delete to return isNew=false")
}
if lease1 != lease2 {
t.Error("expected same lease to be returned")
}
// The Started channel should be closed (from the first run)
select {
case <-lease2.Started:
// Expected - channel is closed
default:
t.Error("expected Started channel to be closed")
}
// Now the first goroutine's defer deletes the lease
mgr.Delete(leaseID1, objectName1)
// The lease should not still exist because same service was processed twice, so we do not increment the counter
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease tonot exist")
}
// Second delete does nothing
mgr.Delete(leaseID2, objectName1)
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to not exist")
}
}
// TestManager_NonCommonLease_MultipleAdds tests that multiple Adds for a non-common
// lease service increment the counter correctly.
func TestManager_NonCommonLease_MultipleAdds(t *testing.T) {
mgr := NewManager()
svc := createTestService("traefik", "traefik", nil) // No common lease annotation
// First Add
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
if !isNew1 {
t.Error("expected first add to return isNew=true")
}
// Close Started to simulate leadership acquired
close(lease1.Started)
// Second Add (simulating another goroutine or restart attempt)
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease1.Add(objectName1)
if isNew2 {
t.Error("expected second add to return isNew=false")
}
if lease1 != lease2 {
t.Error("expected same lease")
}
// Third Add
ctx3, leaseID3 := getSvcData(svc)
lease3 := mgr.Add(ctx3, leaseID3)
isNew3 := lease1.Add(objectName1)
if isNew3 {
t.Error("expected third add to return isNew=false")
}
if lease1 != lease3 {
t.Error("expected same lease")
}
// Need one delete to remove the lease, another delete runs do nothing
mgr.Delete(leaseID1, objectName1)
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to be deleted")
}
mgr.Delete(leaseID2, objectName1)
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to be deleted")
}
mgr.Delete(leaseID3, objectName1)
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to be deleted")
}
}
// TestManager_LeaseContextCancelledBeforeStarted tests the scenario where
// the lease context is cancelled before the Started channel is closed.
// This can happen if leadership is never acquired and the context times out.
func TestManager_LeaseContextCancelledBeforeStarted(t *testing.T) {
mgr := NewManager()
svc := createTestService("traefik", "traefik", nil)
// First Add
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
if !isNew1 {
t.Fatal("expected first add to return isNew=true")
}
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease1.Add(objectName1)
if isNew2 {
t.Error("expected second add to return isNew=false")
}
// Verify Started is not closed yet
select {
case <-lease2.Started:
t.Error("expected Started channel to be open")
default:
// Expected
}
// Cancel the lease context (simulating timeout or leadership loss before acquiring)
lease1.Cancel()
// Verify context is cancelled
select {
case <-lease2.Ctx.Done():
// Expected
case <-time.After(100 * time.Millisecond):
t.Error("expected context to be cancelled")
}
// Delete should still work
mgr.Delete(leaseID1, objectName1)
mgr.Delete(leaseID2, objectName1)
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to be removed")
}
}
// TestManager_RestartAfterLeaseContextCancelled tests that after the lease
// context is cancelled and the lease is deleted, a new lease can be created.
func TestManager_RestartAfterLeaseContextCancelled(t *testing.T) {
mgr := NewManager()
svc := createTestService("traefik", "traefik", nil)
// First Add
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
_ = lease1.Add(objectName1)
// Cancel context before Started is closed
lease1.Cancel()
// Delete the lease
mgr.Delete(leaseID1, objectName1)
// Verify lease is gone
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to be removed after delete")
}
// Add again - should create new lease
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease1.Add(objectName1)
if !isNew2 {
t.Error("expected new lease after delete")
}
// Verify new lease has fresh context and Started channel
select {
case <-lease2.Ctx.Done():
t.Error("expected new lease context to be active")
default:
// Expected
}
select {
case <-lease2.Started:
t.Error("expected new lease Started channel to be open")
default:
// Expected
}
}
// TestManager_NonCommonLease_WaitForLeaseContextDone tests the scenario where
// a non-common lease service calls Add while another leader election is running.
// The caller should wait for the lease context to be done before returning.
// This test verifies the fix for the tight spin loop issue.
func TestManager_NonCommonLease_WaitForLeaseContextDone(t *testing.T) {
mgr := NewManager()
svc := createTestService("egress-service", "default", nil) // Non-common lease
// First Add - simulates the first leader election starting
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
if !isNew1 {
t.Fatal("expected first add to return isNew=true")
}
// Simulate leadership acquired
close(lease1.Started)
// Second Add - simulates another goroutine trying to start leader election
// This should return isNew=false
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease1.Add(objectName1)
if isNew2 {
t.Error("expected second add to return isNew=false")
}
if lease1 != lease2 {
t.Error("expected same lease to be returned")
}
// Verify Started channel is closed (leadership was acquired by first)
select {
case <-lease2.Started:
// Expected - channel is closed
default:
t.Error("expected Started channel to be closed")
}
// In the actual code (leader.go), when isNew=false for non-common lease,
// the code waits on either svcCtx.Ctx.Done() or svcLease.Ctx.Done()
// Here we verify that the lease context gets cancelled when we delete the lease
// Start a goroutine that waits for the lease context to be done
// This simulates what the leader.go code does
waitDone := make(chan struct{})
go func() {
select {
case <-lease2.Ctx.Done():
close(waitDone)
case <-time.After(1 * time.Second):
// Timeout - test will fail
}
}()
// Verify the goroutine is still waiting (lease context not yet cancelled)
select {
case <-waitDone:
t.Fatal("goroutine should still be waiting")
case <-time.After(50 * time.Millisecond):
// Expected - still waiting
}
// Now simulate the first leader election ending (defer deletes the lease)
mgr.Delete(leaseID1, objectName1)
// The lease context should now be cancelled (because counter went to 0)
// But we added twice, so we need to delete twice
mgr.Delete(leaseID2, objectName1)
// Now the goroutine should have completed
select {
case <-waitDone:
// Expected - lease context was cancelled
case <-time.After(200 * time.Millisecond):
t.Error("expected goroutine to complete after lease context cancelled")
}
// Verify lease is removed
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to be removed")
}
}
// TestManager_NonCommonLease_SpinLoopPrevention tests that the fix prevents
// a tight spin loop when a non-common lease service repeatedly calls Add
// while leader election is running. The key behavior is that when isNew=false,
// the lease context should be used to block until the leader election ends.
func TestManager_NonCommonLease_SpinLoopPrevention(t *testing.T) {
mgr := NewManager()
svc := createTestService("egress-service", "default", nil) // Non-common lease
// First Add - leader election starts
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
isNew1 := lease1.Add(objectName1)
if !isNew1 {
t.Fatal("expected first add to return isNew=true")
}
close(lease1.Started)
// Track how many times Add is called in a tight loop
// In the buggy code, this would spin forever
// In the fixed code, Add returns isNew=false and the caller blocks on lease.Ctx.Done()
addCount := 0
done := make(chan struct{})
go func() {
for i := 0; i < 100; i++ {
objectName1 := ServiceNamespacedName(svc)
ctxTmp, leaseIDTmp := getSvcData(svc)
leaseTmp := mgr.Add(ctxTmp, leaseIDTmp)
isNewTmp := leaseTmp.Add(objectName1)
addCount++
if isNewTmp {
// This shouldn't happen while the first lease exists
t.Error("unexpected isNew=true")
break
}
// In the fixed code, we would block here on lease.Ctx.Done()
// For this test, we just verify that isNew=false is returned
// and the same lease is returned each time
if leaseTmp != lease1 {
t.Error("expected same lease")
break
}
}
close(done)
}()
// Wait for the loop to complete
select {
case <-done:
// Expected
case <-time.After(1 * time.Second):
t.Fatal("loop timed out")
}
// All 100 adds should have completed (returning isNew=false)
if addCount != 100 {
t.Errorf("expected 100 adds, got %d", addCount)
}
mgr.Delete(leaseID1, objectName1)
if mgr.Get(getSvcID(svc)) != nil {
t.Error("expected lease to be removed after first delete")
}
}
// TestManager_NonCommonLease_ServiceContextCancellation tests that when
// a service is deleted (svcCtx.Ctx cancelled), the waiting goroutine
// should also unblock. This is the other exit path from the wait.
func TestManager_NonCommonLease_ServiceContextCancellation(t *testing.T) {
mgr := NewManager()
svc := createTestService("egress-service", "default", nil)
// First Add - leader election starts
objectName1 := ServiceNamespacedName(svc)
ctx1, leaseID1 := getSvcData(svc)
lease1 := mgr.Add(ctx1, leaseID1)
_ = lease1.Add(objectName1)
close(lease1.Started)
// Second Add - returns isNew=false
ctx2, leaseID2 := getSvcData(svc)
lease2 := mgr.Add(ctx2, leaseID2)
isNew2 := lease2.Add(objectName1)
if isNew2 {
t.Error("expected isNew=false")
}
// Create a simulated service context
svcCtx, svcCancel := context.WithCancel(context.Background())
// Start a goroutine that waits on either svcCtx or lease context
// This simulates the behavior in leader.go
waitDone := make(chan string)
go func() {
select {
case <-svcCtx.Done():
waitDone <- "svcCtx"
case <-lease2.Ctx.Done():
waitDone <- "leaseCtx"
case <-time.After(1 * time.Second):
waitDone <- "timeout"
}
}()
// Cancel the service context (simulates service deletion)
svcCancel()
// The goroutine should unblock via svcCtx.Done()
select {
case result := <-waitDone:
if result != "svcCtx" {
t.Errorf("expected to unblock via svcCtx, got %s", result)
}
case <-time.After(200 * time.Millisecond):
t.Error("goroutine should have unblocked")
}
}

View File

@@ -16,7 +16,6 @@ import (
"github.com/cloudflare/ipvs/netmask"
"github.com/kube-vip/kube-vip/pkg/backend"
"github.com/kube-vip/kube-vip/pkg/sysctl"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
)
@@ -53,53 +52,47 @@ type IPVSLoadBalancer struct {
lock sync.Mutex
stop chan struct{}
networkInterface string
killFunc func()
leaderCancel context.CancelFunc
signal chan os.Signal
address string
family ipvs.AddressFamily
}
func NewIPVSLB(ctx context.Context, address string, port uint16, forwardingMethod string, backendHealthCheckInterval int,
networkInterface string, killFunc func(), wg *sync.WaitGroup) (*IPVSLoadBalancer, error) {
func NewIPVSLB(address string, port uint16, forwardingMethod string, backendHealthCheckInterval int, networkInterface string, leaderCancel context.CancelFunc, signal chan os.Signal) (*IPVSLoadBalancer, error) {
log.Info("Starting IPVS LoadBalancer", "address", address)
// Create IPVS client
c, err := ipvs.New()
if err != nil {
log.Error("ensure IPVS kernel modules are loaded")
log.Error("error starting IPVS", "err", err)
return nil, fmt.Errorf("starting IPVS: %w", err)
log.Error("Error starting IPVS", "err", err)
panic("")
}
i, err := c.Info()
if err != nil {
log.Error("ensure IPVS kernel modules are loaded")
log.Error("error retrieving IPVS info", "err", err)
log.Error("Error retrieving IPVS info", "err", err)
if errors.Is(err, os.ErrPermission) {
log.Error("no permission to get IPVS info - please ensure that kube-vip is running with proper capabilities/privileged mode")
}
return nil, fmt.Errorf("retrieving IPVS: %w", err)
panic("")
}
log.Info("IPVS Loadbalancer enabled", "version", fmt.Sprintf("%d.%d.%d", i.Version[0], i.Version[1], i.Version[2]))
ip, family := ipAndFamily(address)
if strings.ToLower(forwardingMethod) == "masquerade" {
if err := enableProcSys("/proc/sys/net/ipv4/vs/conntrack", "net.ipv4.vs.conntrack"); err != nil {
return nil, err
}
enableProcSys("/proc/sys/net/ipv4/vs/conntrack", "net.ipv4.vs.conntrack")
if family == ipvs.INET6 {
if err := enableProcSys("/proc/sys/net/ipv6/conf/all/forwarding", "net.ipv6.conf.all.forwarding"); err != nil {
return nil, err
}
enableProcSys("/proc/sys/net/ipv6/conf/all/forwarding", "net.ipv6.conf.all.forwarding")
} else {
if err := enableProcSys("/proc/sys/net/ipv4/ip_forward", "net.ipv4.ip_forward"); err != nil {
return nil, err
}
enableProcSys("/proc/sys/net/ipv4/ip_forward", "net.ipv4.ip_forward")
}
}
netMask := netmask.MaskFrom(31, vip.DefaultMaskIPv4) // For ipv4
netMask := netmask.MaskFrom(31, 32) // For ipv4
if family == ipvs.INET6 {
netMask = netmask.MaskFrom(128, vip.DefaultMaskIPv6) // For ipv6
netMask = netmask.MaskFrom(128, 128) // For ipv6
}
// Generate out API Server LoadBalancer instance
@@ -140,29 +133,29 @@ func NewIPVSLB(ctx context.Context, address string, port uint16, forwardingMetho
forwardingMethod: m,
interval: backendHealthCheckInterval,
backendMap: make(backend.Map),
stop: make(chan struct{}),
networkInterface: networkInterface,
killFunc: killFunc,
leaderCancel: leaderCancel,
signal: signal,
address: address,
family: family,
}
wg.Go(func() {
lb.healthCheck(ctx)
})
go lb.healthCheck()
// Return our created load-balancer
return lb, nil
}
func enableProcSys(path, name string) error {
func enableProcSys(path, name string) {
isSet, err := sysctl.EnableProcSys(path)
if err != nil {
return fmt.Errorf("ensuring %s enabled: %w", name, err)
log.Error(fmt.Sprintf("ensuring %s enabled", name), "err", err)
panic("")
}
if isSet {
log.Info(fmt.Sprintf("sysctl set %s to 1", name))
}
return nil
}
func (lb *IPVSLoadBalancer) RemoveIPVSLB() error {
@@ -234,7 +227,7 @@ func (lb *IPVSLoadBalancer) addBackend(address string, port uint16) error {
// Fatal error at this point as IPVS is probably not working
log.Error("Unable to create an IPVS service, ensure IPVS kernel modules are loaded")
log.Error("IPVS service", "err", err)
return utils.NewPanicError(fmt.Sprintf("unable to create an IPVS service - %s", err))
panic("")
}
log.Info("load-Balancer services created", "address", lb.addrString(), "port", lb.Port)
@@ -319,8 +312,8 @@ func ipAndFamily(address string) (netip.Addr, ipvs.AddressFamily) {
return netip.AddrFrom4([4]byte(ipAddr.To4())), ipvs.INET
}
func (lb *IPVSLoadBalancer) healthCheck(ctx context.Context) {
backend.Watch(ctx, func() {
func (lb *IPVSLoadBalancer) healthCheck() {
backend.Watch(func() {
lb.lock.Lock()
defer lb.lock.Unlock()
for backend, oldStatus := range lb.backendMap {
@@ -345,13 +338,17 @@ func (lb *IPVSLoadBalancer) healthCheck(ctx context.Context) {
lb.backendMap[backend] = newStatus
}
if lb.forwardingMethod == ipvs.Local && !lb.localBackendExists() {
if lb.killFunc != nil {
lb.killFunc()
if lb.signal != nil {
close(lb.signal)
}
if lb.leaderCancel != nil {
lb.leaderCancel()
}
}
}
}
}, lb.interval)
}, lb.interval, lb.stop)
}
func (lb *IPVSLoadBalancer) isLocal(address string) (bool, error) {
@@ -361,7 +358,7 @@ func (lb *IPVSLoadBalancer) isLocal(address string) (bool, error) {
}
family := netlink.FAMILY_V6
if utils.IsIPv4(address) {
if vip.IsIPv4(address) {
family = netlink.FAMILY_V4
}

View File

@@ -6,7 +6,6 @@ import (
"testing"
"github.com/cloudflare/ipvs"
"github.com/kube-vip/kube-vip/pkg/utils"
)
func Test_ipAndFamily(t *testing.T) {
@@ -20,7 +19,7 @@ func Test_ipAndFamily(t *testing.T) {
want1 ipvs.AddressFamily
}{
{
name: utils.IPv4Family,
name: "IPv4",
args: args{
address: "192.168.0.20",
},
@@ -28,7 +27,7 @@ func Test_ipAndFamily(t *testing.T) {
want1: ipvs.INET,
},
{
name: utils.IPv6Family,
name: "IPv6",
args: args{
address: "ff02::3",
},

30
pkg/manager/cluster.go Normal file
View File

@@ -0,0 +1,30 @@
package manager
import (
"github.com/pkg/errors"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/etcd"
)
func initClusterManager(sm *Manager) (*cluster.Manager, error) {
m := &cluster.Manager{
SignalChan: sm.signalChan,
}
switch sm.config.LeaderElectionType {
case "kubernetes", "":
m.KubernetesClient = sm.clientSet
m.RetryWatcherClient = sm.rwClientSet
case "etcd":
client, err := etcd.NewClient(sm.config)
if err != nil {
return nil, err
}
m.EtcdClient = client
default:
return nil, errors.Errorf("invalid LeaderElectionMode %s not supported", sm.config.LeaderElectionType)
}
return m, nil
}

View File

@@ -2,14 +2,13 @@ package manager
import (
"context"
"errors"
"fmt"
"net/http"
"os"
"os/signal"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
@@ -17,25 +16,19 @@ import (
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/manager/worker"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/node"
"github.com/kube-vip/kube-vip/pkg/services"
"github.com/kube-vip/kube-vip/pkg/upnp"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
const plunderLock = "plndr-svcs-lock"
// Manager degines the manager of the load-balancing services
type Manager struct {
clientSet *kubernetes.Clientset
@@ -52,7 +45,8 @@ type Manager struct {
// This channel is used to catch an OS signal and trigger a shutdown
signalChan chan os.Signal
sigint sync.Once
// This channel is used to signal a shutdown
shutdownChan chan struct{}
svcProcessor *services.Processor
@@ -72,20 +66,6 @@ type Manager struct {
// This tracks VIPs and performs ARP/NDP advertisement.
arpMgr *arp.Manager
// This tracks node labels and performs label management
// implementation will be decided in constructor
// based on config.EnableNodeLabeling
nodeLabelManager node.LabelManager
// This variable reports if manager is being closed
closing atomic.Bool
// Will be used for leaderelection when required
electionMgr *election.Manager
// Will handle leases
leaseMgr *lease.Manager
}
// New will create a new managing object
@@ -201,33 +181,21 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(signalChan, syscall.SIGTERM)
// Add Notification for SIGUSR1 (for configuration dump)
signal.Notify(signalChan, syscall.SIGUSR1)
// All watchers and other goroutines should have an additional goroutine that blocks on this, to shut things down
shutdownChan := make(chan struct{})
intfMgr := networkinterface.NewManager()
arpMgr := arp.NewManager(config)
// create the node label manager
// constructor will decide if it should be a noop or not
nodeLabelManager := node.NewManager(config, clientset)
var bgpServer *bgp.Server
if config.EnableBGP {
bgpServer, err = bgp.NewBGPServer(config.BGPConfig)
bgpServer, err = bgp.NewBGPServer(&config.BGPConfig)
if err != nil {
return nil, fmt.Errorf("creating BGP server: %w", err)
}
}
electionMgr, err := election.NewManager(config, clientset, rwClientSet)
if err != nil {
return nil, fmt.Errorf("creating election manager: %w", err)
}
leaseMgr := lease.NewManager()
svcProcessor := services.NewServicesProcessor(config, bgpServer, clientset, rwClientSet,
intfMgr, arpMgr, nodeLabelManager, electionMgr, leaseMgr)
svcProcessor := services.NewServicesProcessor(config, bgpServer, clientset, rwClientSet, shutdownChan, intfMgr, arpMgr)
return &Manager{
clientSet: clientset,
@@ -246,21 +214,29 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
Name: "bgp_session_info",
Help: "Display state of session by setting metric for label value with current state to 1",
}, []string{"state", "peer"}),
signalChan: signalChan,
svcProcessor: svcProcessor,
intfMgr: intfMgr,
arpMgr: arpMgr,
bgpServer: bgpServer,
nodeLabelManager: nodeLabelManager,
electionMgr: electionMgr,
leaseMgr: leaseMgr,
signalChan: signalChan,
shutdownChan: shutdownChan,
svcProcessor: svcProcessor,
intfMgr: intfMgr,
arpMgr: arpMgr,
bgpServer: bgpServer,
}, nil
}
// Start will begin the Manager, which will start services and watch the configmap
func (sm *Manager) Start(ctx context.Context) error {
wg := sync.WaitGroup{}
defer wg.Wait()
func (sm *Manager) Start() error {
// listen for interrupts or the Linux SIGTERM signal and cancel
// our context, which the leader election code will observe and
// step down
sm.signalChan = make(chan os.Signal, 1)
// Add Notification for Userland interrupt
signal.Notify(sm.signalChan, syscall.SIGINT)
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(sm.signalChan, syscall.SIGTERM)
// All watchers and other goroutines should have an additional goroutine that blocks on this, to shut things down
sm.shutdownChan = make(chan struct{})
// HealthCheck
if sm.config.HealthCheckPort != 0 {
@@ -270,7 +246,7 @@ func (sm *Manager) Start(ctx context.Context) error {
http.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
fmt.Fprintf(w, "OK")
})
wg.Go(func() {
go func() {
server := &http.Server{
Addr: fmt.Sprintf(":%d", sm.config.HealthCheckPort),
ReadHeaderTimeout: 3 * time.Second,
@@ -279,19 +255,25 @@ func (sm *Manager) Start(ctx context.Context) error {
if err != nil {
log.Error("healthcheck", "unable to start", err)
}
})
}()
}
// on exit, clean up the node labels
defer func() {
if err := sm.nodeLabelManager.CleanUpLabels(10 * time.Second); err != nil {
log.Error("CleanUpNodeLabels", "unable to cleanup node labels", err)
// If BGP is enabled then we start a server instance that will broadcast VIPs
if sm.config.EnableBGP {
// If Annotations have been set then we will look them up
err := sm.parseAnnotations()
if err != nil {
return err
}
}()
log.Info("Starting Kube-vip Manager with the BGP engine")
return sm.startBGP()
}
if sm.config.EnableARP || sm.config.EnableWireguard {
if sm.config.EnableUPNP {
clients := upnp.GetConnectionClients(ctx)
clients := upnp.GetConnectionClients(context.TODO())
if len(clients) == 0 {
log.Error("Error Enabling UPNP. No Clients found")
// Set the struct to false so nothing should use it in future
@@ -305,145 +287,50 @@ func (sm *Manager) Start(ctx context.Context) error {
log.Info("Found UPNP IGD2 Gateway address", "ip", ip)
}
}
// TODO: It would be nice to run the UPNP refresh only on the leader.
wg.Go(func() {
sm.svcProcessor.RefreshUPNPForwards(ctx)
})
}
// TODO: It would be nice to run the UPNP refresh only on the leader.
go sm.svcProcessor.RefreshUPNPForwards()
}
// If BGP is enabled then we start a server instance that will broadcast VIPs
if sm.config.EnableBGP {
// If Annotations have been set then we will look them up
err := sm.parseAnnotations(ctx)
if err != nil {
return err
}
// If ARP is enabled then we start a LeaderElection that will use ARP to advertise VIPs
if sm.config.EnableARP {
log.Info("Starting Kube-vip Manager with the ARP engine")
return sm.startARP(sm.config.NodeName)
}
return sm.startMode(ctx)
}
// Start will begin the Manager, which will start services and watch the configmap
func (sm *Manager) startMode(ctx context.Context) error {
var cpCluster *cluster.Cluster
var err error
w := worker.New(sm.arpMgr, sm.intfMgr, sm.config, &sm.closing, sm.Kill,
sm.svcProcessor, &sm.mutex, sm.clientSet, sm.bgpServer, sm.bgpSessionInfoGauge,
sm.electionMgr, sm.leaseMgr)
// use a Go context so we can tell the leaderelection code when we
// want to step down
wg := sync.WaitGroup{}
modeCtx, cancel := context.WithCancel(ctx)
defer func() {
// wait for gorutines then cancel context just in case
wg.Wait()
w.Cleanup()
cancel()
log.Info("Shutting down Kube-Vip")
}()
log.Info("starting Kube-vip Manager", "mode", w.Name())
if err := w.Configure(modeCtx, &wg); err != nil {
return fmt.Errorf("failed to configure %s mode: %w", w.Name(), err)
if sm.config.EnableWireguard {
log.Info("Starting Kube-vip Manager with the Wireguard engine")
return sm.startWireguard(sm.config.NodeName)
}
if sm.config.EnableControlPlane {
err = w.InitControlPlane()
if err != nil {
return err
}
}
// Shutdown function that will wait on this signal, unless we call it ourselves
wg.Go(func() {
sm.waitForShutdown(modeCtx, cancel, cpCluster)
})
if sm.config.EnableControlPlane {
wg.Go(func() {
w.StartControlPlane(modeCtx, sm.electionMgr)
})
}
if sm.config.EnableServices {
// This will tidy any dangling kube-vip iptables rules
if sm.config.EgressClean {
err := nftables.ClearTables()
if err != nil {
log.Warn("[egress]", "mode", "nftables-internal", "clearing error", err)
} else {
log.Info("[egress]", "mode", "nftables-internal", "tables", "cleared")
}
// TODO: Deprecate the iptables code v1.2.x
err = vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
if err != nil {
log.Info("[egress]", "legacy-iptables", sm.config.EgressWithNftables, "mode", "IPv4", "error", err)
}
err = vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv6)
if err != nil {
log.Info("[egress]", "legacy-iptables", sm.config.EgressWithNftables, "mode", "IPv6", "error", err)
}
}
w.ConfigureServices()
for {
select {
case <-modeCtx.Done():
return nil
default:
if err = w.StartServices(modeCtx); err != nil {
if errors.Is(err, &utils.PanicError{}) {
sm.Kill()
return fmt.Errorf("failed to reconcile services, non-recoverable error: %w", err)
} else {
log.Error("failed to reconcile services, restarting", "error", err)
}
}
}
}
if sm.config.EnableRoutingTable {
log.Info("Starting Kube-vip Manager with the Routing Table engine")
return sm.startTableMode(sm.config.NodeName)
}
log.Error("prematurely exiting Load-balancer as no modes [ARP/BGP/Wireguard] are enabled")
return nil
}
func (sm *Manager) parseAnnotations(ctx context.Context) error {
func returnNameSpace() (string, error) {
if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
return ns, nil
}
return "", err
}
return "", fmt.Errorf("unable to find Namespace")
}
func (sm *Manager) parseAnnotations() error {
if sm.config.Annotations == "" {
log.Debug("No Node annotations to parse")
return nil
}
err := sm.annotationsWatcher(ctx)
err := sm.annotationsWatcher()
if err != nil {
return err
}
return nil
}
func (sm *Manager) waitForShutdown(ctx context.Context, cancel context.CancelFunc, cpCluster *cluster.Cluster) {
for {
sig := <-sm.signalChan
switch sig {
case syscall.SIGUSR1:
log.Info("Received SIGUSR1, dumping configuration")
sm.dumpConfiguration(ctx)
case syscall.SIGINT, syscall.SIGTERM:
sm.closing.Store(true)
log.Info("Received kube-vip termination, signaling shutdown")
if cpCluster != nil {
cpCluster.Stop()
}
// Cancel the context, which will in turn cancel the leadership and all goroutines
cancel()
return
}
}
}
func (sm *Manager) Kill() {
sm.sigint.Do(func() {
sm.signalChan <- syscall.SIGINT
})
}

159
pkg/manager/manager_arp.go Normal file
View File

@@ -0,0 +1,159 @@
package manager
import (
"context"
"syscall"
"time"
log "log/slog"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/vip"
)
// Start will begin the Manager, which will start services and watch the configmap
func (sm *Manager) startARP(id string) error {
var cpCluster *cluster.Cluster
var ns string
var err error
// use a Go context so we can tell the leaderelection code when we
// want to step down
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
log.Info("Start ARP/NDP advertisement")
go sm.arpMgr.StartAdvertisement(ctx)
// Shutdown function that will wait on this signal, unless we call it ourselves
go func() {
<-sm.signalChan
log.Info("Received kube-vip termination, signaling shutdown")
if sm.config.EnableControlPlane {
cpCluster.Stop()
}
// Close all go routines
close(sm.shutdownChan)
// Cancel the context, which will in turn cancel the leadership
cancel()
}()
if sm.config.EnableControlPlane {
cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
if err != nil {
return err
}
clusterManager, err := initClusterManager(sm)
if err != nil {
return err
}
go func() {
err := cpCluster.StartCluster(sm.config, clusterManager, nil)
if err != nil {
log.Error("starting control plane", "err", err)
// Trigger the shutdown of this manager instance
sm.signalChan <- syscall.SIGINT
}
}()
// Check if we're also starting the services, if not we can sit and wait on the closing channel and return here
if !sm.config.EnableServices {
<-sm.shutdownChan
log.Info("Shutting down Kube-Vip")
return nil
}
ns = sm.config.Namespace
} else {
ns, err = returnNameSpace()
if err != nil {
log.Warn("unable to auto-detect namespace, dropping to config", "namespace", sm.config.Namespace)
ns = sm.config.Namespace
}
}
// This will tidy any dangling kube-vip iptables rules
if sm.config.EgressClean {
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
}
// Start a services watcher (all kube-vip pods will watch services), upon a new service
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
} else {
log.Info("beginning services leadership", "namespace", ns, "lock name", sm.config.ServicesLeaseName, "id", id)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: sm.config.ServicesLeaseName,
Namespace: ns,
},
Client: sm.clientSet.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: id,
},
}
// start the leader election code loop
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(sm.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(sm.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
log.Error("service watcher", "err", err)
panic("") // TODO: - emulating log.fatal here
}
},
OnStoppedLeading: func() {
// we can do cleanup here
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "new leader", id)
sm.svcProcessor.Stop()
log.Error("lost leadership, restarting kube-vip")
panic("") // TODO: - emulating log.fatal here
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if sm.config.EnableNodeLabeling {
applyNodeLabel(sm.clientSet, sm.config.Address, id, identity)
}
if identity == id {
// I just got the lock
return
}
log.Info("new leader elected", "new leader", identity)
},
},
})
}
return nil
}

116
pkg/manager/manager_bgp.go Normal file
View File

@@ -0,0 +1,116 @@
package manager
import (
"context"
"fmt"
"syscall"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/cluster"
api "github.com/osrg/gobgp/v3/api"
"github.com/prometheus/client_golang/prometheus"
)
// Start will begin the Manager, which will start services and watch the configmap
func (sm *Manager) startBGP() error {
var cpCluster *cluster.Cluster
// var ns string
var err error
if sm.bgpServer == nil {
sm.bgpServer, err = bgp.NewBGPServer(&sm.config.BGPConfig)
if err != nil {
return fmt.Errorf("creating BGP server: %w", err)
}
}
log.Info("Starting the BGP server to advertise VIP routes to BGP peers")
if err := sm.bgpServer.Start(func(p *api.WatchEventResponse_PeerEvent) {
ipaddr := p.GetPeer().GetState().GetNeighborAddress()
port := uint64(179)
peerDescription := fmt.Sprintf("%s:%d", ipaddr, port)
for stateName, stateValue := range api.PeerState_SessionState_value {
metricValue := 0.0
if stateValue == int32(p.GetPeer().GetState().GetSessionState().Number()) {
metricValue = 1
}
sm.bgpSessionInfoGauge.With(prometheus.Labels{
"state": stateName,
"peer": peerDescription,
}).Set(metricValue)
}
}); err != nil {
return fmt.Errorf("starting BGP server: %w", err)
}
// use a Go context so we can tell the leaderelection code when we
// want to step down
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Defer a function to check if the bgpServer has been created and if so attempt to close it
defer func() {
if sm.bgpServer != nil {
sm.bgpServer.Close()
}
}()
// Shutdown function that will wait on this signal, unless we call it ourselves
go func() {
<-sm.signalChan
log.Info("Received termination, signaling shutdown")
if sm.config.EnableControlPlane {
if cpCluster != nil {
cpCluster.Stop()
}
}
// Cancel the context, which will in turn cancel the leadership
cancel()
}()
if sm.config.EnableControlPlane {
cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
if err != nil {
return err
}
clusterManager, err := initClusterManager(sm)
if err != nil {
return err
}
go func() {
if sm.config.EnableLeaderElection {
err = cpCluster.StartCluster(sm.config, clusterManager, sm.bgpServer)
} else {
err = cpCluster.StartVipService(sm.config, clusterManager, sm.bgpServer)
}
if err != nil {
log.Error("Control Plane", "err", err)
// Trigger the shutdown of this manager instance
sm.signalChan <- syscall.SIGINT
}
}()
// Check if we're also starting the services, if not we can sit and wait on the closing channel and return here
if !sm.config.EnableServices {
<-sm.signalChan
log.Info("Shutting down Kube-Vip")
return nil
}
}
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
return err
}
log.Info("Shutting down Kube-Vip")
return nil
}

View File

@@ -1,235 +0,0 @@
package manager
import (
"context"
"fmt"
"os"
"time"
"github.com/kube-vip/kube-vip/pkg/nftables"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// dumpConfiguration prints the current configuration to stdout when SIGUSR1 is received
func (sm *Manager) dumpConfiguration(ctx context.Context) {
sm.mutex.Lock()
defer sm.mutex.Unlock()
fmt.Printf("\n")
fmt.Printf("================================================================================\n")
fmt.Printf(" KUBE-VIP CONFIGURATION DUMP\n")
fmt.Printf("================================================================================\n")
fmt.Printf("Timestamp: %s\n", time.Now().Format(time.RFC3339))
fmt.Printf("Node Name: %s\n", sm.config.NodeName)
fmt.Printf("Process ID: %d\n", os.Getpid())
fmt.Printf("================================================================================\n")
fmt.Printf("\n")
sm.dumpConfigSection()
sm.dumpBGPSection()
sm.dumpARPSection()
sm.dumpServicesSection(ctx)
sm.dumpNetworkInterfacesSection()
sm.dumpLeaderElectionSection()
sm.dumpRuntimeSection()
sm.dumpNFTablesSection()
fmt.Printf("================================================================================\n")
fmt.Printf(" END OF CONFIGURATION DUMP\n")
fmt.Printf("================================================================================\n")
fmt.Printf("\n")
}
func (sm *Manager) dumpConfigSection() {
fmt.Printf("--- BASIC CONFIGURATION ---\n")
fmt.Printf("VIP: %s\n", sm.config.Address)
fmt.Printf("VIP Subnet: %s\n", sm.config.VIPSubnet)
fmt.Printf("Port: %d\n", sm.config.Port)
fmt.Printf("Namespace: %s\n", sm.config.Namespace)
fmt.Printf("Service Namespace: %s\n", sm.config.ServiceNamespace)
fmt.Printf("Interface: %s\n", sm.config.Interface)
fmt.Printf("Services Interface: %s\n", sm.config.ServicesInterface)
fmt.Printf("Single Node Mode: %t\n", sm.config.SingleNode)
fmt.Printf("Start As Leader: %t\n", sm.config.StartAsLeader)
fmt.Printf("\n")
}
func (sm *Manager) dumpBGPSection() {
fmt.Printf("--- BGP CONFIGURATION ---\n")
fmt.Printf("BGP Enabled: %t\n", sm.config.EnableBGP)
if sm.config.EnableBGP {
fmt.Printf("BGP AS: %d\n", sm.config.BGPConfig.AS)
fmt.Printf("BGP Router ID: %s\n", sm.config.BGPConfig.RouterID)
fmt.Printf("BGP Source IP: %s\n", sm.config.BGPConfig.SourceIP)
fmt.Printf("BGP Source Interface: %s\n", sm.config.BGPConfig.SourceIF)
fmt.Printf("BGP Hold Time: %d\n", sm.config.BGPConfig.HoldTime)
fmt.Printf("BGP Keepalive Interval: %d\n", sm.config.BGPConfig.KeepaliveInterval)
fmt.Printf("BGP Peers: %d\n", len(sm.config.BGPConfig.Peers))
for i, peer := range sm.config.BGPConfig.Peers {
fmt.Printf(" Peer %d: %s:%d (AS: %d, MultiHop: %t)\n",
i+1, peer.Address, peer.Port, peer.AS, peer.MultiHop)
}
}
fmt.Printf("\n")
}
func (sm *Manager) dumpARPSection() {
fmt.Printf("--- ARP/NDP CONFIGURATION ---\n")
fmt.Printf("ARP Enabled: %t\n", sm.config.EnableARP)
if sm.config.EnableARP {
fmt.Printf("ARP Broadcast Rate: %d\n", sm.config.ArpBroadcastRate)
}
fmt.Printf("Wireguard Enabled: %t\n", sm.config.EnableWireguard)
fmt.Printf("Routing Table Enabled: %t\n", sm.config.EnableRoutingTable)
if sm.config.EnableRoutingTable {
fmt.Printf("Routing Table ID: %d\n", sm.config.RoutingTableID)
fmt.Printf("Routing Protocol: %d\n", sm.config.RoutingProtocol)
fmt.Printf("Clean Routing Table: %t\n", sm.config.CleanRoutingTable)
}
fmt.Printf("\n")
}
func (sm *Manager) dumpServicesSection(ctx context.Context) {
fmt.Printf("--- SERVICES CONFIGURATION ---\n")
fmt.Printf("Services Enabled: %t\n", sm.config.EnableServices)
if sm.config.EnableServices {
fmt.Printf("Services Election: %t\n", sm.config.EnableServicesElection)
fmt.Printf("Load Balancer Class Only: %t\n", sm.config.LoadBalancerClassOnly)
fmt.Printf("Load Balancer Class Name: %s\n", sm.config.LoadBalancerClassName)
fmt.Printf("Disable Service Updates: %t\n", sm.config.DisableServiceUpdates)
fmt.Printf("Enable Endpoints: %t\n", sm.config.EnableEndpoints)
fmt.Printf("Service Security Enabled: %t\n", sm.config.EnableServiceSecurity)
if sm.svcProcessor != nil {
instances := sm.svcProcessor.ServiceInstances
fmt.Printf("Kube-vip Active Service Instances: %d\n", len(instances))
for i, inst := range instances {
if inst.ServiceSnapshot != nil {
svc := inst.ServiceSnapshot
vipConfigs := ""
for j, cfg := range svc.Status.LoadBalancer.Ingress {
if j > 0 {
vipConfigs += ", "
}
vipConfigs += cfg.IP
}
fmt.Printf(" Service %d: %s/%s (Type: %s, VIPs: %s)\n",
i+1, svc.Namespace, svc.Name, svc.Spec.Type, vipConfigs)
}
}
}
if sm.clientSet != nil {
fmt.Println()
// Kubernetes configuration
fmt.Println("--- KUBERNETES CONFIGURATION (SERVICES/ENDPOINTSLICES) ---")
fmt.Println("Service Configuration:")
svcList, err := sm.clientSet.CoreV1().Services(v1.NamespaceAll).List(ctx, metav1.ListOptions{})
if err != nil {
fmt.Println("Unable to retrieve all Services")
} else {
for x := range svcList.Items {
// Build all addresses
vipConfigs := ""
for j, cfg := range svcList.Items[x].Status.LoadBalancer.Ingress {
if j > 0 {
vipConfigs += ", "
}
vipConfigs += cfg.IP
}
fmt.Printf("Name=%s, UUID=%s, Addresses=%s\n", svcList.Items[x].Name, string(svcList.Items[x].UID), vipConfigs)
}
fmt.Println()
}
fmt.Println("EndpointSlice Configuration (note endpoint names have -XXXXX prefixed):")
epList, err := sm.clientSet.DiscoveryV1().EndpointSlices(v1.NamespaceAll).List(ctx, metav1.ListOptions{})
if err != nil {
fmt.Println("Unable to retrieve all EndpointSlices")
} else {
for x := range epList.Items {
// Build all addresses
fmt.Printf(" Endpoint Slice Name: %s\n", epList.Items[x].Name)
for _, ep := range epList.Items[x].Endpoints {
endpoints := ""
for i, addresses := range ep.Addresses {
if i > 0 {
endpoints += ", "
}
endpoints += addresses
}
nodeName := "Unknown"
targetPod := "Unknown"
if ep.NodeName != nil {
nodeName = *ep.NodeName
}
if ep.TargetRef != nil {
targetPod = ep.TargetRef.Name
}
fmt.Printf("\tNode: %s, Target Pod:%s, Addresses: %s\n", nodeName, targetPod, endpoints)
}
}
}
}
}
fmt.Printf("\n")
}
func (sm *Manager) dumpNetworkInterfacesSection() {
fmt.Printf("--- NETWORK INTERFACES ---\n")
fmt.Printf("Network Interface Manager: %t\n", sm.intfMgr != nil)
fmt.Printf("ARP Manager: %t\n", sm.arpMgr != nil)
fmt.Printf("\n")
}
func (sm *Manager) dumpLeaderElectionSection() {
fmt.Printf("--- LEADER ELECTION CONFIGURATION ---\n")
fmt.Printf("Control Plane Enabled: %t\n", sm.config.EnableControlPlane)
if sm.config.EnableControlPlane {
fmt.Printf("Detect Control Plane: %t\n", sm.config.DetectControlPlane)
}
fmt.Printf("Leader Election Type: %s\n", sm.config.LeaderElectionType)
fmt.Printf("Leader Election Enabled: %t\n", sm.config.EnableLeaderElection)
if sm.config.EnableLeaderElection {
fmt.Printf("Lease Name: %s\n", sm.config.LeaseName)
fmt.Printf("Lease Duration: %d seconds\n", sm.config.LeaseDuration)
fmt.Printf("Renew Deadline: %d seconds\n", sm.config.RenewDeadline)
fmt.Printf("Retry Period: %d seconds\n", sm.config.RetryPeriod)
}
fmt.Printf("Services Lease Name: %s\n", sm.config.ServicesLeaseName)
fmt.Printf("Node Labeling Enabled: %t\n", sm.config.EnableNodeLabeling)
fmt.Printf("\n")
}
func (sm *Manager) dumpRuntimeSection() {
fmt.Printf("--- RUNTIME STATISTICS ---\n")
fmt.Printf("Load Balancer Enabled: %t\n", sm.config.EnableLoadBalancer)
if sm.config.EnableLoadBalancer {
fmt.Printf("Load Balancer Port: %d\n", sm.config.LoadBalancerPort)
fmt.Printf("Load Balancer Forwarding Method: %s\n", sm.config.LoadBalancerForwardingMethod)
fmt.Printf("Load Balancers Configured: %d\n", len(sm.config.LoadBalancers))
}
fmt.Printf("Prometheus HTTP Server: %s\n", sm.config.PrometheusHTTPServer)
fmt.Printf("Health Check Port: %d\n", sm.config.HealthCheckPort)
fmt.Printf("UPNP Enabled: %t\n", sm.config.EnableUPNP)
fmt.Printf("Egress Clean Enabled: %t\n", sm.config.EgressClean)
if sm.config.EgressClean {
fmt.Printf("Egress with nftables: %t\n", sm.config.EgressWithNftables)
fmt.Printf("Egress Pod CIDR: %s\n", sm.config.EgressPodCidr)
fmt.Printf("Egress Service CIDR: %s\n", sm.config.EgressServiceCidr)
}
fmt.Printf("\n")
}
func (sm *Manager) dumpNFTablesSection() {
fmt.Printf("--- NFTABLES CONFIGURATION ---\n")
chains, err := nftables.ListChains()
if err != nil {
fmt.Printf("Unable to retrieve NFTables chains, error=%s", err)
}
for x := range chains {
fmt.Printf("Chain: %s\n", chains[x])
}
fmt.Println()
}

View File

@@ -1,239 +0,0 @@
package manager
import (
"bytes"
"context"
"io"
"os"
"testing"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/stretchr/testify/assert"
)
func TestDumpConfiguration(t *testing.T) {
config := &kubevip.Config{
Address: "192.168.1.100",
Interface: "eth0",
Port: 6443,
EnableARP: true,
EnableBGP: false,
EnableControlPlane: true,
EnableServices: true,
LeaderElectionType: "kubernetes",
Namespace: "kube-system",
NodeName: "test-node",
KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
EnableLeaderElection: true,
LeaseName: "test-lease",
},
}
mgr := &Manager{
config: config,
}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpConfiguration(context.TODO())
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "KUBE-VIP CONFIGURATION DUMP", "should contain header")
assert.Contains(t, output, "Node Name: test-node", "should contain node name")
assert.Contains(t, output, "VIP: 192.168.1.100", "should contain VIP address")
assert.Contains(t, output, "Interface: eth0", "should contain interface")
assert.Contains(t, output, "Port: 6443", "should contain port")
assert.Contains(t, output, "ARP Enabled: true", "should contain ARP status")
assert.Contains(t, output, "BGP Enabled: false", "should contain BGP status")
}
func TestDumpConfigSection(t *testing.T) {
config := &kubevip.Config{
Address: "192.168.1.100",
Interface: "eth0",
Port: 6443,
VIPSubnet: "/24",
EnableARP: true,
EnableBGP: false,
EnableControlPlane: true,
EnableServices: true,
LeaderElectionType: "kubernetes",
Namespace: "kube-system",
KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
EnableLeaderElection: true,
LeaseName: "test-lease",
},
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpConfigSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "--- BASIC CONFIGURATION ---")
assert.Contains(t, output, "VIP: 192.168.1.100")
assert.Contains(t, output, "VIP Subnet: /24")
assert.Contains(t, output, "Interface: eth0")
assert.Contains(t, output, "Port: 6443")
assert.Contains(t, output, "Namespace: kube-system")
assert.Contains(t, output, "Single Node Mode: false")
assert.Contains(t, output, "Start As Leader: false")
}
func TestDumpBGPSection(t *testing.T) {
t.Run("BGP disabled", func(t *testing.T) {
config := &kubevip.Config{
EnableBGP: false,
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpBGPSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "BGP Enabled: false")
})
t.Run("BGP enabled", func(t *testing.T) {
config := &kubevip.Config{
EnableBGP: true,
BGPConfig: kubevip.BGPConfig{
RouterID: "192.168.1.1",
AS: 65000,
Peers: []kubevip.BGPPeer{
{Address: "192.168.1.2", AS: 65001},
{Address: "192.168.1.3", AS: 65002},
},
},
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpBGPSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "BGP Enabled: true")
assert.Contains(t, output, "BGP Router ID: 192.168.1.1")
assert.Contains(t, output, "BGP AS: 65000")
assert.Contains(t, output, "BGP Peers: 2")
})
}
func TestDumpARPSection(t *testing.T) {
t.Run("ARP disabled", func(t *testing.T) {
config := &kubevip.Config{
EnableARP: false,
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpARPSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "ARP Enabled: false")
})
t.Run("ARP enabled", func(t *testing.T) {
config := &kubevip.Config{
EnableARP: true,
ArpBroadcastRate: 5,
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpARPSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "ARP Enabled: true")
assert.Contains(t, output, "ARP Broadcast Rate: 5")
})
}
func TestDumpRuntimeSection(t *testing.T) {
config := &kubevip.Config{
EnableLoadBalancer: false,
PrometheusHTTPServer: "",
HealthCheckPort: 0,
EnableUPNP: false,
EgressClean: false,
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpRuntimeSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "--- RUNTIME STATISTICS ---", "should contain runtime section header")
assert.Contains(t, output, "Load Balancer Enabled: false", "should contain load balancer status")
assert.Contains(t, output, "UPNP Enabled: false", "should contain UPNP status")
}

View File

@@ -0,0 +1,185 @@
package manager
import (
"context"
"fmt"
"syscall"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
)
// Start will begin the Manager, which will start services and watch the configmap
func (sm *Manager) startTableMode(id string) error {
var cpCluster *cluster.Cluster
var err error
// use a Go context so we can tell the leaderelection code when we
// want to step down
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
log.Info("destination for routes", "table", sm.config.RoutingTableID, "protocol", sm.config.RoutingProtocol)
if sm.config.CleanRoutingTable {
go func() {
// we assume that after 10s all services should be configured so we can delete redundant routes
time.Sleep(time.Second * 10)
if err := sm.cleanRoutes(); err != nil {
log.Error("error checking for old routes", "err", err)
}
}()
}
if sm.config.EgressClean {
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv6)
log.Debug("IPtables rules cleaned on startup")
}
// Shutdown function that will wait on this signal, unless we call it ourselves
go func() {
<-sm.signalChan
log.Info("Received kube-vip termination, signaling shutdown")
if sm.config.EnableControlPlane {
cpCluster.Stop()
}
// Cancel the context, which will in turn cancel the leadership
cancel()
}()
if sm.config.EnableControlPlane {
cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
if err != nil {
return fmt.Errorf("cluster initialization error: %w", err)
}
clusterManager, err := initClusterManager(sm)
if err != nil {
return fmt.Errorf("cluster manager initialization error: %w", err)
}
if err := cpCluster.StartVipService(sm.config, clusterManager, nil); err != nil {
log.Error("Control Plane", "err", err)
// Trigger the shutdown of this manager instance
sm.signalChan <- syscall.SIGINT
}
} else {
ns, err := returnNameSpace()
if err != nil {
log.Warn("unable to auto-detect namespace", "dropping to", sm.config.Namespace)
ns = sm.config.Namespace
}
// Start a services watcher (all kube-vip pods will watch services), upon a new service
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
} else if sm.config.EnableLeaderElection {
log.Info("beginning services leadership", "namespace", ns, "lock name", plunderLock, "id", id)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: plunderLock,
Namespace: ns,
},
Client: sm.clientSet.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: id,
},
}
// start the leader election code loop
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(sm.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(sm.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
log.Error(err.Error())
panic("")
}
},
OnStoppedLeading: func() {
// we can do cleanup here
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "id", id)
sm.svcProcessor.Stop()
log.Error("lost leadership, restarting kube-vip")
panic("")
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if identity == id {
// I just got the lock
return
}
log.Info("new leader elected", "id", identity)
},
},
})
} else {
log.Info("beginning watching services without leader election")
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
log.Error("Cannot watch services", "err", err)
}
}
}
return nil
}
func (sm *Manager) cleanRoutes() error {
sm.mutex.Lock()
defer sm.mutex.Unlock()
routes, err := vip.ListRoutes(sm.config.RoutingTableID, sm.config.RoutingProtocol)
if err != nil {
return fmt.Errorf("error getting routes: %w", err)
}
for i := range routes {
found := false
if sm.config.EnableControlPlane {
found = (routes[i].Dst.IP.String() == sm.config.Address)
} else {
found = endpoints.CountRouteReferences(&routes[i], &sm.svcProcessor.ServiceInstances) > 0
}
if !found {
err = netlink.RouteDel(&(routes[i]))
if err != nil {
log.Error("[route] deletion", "route", routes[i], "err", err)
}
log.Debug("[route] deletion", "route", routes[i])
}
}
return nil
}

View File

@@ -0,0 +1,122 @@
package manager
import (
"context"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/wireguard"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
)
// Start will begin the Manager, which will start services and watch the configmap
func (sm *Manager) startWireguard(id string) error {
var ns string
var err error
// use a Go context so we can tell the leaderelection code when we
// want to step down
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
log.Info("reading wireguard peer configuration from Kubernetes secret")
s, err := sm.clientSet.CoreV1().Secrets(sm.config.Namespace).Get(ctx, "wireguard", metav1.GetOptions{})
if err != nil {
return err
}
// parse all the details needed for Wireguard
peerPublicKey := s.Data["peerPublicKey"]
peerEndpoint := s.Data["peerEndpoint"]
privateKey := s.Data["privateKey"]
// Configure the interface to join the Wireguard VPN
err = wireguard.ConfigureInterface(string(privateKey), string(peerPublicKey), string(peerEndpoint))
if err != nil {
return err
}
// Shutdown function that will wait on this signal, unless we call it ourselves
go func() {
<-sm.signalChan
log.Info("Received termination, signaling shutdown")
// Cancel the context, which will in turn cancel the leadership
cancel()
}()
ns, err = returnNameSpace()
if err != nil {
log.Warn("unable to auto-detect namespace", "dropping to", sm.config.Namespace)
ns = sm.config.Namespace
}
// Start a services watcher (all kube-vip pods will watch services), upon a new service
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
} else {
log.Info("beginning services leadership", "namespace", ns, "lock name", plunderLock, "id", id)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: plunderLock,
Namespace: ns,
},
Client: sm.clientSet.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: id,
},
}
// start the leader election code loop
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(sm.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(sm.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
log.Error(err.Error())
panic("")
}
},
OnStoppedLeading: func() {
// we can do cleanup here
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "id", id)
sm.svcProcessor.Stop()
log.Error("lost leadership, restarting kube-vip")
panic("")
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if identity == id {
// I just got the lock
return
}
log.Info("new leader elected", "id", identity)
},
},
})
}
return nil
}

View File

@@ -1,4 +1,4 @@
package worker
package manager
import (
"context"
@@ -7,12 +7,16 @@ import (
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
)
const (
nodeLabelIndex = "kube-vip.io/has-ip"
nodeLabelJSONPath = `kube-vip.io~1has-ip`
)
type patchStringLabel struct {
Op string `json:"op"`
Path string `json:"path"`
@@ -21,7 +25,8 @@ type patchStringLabel struct {
// applyNodeLabel add/remove node label `kube-vip.io/has-ip=<VIP-Address>` to/from
// the node where the virtual IP was added to/removed from.
func applyNodeLabel(ctx context.Context, clientSet *kubernetes.Clientset, address, id, identity string) {
func applyNodeLabel(clientSet *kubernetes.Clientset, address, id, identity string) {
ctx := context.Background()
node, err := clientSet.CoreV1().Nodes().Get(ctx, id, metav1.GetOptions{})
if err != nil {
log.Error("can't query node labels", "node", id, "err", err)
@@ -30,8 +35,8 @@ func applyNodeLabel(ctx context.Context, clientSet *kubernetes.Clientset, addres
log.Debug(fmt.Sprintf("node %s labels: %+v", id, node.Labels))
value, ok := node.Labels[kubevip.HasIP]
path := fmt.Sprintf("/metadata/labels/%s", kubevip.HasIPJSONPath)
value, ok := node.Labels[nodeLabelIndex]
path := fmt.Sprintf("/metadata/labels/%s", nodeLabelJSONPath)
log.Debug(fmt.Sprintf("Received identity: %s - id: %s", identity, id))
if ok && value == address {
log.Debug(fmt.Sprintf("removing node label `has-ip=%s` on %s", address, id))

View File

@@ -9,7 +9,7 @@ func (sm *Manager) PrometheusCollector() []prometheus.Collector {
collectors = append(collectors, sm.svcProcessor.CountServiceWatchEvent)
}
if sm.bgpServer != nil {
collectors = append(collectors, sm.bgpSessionInfoGauge)
collectors = append(collectors, sm.bgpServer.BGPSessionInfoGauge)
}
return collectors
}

View File

@@ -10,7 +10,7 @@ import (
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/davecgh/go-spew/spew"
v1 "k8s.io/api/core/v1"
@@ -25,7 +25,7 @@ import (
// This file handles the watching of node annotations for configuration, it will exit once the annotations are
// present
func (sm *Manager) annotationsWatcher(ctx context.Context) error {
func (sm *Manager) annotationsWatcher() error {
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
log.Info("Kube-Vip is waiting for annotation prefix to be present on this node", "prefix", sm.config.Annotations)
@@ -36,7 +36,7 @@ func (sm *Manager) annotationsWatcher(ctx context.Context) error {
// First we'll check the annotations for the node and if
// they aren't what are expected, we'll drop into the watch until they are
nodeList, err := sm.clientSet.CoreV1().Nodes().List(ctx, listOptions)
nodeList, err := sm.clientSet.CoreV1().Nodes().List(context.Background(), listOptions)
if err != nil {
return err
}
@@ -57,19 +57,30 @@ func (sm *Manager) annotationsWatcher(ctx context.Context) error {
// they're as needed
log.Warn(err.Error())
rw, err := watchtools.NewRetryWatcherWithContext(ctx, node.ResourceVersion, &cache.ListWatch{
// TODO, will need refactoring as part of rikatz work
rw, err := watchtools.NewRetryWatcherWithContext(context.TODO(), node.ResourceVersion, &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.rwClientSet.CoreV1().Nodes().Watch(ctx, listOptions)
return sm.rwClientSet.CoreV1().Nodes().Watch(context.Background(), listOptions)
},
})
if err != nil {
return fmt.Errorf("error creating annotations watcher: %s", err.Error())
}
defer func() {
rw.Stop()
log.Debug("[annotations] watcher stopped")
exitFunction := make(chan struct{})
go func() {
select {
case <-sm.shutdownChan:
log.Debug("[annotations] shutdown called")
// Stop the retry watcher
rw.Stop()
return
case <-exitFunction:
log.Debug("[annotations] function ending")
// Stop the retry watcher
rw.Stop()
return
}
}()
ch := rw.ResultChan()
@@ -92,8 +103,7 @@ func (sm *Manager) annotationsWatcher(ctx context.Context) error {
sm.config.BGPConfig = bgpConfig
sm.config.BGPPeerConfig = bgpPeer
log.Info("[annotations] exiting Annotations watcher - annotations found")
return nil
rw.Stop()
case watch.Deleted:
node, ok := event.Object.(*v1.Node)
if !ok {
@@ -120,8 +130,10 @@ func (sm *Manager) annotationsWatcher(ctx context.Context) error {
default:
}
}
log.Info("[annotations] exiting annotations watcher")
close(exitFunction)
log.Info("Exiting Annotations watcher")
return nil
}
// parseNodeAnnotations parses the annotations on the node and updates the configuration
@@ -139,8 +151,8 @@ func (sm *Manager) annotationsWatcher(ctx context.Context) error {
// * `<info>` is the relevant information, such as `node-asn` or `peer-ip`
// * `{{n}}` is the number of the peer, always starting with `0`
// * kube-vip is only designed to manage one peer, just look for {{n}} == 0
func parseBgpAnnotations(bgpConfig kubevip.BGPConfig, node *v1.Node, prefix string) (kubevip.BGPConfig, kubevip.BGPPeer, error) {
bgpPeer := kubevip.BGPPeer{}
func parseBgpAnnotations(bgpConfig bgp.Config, node *v1.Node, prefix string) (bgp.Config, bgp.Peer, error) {
bgpPeer := bgp.Peer{}
nodeASN := ""
for k, v := range node.Annotations {
@@ -204,7 +216,7 @@ func parseBgpAnnotations(bgpConfig kubevip.BGPConfig, node *v1.Node, prefix stri
peerIPs := strings.Split(peerIPString, ",")
bgpConfig.Peers = make([]kubevip.BGPPeer, 0, len(peerIPs))
bgpConfig.Peers = make([]bgp.Peer, 0, len(peerIPs))
for _, peerIP := range peerIPs {
ipAddr := strings.TrimSpace(peerIP)

View File

@@ -4,7 +4,7 @@ import (
"reflect"
"testing"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -15,7 +15,7 @@ func TestParseBgpAnnotations(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{Name: "test", Annotations: map[string]string{}},
}
bgpConfigBase := kubevip.BGPConfig{
bgpConfigBase := bgp.Config{
HoldTime: 15,
KeepaliveInterval: 5,
}
@@ -54,7 +54,7 @@ func TestParseBgpAnnotations(t *testing.T) {
t.Fatal("Parsing BGP annotations should return nil when minimum config is met")
}
bgpPeers := []kubevip.BGPPeer{
bgpPeers := []bgp.Peer{
{Address: "10.0.0.1", AS: uint32(64000), Password: "password"},
{Address: "10.0.0.2", AS: uint32(64000), Password: "password"},
{Address: "10.0.0.3", AS: uint32(64000), Password: "password"},
@@ -77,7 +77,7 @@ func TestParseNewBgpAnnotations(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{Name: "test", Annotations: map[string]string{}},
}
bgpConfigBase := kubevip.BGPConfig{
bgpConfigBase := bgp.Config{
HoldTime: 15,
KeepaliveInterval: 5,
}
@@ -99,7 +99,7 @@ func TestParseNewBgpAnnotations(t *testing.T) {
t.Fatalf("Parsing BGP annotations should return nil when minimum config is met [%v]", err)
}
bgpPeers := []kubevip.BGPPeer{
bgpPeers := []bgp.Peer{
{Address: "10.0.0.1", AS: uint32(64000), Password: "password"},
{Address: "10.0.0.2", AS: uint32(64000), Password: "password"},
{Address: "10.0.0.3", AS: uint32(64000), Password: "password"},
@@ -121,15 +121,15 @@ func Test_parseBgpAnnotations(t *testing.T) {
tests := []struct {
name string
args args
want kubevip.BGPConfig
want1 kubevip.BGPPeer
want bgp.Config
want1 bgp.Peer
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, got1, err := parseBgpAnnotations(kubevip.BGPConfig{}, tt.args.node, tt.args.prefix)
got, got1, err := parseBgpAnnotations(bgp.Config{}, tt.args.node, tt.args.prefix)
if (err != nil) != tt.wantErr {
t.Errorf("parseBgpAnnotations() error = %v, wantErr %v", err, tt.wantErr)
return

View File

@@ -1,69 +0,0 @@
package worker
import (
"context"
log "log/slog"
"sync"
"sync/atomic"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/services"
"k8s.io/client-go/kubernetes"
)
type ARP struct {
Common
}
func NewARP(arpMgr *arp.Manager, intfMgr *networkinterface.Manager,
config *kubevip.Config, closing *atomic.Bool, killFunc func(),
svcProcessor *services.Processor, mutex *sync.Mutex, clientSet *kubernetes.Clientset,
electionMgr *election.Manager, leaseMgr *lease.Manager) *ARP {
return &ARP{
Common: *newCommon(arpMgr, intfMgr, config, closing, killFunc,
svcProcessor, mutex, clientSet, electionMgr, leaseMgr),
}
}
func (a *ARP) Configure(ctx context.Context, wg *sync.WaitGroup) error {
log.Info("Start ARP/NDP advertisement Global")
wg.Go(func() {
a.arpMgr.StartAdvertisement(ctx)
})
return nil
}
func (a *ARP) StartControlPlane(ctx context.Context, electionManager *election.Manager) {
err := a.cpCluster.StartCluster(ctx, a.config, electionManager, nil, a.leaseMgr, a.killFunc)
if err != nil {
log.Error("starting control plane", "err", err)
}
// Trigger the shutdown of this manager instance
a.killFunc()
}
func (a *ARP) ConfigureServices() {
}
func (a *ARP) StartServices(ctx context.Context) error {
// Start a services watcher (all kube-vip pods will watch services), upon a new service
// a lock based upon that service is created that they will all leaderElection on
if a.config.EnableServicesElection {
if err := a.PerServiceLeader(ctx); err != nil {
return err
}
} else {
a.GlobalLeader(ctx, a.config.ServicesLeaseName)
}
return nil
}
func (a *ARP) Name() string {
return "ARP"
}

View File

@@ -1,128 +0,0 @@
package worker
import (
"context"
"fmt"
log "log/slog"
"sync"
"sync/atomic"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/services"
api "github.com/osrg/gobgp/v3/api"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/client-go/kubernetes"
)
type BGP struct {
Common
bgpServer *bgp.Server
bgpSessionInfoGauge *prometheus.GaugeVec
}
func NewBGP(arpMgr *arp.Manager, intfMgr *networkinterface.Manager,
config *kubevip.Config, closing *atomic.Bool, killFunc func(),
svcProcessor *services.Processor, mutex *sync.Mutex, clientSet *kubernetes.Clientset,
bgpServer *bgp.Server, bgpSessionInfoGauge *prometheus.GaugeVec,
electionMgr *election.Manager, leaseMgr *lease.Manager) *BGP {
return &BGP{
Common: *newCommon(arpMgr, intfMgr, config, closing, killFunc,
svcProcessor, mutex, clientSet, electionMgr, leaseMgr),
bgpServer: bgpServer,
bgpSessionInfoGauge: bgpSessionInfoGauge,
}
}
func (b *BGP) Configure(ctx context.Context, _ *sync.WaitGroup) error {
var err error
if b.bgpServer == nil {
b.bgpServer, err = bgp.NewBGPServer(b.config.BGPConfig)
if err != nil {
return fmt.Errorf("creating BGP server: %w", err)
}
}
log.Info("Starting the BGP server to advertise VIP routes to BGP peers")
if err := b.bgpServer.Start(ctx, func(p *api.WatchEventResponse_PeerEvent) {
ipaddr := p.GetPeer().GetState().GetNeighborAddress()
port := uint64(179)
peerDescription := fmt.Sprintf("%s:%d", ipaddr, port)
for stateName, stateValue := range api.PeerState_SessionState_value {
metricValue := 0.0
if stateValue == int32(p.GetPeer().GetState().GetSessionState().Number()) {
metricValue = 1
}
b.bgpSessionInfoGauge.With(prometheus.Labels{
"state": stateName,
"peer": peerDescription,
}).Set(metricValue)
}
}); err != nil {
return fmt.Errorf("starting BGP server: %w", err)
}
return nil
}
func (b *BGP) Cleanup() {
// Defer a function to check if the bgpServer has been created and if so attempt to close it
if b.bgpServer != nil {
b.bgpServer.Close()
}
}
func (b *BGP) StartControlPlane(ctx context.Context, electionManager *election.Manager) {
var err error
if b.config.EnableLeaderElection {
err = b.cpCluster.StartCluster(ctx, b.config, electionManager, b.bgpServer, b.leaseMgr, b.killFunc)
} else {
err = b.cpCluster.StartVipService(ctx, b.config, electionManager, b.bgpServer, b.killFunc)
}
if err != nil {
log.Error("Control Plane", "err", err)
// Trigger the shutdown of this manager instance
b.killFunc()
}
}
func (b *BGP) ConfigureServices() {
// No configuration required
}
func (b *BGP) StartServices(ctx context.Context) error {
if b.config.EnableServicesElection {
if err := b.PerServiceLeader(ctx); err != nil {
return err
}
} else {
if err := b.ServicesNoLeader(ctx); err != nil {
return err
}
}
return nil
}
func (b *BGP) ServicesGlobalLeader(ctx context.Context, id string) {
// NOT IMPLEMENTED
}
func (b *BGP) ServicesNoLeader(ctx context.Context) error {
log.Info("beginning watching services without leader election")
err := b.svcProcessor.ServicesWatcher(ctx, b.svcProcessor.SyncServices)
if err != nil {
return err
}
return nil
}
func (b *BGP) Name() string {
return "BGP"
}

View File

@@ -1,216 +0,0 @@
package worker
import (
"context"
"fmt"
log "log/slog"
"sync"
"sync/atomic"
"time"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/services"
"k8s.io/client-go/kubernetes"
)
type Common struct {
arpMgr *arp.Manager
cpCluster *cluster.Cluster
intfMgr *networkinterface.Manager
config *kubevip.Config
closing *atomic.Bool
killFunc func()
svcProcessor *services.Processor
mutex *sync.Mutex
clientSet *kubernetes.Clientset
electionMgr *election.Manager
leaseMgr *lease.Manager
}
func newCommon(arpMgr *arp.Manager, intfMgr *networkinterface.Manager,
config *kubevip.Config, closing *atomic.Bool, killFunc func(),
svcProcessor *services.Processor, mutex *sync.Mutex, clientSet *kubernetes.Clientset,
electionMgr *election.Manager, leaseMgr *lease.Manager) *Common {
return &Common{
arpMgr: arpMgr,
intfMgr: intfMgr,
config: config,
closing: closing,
killFunc: killFunc,
svcProcessor: svcProcessor,
mutex: mutex,
clientSet: clientSet,
electionMgr: electionMgr,
leaseMgr: leaseMgr,
}
}
func (c *Common) InitControlPlane() error {
var err error
c.cpCluster, err = cluster.InitCluster(c.config, false, c.intfMgr, c.arpMgr)
if err != nil {
return fmt.Errorf("cluster initialization error: %w", err)
}
return nil
}
func (c *Common) PerServiceLeader(ctx context.Context) error {
log.Info("beginning watching services, leaderelection will happen for every service")
err := c.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
return nil
}
func (c *Common) GlobalLeader(ctx context.Context, leaseName string) {
c.runGlobalElection(ctx, c, leaseName, c.config, c.electionMgr)
}
func (c *Common) ServicesNoLeader(ctx context.Context) error {
log.Info("beginning watching services without leader election")
err := c.svcProcessor.ServicesWatcher(ctx, c.svcProcessor.SyncServices)
if err != nil {
return fmt.Errorf("error while watching services: %w", err)
}
return nil
}
func (c *Common) Cleanup() {
// NOT IMPLEMENTED
}
func (c *Common) OnStartedLeading(ctx context.Context) {
err := c.svcProcessor.ServicesWatcher(ctx, c.svcProcessor.SyncServices)
if err != nil {
log.Error("service watcher", "err", err)
c.killFunc()
}
}
func (c *Common) OnStoppedLeading() {
// we can do cleanup here
c.mutex.Lock()
defer c.mutex.Unlock()
log.Info("leader lost", "former leader", c.config.NodeName)
c.svcProcessor.Stop()
log.Error("lost services leadership, restarting kube-vip")
c.killFunc()
}
func (c *Common) OnNewLeader(identity string) {
// we're notified when new leader elected
if c.config.EnableNodeLabeling {
labelCtx, cancel := context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
applyNodeLabel(labelCtx, c.clientSet, c.config.Address, c.config.NodeName, identity)
}
if identity == c.config.NodeName {
// I just got the lock
return
}
log.Info("new leader elected", "new leader", identity)
}
func (c *Common) runGlobalElection(ctx context.Context, a election.Actions, leaseName string,
config *kubevip.Config, electionManager *election.Manager) {
ns, leaseName := lease.NamespaceName(leaseName, config)
leaseID := lease.NewID(config.LeaderElectionType, ns, leaseName)
objectName := lease.ObjectName(leaseID, "svcs0")
// objLease, isNew, isSharedLease := c.leaseMgr.Add(leaseID, objectName)
objLease := c.leaseMgr.Add(ctx, leaseID)
isNew := objLease.Add(objectName)
// this service was already processed so we do not need to do anything
if !isNew {
log.Debug("this election was already done, waiting for it to finish", "lease", c.config.ServicesLeaseName)
// Wait for either the service context or lease context to be done
select {
case <-ctx.Done():
// Service was deleted
c.leaseMgr.Delete(leaseID, objectName)
case <-objLease.Ctx.Done():
// Leader election ended (leadership lost or context cancelled)
}
return
}
objLease.Lock()
defer func() {
objLease.Unlock()
}()
if objLease.Elected.Load() {
objLease.Unlock()
log.Debug("this election was already done, shared lease", "lease", leaseID.Name())
// wait for leader election to start or context to be done
select {
case <-objLease.Started:
case <-objLease.Ctx.Done():
// Lease was cancelled (e.g., leader election ended), return immediately
// This allows the restart loop to create a fresh lease
log.Debug("lease context cancelled before leader election started", "lease", leaseID.Name())
return
}
a.OnStartedLeading(objLease.Ctx)
log.Debug("waiting for lease to finish", "lease", leaseID.Name())
// wait for leaderelection to be finished
<-objLease.Ctx.Done()
// we can do cleanup here
a.OnStoppedLeading()
log.Error("lost leadership, restarting kube-vip", "lease", leaseID.Name())
c.killFunc()
return
}
// For new leases (not shared), ensure cleanup when the leader election ends
// This is critical for the restartable service watcher to be able to restart
// the leader election after leadership loss
defer func() {
// Delete the lease from the manager so subsequent calls can create a fresh lease
// This handles the case where leader election ends due to:
// 1. Leadership loss (e.g., network timeout)
// 2. Context cancellation
// 3. Any other reason RunOrDie returns
c.leaseMgr.Delete(leaseID, objectName)
}()
run := &election.RunConfig{
Config: config,
LeaseID: leaseID,
LeaseAnnotations: map[string]string{},
Mgr: electionManager,
OnStartedLeading: func(ctx context.Context) {
objLease.Elected.Store(true)
objLease.Unlock()
close(objLease.Started)
a.OnStartedLeading(ctx)
},
OnStoppedLeading: func() {
objLease.Elected.Store(false)
a.OnStoppedLeading()
},
OnNewLeader: a.OnNewLeader,
}
if err := election.RunOrDie(ctx, run, config); err != nil {
log.Error("leaderelection failed", "err", err, "id", config.NodeName, "name", leaseID.Name())
}
}

View File

@@ -1,113 +0,0 @@
package worker
import (
"context"
"fmt"
log "log/slog"
"sync"
"sync/atomic"
"time"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/services"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
"k8s.io/client-go/kubernetes"
)
type Table struct {
Common
}
func NewTable(arpMgr *arp.Manager, intfMgr *networkinterface.Manager,
config *kubevip.Config, closing *atomic.Bool, killFUnc func(),
svcProcessor *services.Processor, mutex *sync.Mutex, clientSet *kubernetes.Clientset,
electionMgr *election.Manager, leaseMgr *lease.Manager) *Table {
return &Table{
Common: *newCommon(arpMgr, intfMgr, config, closing, killFUnc,
svcProcessor, mutex, clientSet, electionMgr, leaseMgr),
}
}
func (t *Table) Configure(ctx context.Context, wg *sync.WaitGroup) error {
log.Info("destination for routes", "table", t.config.RoutingTableID, "protocol", t.config.RoutingProtocol)
if t.config.CleanRoutingTable {
wg.Go(func() {
// we assume that after 10s all services should be configured so we can delete redundant routes
time.Sleep(time.Second * 10)
if err := t.cleanRoutes(); err != nil {
log.Error("error checking for old routes", "err", err)
}
})
}
return nil
}
func (t *Table) StartControlPlane(ctx context.Context, electionManager *election.Manager) {
if err := t.cpCluster.StartVipService(ctx, t.config, electionManager, nil, t.killFunc); err != nil {
log.Error("Control Plane", "err", err)
// Trigger the shutdown of this manager instance
t.killFunc()
} else {
log.Debug("start VipServer for cluster manager successful")
}
}
func (t *Table) ConfigureServices() {
// No configuration required
}
func (t *Table) StartServices(ctx context.Context) error {
log.Debug("starting Services")
if t.config.EnableServicesElection {
if err := t.PerServiceLeader(ctx); err != nil {
return err
}
} else if t.config.EnableLeaderElection {
t.GlobalLeader(ctx, t.config.ServicesLeaseName)
} else {
if err := t.ServicesNoLeader(ctx); err != nil {
return err
}
}
return nil
}
func (t *Table) Name() string {
return "Routing Table"
}
func (t *Table) cleanRoutes() error {
t.mutex.Lock()
defer t.mutex.Unlock()
routes, err := vip.ListRoutes(t.config.RoutingTableID, t.config.RoutingProtocol)
if err != nil {
return fmt.Errorf("error getting routes: %w", err)
}
for i := range routes {
found := false
if t.config.EnableControlPlane {
found = (routes[i].Dst.IP.String() == t.config.Address)
} else {
found = endpoints.CountRouteReferences(&routes[i], &t.svcProcessor.ServiceInstances) > 0
}
if !found {
err = netlink.RouteDel(&(routes[i]))
if err != nil {
log.Error("[route] deletion", "route", routes[i], "err", err)
}
log.Debug("[route] deletion", "route", routes[i])
}
}
return nil
}

View File

@@ -1,274 +0,0 @@
package worker
import (
"context"
"fmt"
log "log/slog"
"os"
"sync"
"sync/atomic"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/services"
"github.com/kube-vip/kube-vip/pkg/sysctl"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/wireguard"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
)
type WireGuard struct {
Common
tunnelMgr *wireguard.TunnelManager
kubeAPIHost string
kubeAPIPort string
endpointWatcherCtx context.Context
endpointWatcherStop context.CancelFunc
endpointWatcherWg sync.WaitGroup
}
func NewWireGuard(arpMgr *arp.Manager, intfMgr *networkinterface.Manager,
config *kubevip.Config, closing *atomic.Bool, killFUnc func(),
svcProcessor *services.Processor, mutex *sync.Mutex, clientSet *kubernetes.Clientset,
electionMgr *election.Manager, leaseMgr *lease.Manager) *WireGuard {
return &WireGuard{
Common: *newCommon(arpMgr, intfMgr, config, closing, killFUnc,
svcProcessor, mutex, clientSet, electionMgr, leaseMgr),
}
}
func (w *WireGuard) Configure(ctx context.Context, _ *sync.WaitGroup) error {
log.Info("reading wireguard tunnel configurations from Kubernetes secret")
tunnelMgr := wireguard.NewTunnelManager()
err := tunnelMgr.LoadConfigurationsFromSecret(ctx, w.clientSet, w.config.Namespace, "wireguard")
if err != nil {
return fmt.Errorf("failed to load WireGuard tunnel configurations: %w", err)
}
// Clean up any stale resources from previous runs (crash recovery for hostNetwork: true)
// Must be called AFTER loading configs so we know which interfaces/ports to clean
if err := tunnelMgr.CleanupStaleResources(); err != nil {
log.Warn("failed to cleanup stale resources", "err", err)
// Continue anyway - the cleanup is best-effort
}
if _, err := sysctl.EnableProcSys("/proc/sys/net/ipv4/conf/all/src_valid_mark"); err != nil {
return fmt.Errorf("net.ipv4.conf.all.src_valid_mark is disabled and could not be enabled %w", err)
}
if _, err := sysctl.EnableProcSys("/proc/sys/net/ipv4/conf/all/route_localnet"); err != nil {
return fmt.Errorf("net.ipv4.conf.all.route_localnet is disabled and could not be enabled %w", err)
}
w.tunnelMgr = tunnelMgr
configuredVIPs := tunnelMgr.ListConfiguredTunnels()
log.Info("loaded WireGuard tunnel configurations", "vips", configuredVIPs)
return nil
}
func (w *WireGuard) InitControlPlane() error {
// Get Kubernetes service IP and port from environment
w.kubeAPIHost = os.Getenv("KUBERNETES_SERVICE_HOST")
w.kubeAPIPort = os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")
if w.kubeAPIHost == "" || w.kubeAPIPort == "" {
return fmt.Errorf("KUBERNETES_SERVICE_HOST or KUBERNETES_SERVICE_PORT_HTTPS not set")
}
return nil
}
func (w *WireGuard) StartControlPlane(ctx context.Context, electionManager *election.Manager) {
if !w.tunnelMgr.HasConfigForVIP(w.config.VIP) {
log.Error("no WireGuard tunnel configuration found for control plane VIP", "vip", w.config.VIP)
return
}
w.runGlobalElection(ctx, w, w.config.LeaseName, w.config, electionManager)
}
func (w *WireGuard) ConfigureServices() {
w.svcProcessor.TunnelMgr = w.tunnelMgr
}
func (w *WireGuard) StartServices(ctx context.Context) error {
if w.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err := w.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
}
return nil
}
func (w *WireGuard) Name() string {
return "WireGuard"
}
func (w *WireGuard) OnStartedLeading(ctx context.Context) {
// Bring up the WireGuard tunnel for control plane VIP
err := w.tunnelMgr.BringUpTunnelForVIP(w.config.VIP)
if err != nil {
log.Error("could not start wireguard tunnel for control plane", "vip", w.config.VIP, "err", err)
_ = w.tunnelMgr.TearDownTunnelForVIP(w.config.VIP)
w.killFunc()
return
}
// Get the tunnel to access its configuration
wg := w.tunnelMgr.GetTunnelForVIP(w.config.VIP)
if wg == nil {
log.Error("failed to get wireguard tunnel after bringing up", "vip", w.config.VIP)
w.killFunc()
return
}
tunnelConfig := w.tunnelMgr.GetConfigForVIP(w.config.VIP)
if tunnelConfig == nil {
log.Error("failed to get tunnel configuration", "vip", w.config.VIP)
_ = w.tunnelMgr.TearDownTunnelForVIP(w.config.VIP)
w.killFunc()
return
}
// Start endpoint watcher - DNAT rules will be applied when endpoints arrive
w.endpointWatcherCtx, w.endpointWatcherStop = context.WithCancel(ctx)
w.endpointWatcherWg.Go(func() {
w.watchKubernetesEndpoints(w.endpointWatcherCtx, tunnelConfig)
})
if w.config.EnableServices && !w.config.EnableServicesElection {
if err := w.svcProcessor.ServicesWatcher(ctx, w.svcProcessor.SyncServices); err != nil {
log.Error("failed to start services watcher", "err", err)
}
}
}
// watchKubernetesEndpoints watches the kubernetes service EndpointSlices for changes
// and updates the DNAT rules when API server endpoints change (e.g., when an API server goes down)
func (w *WireGuard) watchKubernetesEndpoints(ctx context.Context, tunnelConfig *wireguard.TunnelConfig) {
log.Info("starting kubernetes endpoint watcher for control plane")
kubeSvc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "kubernetes",
Namespace: "default",
},
}
provider := providers.NewEndpointslices()
rw, err := provider.CreateRetryWatcher(ctx, w.clientSet, kubeSvc)
if err != nil {
log.Error("failed to create kubernetes endpoint watcher", "err", err)
return
}
defer rw.Stop()
for event := range rw.ResultChan() {
select {
case <-ctx.Done():
log.Info("kubernetes endpoint watcher stopped")
return
default:
}
switch event.Type {
case watch.Added, watch.Modified, watch.Deleted:
if err := provider.LoadObject(event.Object, func() {}); err != nil {
log.Error("failed to load endpoint object", "err", err)
continue
}
endpoints, _ := provider.GetAllEndpoints()
log.Info("kubernetes endpoints changed, updating DNAT rules", "eventType", event.Type, "endpoints", endpoints)
if err := w.updateControlPlaneDNAT(tunnelConfig, endpoints); err != nil {
log.Error("failed to update control plane DNAT rules", "err", err)
}
case watch.Error:
log.Warn("kubernetes endpoint watch error", "event", event)
}
}
}
// updateControlPlaneDNAT updates the DNAT rules for the control plane with the given endpoints
func (w *WireGuard) updateControlPlaneDNAT(tunnelConfig *wireguard.TunnelConfig, endpoints []string) error {
if len(endpoints) == 0 {
log.Warn("no kubernetes API server endpoints available")
// Don't delete rules - keep routing to last known endpoints
return nil
}
// Build targets with default port 6443
targets := make([]nftables.DNATTarget, len(endpoints))
for i, ep := range endpoints {
targets[i] = nftables.DNATTarget{IP: ep, Port: 6443}
}
vipIP := utils.StripCIDR(w.config.VIP)
err := nftables.ApplyDNAT(
tunnelConfig.InterfaceName,
vipIP,
6443,
targets,
"controlplane",
v1.ProtocolTCP,
false,
tunnelConfig.ListenPort,
)
if err != nil {
return fmt.Errorf("failed to apply updated DNAT rule: %w", err)
}
log.Info("control plane DNAT rules updated", "targetCount", len(targets))
return nil
}
func (w *WireGuard) OnStoppedLeading() {
// we can do cleanup here
w.mutex.Lock()
defer w.mutex.Unlock()
log.Info("leader lost", "id", w.config.NodeName)
// Stop the kubernetes endpoint watcher and wait for it to finish
if w.endpointWatcherStop != nil {
w.endpointWatcherStop()
w.endpointWatcherWg.Wait()
}
log.Info("deleting nftables DNAT chains")
err := nftables.DeleteIngressChains(false, "controlplane")
if err != nil {
log.Error("could not delete DNAT ingress chains", "err", err)
} else {
log.Info("nftables DNAT chains deleted successfully")
}
// Tear down all tunnels (control plane + services)
err = w.tunnelMgr.TearDownAllTunnels()
if err != nil {
log.Error("failed to tear down tunnels", "err", err)
}
if w.config.EnableServices && !w.config.EnableServicesElection {
w.svcProcessor.Stop()
}
log.Error("lost control plane leadership, restarting kube-vip")
w.killFunc()
}
func (w *WireGuard) OnNewLeader(identity string) {
// we're notified when new leader elected
if identity == w.config.NodeName {
// I just got the lock
return
}
// safety check - tear down tunnel if we're not the leader
_ = w.tunnelMgr.TearDownTunnelForVIP(w.config.VIP)
log.Info("new leader elected", "id", identity)
}

View File

@@ -1,56 +0,0 @@
package worker
import (
"context"
"sync"
"sync/atomic"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/services"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/client-go/kubernetes"
)
type Worker interface {
Configure(context.Context, *sync.WaitGroup) error
InitControlPlane() error
StartControlPlane(context.Context, *election.Manager)
ConfigureServices()
StartServices(ctx context.Context) error
Name() string
Cleanup()
}
func New(arpMgr *arp.Manager, intfMgr *networkinterface.Manager,
config *kubevip.Config, closing *atomic.Bool, killFunc func(),
svcProcessor *services.Processor, mutex *sync.Mutex, clientSet *kubernetes.Clientset,
bgpServer *bgp.Server, bgpSessionInfoGauge *prometheus.GaugeVec,
electionMgr *election.Manager, leaseMgr *lease.Manager) Worker {
if config.EnableARP {
return NewARP(arpMgr, intfMgr, config, closing, killFunc,
svcProcessor, mutex, clientSet, electionMgr, leaseMgr)
}
if config.EnableBGP {
return NewBGP(arpMgr, intfMgr, config, closing, killFunc,
svcProcessor, mutex, clientSet, bgpServer, bgpSessionInfoGauge,
electionMgr, leaseMgr)
}
if config.EnableRoutingTable {
return NewTable(arpMgr, intfMgr, config, closing, killFunc,
svcProcessor, mutex, clientSet, electionMgr, leaseMgr)
}
if config.EnableWireguard {
return NewWireGuard(arpMgr, intfMgr, config, closing, killFunc,
svcProcessor, mutex, clientSet, electionMgr, leaseMgr)
}
return nil
}

View File

@@ -1,7 +1,6 @@
package networkinterface
import (
log "log/slog"
"sync"
"github.com/vishvananda/netlink"
@@ -24,12 +23,6 @@ func NewManager() *Manager {
func (m *Manager) Get(intf netlink.Link) *Link {
if l, ok := m.interfaces[intf.Attrs().Name]; ok {
updated, err := netlink.LinkByName(l.Intf.Attrs().Name)
if err != nil {
log.Error("failed to get interface %q: %w", l.Intf.Attrs().Name, err)
return nil
}
l.Intf = updated
return l
}
result := &Link{

File diff suppressed because it is too large Load Diff

View File

@@ -1,165 +0,0 @@
package labeler
import (
"context"
"encoding/hex"
"encoding/json"
"fmt"
"net"
"strings"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
)
const (
// labelName is the name of the label that will be added to the node
// it is prefix for the label key before "/"
labelName = "service-provided.kube-vip.io"
)
// labelOperation is the operation to perform on the node labels
type labelOperation string
// labelOperation constants
const (
labelOperationRemove labelOperation = "remove"
labelOperationAdd labelOperation = "add"
)
// NewManager creates a new Label Manager for the given node
func NewManager(nodeName string, clientSet *kubernetes.Clientset) *Manager {
return &Manager{
nodeName: nodeName,
clientSet: clientSet,
}
}
// Manager is the label Manager for the node
type Manager struct {
// nodeName is the name of the node to manage
nodeName string
// clientSet is the Kubernetes client set to use
clientSet *kubernetes.Clientset
}
// AddLabel a new label to the node
func (m *Manager) AddLabel(ctx context.Context, svc *corev1.Service) error {
log.Debug("[service] add label to node", "namespace", svc.Namespace, "name", svc.Name)
labelKey, labelValue := generateNodeLabelKeyValue(svc)
return m.patchNode(ctx, labelOperationAdd, map[string]string{labelKey: labelValue})
}
// RemoveLabel a label from the node
func (m *Manager) RemoveLabel(ctx context.Context, svc *corev1.Service) error {
log.Debug("[service] delete label from node", "namespace", svc.Namespace, "name", svc.Name)
labelKey, _ := generateNodeLabelKeyValue(svc)
return m.patchNode(ctx, labelOperationRemove, map[string]string{labelKey: ""})
}
// clean up the node labels
func (m *Manager) CleanUpLabels(timeout time.Duration) error {
log.Debug("cleaning up labels for node", "node", m.nodeName, "timeout", timeout)
// create new context for labels cleanup (independent)
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
// get the node
node, err := m.clientSet.CoreV1().Nodes().Get(ctx, m.nodeName, metav1.GetOptions{})
if err != nil {
return errors.Wrapf(err, "failed to get node %s", m.nodeName)
}
// collect all labels with the prefix to remove
labels := map[string]string{}
for k := range node.Labels {
if strings.HasPrefix(k, labelName) {
labels[k] = ""
}
}
if len(labels) == 0 {
log.Debug("no labels to remove for node", "node", m.nodeName)
return nil
}
// patch the node with the labels to remove
return m.patchNode(ctx, labelOperationRemove, labels)
}
// generateNodeLabelKeyValue generates a label key and value for the given service
func generateNodeLabelKeyValue(svc *corev1.Service) (string, string) {
addresses, _ := instance.FetchServiceAddresses(svc)
sanitized := make([]string, len(addresses))
for i, addr := range addresses {
sanitized[i] = sanitizeIPForLabel(addr)
}
return fmt.Sprintf("%s/%s.%s", labelName, svc.Name, svc.Namespace), strings.Join(sanitized, ",")
}
// Helper function to convert IPv6 hex address without colons
func sanitizeIPForLabel(addr string) string {
ip := net.ParseIP(addr)
if ip == nil || ip.To4() != nil {
return addr
}
return hex.EncodeToString(ip.To16())
}
// patchNode patches the node with the given labels
func (m *Manager) patchNode(ctx context.Context, operation labelOperation, labels map[string]string) error {
type patchStringLabel struct {
Op string `json:"op"`
Path string `json:"path"`
Value string `json:"value"`
}
patchLabels := []patchStringLabel{}
// generate the patch
for k, v := range labels {
patchLabels = append(patchLabels, patchStringLabel{
Op: string(operation),
// replace all slashes with ~1
Path: fmt.Sprintf("/metadata/labels/%s", strings.ReplaceAll(k, "/", "~1")),
Value: v,
})
}
patchData, err := json.Marshal(patchLabels)
if err != nil {
log.Debug("node patch marshaling failed", "err", err, "labels", labels, "patch", patchLabels)
return errors.Wrapf(err, "node patch marshaling failed for labels %v", labels)
}
log.Debug("patching node",
"node", m.nodeName,
"patch", string(patchData),
"operation", operation,
"labels", labels,
"clientSetNil", m.clientSet == nil)
if m.clientSet == nil {
return errors.New("kubernetes client is not initialized")
}
// patch node
node, err := m.clientSet.CoreV1().Nodes().Patch(ctx, m.nodeName, types.JSONPatchType, patchData, metav1.PatchOptions{})
if err != nil {
log.Debug("node patching failed", "err", err, "patchData", patchData)
return errors.Wrapf(err, "node patching failed with patch %s", string(patchData))
}
log.Debug("updated", "node", m.nodeName, "labels", node.Labels)
return nil
}

View File

@@ -1,27 +0,0 @@
package noop
import (
"context"
"time"
corev1 "k8s.io/api/core/v1"
)
// NewManager creates a new NoOp label manager
func NewManager() *Manager {
return &Manager{}
}
type Manager struct{}
func (m *Manager) AddLabel(_ context.Context, _ *corev1.Service) error {
return nil
}
func (m *Manager) RemoveLabel(_ context.Context, _ *corev1.Service) error {
return nil
}
func (m *Manager) CleanUpLabels(_ time.Duration) error {
return nil
}

View File

@@ -1,49 +0,0 @@
package node
import (
"context"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/node/labeler"
"github.com/kube-vip/kube-vip/pkg/node/noop"
v1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
)
// LabelManager is the interface for the node label manager
type LabelManager interface {
// AddLabel adds a label to the node for the given service
AddLabel(ctx context.Context, svc *v1.Service) error
// RemoveLabel removes the label from the node for the given service
RemoveLabel(ctx context.Context, svc *v1.Service) error
// CleanUpLabels removes all labels from the node
CleanUpLabels(timeout time.Duration) error
}
// NewManager creates a new Label Manager for the given node
// NoOp implementation is returned if node labeling is disabled, or if
// running in control plane mode, or if the client is not ready
func NewManager(config *kubevip.Config, clientSet *kubernetes.Clientset) LabelManager {
if !config.EnableNodeLabeling {
return noop.NewManager()
}
if config.EnableControlPlane {
log.Debug("Skip node labeling, control plane mode enabled")
return noop.NewManager()
}
if clientSet == nil {
log.Debug("Skip node labeling, client is not ready")
return noop.NewManager()
}
log.Debug("Node labeling enabled")
return labeler.NewManager(config.NodeName, clientSet)
}

View File

@@ -3,9 +3,6 @@ package servicecontext
import (
"context"
"sync"
"sync/atomic"
"github.com/kube-vip/kube-vip/pkg/lease"
)
type Context struct {
@@ -14,9 +11,6 @@ type Context struct {
IsActive bool
IsWatched bool
ConfiguredNetworks sync.Map
Lease *lease.Lease
HasEndpoints atomic.Bool
LeaderCancel context.CancelFunc
}
func New(ctx context.Context) *Context {

36
pkg/services/context.go Normal file
View File

@@ -0,0 +1,36 @@
package services
import (
"context"
"sync"
)
type Context struct {
Ctx context.Context
Cancel context.CancelFunc
IsActive bool
IsWatched bool
ConfiguredNetworks sync.Map
}
func NewContext(ctx context.Context) *Context {
svcCtx, svcCancel := context.WithCancel(ctx)
return &Context{
Ctx: svcCtx,
Cancel: svcCancel,
}
}
func (ctx *Context) HasConfiguredNetworks() bool {
cnt := 0
ctx.ConfiguredNetworks.Range(func(_ any, _ any) bool {
cnt++
return cnt < 1
})
return cnt > 0
}
func (ctx *Context) IsNetworkConfigured(ip string) bool {
_, exists := ctx.ConfiguredNetworks.Load(ip)
return exists
}

View File

@@ -13,7 +13,6 @@ import (
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -78,12 +77,12 @@ func (p *Processor) nftablesCheck() error {
func getSameFamilyCidr(sourceCidrs, ip string) string { //Todo: not sure how this ever worked
cidrs := strings.Split(sourceCidrs, ",")
isV6 := utils.IsIPv6(ip)
isV6 := vip.IsIPv6(ip)
matchingFamily := []string{}
for _, cidr := range cidrs {
// Is the ip an IPv6 address
if isV6 {
if utils.IsIPv6CIDR(cidr) {
if vip.IsIPv6CIDR(cidr) {
matchingFamily = append(matchingFamily, cidr)
selectedCIDR, err := checkCIDR(ip, cidr)
if err != nil {
@@ -95,7 +94,7 @@ func getSameFamilyCidr(sourceCidrs, ip string) string { //Todo: not sure how thi
}
}
} else {
if utils.IsIPv4CIDR(cidr) {
if vip.IsIPv4CIDR(cidr) {
matchingFamily = append(matchingFamily, cidr)
selectedCidr, err := checkCIDR(ip, cidr)
if err != nil {
@@ -135,7 +134,7 @@ func checkCIDR(ip, cidr string) (string, error) {
return "", nil
}
func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace, serviceUUID string, annotations map[string]string) error {
func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string, annotations map[string]string) error {
var podCidr, serviceCidr string
var autoServiceCIDR, autoPodCIDR string
var discoverErr error
@@ -147,7 +146,7 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
internalEgress := annotations[kubevip.EgressInternal]
if p.config.EgressPodCidr == "" || p.config.EgressServiceCidr == "" {
autoServiceCIDR, autoPodCIDR, discoverErr = p.AutoDiscoverCIDRs(ctx)
autoServiceCIDR, autoPodCIDR, discoverErr = p.AutoDiscoverCIDRs()
}
if discoverErr != nil {
@@ -164,7 +163,7 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
if podCidr == "" {
// There's no default IPv6 pod CIDR, therefore we silently back off if CIDR s not specified.
if !utils.IsIPv4(podIP) {
if !vip.IsIPv4(podIP) {
return fmt.Errorf("error with the CIDR [%s]", podIP)
}
podCidr = defaultPodCIDR
@@ -180,7 +179,7 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
if serviceCidr == "" {
// There's no default IPv6 service CIDR, therefore we silently back off if CIDR s not specified.
if !utils.IsIPv4(vipIP) {
if !vip.IsIPv4(vipIP) {
return nil
}
serviceCidr = defaultServiceCIDR
@@ -189,23 +188,23 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
log.Info("[Egress]", "podCIDR", podCidr, "serviceCIDR", serviceCidr, "vip", serviceCidr, "pod", podIP)
// checking if all addresses are of the same IP family
if utils.IsIPv4(podIP) != utils.IsIPv4CIDR(podCidr) {
if vip.IsIPv4(podIP) != vip.IsIPv4CIDR(podCidr) {
log.Error("[Egress] family is not matching. Backing off...", "pod", podIP, "podCIDR", podCidr)
return nil
}
if utils.IsIPv4(vipIP) != utils.IsIPv4CIDR(serviceCidr) {
if vip.IsIPv4(vipIP) != vip.IsIPv4CIDR(serviceCidr) {
log.Error("[Egress] family is not matching. Backing off...", "pod", podIP, "serviceCIDR", serviceCidr)
return nil
}
if utils.IsIPv4(vipIP) != utils.IsIPv4(podIP) {
if vip.IsIPv4(vipIP) != vip.IsIPv4(podIP) {
log.Error("[Egress] family is not matching. Backing off...", "pod", podIP, "vipIP", vipIP)
return nil
}
protocol := iptables.ProtocolIPv4
if utils.IsIPv6(vipIP) {
if vip.IsIPv6(vipIP) {
protocol = iptables.ProtocolIPv6
}
@@ -224,15 +223,8 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
}
// Add networks that we specifically should only SNAT for
allowCIDRs := []string{}
if allowedNetworks != "" {
networks := strings.Split(strings.TrimSpace(allowedNetworks), ",") //Remove whitespace characters and then create an array from the CIDRs
allowCIDRs = append(allowCIDRs, networks...)
}
// Apply the SNAT rules
err := nftables.ApplySNAT(podIP, vipIP, serviceUUID, destinationPorts, ignoreCIDRs, allowCIDRs, utils.IsIPv6(vipIP))
err := nftables.ApplySNAT(podIP, vipIP, serviceUUID, destinationPorts, ignoreCIDRs, vip.IsIPv6(vipIP))
if err != nil {
return fmt.Errorf("error performing netlink nftables [%s]", err)
}
@@ -276,9 +268,9 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
}
}
mask := fmt.Sprintf("/%d", vip.DefaultMaskIPv4)
if !utils.IsIPv4(podIP) {
mask = fmt.Sprintf("/%d", vip.DefaultMaskIPv6)
mask := "/32"
if !vip.IsIPv4(podIP) {
mask = "/128"
}
if allowedNetworks != "" {
@@ -338,12 +330,12 @@ func (p *Processor) configureEgress(ctx context.Context, vipIP, podIP, namespace
return nil
}
func (p *Processor) AutoDiscoverCIDRs(ctx context.Context) (serviceCIDR, podCIDR string, err error) {
func (p *Processor) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error) {
log.Debug("Trying to automatically discover Service and Pod CIDRs")
options := v1.ListOptions{
LabelSelector: "component=kube-controller-manager",
}
podList, err := p.clientSet.CoreV1().Pods("kube-system").List(ctx, options)
podList, err := p.clientSet.CoreV1().Pods("kube-system").List(context.TODO(), options)
if err != nil {
return "", "", fmt.Errorf("[Egress] Unable to get kube-controller-manager pod: %w", err)
}

View File

@@ -4,15 +4,24 @@ import (
"context"
"fmt"
"sync"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
)
var (
svcLocks map[string]*sync.Mutex
)
func init() {
svcLocks = make(map[string]*sync.Mutex)
}
// The StartServicesWatchForLeaderElection function will start a services watcher, the
func (p *Processor) StartServicesWatchForLeaderElection(ctx context.Context) error {
err := p.ServicesWatcher(ctx, p.StartServicesLeaderElection)
@@ -20,13 +29,12 @@ func (p *Processor) StartServicesWatchForLeaderElection(ctx context.Context) err
return err
}
if p.config.EnableRoutingTable {
for _, instance := range p.ServiceInstances {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
_ = cluster.Network[i].DeleteRoute()
}
for _, instance := range p.ServiceInstances {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
_ = cluster.Network[i].DeleteRoute()
}
cluster.Stop()
}
}
@@ -36,145 +44,84 @@ func (p *Processor) StartServicesWatchForLeaderElection(ctx context.Context) err
}
// The startServicesWatchForLeaderElection function will start a services watcher, the
func (p *Processor) StartServicesLeaderElection(svcCtx *servicecontext.Context, service *v1.Service, _ *sync.WaitGroup) error {
if svcCtx == nil {
return fmt.Errorf("no context context for service %q with UID %q: nil context", service.Name, service.UID)
}
leaseNamespace, serviceLease := lease.ServiceName(service)
id := lease.NewID(p.config.LeaderElectionType, leaseNamespace, serviceLease)
objectName := lease.ServiceNamespacedName(service)
svcLease := p.leaseMgr.Get(id)
if svcLease == nil {
return fmt.Errorf("no existing lease found for service %q with UID %q", service.Name, service.UID)
}
isNew := svcLease.Add(objectName)
// this service was already processed so we do not need to do anything
if !isNew && svcLease.Elected.Load() {
log.Debug("this service was already handled, waiting for it to finish", "service", service.Name, "uid", service.UID)
// Wait for either the service context or lease context to be done
select {
case <-svcCtx.Ctx.Done():
case <-svcLease.Ctx.Done():
}
return nil
}
svcLease.Lock()
defer func() {
svcLease.Unlock()
}()
wg := sync.WaitGroup{}
defer wg.Wait()
// Start a goroutine that will delete the lease when the service context is cancelled.
// This is important for proper cleanup when a service is deleted - it ensures that
// the lease context (svcLease.Ctx) gets cancelled, which causes RunOrDie to return.
// Without this, RunOrDie would continue running until leadership is naturally lost.
wg.Go(func() {
<-svcCtx.Ctx.Done()
p.leaseMgr.Delete(id, objectName)
})
// this service is sharing lease with another service
if svcLease.Elected.Load() {
svcLease.Unlock()
// wait for leader election to start or context to be done
select {
case <-svcLease.Started:
case <-svcLease.Ctx.Done():
// Lease was cancelled (e.g., leader election ended), return immediately
// This allows the restart loop to create a fresh lease
log.Debug("lease context cancelled before leader election started", "service", service.Name, "uid", service.UID)
svcCtx.IsActive = false
return nil
}
// Common lease handling: sync the service and wait for context cancellation
if !svcCtx.IsActive {
if err := p.SyncServices(svcCtx, service, &wg); err != nil {
log.Error("service sync", "err", err, "uid", service.UID)
svcLease.Cancel()
}
svcCtx.IsActive = true
}
// Block until service context is cancelled
<-svcCtx.Ctx.Done()
if svcCtx.IsActive {
// we have no context left here so we use a new one
if err := p.deleteService(context.TODO(), service.UID); err != nil {
log.Error("service deletion", "uid", service.UID, "err", err)
}
}
// Mark this service is inactive
svcCtx.IsActive = false
// wait for leaderelection to be finished
<-svcLease.Ctx.Done()
return nil
}
func (p *Processor) StartServicesLeaderElection(ctx context.Context, service *v1.Service) error {
serviceLease := fmt.Sprintf("kubevip-%s", service.Name)
log.Info("new leader election", "service", service.Name, "namespace", service.Namespace, "lock_name", serviceLease, "host_id", p.config.NodeName)
leaderCtx, leaderCancel := context.WithCancel(svcLease.Ctx)
svcCtx.LeaderCancel = leaderCancel
run := election.RunConfig{
Config: p.config,
LeaseID: id,
Mgr: p.electionMgr,
LeaseAnnotations: map[string]string{},
OnStartedLeading: func(_ context.Context) {
svcLease.Elected.Store(true)
svcLease.Unlock()
close(svcLease.Started)
// Mark this service as active (as we've started leading)
// we run this in background as it's blocking
svcCtx.IsActive = true
if err := p.SyncServices(svcCtx, service, &wg); err != nil {
log.Error("service sync", "uid", service.UID, "err", err)
leaderCancel()
}
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: serviceLease,
Namespace: service.Namespace,
},
OnStoppedLeading: func() {
// we can do cleanup here
svcLease.Elected.Store(false)
log.Info("leadership lost", "service", service.Name, "uid", service.UID, "leader", p.config.NodeName)
if svcCtx.IsActive {
log.Debug("deleting service due to lost leadership", "uid", service.UID)
if err := p.deleteService(svcLease.Ctx, service.UID); err != nil {
log.Error("service deletion", "err", err)
Client: p.clientSet.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: p.config.NodeName,
},
}
childCtx, childCancel := context.WithCancel(ctx)
defer childCancel()
if _, ok := svcLocks[serviceLease]; !ok {
svcLocks[serviceLease] = new(sync.Mutex)
}
svcLocks[serviceLease].Lock()
defer svcLocks[serviceLease].Unlock()
svcCtx, err := p.getServiceContext(service.UID)
if err != nil {
return fmt.Errorf("failed to get context for service %q with UID %q: %w", service.Name, service.UID, err)
}
if svcCtx == nil {
return fmt.Errorf("failed to get context for service %q with UID %q: nil context", service.Name, service.UID)
}
svcCtx.IsActive = true
// start the leader election code loop
leaderelection.RunOrDie(childCtx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(p.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(p.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(p.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
// Mark this service as active (as we've started leading)
// we run this in background as it's blocking
if err := p.SyncServices(ctx, service); err != nil {
log.Error("service sync", "err", err)
childCancel()
}
}
// Mark this service is inactive
svcCtx.IsActive = false
svcLease.Started = make(chan any)
leaderCancel()
},
OnStoppedLeading: func() {
// we can do cleanup here
log.Info("leadership lost", "service", service.Name, "leader", p.config.NodeName)
if svcCtx.IsActive {
if err := p.deleteService(service.UID); err != nil {
log.Error("service deletion", "err", err)
}
}
// Mark this service is inactive
svcCtx.IsActive = false
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if identity == p.config.NodeName {
// I just got the lock
return
}
log.Info("new leader", "leader", identity)
},
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if identity == p.config.NodeName {
// I just got the lock
return
}
log.Info("new leader", "leader", identity, "service", service.Name, "uid", service.UID)
},
}
if err := election.RunOrDie(leaderCtx, &run, p.config); err != nil {
return fmt.Errorf("services election failed: %w", err)
}
log.Info("stopping leader election", "service", service.Name, "uid", service.UID)
})
log.Info("stopping leader election", "service", service.Name)
return nil
}

View File

@@ -2,7 +2,6 @@ package services
import (
"context"
"errors"
"fmt"
log "log/slog"
"reflect"
@@ -10,17 +9,13 @@ import (
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/kube-vip/kube-vip/pkg/wireguard"
"github.com/prometheus/client_golang/prometheus"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
@@ -43,34 +38,19 @@ type Processor struct {
clientSet *kubernetes.Clientset
rwClientSet *kubernetes.Clientset
shutdownChan chan struct{}
// This is a prometheus counter used to count the number of events received
// from the service watcher
CountServiceWatchEvent *prometheus.CounterVec
intfMgr *networkinterface.Manager
arpMgr *arp.Manager
leaseMgr *lease.Manager
// nodeLabelManager is the manager for the node labels
nodeLabelManager labelManager
electionMgr *election.Manager
// TunnelMgr manages multiple WireGuard tunnels (one per service VIP)
TunnelMgr *wireguard.TunnelManager
}
// labelManager is the interface for the node label manager to add/remove labels
type labelManager interface {
AddLabel(ctx context.Context, svc *v1.Service) error
RemoveLabel(ctx context.Context, svc *v1.Service) error
}
func NewServicesProcessor(config *kubevip.Config, bgpServer *bgp.Server,
clientSet *kubernetes.Clientset, rwClientSet *kubernetes.Clientset,
intfMgr *networkinterface.Manager, arpMgr *arp.Manager, nodeLabelManager labelManager,
electionMgr *election.Manager, leaseMgr *lease.Manager) *Processor {
clientSet *kubernetes.Clientset, rwClientSet *kubernetes.Clientset, shutdownChan chan struct{},
intfMgr *networkinterface.Manager, arpMgr *arp.Manager) *Processor {
lbClassFilterFunc := lbClassFilter
if config.LoadBalancerClassLegacyHandling {
lbClassFilterFunc = lbClassFilterLegacy
@@ -83,6 +63,7 @@ func NewServicesProcessor(config *kubevip.Config, bgpServer *bgp.Server,
bgpServer: bgpServer,
clientSet: clientSet,
rwClientSet: rwClientSet,
shutdownChan: shutdownChan,
CountServiceWatchEvent: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "kube_vip",
Subsystem: "manager",
@@ -90,88 +71,58 @@ func NewServicesProcessor(config *kubevip.Config, bgpServer *bgp.Server,
Help: "Count all events fired by the service watcher categorised by event type",
}, []string{"type"}),
intfMgr: intfMgr,
arpMgr: arpMgr,
leaseMgr: leaseMgr,
nodeLabelManager: nodeLabelManager,
electionMgr: electionMgr,
TunnelMgr: wireguard.NewTunnelManager(),
intfMgr: intfMgr,
arpMgr: arpMgr,
}
}
func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceFunc func(*servicecontext.Context, *v1.Service, *sync.WaitGroup) error, wg *sync.WaitGroup) error {
func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceFunc func(context.Context, *v1.Service) error) (bool, error) {
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
svc, ok := event.Object.(*v1.Service)
if !ok {
return fmt.Errorf("unable to parse Kubernetes services from API watcher")
return false, fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
return nil
return true, nil
}
// Check if we ignore this service
if svc.Annotations[kubevip.LoadbalancerIgnore] == "true" {
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Info("ignore annotation for kube-vip", "service name", svc.Name)
return nil
return true, nil
}
// Check the loadBalancer class
if p.lbClassFilter(svc, p.config) {
return nil
return true, nil
}
svcAddresses, svcHostnames := instance.FetchServiceAddresses(svc)
svcAddresses := instance.FetchServiceAddresses(svc)
// We only care about LoadBalancer services that have been allocated an address
if len(svcAddresses) <= 0 && len(svcHostnames) <= 0 {
return nil
}
svcInstance := instance.FindServiceInstance(svc, p.ServiceInstances)
var err error
if svcInstance == nil {
svcInstance, err = instance.NewInstance(ctx, svc, p.config, p.intfMgr, p.arpMgr, wg)
if err != nil {
return fmt.Errorf("unalbe to create instance for service %s/%s", svc.Namespace, svc.Name)
}
p.ServiceInstances = append(p.ServiceInstances, svcInstance)
}
_, usesCommonLease := svc.Annotations[kubevip.ServiceLease]
if usesCommonLease && svc.Spec.ExternalTrafficPolicy != v1.ServiceExternalTrafficPolicyTypeCluster {
return fmt.Errorf("annotation %q cannot be used with service traffic policy other than %q, service %s/%s",
kubevip.ServiceLease, v1.ServiceExternalTrafficPolicyTypeCluster, svc.Namespace, svc.Name)
if len(svcAddresses) <= 0 {
return true, nil
}
svcCtx, err := p.getServiceContext(svc.UID)
if err != nil {
return fmt.Errorf("failed to get service context: %w", err)
return false, fmt.Errorf("failed to get service context: %w", err)
}
// The modified event should only be triggered if the service has been modified (i.e. moved somewhere else)
if event.Type == watch.Modified {
i := svcInstance
shouldGarbageCollect := false
i := instance.FindServiceInstance(svc, p.ServiceInstances)
var originalService []string
shouldGarbageCollect := true
if i != nil {
originalServiceAddresses, originalServiceHostnames := instance.FetchServiceAddresses(i.ServiceSnapshot)
shouldGarbageCollect =
// Service addresses changed
!reflect.DeepEqual(originalServiceAddresses, svcAddresses) ||
// Service hostnames changed
!reflect.DeepEqual(originalServiceHostnames, svcHostnames) ||
// ExternalTrafficPolicy changed
svc.Spec.ExternalTrafficPolicy != i.ServiceSnapshot.Spec.ExternalTrafficPolicy ||
// IP stack configuration changed
!reflect.DeepEqual(svc.Spec.IPFamilies, i.ServiceSnapshot.Spec.IPFamilies) ||
*svc.Spec.IPFamilyPolicy != *i.ServiceSnapshot.Spec.IPFamilyPolicy ||
// DDNS was disabled/enabled
svc.Annotations[kubevip.ServiceDDNS] != i.ServiceSnapshot.Annotations[kubevip.ServiceDDNS]
originalService = instance.FetchServiceAddresses(i.ServiceSnapshot)
shouldGarbageCollect = !reflect.DeepEqual(originalService, svcAddresses)
}
if shouldGarbageCollect {
for _, addr := range svcAddresses {
// log.Debugf("(svcs) Retrieving local addresses, to ensure that this modified address doesn't exist: %s", addr)
// log.Debugf("(svcs) Retreiving local addresses, to ensure that this modified address doesn't exist: %s", addr)
f, err := vip.GarbageCollect(p.config.Interface, addr, p.intfMgr)
if err != nil {
log.Error("(svcs) cleaning existing address error", "err", err)
@@ -180,23 +131,30 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
log.Warn("(svcs) already found existing config", "address", addr, "adapter", p.config.Interface)
}
}
// This service has been modified, but it was also active.
if svcCtx != nil && svcCtx.IsActive {
log.Warn("(svcs) The load balancer has changed, cancelling original load balancer")
//Set it to inactive
svcCtx.IsActive = false
svcCtx.Cancel()
log.Warn("(svcs) waiting for load balancer to finish")
<-svcCtx.Ctx.Done()
}
// This service has been modified, but it was also active.
if svcCtx != nil && svcCtx.IsActive {
if i != nil {
originalService := instance.FetchServiceAddresses(i.ServiceSnapshot)
newService := instance.FetchServiceAddresses(svc)
if !reflect.DeepEqual(originalService, newService) {
if err := p.deleteService(ctx, svc.UID); err != nil {
log.Error("(svc) unable to remove", "service", svc.UID)
// Calls the cancel function of the context
if svcCtx != nil {
log.Warn("(svcs) The load balancer has changed, cancelling original load balancer")
svcCtx.Cancel()
log.Warn("(svcs) waiting for load balancer to finish")
<-svcCtx.Ctx.Done()
}
err = p.deleteService(svc.UID)
if err != nil {
log.Error("(svc) unable to remove", "service", svc.UID)
}
p.svcMap.Delete(svc.UID)
}
// in theory this should never fail
p.svcMap.Delete(svc.UID)
// Reset the the svcCtx when it was garbage collected
// As the next function will create a new context when nil
svcCtx = nil
}
}
}
@@ -208,51 +166,14 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
//
if svcCtx == nil || svcCtx != nil && !svcCtx.IsActive {
ips, hostnames := instance.FetchServiceAddresses(svc)
log.Debug("(svcs) has been added/modified with addresses", "service name", svc.Name, "ips", ips, "hostnames", hostnames)
log.Debug("(svcs) has been added/modified with addresses", "service name", svc.Name, "ip", instance.FetchServiceAddresses(svc))
if svcCtx == nil {
log.Debug("new context for service", "namespace", svc.Namespace, "name", svc.Name)
ns, name := lease.ServiceName(svc)
leaseID := lease.NewID(p.config.LeaderElectionType, ns, name)
lease := p.leaseMgr.Add(ctx, leaseID)
svcCtx = servicecontext.New(lease.Ctx)
svcCtx = servicecontext.New(ctx)
p.svcMap.Store(svc.UID, svcCtx)
}
// WireGuard services always need endpoint watching for DNAT rule updates
// This is independent of leader election settings (which are for control plane HA)
if p.config.EnableWireguard && !svcCtx.IsWatched {
// Call serviceFunc first to set up the WireGuard tunnel
err = serviceFunc(svcCtx, svc, wg)
if err != nil {
log.Error(err.Error())
if errors.Is(err, &utils.PanicError{}) {
return err
}
}
wg.Go(func() {
defer func() {
if svcCtx != nil {
svcCtx.IsWatched = false
}
}()
// Start endpoint watcher for WireGuard services (uses EndpointSlices by default)
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
} else {
provider = providers.NewEndpointslices()
}
if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
})
svcCtx.IsWatched = true
} else if p.config.EnableServicesElection || // Service Election
if p.config.EnableServicesElection || // Service Election
((p.config.EnableRoutingTable || p.config.EnableBGP) && // Routing table mode or BGP
(!p.config.EnableLeaderElection && !p.config.EnableServicesElection)) { // No leaderelection or services election
@@ -263,22 +184,13 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
if !svcCtx.IsWatched {
// background the endpoint watcher
if (p.config.EnableRoutingTable || p.config.EnableBGP) && (!p.config.EnableLeaderElection && !p.config.EnableServicesElection) {
err = serviceFunc(svcCtx, svc, wg)
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
if errors.Is(err, &utils.PanicError{}) {
return err
}
}
}
wg.Go(func() {
defer func() {
if svcCtx != nil {
svcCtx.IsWatched = false
}
}()
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Add Endpoint or EndpointSlices watcher
var provider providers.Provider
@@ -291,27 +203,18 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
log.Error(err.Error())
}
}
})
}()
// We're now watching this service
svcCtx.IsWatched = true
}
} else if (p.config.EnableBGP || p.config.EnableRoutingTable) && (!p.config.EnableLeaderElection && !p.config.EnableServicesElection) {
err = serviceFunc(svcCtx, svc, wg)
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
if errors.Is(err, &utils.PanicError{}) {
return err
}
}
wg.Go(func() {
defer func() {
if svcCtx != nil {
svcCtx.IsWatched = false
}
}()
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
// Add Endpoint watcher
var provider providers.Provider
@@ -324,72 +227,61 @@ func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceF
log.Error(err.Error())
}
}
})
}()
// We're now watching this service
svcCtx.IsWatched = true
} else {
wg.Go(func() {
go func() {
for {
select {
case <-svcCtx.Ctx.Done():
log.Warn("(svcs) restartable service watcher ending", "uid", svc.UID)
return
default:
if !svcCtx.IsActive {
log.Info("(svcs) restartable service watcher starting", "uid", svc.UID)
err = serviceFunc(svcCtx, svc, wg)
if err != nil {
log.Error(err.Error())
if errors.Is(err, &utils.PanicError{}) {
svcCtx.Cancel()
}
}
log.Info("(svcs) restartable service watcher done", "uid", svc.UID)
log.Info("(svcs) restartable service watcher starting", "uid", svc.UID)
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
}
})
}()
}
} else {
// Increment the waitGroup before the service Func is called (Done is completed in there)
err = serviceFunc(svcCtx, svc, wg)
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
if errors.Is(err, &utils.PanicError{}) {
return err
}
}
}
if !p.config.EnableServicesElection {
log.Debug("Service now active", "name", svc.Name, "uid", svc.UID)
svcCtx.IsActive = true
}
svcCtx.IsActive = true
}
return nil
return false, nil
}
func (p *Processor) Delete(event watch.Event) error {
func (p *Processor) Delete(event watch.Event) (bool, error) {
svc, ok := event.Object.(*v1.Service)
if !ok {
return fmt.Errorf("(svcs) unable to parse Kubernetes services from API watcher")
return false, fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
svcCtx, err := p.getServiceContext(svc.UID)
if err != nil {
return fmt.Errorf("(svcs) unable to get context: %w", err)
return false, fmt.Errorf("(svcs) unable to get context: %w", err)
}
if svcCtx != nil {
if svcCtx != nil && svcCtx.IsActive {
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
return nil
return true, nil
}
// We can ignore this service
if svc.Annotations[kubevip.LoadbalancerIgnore] == "true" {
log.Info("(svcs) ignore annotation for kube-vip", "service name", svc.Name)
return nil
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Info("(svcs)ignore annotation for kube-vip", "service name", svc.Name)
return true, nil
}
// If no leader election is enabled, delete routes here
@@ -400,26 +292,31 @@ func (p *Processor) Delete(event watch.Event) error {
}
}
if svcCtx.IsActive && !p.config.EnableServicesElection {
// If this is an active service then and additional leaderElection will handle stopping
err = p.deleteService(svcCtx.Ctx, svc.UID)
if err != nil {
log.Error(err.Error())
}
svcCtx.IsActive = false
// If this is an active service then and additional leaderElection will handle stopping
err = p.deleteService(svc.UID)
if err != nil {
log.Error(err.Error())
}
// Calls the cancel function of the context
log.Warn("(svcs) The load balancer was deleted, cancelling context", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
log.Warn("(svcs) The load balancer was deleted, cancelling context")
svcCtx.Cancel()
log.Warn("(svcs) waiting for load balancer to finish", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
log.Warn("(svcs) waiting for load balancer to finish")
<-svcCtx.Ctx.Done()
p.svcMap.Delete(svc.UID)
}
if p.config.EnableLeaderElection && !p.config.EnableServicesElection {
if p.config.EnableBGP {
endpoints.ClearBGPHosts(svc, &p.ServiceInstances, p.bgpServer)
} else if p.config.EnableRoutingTable {
endpoints.ClearRoutes(svc, &p.ServiceInstances)
}
}
log.Info("(svcs) deleted", "service name", svc.Name, "namespace", svc.Namespace)
return nil
return true, nil
}
func (p *Processor) Stop() {

View File

@@ -3,10 +3,8 @@ package services
import (
"context"
"fmt"
"net"
"slices"
"strings"
"sync"
"time"
log "log/slog"
@@ -14,7 +12,6 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
@@ -25,9 +22,7 @@ import (
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/upnp"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
@@ -37,104 +32,72 @@ const (
ActionDelete ServiceInstanceAction = "delete"
ActionAdd ServiceInstanceAction = "add"
ActionNone ServiceInstanceAction = "none"
// Default UPNP lease requested is 3600 seconds.
defaultUPNPLeaseDuration = 1 * time.Hour
)
func (p *Processor) SyncServices(ctx *servicecontext.Context, svc *v1.Service, wg *sync.WaitGroup) error {
log.Debug("[STARTING] Service Sync", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
func (p *Processor) SyncServices(ctx context.Context, svc *v1.Service) error {
log.Debug("[STARTING] Service Sync", "namespace", svc.Namespace, "name", svc.Name)
// Iterate through the synchronising services
action, instance := p.getServiceInstanceAction(svc)
action := p.getServiceInstanceAction(svc)
switch action {
case ActionDelete:
// remove the label from the node before deleting the service
if err := p.nodeLabelManager.RemoveLabel(ctx.Ctx, svc); err != nil {
return fmt.Errorf("error removing label from node: %w", err)
}
log.Debug("[service] delete", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
if err := p.deleteService(ctx.Ctx, svc.UID); err != nil {
log.Debug("[service] delete", "namespace", svc.Namespace, "name", svc.Name)
if err := p.deleteService(svc.UID); err != nil {
return fmt.Errorf("error deleting service %s/%s: %w", svc.Namespace, svc.Name, err)
}
case ActionAdd:
log.Debug("[service] add", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
if err := p.addService(ctx.Ctx, instance, svc, wg); err != nil {
log.Debug("[service] add", "namespace", svc.Namespace, "name", svc.Name)
if err := p.addService(ctx, svc); err != nil {
return fmt.Errorf("error adding service %s/%s: %w", svc.Namespace, svc.Name, err)
}
// add the label to the node after adding the service
if err := p.nodeLabelManager.AddLabel(ctx.Ctx, svc); err != nil {
return fmt.Errorf("error adding label to node: %w", err)
}
case ActionNone:
log.Debug("[service] no action", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
log.Debug("[service] no action", "namespace", svc.Namespace, "name", svc.Name)
}
log.Debug("[FINISHED] Service Sync", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
log.Debug("[FINISHED] Service Sync", "namespace", svc.Namespace, "name", svc.Name)
return nil
}
func (p *Processor) getServiceInstanceAction(svc *v1.Service) (ServiceInstanceAction, *instance.Instance) {
func (p *Processor) getServiceInstanceAction(svc *v1.Service) ServiceInstanceAction {
// protect against multiple calls
// get the annotations or legacy values from manual configuration
addresses, hostnames := instance.FetchServiceAddresses(svc)
// get the status information of the LB Service
statusAddresses, _ := instance.FetchLoadBalancerIngress(svc)
addresses := instance.FetchServiceAddresses(svc)
ingressIPs := instance.FetchLoadBalancerIngressAddresses(svc)
p.mutex.Lock()
defer p.mutex.Unlock()
for _, instance := range p.ServiceInstances {
if instance != nil && instance.ServiceSnapshot.UID == svc.UID {
if !instance.AddCalled {
return ActionAdd, instance
}
for _, address := range addresses {
// handle the case where the service instance needs to be deleted
if instance.IsDHCPv4 {
if instance.IsDHCP {
if address != "0.0.0.0" {
return ActionDelete, instance
return ActionDelete
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(statusAddresses, instance.DHCPInterfaceIPv4) {
return ActionDelete, instance
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, instance.DHCPInterfaceIP) {
return ActionDelete
}
} else {
}
if !instance.IsDHCP {
if address == "0.0.0.0" {
return ActionDelete, instance
return ActionDelete
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(statusAddresses, address) {
return ActionDelete, instance
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, address) {
return ActionDelete
}
}
if instance.IsDHCPv6 {
if address != "::" {
return ActionDelete, instance
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(statusAddresses, instance.DHCPInterfaceIPv6) {
return ActionDelete, instance
}
} else {
if address == "::" {
return ActionDelete, instance
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(statusAddresses, address) {
return ActionDelete, instance
}
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !comparePortsAndPortStatuses(svc) {
return ActionDelete, instance
if !comparePortsAndPortStatuses(svc) {
return ActionDelete
}
}
// If we reach here, it means the service instance matches the service UID and is not a DHCP service, so we can return "no action"
return ActionNone, instance
return ActionNone
}
}
if len(addresses) > 0 || len(hostnames) > 0 {
log.Debug("no matching service instance found", "service", svc.Name, "namespace", svc.Namespace, "uid", svc.UID, "addresses", addresses, "hostnames", hostnames)
return ActionAdd, nil // If no matching instance is found, we need to add a new service instance
if len(addresses) > 0 {
log.Debug("No matching service instance found", "service", svc.Name, "namespace", svc.Namespace, "addresses", addresses)
return ActionAdd // If no matching instance is found, we need to add a new service instance
}
return ActionNone, nil
return ActionNone
}
func comparePortsAndPortStatuses(svc *v1.Service) bool {
@@ -153,97 +116,51 @@ func comparePortsAndPortStatuses(svc *v1.Service) bool {
return true
}
func (p *Processor) addService(ctx context.Context, newService *instance.Instance, svc *v1.Service, wg *sync.WaitGroup) error {
func (p *Processor) addService(ctx context.Context, svc *v1.Service) error {
// protect against addService while reading
p.mutex.Lock()
defer p.mutex.Unlock()
startTime := time.Now()
var err error
if newService == nil {
newService, err = instance.NewInstance(ctx, svc, p.config, p.intfMgr, p.arpMgr, wg)
if err != nil {
return err
}
p.ServiceInstances = append(p.ServiceInstances, newService)
newService, err := instance.NewInstance(svc, p.config, p.intfMgr, p.arpMgr)
if err != nil {
return err
}
for x := range newService.VIPConfigs {
log.Debug("starting loadbalancer for service", "name", svc.Name, "namespace", svc.Namespace, "uid", svc.UID)
if err := newService.Clusters[x].StartLoadBalancerService(ctx, newService.VIPConfigs[x], p.bgpServer, svc.Name, p.CountRouteReferences, wg); err != nil {
return fmt.Errorf("failed to start lb: %w", err)
}
log.Debug("starting loadbalancer for service", "name", svc.Name, "namespace", svc.Namespace)
newService.Clusters[x].StartLoadBalancerService(ctx, newService.VIPConfigs[x], p.bgpServer, svc.Name, p.CountRouteReferences)
}
p.upnpMap(ctx, newService)
if newService.IsDHCPv4 {
wg.Go(func() {
index := -1
for i := range newService.VIPConfigs {
ip := net.ParseIP(newService.VIPConfigs[i].VIP)
if ip.To4() != nil {
index = i
break
}
}
if index == -1 {
log.Error("unable to find proper VIPConfig for the DHCPv4")
} else {
for ip := range newService.DHCPv4Client.IPChannel() {
log.Debug("IP changed", "ip", ip)
newService.VIPConfigs[index].VIP = ip
newService.DHCPInterfaceIPv4 = ip
if !p.config.DisableServiceUpdates {
if err := p.updateStatus(ctx, newService); err != nil {
log.Warn("updating svc", "err", err)
}
if newService.IsDHCP && len(newService.VIPConfigs) == 1 {
go func() {
for ip := range newService.DHCPClient.IPChannel() {
log.Debug("IP changed", "ip", ip)
newService.VIPConfigs[0].VIP = ip
newService.DHCPInterfaceIP = ip
if !p.config.DisableServiceUpdates {
if err := p.updateStatus(newService); err != nil {
log.Warn("updating svc", "err", err)
}
}
log.Debug("IPv4 update channel closed, stopping")
}
})
log.Debug("IP update channel closed, stopping")
}()
}
if newService.IsDHCPv6 {
wg.Go(func() {
index := -1
for i := range newService.VIPConfigs {
ip := net.ParseIP(newService.VIPConfigs[i].VIP)
if ip.To4() == nil {
index = i
break
}
}
if index == -1 {
log.Error("unable to find proper VIPConfig for the DHCPv6")
} else {
for ip := range newService.DHCPv4Client.IPChannel() {
log.Debug("IP changed", "ip", ip)
newService.VIPConfigs[index].VIP = ip
newService.DHCPInterfaceIPv6 = ip
if !p.config.DisableServiceUpdates {
if err := p.updateStatus(ctx, newService); err != nil {
log.Warn("updating svc", "err", err)
}
}
}
log.Debug("IPv6 update channel closed, stopping")
}
})
}
p.ServiceInstances = append(p.ServiceInstances, newService)
if !p.config.DisableServiceUpdates {
log.Debug("[service] update", "namespace", newService.ServiceSnapshot.Namespace, "name", newService.ServiceSnapshot.Name)
if err := p.updateStatus(ctx, newService); err != nil {
if err := p.updateStatus(newService); err != nil {
log.Error("[service] updating status", "namespace", newService.ServiceSnapshot.Namespace, "name", newService.ServiceSnapshot.Name, "err", err)
}
}
serviceIPs, _ := instance.FetchServiceAddresses(svc)
serviceIPs := instance.FetchServiceAddresses(svc)
// Check if we need to flush any conntrack connections (due to some dangling conntrack connections)
if svc.Annotations[kubevip.FlushContrack] == "true" {
@@ -268,13 +185,13 @@ func (p *Processor) addService(ctx context.Context, newService *instance.Instanc
// Ensure that kernel modules are loaded and report back missing modules.
err = p.nftablesCheck()
if err != nil {
log.Warn("[service] configuring nft egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
} else {
// Ensure that kernel modules are loaded and report back missing modules.
err = p.iptablesCheck()
if err != nil {
log.Warn("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
var podIP string
@@ -285,14 +202,14 @@ func (p *Processor) addService(ctx context.Context, newService *instance.Instanc
// Does the service have an active IPv6 endpoint
if svc.Annotations[kubevip.ActiveEndpointIPv6] != "" {
for _, serviceIP := range serviceIPs {
if !p.config.EnableEndpoints && utils.IsIPv6(serviceIP) {
if !p.config.EnableEndpoints && vip.IsIPv6(serviceIP) {
podIP = svc.Annotations[kubevip.ActiveEndpointIPv6]
err = p.configureEgress(ctx, serviceIP, podIP, svc.Namespace, string(svc.UID), svc.Annotations)
err = p.configureEgress(serviceIP, podIP, svc.Namespace, string(svc.UID), svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Warn("[service] configuring egress IPv6", "service", svc.Name, "namespace", svc.Namespace, "err", err)
log.Error("[service] configuring egress IPv6", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
@@ -300,13 +217,13 @@ func (p *Processor) addService(ctx context.Context, newService *instance.Instanc
} else if svc.Annotations[kubevip.ActiveEndpoint] != "" { // Not expected to be IPv6, so should be an IPv4 address
for _, serviceIP := range serviceIPs {
podIPs := svc.Annotations[kubevip.ActiveEndpoint]
if !p.config.EnableEndpoints && utils.IsIPv6(serviceIP) {
if !p.config.EnableEndpoints && vip.IsIPv6(serviceIP) {
podIPs = svc.Annotations[kubevip.ActiveEndpointIPv6]
}
err = p.configureEgress(ctx, serviceIP, podIPs, svc.Namespace, string(svc.UID), svc.Annotations)
err = p.configureEgress(serviceIP, podIPs, svc.Namespace, string(svc.UID), svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Warn("[service] configuring egress IPv4", "service", svc.Name, "namespace", svc.Namespace, "err", err)
log.Error("[service] configuring egress IPv4", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
@@ -317,29 +234,20 @@ func (p *Processor) addService(ctx context.Context, newService *instance.Instanc
} else {
provider = providers.NewEndpointslices()
}
err = provider.UpdateServiceAnnotation(ctx, svc.Annotations[kubevip.ActiveEndpoint], svc.Annotations[kubevip.ActiveEndpointIPv6], svc, p.clientSet)
err = provider.UpdateServiceAnnotation(svc.Annotations[kubevip.ActiveEndpoint], svc.Annotations[kubevip.ActiveEndpointIPv6], svc, p.clientSet)
if err != nil {
log.Warn("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
// Configure WireGuard DNAT rules if WireGuard is enabled
if p.config.EnableWireguard {
log.Debug("[service] configuring WireGuard DNAT rules", "service", svc.Name, "namespace", svc.Namespace)
if err := p.addServiceWireguard(ctx, svc); err != nil {
log.Warn("[service] failed to configure WireGuard DNAT", "service", svc.Name, "namespace", svc.Namespace, "err", err)
// Don't fail the entire service if WireGuard config fails
}
}
finishTime := time.Since(startTime)
log.Info("[service]", "service", svc.Name, "namespace", svc.Namespace, "synchronised in", fmt.Sprintf("%dms", finishTime.Milliseconds()))
return nil
}
func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
func (p *Processor) deleteService(uid types.UID) error {
// protect multiple calls
p.mutex.Lock()
defer p.mutex.Unlock()
@@ -348,7 +256,7 @@ func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
var serviceInstance *instance.Instance
found := false
for x := range p.ServiceInstances {
log.Debug("[service] lookup", "target UID", uid, "found UID", p.ServiceInstances[x].ServiceSnapshot.UID, "name", p.ServiceInstances[x].ServiceSnapshot.Name, "namespace", p.ServiceInstances[x].ServiceSnapshot.Namespace)
log.Debug("[service] lookup", "target UID", uid, "found UID ", p.ServiceInstances[x].ServiceSnapshot.UID, "name", p.ServiceInstances[x].ServiceSnapshot.Name, "namespace", p.ServiceInstances[x].ServiceSnapshot.Namespace)
// Add the running services to the new array
if p.ServiceInstances[x].ServiceSnapshot.UID != uid {
updatedInstances = append(updatedInstances, p.ServiceInstances[x])
@@ -358,12 +266,10 @@ func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
serviceInstance = p.ServiceInstances[x]
}
}
// If we've been through all services and not found the correct one then error
if !found {
// TODO: - fix UX
// return fmt.Errorf("unable to find/stop service [%s]", uid)
log.Warn("unable to find/stop service", "uid", uid)
return nil
}
@@ -373,17 +279,15 @@ func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
}
}
// Determine if this VIP is shared with other loadbalancers
// Determine if this this VIP is shared with other loadbalancers
shared := false
vipSet := make(map[string]interface{})
for x := range updatedInstances {
vips, _ := instance.FetchServiceAddresses(updatedInstances[x].ServiceSnapshot)
for _, vip := range vips { //updatedInstances[x].ServiceSnapshot.Spec.LoadBalancerIP {
for _, vip := range instance.FetchServiceAddresses(updatedInstances[x].ServiceSnapshot) { //updatedInstances[x].ServiceSnapshot.Spec.LoadBalancerIP {
vipSet[vip] = nil
}
}
vips, _ := instance.FetchServiceAddresses(serviceInstance.ServiceSnapshot)
for _, vip := range vips {
for _, vip := range instance.FetchServiceAddresses(serviceInstance.ServiceSnapshot) {
if _, found := vipSet[vip]; found {
shared = true
}
@@ -392,15 +296,8 @@ func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
for x := range serviceInstance.Clusters {
serviceInstance.Clusters[x].Stop()
}
if serviceInstance.IsDHCPv4 || serviceInstance.IsDHCPv6 {
if serviceInstance.IsDHCPv4 {
serviceInstance.DHCPv4Client.Stop()
}
if serviceInstance.IsDHCPv6 {
serviceInstance.DHCPv6Client.Stop()
}
if serviceInstance.IsDHCP {
serviceInstance.DHCPClient.Stop()
macvlan, err := netlink.LinkByName(serviceInstance.DHCPInterface)
if err != nil {
return fmt.Errorf("[service] error finding VIP Interface: %v", err)
@@ -411,15 +308,9 @@ func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
return fmt.Errorf("[service] error deleting DHCP Link : %v", err)
}
}
if p.config.EnableBGP {
endpoints.ClearBGPHostsByInstance(ctx, serviceInstance, p.bgpServer)
}
if p.config.EnableRoutingTable && (p.config.EnableLeaderElection || p.config.EnableServicesElection) {
if errs := endpoints.ClearRoutesByInstance(serviceInstance.ServiceSnapshot, serviceInstance, &p.ServiceInstances); len(errs) > 0 {
for _, err := range errs {
log.Error("unable to clear routes", "err", err)
}
for i := range serviceInstance.VIPConfigs {
if serviceInstance.VIPConfigs[i].EnableBGP {
endpoints.ClearBGPHostsByInstance(serviceInstance, p.bgpServer)
}
}
@@ -438,183 +329,11 @@ func (p *Processor) deleteService(ctx context.Context, uid types.UID) error {
// Update the service array
p.ServiceInstances = updatedInstances
// Clean up WireGuard DNAT rules if WireGuard is enabled
if p.config.EnableWireguard {
log.Debug("[service] cleaning up WireGuard DNAT rules", "uid", uid, "name", serviceInstance.ServiceSnapshot.Name)
p.deleteServiceWireguard(ctx, serviceInstance.ServiceSnapshot)
}
log.Info("Removed instance from manager", "uid", uid, "name", serviceInstance.ServiceSnapshot.Name, "remaining advertised services", len(p.ServiceInstances))
log.Info("Removed instance from manager", "uid", uid, "remaining advertised services", len(p.ServiceInstances))
return nil
}
func (p *Processor) updateEgressConfiguration(ctx context.Context, svc *v1.Service) error {
p.mutex.Lock()
defer p.mutex.Unlock()
i := instance.FindServiceInstance(svc, p.ServiceInstances)
if i == nil {
return fmt.Errorf("service instance not found for %s/%s", svc.Namespace, svc.Name)
}
oldIPv4 := i.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint]
newIPv4 := svc.Annotations[kubevip.ActiveEndpoint]
oldIPv6 := i.ServiceSnapshot.Annotations[kubevip.ActiveEndpointIPv6]
newIPv6 := svc.Annotations[kubevip.ActiveEndpointIPv6]
// Skip update if endpoints haven't changed
if oldIPv4 == newIPv4 && oldIPv6 == newIPv6 {
return nil
}
log.Info("[service] updating egress configuration",
"service", svc.Name,
"namespace", svc.Namespace,
"old_ipv4", oldIPv4,
"new_ipv4", newIPv4,
"old_ipv6", oldIPv6,
"new_ipv6", newIPv6)
// Remove old egress rules if they exist
if oldIPv4 != "" || oldIPv6 != "" {
oldEndpoint := oldIPv4
if oldEndpoint == "" {
oldEndpoint = oldIPv6
}
serviceIPs, _ := instance.FetchServiceAddresses(i.ServiceSnapshot)
for _, serviceIP := range serviceIPs {
if err := egress.Teardown(
oldEndpoint,
serviceIP,
i.ServiceSnapshot.Namespace,
string(i.ServiceSnapshot.UID),
i.ServiceSnapshot.Annotations,
p.config.EgressWithNftables,
); err != nil {
log.Warn("[service] removing old egress rules", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
// Apply new egress rules with updated endpoint
serviceIPs, _ := instance.FetchServiceAddresses(svc)
errList := []error{}
// Check if egress should be IPv6
if svc.Annotations[kubevip.EgressIPv6] == "true" {
// Does the service have an active IPv6 endpoint
if newIPv6 != "" {
for _, serviceIP := range serviceIPs {
if !p.config.EnableEndpoints && utils.IsIPv6(serviceIP) {
podIP := newIPv6
err := p.configureEgress(ctx, serviceIP, podIP, svc.Namespace, string(svc.UID), svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Warn("[service] configuring egress IPv6", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
}
} else if newIPv4 != "" { // Not expected to be IPv6, so should be an IPv4 address
for _, serviceIP := range serviceIPs {
podIPs := newIPv4
if !p.config.EnableEndpoints && utils.IsIPv6(serviceIP) {
podIPs = newIPv6
}
err := p.configureEgress(ctx, serviceIP, podIPs, svc.Namespace, string(svc.UID), svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Warn("[service] configuring egress IPv4", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
if len(errList) > 0 {
return fmt.Errorf("errors configuring egress: %v", errList)
}
// Update the service snapshot to reflect the new state
// NOTE: Do NOT call UpdateServiceAnnotation here - the annotation was already updated
// by the endpoint processor, which is why we're being called in the first place.
// Calling it again would create an infinite loop of Modified events.
// svc is already a DeepCopy from the endpoint processor, so no need to copy again.
i.ServiceSnapshot = svc
log.Info("[service] egress configuration updated successfully", "service", svc.Name, "namespace", svc.Namespace)
return nil
}
// upnpLeaseDurationForService determines the UPNP lease duration for a given service, based on its annotations.
//
// The default lease duration is set to 1 hour, maintaining the default of 3600 seconds that was previously passed. If
// the service has an annotation of [kubevip.UpnpLeaseDuration], the function attempts to parse its value as a
// [time.Duration] using [time.ParseDuration].
//
// If parsing is successful, the lease duration is updated accordingly; otherwise, a warning is logged and the default
// duration is retained.
//
// Overriding the default lease duration can be useful for services that require longer or shorter UPNP port mappings,
// or for buggy UPNP implementations that may not handle renewals correctly. At least one router's implementation
// completely times out the mapping very shortly after creation if it is set to 3600 or 7200, but works fine if 0 is
// used.
//
// This function must therefore explicitly permit duration of 0, and callers and the underlying library must pass that
// value in XML correctly. A duration of 0 indicates to the UPNP gateway that the mapping should be permanent.
//
// It may be useful to update this function to read a global configuration option as well. This helper could also take
// in v1.Service instead of [instance.Instance], but the latter is more convenient for callers.
//
// Example where 0 was observed to stay on the problematic router: miniupnpc's test client, upnpc v2.2.4.
func upnpLeaseDurationForService(s *instance.Instance) time.Duration {
if s == nil || s.ServiceSnapshot == nil || s.ServiceSnapshot.Annotations == nil {
// No warning output. No annotation is unusual but perfectly ok.
return defaultUPNPLeaseDuration
}
// Constant is named `UpnpLeaseDuration` for consistency with `UpnpEnabled`. According to Go naming conventions
// regarding use of acronyms, `UPNPLeaseDuration` would be preferred. Cleanup of both of these is left for a future
// refactor as these are public symbols and might be used elsewhere in the ecosystem.
val, ok := s.ServiceSnapshot.Annotations[kubevip.UpnpLeaseDuration]
if !ok {
// No warning output. No annotation is common and perfectly ok.
return defaultUPNPLeaseDuration
}
if val == "" {
log.Warn("[UPNP] Lease duration annotation is empty, using default of 1 hour", "service", s.ServiceSnapshot.Name)
return defaultUPNPLeaseDuration
}
parsed, err := time.ParseDuration(val)
if err != nil {
log.Warn("[UPNP] Unable to parse lease duration from annotation, using default of 1 hour", "service", s.ServiceSnapshot.Name, "err", err)
return defaultUPNPLeaseDuration
}
if parsed < 0 {
log.Warn("[UPNP] Lease duration from annotation is negative, using default of 1 hour", "service", s.ServiceSnapshot.Name)
return defaultUPNPLeaseDuration
}
return parsed
}
// upnpLeaseDurationForServiceSec returns the UPNP lease duration for a service in uint32 seconds, as expected by the
// helper library. This is a convenience wrapper around [upnpLeaseDurationForService], and in case
// upnpLeaseDurationForService returns a duration that maps to a negative value of seconds or invalid float of seconds,
// it will return the default lease duration in seconds instead. (Technically, it will check for a reasonable range of
// seconds, e.g. ~10 years-ish.)
func upnpLeaseDurationForServiceSec(s *instance.Instance) uint32 {
duration := upnpLeaseDurationForService(s)
seconds := duration.Seconds()
// Check if within range.
if seconds >= 0 && seconds <= float64(10*365*24*60*60) {
return uint32(seconds)
}
return uint32(defaultUPNPLeaseDuration.Seconds())
}
// Set up UPNP forwards for a service
// We first try to use the more modern Pinhole API introduced in UPNPv2 and fall back to UPNPv2 Port Forwarding if no forward was successful
func (p *Processor) upnpMap(ctx context.Context, s *instance.Instance) {
@@ -631,23 +350,17 @@ func (p *Processor) upnpMap(ctx context.Context, s *instance.Instance) {
gateways := upnp.GetGatewayClients(ctx)
// Determine desired UPNP TTL / "lease duration". Passed into the library as integer seconds from now, as the
// underlying XML API wants integer seconds.
leaseDurationSec := upnpLeaseDurationForServiceSec(s)
// Reset Gateway IPs to remove stale addresses
s.UPNPGatewayIPs = make([]string, 0)
vips, _ := instance.FetchServiceAddresses(s.ServiceSnapshot)
for _, vip := range vips {
for _, vip := range instance.FetchServiceAddresses(s.ServiceSnapshot) {
for _, port := range s.ServiceSnapshot.Spec.Ports {
for _, gw := range gateways {
log.Info("[UPNP] Adding map", "vip", vip, "port", port.Port, "service", s.ServiceSnapshot.Name, "gateway", gw.WANIPv6FirewallControlClient.Location)
forwardSucessful := false
if gw.WANIPv6FirewallControlClient != nil {
log.Info("[UPNP] Adding map", "vip", vip, "port", port.Port, "service", s.ServiceSnapshot.Name, "gateway", gw.WANIPv6FirewallControlClient.Location, "leaseDurationSec", leaseDurationSec)
pinholeID, pinholeErr := gw.WANIPv6FirewallControlClient.AddPinholeCtx(ctx, "0.0.0.0", uint16(port.Port), vip, uint16(port.Port), upnp.MapProtocolToIANA(string(port.Protocol)), leaseDurationSec) //nolint TODO
pinholeID, pinholeErr := gw.WANIPv6FirewallControlClient.AddPinholeCtx(ctx, "0.0.0.0", uint16(port.Port), vip, uint16(port.Port), upnp.MapProtocolToIANA(string(port.Protocol)), 3600) //nolint TODO
if pinholeErr == nil {
forwardSucessful = true
log.Info("[UPNP] Service should be accessible externally", "port", port.Port, "pinhold ID", pinholeID)
@@ -658,17 +371,9 @@ func (p *Processor) upnpMap(ctx context.Context, s *instance.Instance) {
}
// Fallback to PortForward
if !forwardSucessful {
log.Info("[UPNP] Adding map", "vip", vip, "port", port.Port, "service", s.ServiceSnapshot.Name, "leaseDurationSec", leaseDurationSec)
portMappingErr := gw.ConnectionClient.AddPortMapping("0.0.0.0", uint16(port.Port), strings.ToUpper(string(port.Protocol)), uint16(port.Port), vip, true, s.ServiceSnapshot.Name, leaseDurationSec) //nolint TODO
portMappingErr := gw.ConnectionClient.AddPortMapping("0.0.0.0", uint16(port.Port), strings.ToUpper(string(port.Protocol)), uint16(port.Port), vip, true, s.ServiceSnapshot.Name, 3600) //nolint TODO
if portMappingErr == nil {
ip, err := gw.ConnectionClient.GetExternalIPAddress()
if err != nil {
// Log the error but continue on the off chance the mapping was successful
log.Error("[UPNP] Unable to get external IP address from gateway", "service", s.ServiceSnapshot.Name, "port", port.Port, "err", err)
} else {
log.Info("[UPNP] Service should be accessible externally", "service", s.ServiceSnapshot.Name, "port", port.Port, "externalip", ip)
}
log.Info("[UPNP] Service should be accessible externally", "port", port.Port)
forwardSucessful = true
} else {
//TODO: Cleanup
@@ -691,7 +396,7 @@ func (p *Processor) upnpMap(ctx context.Context, s *instance.Instance) {
s.UPNPGatewayIPs = slices.Compact(s.UPNPGatewayIPs)
}
func (p *Processor) updateStatus(ctx context.Context, i *instance.Instance) error {
func (p *Processor) updateStatus(i *instance.Instance) error {
// let's retry status update every 10ms for 30s
retryConfig := wait.Backoff{
Steps: 3000,
@@ -703,7 +408,7 @@ func (p *Processor) updateStatus(ctx context.Context, i *instance.Instance) erro
err := retry.OnError(retryConfig, func(error) bool { return true }, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := p.clientSet.CoreV1().Services(i.ServiceSnapshot.Namespace).Get(ctx, i.ServiceSnapshot.Name, metav1.GetOptions{})
currentService, err := p.clientSet.CoreV1().Services(i.ServiceSnapshot.Namespace).Get(context.TODO(), i.ServiceSnapshot.Name, metav1.GetOptions{})
if err != nil {
return err
}
@@ -713,24 +418,14 @@ func (p *Processor) updateStatus(ctx context.Context, i *instance.Instance) erro
currentServiceCopy.Annotations = make(map[string]string)
}
// If we're using ARP then we can only broadcast the VIP from one place, also useful for other software when running BGP, add an annotation to the service
if p.config.EnableARP || p.config.EnableBGP {
// If we're using ARP then we can only broadcast the VIP from one place, add an annotation to the service
if p.config.EnableARP {
// Add the current host
currentServiceCopy.Annotations[kubevip.VipHost] = p.config.NodeName
}
if i.DHCPInterfaceHwaddr != "" || i.DHCPInterfaceIPv4 != "" || i.DHCPInterfaceIPv6 != "" {
if i.DHCPInterfaceHwaddr != "" || i.DHCPInterfaceIP != "" {
currentServiceCopy.Annotations[kubevip.HwAddrKey] = i.DHCPInterfaceHwaddr
dhcpInterfaceIP := ""
if i.DHCPInterfaceIPv4 != "" {
dhcpInterfaceIP = i.DHCPInterfaceIPv4
if i.DHCPInterfaceIPv6 != "" {
dhcpInterfaceIP += ","
}
}
if i.DHCPInterfaceIPv6 != "" {
dhcpInterfaceIP += i.DHCPInterfaceIPv6
}
currentServiceCopy.Annotations[kubevip.RequestedIP] = dhcpInterfaceIP
currentServiceCopy.Annotations[kubevip.RequestedIP] = i.DHCPInterfaceIP
}
if currentService.Annotations["development.kube-vip.io/synthetic-api-server-error-on-update"] == "true" {
@@ -739,7 +434,7 @@ func (p *Processor) updateStatus(ctx context.Context, i *instance.Instance) erro
}
if !cmp.Equal(currentService, currentServiceCopy) {
currentService, err = p.clientSet.CoreV1().Services(currentServiceCopy.Namespace).Update(ctx, currentServiceCopy, metav1.UpdateOptions{})
currentService, err = p.clientSet.CoreV1().Services(currentServiceCopy.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("updating Spec", "service", i.ServiceSnapshot.Name, "err", err)
return err
@@ -757,8 +452,8 @@ func (p *Processor) updateStatus(ctx context.Context, i *instance.Instance) erro
ingresses := []v1.LoadBalancerIngress{}
for _, c := range i.VIPConfigs {
if !utils.IsIP(c.VIP) {
ips, err := utils.LookupHost(c.VIP, c.DNSMode, *i.ServiceSnapshot.Spec.IPFamilyPolicy == v1.IPFamilyPolicyRequireDualStack)
if !vip.IsIP(c.VIP) {
ips, err := vip.LookupHost(c.VIP, p.config.DNSMode)
if err != nil {
return err
}
@@ -786,12 +481,10 @@ func (p *Processor) updateStatus(ctx context.Context, i *instance.Instance) erro
}
}
}
log.Debug("LB status", "current", currentService.Status.LoadBalancer.Ingress, "new", ingresses)
if !ingressEqual(currentService.Status.LoadBalancer.Ingress, ingresses) {
if !cmp.Equal(currentService.Status.LoadBalancer.Ingress, ingresses) {
currentService.Status.LoadBalancer.Ingress = ingresses
log.Debug("updating service status", "namespace", currentService.Namespace, "name", currentService.Name, "uid", currentService.UID)
_, err = p.clientSet.CoreV1().Services(currentService.Namespace).UpdateStatus(ctx, currentService, metav1.UpdateOptions{})
if err != nil && !apierrors.IsInvalid(err) {
_, err = p.clientSet.CoreV1().Services(currentService.Namespace).UpdateStatus(context.TODO(), currentService, metav1.UpdateOptions{})
if err != nil {
log.Error("updating Service", "namespace", i.ServiceSnapshot.Namespace, "name", i.ServiceSnapshot.Name, "err", err)
return err
}
@@ -802,38 +495,20 @@ func (p *Processor) updateStatus(ctx context.Context, i *instance.Instance) erro
return err
}
func ingressEqual(a, b []v1.LoadBalancerIngress) bool {
if len(a) != len(b) {
return false
}
for i := range len(a) {
if a[i].IP != b[i].IP || a[i].Hostname != b[i].Hostname ||
!cmp.Equal(a[i].Ports, b[i].Ports) {
return false
}
}
return true
}
func isUPNPEnabled(s *v1.Service) bool {
return metav1.HasAnnotation(s.ObjectMeta, kubevip.UpnpEnabled) && s.Annotations[kubevip.UpnpEnabled] == "true"
}
// Refresh UPNP Port Forwards for all Service Instances registered in the processor
func (p *Processor) RefreshUPNPForwards(ctx context.Context) {
func (p *Processor) RefreshUPNPForwards() {
log.Info("Starting UPNP Port Refresher")
for {
time.Sleep(300 * time.Second)
// Skip logging if no service instances
if len(p.ServiceInstances) == 0 {
continue
}
log.Info("[UPNP] Refreshing Instances", "number of instances", len(p.ServiceInstances))
for i := range p.ServiceInstances {
p.upnpMap(ctx, p.ServiceInstances[i])
if err := p.updateStatus(ctx, p.ServiceInstances[i]); err != nil {
p.upnpMap(context.TODO(), p.ServiceInstances[i])
if err := p.updateStatus(p.ServiceInstances[i]); err != nil {
log.Warn("[UPNP] Error updating service", "ip", p.ServiceInstances[i].ServiceSnapshot.Name, "err", err)
}
}

View File

@@ -1,68 +0,0 @@
package services
import (
"testing"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/kube-vip/kube-vip/pkg/instance"
)
// Test_upnpLeaseDurationForService tests whether the default lease duration is used, and whether the annotation
// overrides it correctly.
//
// For simplicity, this table driven test does not cover all edge cases (passing in nil instance, nil service snapshot,
// nil annotations, etc).
func Test_upnpLeaseDurationForService(t *testing.T) {
const annotation = "kube-vip.io/upnp-lease-duration" // Validating value of kubevip.UpnpLeaseDuration.
tcs := []struct {
name string
annotations map[string]string
want int // in seconds
}{
{
name: "No annotation uses default",
annotations: map[string]string{},
want: 3600,
},
{
name: "Valid annotation overrides default",
annotations: map[string]string{
annotation: "2h",
},
want: 7200,
},
{
name: "Valid short annotation overrides default",
annotations: map[string]string{
annotation: "30m",
},
want: 1800,
},
{
name: "Invalid annotation uses default",
annotations: map[string]string{
annotation: "invalid-duration",
},
want: 3600,
},
}
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
i := &instance.Instance{
ServiceSnapshot: &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Annotations: tc.annotations,
},
},
}
gotDuration := upnpLeaseDurationForService(i)
got := int(gotDuration.Seconds())
if got != tc.want {
t.Errorf("upnpLeaseDurationForService(%+v) = %v, want %v", tc.annotations, got, tc.want)
}
})
}
}

View File

@@ -1,170 +0,0 @@
package services
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/utils"
v1 "k8s.io/api/core/v1"
)
// addServiceWireguard configures a WireGuard tunnel for a service
// The tunnel is brought up here, but DNAT rules are configured by the endpoint watcher
// via wireguardWorker.processInstance() when endpoints become available
func (p *Processor) addServiceWireguard(_ context.Context, svc *v1.Service) error {
if !p.config.EnableWireguard {
return nil
}
// Get service VIPs
serviceIPs, err := utils.FetchServiceIPs(svc)
if err != nil {
return fmt.Errorf("failed to get service IPs for %s/%s: %w", svc.Namespace, svc.Name, err)
}
if len(serviceIPs) == 0 {
return fmt.Errorf("no service IPs found for service %s/%s", svc.Namespace, svc.Name)
}
// For each VIP, bring up the WireGuard tunnel
// DNAT rules will be configured by the endpoint watcher when endpoints are available
var successCount int
var lastErr error
for _, vip := range serviceIPs {
if err := p.setupServiceWireguardTunnel(svc, vip); err != nil {
log.Error("[wireguard] failed to setup tunnel for VIP",
"service", svc.Name,
"namespace", svc.Namespace,
"vip", vip,
"err", err)
lastErr = err
// Continue with other VIPs even if one fails
continue
}
successCount++
}
if successCount == 0 {
return fmt.Errorf("failed to setup WireGuard tunnel for any VIP in service %s/%s: %w", svc.Namespace, svc.Name, lastErr)
}
return nil
}
// setupServiceWireguardTunnel brings up the WireGuard tunnel for a single VIP
// DNAT rules are NOT configured here - they are handled by the endpoint watcher
func (p *Processor) setupServiceWireguardTunnel(svc *v1.Service, vip string) error {
// Check if we have a tunnel configuration for this VIP
if !p.TunnelMgr.HasConfigForVIP(vip) {
return fmt.Errorf("no WireGuard tunnel configuration found for VIP %s", vip)
}
// Get the tunnel configuration to determine the interface name
tunnelConfig := p.TunnelMgr.GetConfigForVIP(vip)
if tunnelConfig == nil {
return fmt.Errorf("failed to get tunnel configuration for VIP %s", vip)
}
// Bring up the WireGuard tunnel for this VIP
if err := p.TunnelMgr.BringUpTunnelForVIP(vip); err != nil {
return fmt.Errorf("failed to bring up WireGuard tunnel for VIP %s: %w", vip, err)
}
log.Info("[wireguard] brought up tunnel for service",
"namespace", svc.Namespace,
"name", svc.Name,
"vip", vip,
"interface", tunnelConfig.InterfaceName)
// DNAT rules will be configured by wireguardWorker.processInstance()
// when the endpoint watcher detects available endpoints
return nil
}
// deleteServiceWireguard removes nftables DNAT rules and tears down WireGuard tunnel for a service
func (p *Processor) deleteServiceWireguard(_ context.Context, svc *v1.Service) {
if !p.config.EnableWireguard {
return
}
serviceID := fmt.Sprintf("%s_%s", svc.Namespace, svc.Name)
serviceID = utils.SanitizeServiceID(serviceID)
log.Info("[wireguard] deleting DNAT rules and tunnel for service",
"namespace", svc.Namespace,
"name", svc.Name,
"serviceID", serviceID)
// Get service IPs
serviceIPs, _ := utils.FetchServiceIPs(svc)
// Delete DNAT chains for each port
for _, port := range svc.Spec.Ports {
if port.Protocol != v1.ProtocolTCP && port.Protocol != v1.ProtocolUDP {
continue
}
portServiceID := fmt.Sprintf("%s_p%d", serviceID, port.Port)
// Try to delete for both IPv4 and IPv6 if we have mixed IPs
hasIPv4 := false
hasIPv6 := false
for _, vip := range serviceIPs {
// Strip CIDR notation before checking IP version
addr := utils.StripCIDR(vip)
if utils.IsIPv6(addr) {
hasIPv6 = true
} else {
hasIPv4 = true
}
}
if hasIPv4 {
if err := nftables.DeleteIngressChains(false, portServiceID); err != nil {
log.Error("[wireguard] failed to delete IPv4 DNAT chains",
"service", svc.Name,
"port", port.Port,
"err", err)
} else {
log.Debug("[wireguard] deleted IPv4 DNAT chains",
"service", svc.Name,
"port", port.Port)
}
}
if hasIPv6 {
if err := nftables.DeleteIngressChains(true, portServiceID); err != nil {
log.Error("[wireguard] failed to delete IPv6 DNAT chains",
"service", svc.Name,
"port", port.Port,
"err", err)
} else {
log.Debug("[wireguard] deleted IPv6 DNAT chains",
"service", svc.Name,
"port", port.Port)
}
}
}
// Tear down the WireGuard tunnel for each VIP
for _, vip := range serviceIPs {
if err := p.TunnelMgr.TearDownTunnelForVIP(vip); err != nil {
log.Error("[wireguard] failed to tear down tunnel",
"service", svc.Name,
"vip", vip,
"err", err)
} else {
log.Info("[wireguard] tore down tunnel",
"service", svc.Name,
"vip", vip)
}
}
log.Info("[wireguard] DNAT rules deleted and tunnels torn down for service",
"namespace", svc.Namespace,
"name", svc.Name)
}

View File

@@ -1,8 +1,8 @@
package services
import (
"context"
"fmt"
"sync"
log "log/slog"
@@ -18,26 +18,46 @@ func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, ser
log.Info("watching", "provider", provider.GetLabel(), "service_name", service.Name, "namespace", service.Namespace)
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := provider.CreateRetryWatcher(svcCtx.Ctx, p.rwClientSet, service)
leaderCtx, cancel := context.WithCancel(svcCtx.Ctx)
defer cancel()
var leaderElectionActive bool
rw, err := provider.CreateRetryWatcher(leaderCtx, p.rwClientSet, service)
if err != nil {
return fmt.Errorf("[%s] error watching endpoints: %w", provider.GetLabel(), err)
}
wg := sync.WaitGroup{}
defer func() {
rw.Stop()
wg.Wait()
exitFunction := make(chan struct{})
go func() {
select {
case <-svcCtx.Ctx.Done():
log.Debug("context cancelled", "provider", provider.GetLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-p.shutdownChan:
log.Debug("shutdown called", "provider", provider.GetLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-exitFunction:
log.Debug("function ending", "provider", provider.GetLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
}
}()
wg.Go(func() {
<-svcCtx.Ctx.Done()
log.Debug("[endpoint watcher] service context cancelled", "provider", provider.GetLabel())
})
ch := rw.ResultChan()
epProcessor := endpoints.NewEndpointProcessor(p.config, provider, p.bgpServer, &p.ServiceInstances, p.leaseMgr, p.TunnelMgr)
epProcessor := endpoints.NewEndpointProcessor(p.config, provider, p.bgpServer, &p.ServiceInstances)
var lastKnownGoodEndpoint string
for event := range ch {
@@ -45,7 +65,7 @@ func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, ser
switch event.Type {
case watch.Added, watch.Modified:
restart, err := epProcessor.AddOrModify(svcCtx, event, &lastKnownGoodEndpoint, service, id, p.StartServicesLeaderElection, &wg, p.clientSet, p.updateEgressConfiguration)
restart, err := epProcessor.AddOrModify(svcCtx, event, &lastKnownGoodEndpoint, service, id, &leaderElectionActive, p.StartServicesLeaderElection, &leaderCtx, &cancel)
if restart {
continue
} else if err != nil {
@@ -53,11 +73,14 @@ func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, ser
}
case watch.Deleted:
if err := epProcessor.Delete(svcCtx.Ctx, service, id); err != nil {
if err := epProcessor.Delete(service, id); err != nil {
return fmt.Errorf("[%s] error while processing delete event: %w", provider.GetLabel(), err)
}
log.Info("[endpoint watcher] stopping watching - endpoint object deleted", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
// Close the goroutine that will end the retry watcher, then exit the endpoint watcher function
close(exitFunction)
log.Info("stopping watching", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil
case watch.Error:
errObject := apierrors.FromObject(event.Object)
@@ -65,6 +88,7 @@ func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, ser
log.Error("watch error", "provider", provider.GetLabel(), "err", statusErr)
}
}
log.Info("[endpoint watcher] stopping watching", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
close(exitFunction)
log.Info("stopping watching", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil //nolint:govet
}

View File

@@ -2,63 +2,24 @@ package services
import (
"context"
"errors"
"fmt"
"sync"
"time"
log "log/slog"
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/trafficmirror"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/prometheus/client_golang/prometheus"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
)
// watchWithAuthRetry retries watchFn with exponential backoff on transient 403 Forbidden
// and 401 Unauthorized errors. On joining control plane nodes with K8s 1.34+, the local
// etcd may still be a learner when kube-vip starts, causing RBAC data to be unavailable.
// These auth errors resolve once etcd is promoted to a full member (typically within seconds).
// Non-auth errors are returned immediately. Context cancellation stops the retry loop.
func watchWithAuthRetry(ctx context.Context, watchFn func(context.Context) (watch.Interface, error)) (watch.Interface, error) {
var w watch.Interface
var lastErr error
err := wait.ExponentialBackoffWithContext(ctx, wait.Backoff{
Duration: 2 * time.Second,
Factor: 2.0,
Jitter: 0.1,
Steps: 10,
Cap: 30 * time.Second,
}, func(ctx context.Context) (bool, error) {
var watchErr error
w, watchErr = watchFn(ctx)
if watchErr == nil {
return true, nil
}
if !apierrors.IsForbidden(watchErr) && !apierrors.IsUnauthorized(watchErr) {
return false, watchErr
}
lastErr = watchErr
log.Warn("(svcs) services watch auth error, retrying", "err", watchErr)
return false, nil
})
if err != nil {
return nil, fmt.Errorf("(svcs) services watch failed after retries: %w (last: %v)", err, lastErr)
}
return w, nil
}
// This function handles the watching of a services endpoints and updates a load balancers endpoint configurations accordingly
func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc func(*servicecontext.Context, *v1.Service, *sync.WaitGroup) error) error {
func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc func(context.Context, *v1.Service) error) error {
// first start port mirroring if enabled
if err := p.startTrafficMirroringIfEnabled(); err != nil {
return err
@@ -82,27 +43,27 @@ func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc func(*servi
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return watchWithAuthRetry(ctx, func(ctx context.Context) (watch.Interface, error) {
return p.rwClientSet.CoreV1().Services(p.config.ServiceNamespace).Watch(ctx, metav1.ListOptions{})
})
return p.rwClientSet.CoreV1().Services(p.config.ServiceNamespace).Watch(ctx, metav1.ListOptions{})
},
})
if err != nil {
return fmt.Errorf("error creating services watcher: %s", err.Error())
}
wg := sync.WaitGroup{}
defer wg.Wait()
watcherCtx, watcherCancel := context.WithCancel(ctx)
defer watcherCancel()
wg.Go(func() {
<-watcherCtx.Done()
log.Debug("(svcs) watcher context cancelled")
rw.Stop()
p.Stop()
})
exitFunction := make(chan struct{})
go func() {
select {
case <-p.shutdownChan:
log.Debug("(svcs) shutdown called")
// Stop the retry watcher
rw.Stop()
return
case <-exitFunction:
log.Debug("(svcs) function ending")
// Stop the retry watcher
rw.Stop()
return
}
}()
ch := rw.ResultChan()
// Used for tracking an active endpoint / pod
@@ -112,20 +73,20 @@ func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc func(*servi
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
if err := p.AddOrModify(watcherCtx, event, serviceFunc, &wg); err != nil {
if errors.Is(err, &utils.PanicError{}) {
return fmt.Errorf("add/modify service error: %w", err)
} else {
log.Error("service watcher event failed", "type", event.Type, "error", err)
}
restart, err := p.AddOrModify(ctx, event, serviceFunc)
if restart {
break
}
if err != nil {
return fmt.Errorf("add/modify service error: %w", err)
}
case watch.Deleted:
if err := p.Delete(event); err != nil {
if errors.Is(err, &utils.PanicError{}) {
return fmt.Errorf("delete service error: %w", err)
} else {
log.Error("service watcher event failed", "type", event.Type, "error", err)
}
restart, err := p.Delete(event)
if restart {
break
}
if err != nil {
return fmt.Errorf("delete service error: %w", err)
}
case watch.Bookmark:
// Un-used
@@ -144,7 +105,7 @@ func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc func(*servi
default:
}
}
close(exitFunction)
log.Warn("Stopping watching services for type: LoadBalancer in all namespaces")
return nil
}

View File

@@ -1,98 +0,0 @@
package services
import (
"context"
"fmt"
"testing"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/watch"
)
// fakeWatchInterface is a minimal watch.Interface for testing.
type fakeWatchInterface struct {
ch chan watch.Event
}
func newFakeWatchInterface() *fakeWatchInterface {
return &fakeWatchInterface{ch: make(chan watch.Event)}
}
func (f *fakeWatchInterface) Stop() { close(f.ch) }
func (f *fakeWatchInterface) ResultChan() <-chan watch.Event { return f.ch }
func TestWatchWithAuthRetry(t *testing.T) {
svcResource := schema.GroupResource{Resource: "services"}
fw := newFakeWatchInterface()
tcs := []struct {
name string
watchFn func(int) (watch.Interface, error)
wantErr bool
wantAttempts int
}{
{
name: "403 Forbidden retried then succeeds",
watchFn: func(attempt int) (watch.Interface, error) {
if attempt <= 2 {
return nil, apierrors.NewForbidden(svcResource, "", nil)
}
return fw, nil
},
wantAttempts: 3,
},
{
name: "401 Unauthorized retried then succeeds",
watchFn: func(attempt int) (watch.Interface, error) {
if attempt <= 2 {
return nil, apierrors.NewUnauthorized("not authorized yet")
}
return fw, nil
},
wantAttempts: 3,
},
{
name: "non-auth error fails immediately",
watchFn: func(_ int) (watch.Interface, error) {
return nil, fmt.Errorf("connection refused")
},
wantErr: true,
wantAttempts: 1,
},
{
name: "immediate success no retry",
watchFn: func(_ int) (watch.Interface, error) {
return fw, nil
},
wantAttempts: 1,
},
}
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
attempts := 0
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
w, err := watchWithAuthRetry(ctx, func(_ context.Context) (watch.Interface, error) {
attempts++
return tc.watchFn(attempts)
})
if tc.wantErr && err == nil {
t.Fatal("expected error, got nil")
}
if !tc.wantErr && err != nil {
t.Fatalf("expected success, got: %v", err)
}
if !tc.wantErr && w != fw {
t.Fatal("returned watcher does not match expected")
}
if attempts != tc.wantAttempts {
t.Errorf("expected %d attempts, got %d", tc.wantAttempts, attempts)
}
})
}
}

View File

@@ -1,81 +0,0 @@
package utils
import (
"fmt"
"log/slog"
"net"
"strings"
"github.com/pkg/errors"
)
const (
IPv4Family = "IPv4"
IPv6Family = "IPv6"
DualFamily = "dual"
)
// LookupHost resolves dnsName and return an IP or an error
func LookupHost(dnsName, dnsMode string, requireDualStack bool) ([]string, error) {
result, err := net.LookupHost(dnsName)
if err != nil {
return nil, err
}
if len(result) == 0 {
return nil, errors.Errorf("empty address for %s", dnsName)
}
addrs := []string{}
switch dnsMode {
case strings.ToLower(IPv4Family), strings.ToLower(IPv6Family), DualFamily:
a, err := getIPbyFamily(result, dnsMode, requireDualStack)
if err != nil {
return nil, err
}
addrs = append(addrs, a...)
default:
addrs = append(addrs, result[0])
}
return addrs, nil
}
func getIPbyFamily(addresses []string, family string, requireDualStack bool) ([]string, error) {
var checkers []func(string) bool
families := []string{}
if family == DualFamily || family == strings.ToLower(IPv4Family) {
checkers = append(checkers, IsIPv4)
families = append(families, IPv4Family)
}
if family == DualFamily || family == strings.ToLower(IPv6Family) {
checkers = append(checkers, IsIPv6)
families = append(families, IPv6Family)
}
addrs := []string{}
for i, c := range checkers {
addr, err := getIPbyChecker(addresses, c)
if err != nil {
if len(checkers) > 1 && !requireDualStack {
slog.Warn("no address found", "family", families[i])
continue
}
return nil, fmt.Errorf("error getting %s address: %w", families[i], err)
}
addrs = append(addrs, addr)
}
if len(addrs) == 0 {
return nil, fmt.Errorf("no addresses found")
}
return addrs, nil
}
func getIPbyChecker(addresses []string, checker func(string) bool) (string, error) {
for _, addr := range addresses {
if checker(addr) {
return addr, nil
}
}
return "", fmt.Errorf("address not found")
}

View File

@@ -1,132 +0,0 @@
package utils
import (
"fmt"
"net"
"strings"
v1 "k8s.io/api/core/v1"
)
// FormatIPWithSubnetMask takes a raw IP address and a subnet mask, and returns a formatted string in CIDR notation.
func FormatIPWithSubnetMask(rawIP string, subnetMask string) (string, error) {
addr := fmt.Sprintf("%s/%s", rawIP, subnetMask)
// Check if the input is valid
_, _, err := net.ParseCIDR(addr)
if err != nil {
return "", fmt.Errorf("invalid CIDR: %q, %w", addr, err)
}
return addr, nil
}
// IsIP returns if address is an IP or not
func IsIP(address string) bool {
ip := net.ParseIP(address)
return ip != nil
}
// IsIPv4 returns true only if address is a valid IPv4 address
func IsIPv4(address string) bool {
ip := net.ParseIP(address)
if ip == nil {
return false
}
return ip.To4() != nil
}
// IsIPv6 returns true only if address is a valid IPv6 address
func IsIPv6(address string) bool {
ip := net.ParseIP(address)
if ip == nil {
return false
}
return ip.To4() == nil
}
func IsIPv4CIDR(cidr string) bool {
ip, _, _ := net.ParseCIDR(cidr)
if ip == nil {
return false
}
return ip.To4() != nil
}
func IsIPv6CIDR(cidr string) bool {
ip, _, _ := net.ParseCIDR(cidr)
if ip == nil {
return false
}
return ip.To4() == nil
}
// StripCIDR removes the CIDR notation (e.g., "/24") from an IP address string.
// If no CIDR notation is present, the original string is returned unchanged.
func StripCIDR(ip string) string {
if idx := strings.Index(ip, "/"); idx >= 0 {
return ip[:idx]
}
return ip
}
// SanitizeServiceID sanitizes a service ID to be valid for nftables chain names.
// Only alphanumeric characters and underscores are allowed; other characters are replaced with underscores.
// The result is truncated to 50 characters to respect nftables name length limits.
func SanitizeServiceID(id string) string {
var result strings.Builder
for _, r := range id {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
result.WriteRune(r)
} else {
result.WriteRune('_')
}
}
sanitized := result.String()
// Ensure it doesn't exceed nftables name length limit
if len(sanitized) > 50 {
sanitized = sanitized[:50]
}
return sanitized
}
// FetchServiceIPs extracts IP addresses from a Kubernetes Service.
// It checks the following sources in order:
// 1. kube-vip.io/loadbalancerIPs annotation (comma-separated list)
// 2. spec.LoadBalancerIP (deprecated but still used)
// 3. status.loadBalancer.ingress
// Returns an error if no IPs are found.
func FetchServiceIPs(service *v1.Service) ([]string, error) {
var ips []string
// Check for loadBalancerIPs annotation first (new style)
if loadBalancerIPs, ok := service.Annotations["kube-vip.io/loadbalancerIPs"]; ok {
for _, ip := range strings.Split(loadBalancerIPs, ",") {
ip = strings.TrimSpace(ip)
if ip != "" {
ips = append(ips, ip)
}
}
}
// Fallback to spec.LoadBalancerIP (deprecated but still used)
if len(ips) == 0 && service.Spec.LoadBalancerIP != "" {
ips = append(ips, service.Spec.LoadBalancerIP)
}
// Check status.loadBalancer.ingress as well
if len(ips) == 0 {
for _, ingress := range service.Status.LoadBalancer.Ingress {
if ingress.IP != "" {
ips = append(ips, ingress.IP)
}
}
}
if len(ips) == 0 {
return nil, fmt.Errorf("no IPs found for service")
}
return ips, nil
}

View File

@@ -1,55 +0,0 @@
package utils
import (
"fmt"
"net"
"github.com/vishvananda/netlink"
)
// GetInterfaceByIP returns the network interface that has the specified IP address assigned.
func GetInterfaceByIP(ipAddr string) (*netlink.Link, error) {
ip := net.ParseIP(ipAddr)
if ip == nil {
return nil, fmt.Errorf("invalid IP address: %s", ipAddr)
}
links, err := netlink.LinkList()
if err != nil {
return nil, fmt.Errorf("failed to list network interfaces: %v", err)
}
for i := range links {
addrs, err := netlink.AddrList(links[i], netlink.FAMILY_ALL)
if err != nil {
return nil, fmt.Errorf("failed to list addresses for interface %s: %v", links[i].Attrs().Name, err)
}
for _, addr := range addrs {
if addr.IP.Equal(ip) {
return &links[i], nil
}
}
}
return nil, fmt.Errorf("no interface found with IP address: %s", ipAddr)
}
// GetNonLinkLocalIP returns the first non link-local IPv4/IPv6 address on the given interface.
func GetNonLinkLocalIP(iface *netlink.Link, family int) (string, error) {
a, err := netlink.AddrList(*iface, family)
if err != nil {
return "", fmt.Errorf("failed to list addresses for interface %s: %v", (*iface).Attrs().Name, err)
}
for _, addr := range a {
if addr.IPNet != nil {
ip := addr.IPNet.IP
if !ip.IsLinkLocalUnicast() {
return ip.String(), nil
}
}
}
return "", fmt.Errorf("failed to find non-local IP on interface: %s", (*iface).Attrs().Name)
}

View File

@@ -1,15 +0,0 @@
package utils
import "fmt"
type PanicError struct {
cause string
}
func (e *PanicError) Error() string {
return fmt.Sprintf("%s - unrecoverable error", e.cause)
}
func NewPanicError(cause string) error {
return &PanicError{cause: cause}
}

View File

@@ -1,6 +1,8 @@
package utils
import (
"fmt"
"net"
"os"
)
@@ -11,3 +13,15 @@ func FileExists(filename string) bool {
}
return !info.IsDir()
}
// FormatIPWithSubnetMask takes a raw IP address and a subnet mask, and returns a formatted string in CIDR notation.
func FormatIPWithSubnetMask(rawIP string, subnetMask string) (string, error) {
addr := fmt.Sprintf("%s/%s", rawIP, subnetMask)
// Check if the input is valid
_, _, err := net.ParseCIDR(addr)
if err != nil {
return "", fmt.Errorf("invalid CIDR: %q, %w", addr, err)
}
return addr, nil
}

View File

@@ -17,32 +17,26 @@ import (
"golang.org/x/sys/unix"
v1 "k8s.io/api/core/v1"
iptables "github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/utils"
)
const (
defaultValidLft = 60
iptablesComment = "%s kube-vip load balancer IP"
iptablesCommentMarkRule = "kube-vip load balancer IP set mark for masquerade"
DefaultMaskIPv4 = 32
DefaultMaskIPv6 = 128
NoLifetime = 0
defaultValidLft = 60
iptablesComment = "%s kube-vip load balancer IP"
iptablesCommentMarkRule = "kube-vip load balancer IP set mark for masquerade"
ignoreServiceSecurityAnnotation = "kube-vip.io/ignore-service-security"
)
// Network is an interface that enable managing operations for a given IP
type Network interface {
AddIP(precheck bool, skipDAD bool, minLifetime ...int) (bool, error)
AddIP(precheck bool) (bool, error)
AddRoute(precheck bool) error
DeleteIP() (bool, error)
DeleteRoute() error
UpdateRoutes() (bool, error)
IsSet() (*netlink.Addr, error)
IsSet() (bool, error)
IP() string
CIDR() string
IPisLinkLocal() bool
@@ -59,8 +53,6 @@ type Network interface {
SetHasEndpoints(value bool)
HasEndpoints() bool
ARPName() string
GetPossibleSubnets() string
DHCPFamily() string
}
// network - This allows network configuration
@@ -87,18 +79,12 @@ type network struct {
ipvsEnabled bool
hasEndpoints bool
possibleSubnets string
// used by DHCP to get address of proper family
dhcpFamily string
}
// NewConfig will attempt to provide an interface to the kernel network configuration
func NewConfig(address string, iface string, loGlobalScope bool, subnet string, isDDNS bool,
dhcpMode string, requireDualStack, isDualStack bool, tableID int, tableType int, routingProtocol int,
dnsMode, forwardMethod, iptablesBackend string, ipvsEnabled, enableSecurity bool,
intfMgr *networkinterface.Manager) ([]Network, error) {
func NewConfig(address string, iface string, loGlobalScope bool, subnet string, isDDNS bool, tableID int, tableType int,
routingProtocol int, dnsMode, forwardMethod, iptablesBackend string,
ipvsEnabled, enableSecurity bool, intfMgr *networkinterface.Manager) ([]Network, error) {
networks := []Network{}
link, err := netlink.LinkByName(iface)
@@ -108,7 +94,7 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
networkLink := intfMgr.Get(link)
if utils.IsIP(address) {
if IsIP(address) {
result := &network{
link: networkLink,
routeTable: tableID,
@@ -117,8 +103,6 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
forwardMethod: forwardMethod,
iptablesBackend: iptablesBackend,
ipvsEnabled: ipvsEnabled,
enableSecurity: enableSecurity,
possibleSubnets: subnet,
}
subnet, err = SelectSubnet(address, subnet)
@@ -150,51 +134,25 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
networks = append(networks, result)
} else {
// try to resolve the address
log.Debug("looking up host", "address", address, "dnsMode", dnsMode)
ips, err := utils.LookupHost(address, dnsMode, requireDualStack)
if (dnsMode == utils.DualFamily && isDDNS && isDualStack) || err != nil {
ips, err := LookupHost(address, dnsMode)
if err != nil {
// return early for ddns if no IP is allocated for the domain
// when leader starts, should do get IP from DHCP for the domain
if isDDNS {
log.Info("isDDNS true", "dhcpMode", dhcpMode)
if strings.EqualFold(dhcpMode, utils.IPv4Family) || strings.EqualFold(dhcpMode, utils.DualFamily) {
result := &network{
link: networkLink,
routeTable: tableID,
routingTableType: tableType,
routingProtocol: routingProtocol,
forwardMethod: forwardMethod,
iptablesBackend: iptablesBackend,
isDDNS: isDDNS,
dnsName: address,
ipvsEnabled: ipvsEnabled,
enableSecurity: enableSecurity,
possibleSubnets: subnet,
dhcpFamily: utils.IPv4Family,
}
networks = append(networks, result)
}
if strings.EqualFold(dhcpMode, utils.IPv6Family) || strings.EqualFold(dhcpMode, utils.DualFamily) {
result := &network{
link: networkLink,
routeTable: tableID,
routingTableType: tableType,
routingProtocol: routingProtocol,
forwardMethod: forwardMethod,
iptablesBackend: iptablesBackend,
isDDNS: isDDNS,
dnsName: address,
ipvsEnabled: ipvsEnabled,
enableSecurity: enableSecurity,
possibleSubnets: subnet,
dhcpFamily: utils.IPv6Family,
}
networks = append(networks, result)
result := &network{
link: networkLink,
routeTable: tableID,
routingTableType: tableType,
routingProtocol: routingProtocol,
forwardMethod: forwardMethod,
iptablesBackend: iptablesBackend,
isDDNS: isDDNS,
dnsName: address,
ipvsEnabled: ipvsEnabled,
enableSecurity: enableSecurity,
}
networks = append(networks, result)
return networks, nil
}
return nil, err
@@ -212,15 +170,11 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
dnsName: address,
ipvsEnabled: ipvsEnabled,
enableSecurity: enableSecurity,
possibleSubnets: subnet,
}
s, err := SelectSubnet(ip, subnet)
if err != nil {
return nil, fmt.Errorf("failed to select subnet: %w", err)
}
// we're able to resolve store this as the initial IP
if result.address, err = netlink.ParseAddr(fmt.Sprintf("%s/%s", ip, s)); err != nil {
if result.address, err = netlink.ParseAddr(fmt.Sprintf("%s/%s", ip, subnet)); err != nil {
return networks, err
}
// set ValidLft so that the VIP expires if the DNS entry is updated, otherwise it'll be refreshed by the DNS prober
@@ -229,11 +183,6 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
// set address as deprecated so it isn't used as source address according to RFC 3484
result.address.PreferedLft = 0
result.dhcpFamily = strings.ToLower(utils.IPv6Family)
if net.ParseIP(ip).To4() != nil {
result.dhcpFamily = strings.ToLower(utils.IPv4Family)
}
networks = append(networks, result)
}
@@ -359,41 +308,23 @@ func (configurator *network) UpdateRoutes() (bool, error) {
}
// AddIP - Add an IP address to the interface
// precheck: if true, check if the IP already exists before adding
// skipDAD: if true, set IFA_F_NODAD flag for IPv6 addresses to skip Duplicate Address Detection
func (configurator *network) AddIP(precheck bool, skipDAD bool, minLifetime ...int) (bool, error) {
func (configurator *network) AddIP(precheck bool) (bool, error) {
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
var existing *netlink.Addr
exists := false
var err error
if precheck {
if existing, err = configurator.IsSet(); err != nil {
if exists, err = configurator.IsSet(); err != nil {
return false, errors.Wrap(err, "could not check if address exists")
}
}
lifetime := NoLifetime
if len(minLifetime) > 0 {
lifetime = minLifetime[0]
}
if existing != nil && existing.ValidLft > lifetime {
if exists {
return false, nil
}
// For IPv6 addresses, optionally set NODAD flag to skip Duplicate Address Detection (DAD)
// This prevents DADFAILED loops when recovering from a previous DADFAILED state
// The flag tells the kernel to skip DAD, which is safe when we're re-adding
// an address that we know should be ours (e.g., after DADFAILED recovery)
if skipDAD && utils.IsIPv6(configurator.address.IP.String()) {
configurator.address.Flags |= unix.IFA_F_NODAD
log.Debug("Setting IFA_F_NODAD flag for IPv6 address to skip DAD", "ip", configurator.address.IP.String())
}
log.Debug("replacing IP", "address", configurator.address)
if err := netlink.AddrReplace(configurator.link.Intf, configurator.address); err != nil {
return false, errors.Wrap(err, fmt.Sprintf("could not add ip to device %q", configurator.link.Intf.Attrs().Name))
return false, errors.Wrap(err, "could not add ip")
}
if err := configurator.configureIPTables(); err != nil {
@@ -404,13 +335,13 @@ func (configurator *network) AddIP(precheck bool, skipDAD bool, minLifetime ...i
}
func (configurator *network) configureIPTables() error {
if configurator.enableSecurity && !configurator.ignoreSecurity && len(configurator.ports) > 0 {
if configurator.enableSecurity && !configurator.ignoreSecurity {
if err := configurator.addIptablesRulesToLimitTrafficPorts(); err != nil {
return errors.Wrap(err, "could not add iptables rules to limit traffic ports")
}
}
// It seems that masquerading is only required with IPv4 for IPVS to work.
// It seems that masquerading is only reuired with IPv4 for IPVS to work.
if configurator.ipvsEnabled && configurator.forwardMethod == "masquerade" && configurator.address.IP.To4() != nil {
if err := configurator.addIptablesRulesForMasquerade(); err != nil {
return errors.Wrap(err, "could not add iptables rules for masquerade")
@@ -421,17 +352,12 @@ func (configurator *network) configureIPTables() error {
}
func (configurator *network) addIptablesRulesToLimitTrafficPorts() error {
vip := configurator.address.IP.String()
opt := iptables.IPFamily(iptables.ProtocolIPv4)
if utils.IsIPv6(vip) {
opt = iptables.IPFamily(iptables.ProtocolIPv6)
}
ipt, err := iptables.New(opt)
ipt, err := iptables.New()
if err != nil {
return errors.Wrap(err, "could not create iptables client")
}
vip := configurator.address.IP.String()
comment := fmt.Sprintf(iptablesComment, configurator.serviceName)
if err := insertCommonIPTablesRules(ipt, vip, comment); err != nil {
return fmt.Errorf("could not add common iptables rules: %w", err)
@@ -564,7 +490,7 @@ func (configurator *network) DeleteIP() (bool, error) {
}
// Nothing to delete
if result == nil {
if !result {
return false, nil
}
@@ -654,7 +580,7 @@ func (configurator *network) IsDADFAIL() bool {
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
if configurator.address == nil || !utils.IsIPv6(configurator.address.IP.String()) {
if configurator.address == nil || !IsIPv6(configurator.address.IP.String()) {
return false
}
@@ -679,31 +605,27 @@ func addressHasDADFAILEDFlag(address netlink.Addr) bool {
}
// isSet - Check to see if VIP is set
func (configurator *network) IsSet() (result *netlink.Addr, err error) {
func (configurator *network) IsSet() (result bool, err error) {
var addresses []netlink.Addr
if configurator.address == nil {
return nil, nil
}
if configurator.address.Mask == nil {
return nil, nil
return false, nil
}
addresses, err = netlink.AddrList(configurator.link.Intf, 0)
if err != nil {
err = errors.Wrap(err, "could not list addresses")
return nil, err
return false, err
}
for _, address := range addresses {
if address.Equal(*configurator.address) {
return &address, nil
return true, nil
}
}
return nil, nil
return false, nil
}
// SetIP updates the IP that is used
@@ -717,25 +639,6 @@ func (configurator *network) SetIP(ip string) error {
if strings.Contains("/", ip) {
return fmt.Errorf("ip should not contain CIDR notation got: %s", ip)
}
if configurator.address == nil {
log.Debug("possible", "subnets", configurator.possibleSubnets)
subnet, err := SelectSubnet(ip, configurator.possibleSubnets)
if err != nil {
return fmt.Errorf("unable to select subnet for IP %q from %q: %w", ip, subnet, err)
}
// Check if the subnet needs overriding
cidr, err := utils.FormatIPWithSubnetMask(ip, subnet)
if err != nil {
return errors.Wrapf(err, "2 could not format address %q with subnetMask %q", ip, subnet)
}
configurator.address, err = netlink.ParseAddr(cidr)
if err != nil {
return errors.Wrapf(err, "could not parse address %q", cidr)
}
}
ones, _ := configurator.address.Mask.Size()
cidr, err := utils.FormatIPWithSubnetMask(ip, strconv.Itoa(ones))
if err != nil {
@@ -766,7 +669,7 @@ func (configurator *network) SetServicePorts(service *v1.Service) {
configurator.ports = service.Spec.Ports
configurator.serviceName = service.Namespace + "/" + service.Name
configurator.ignoreSecurity = service.Annotations[kubevip.ServiceSecurityIgnore] == "true"
configurator.ignoreSecurity = service.Annotations[ignoreServiceSecurityAnnotation] == "true"
}
// IP - return the IP Address
@@ -774,10 +677,6 @@ func (configurator *network) IP() string {
configurator.mu.Lock()
defer configurator.mu.Unlock()
if configurator.address == nil || configurator.address.IP == nil {
return ""
}
return configurator.address.IP.String()
}
@@ -785,10 +684,6 @@ func (configurator *network) CIDR() string {
configurator.mu.Lock()
defer configurator.mu.Unlock()
if configurator.address == nil || configurator.address.IPNet == nil {
return ""
}
return configurator.address.IPNet.String()
}
@@ -862,20 +757,9 @@ func GarbageCollect(adapter, address string, intfMgr *networkinterface.Manager)
}
func (configurator *network) SetMask(mask string) error {
selectedMask := mask
var err error
if mask == "" {
return fmt.Errorf("no mask provided")
}
if configurator.IP() != "" {
selectedMask, err = SelectSubnet(configurator.IP(), mask)
if err != nil {
return fmt.Errorf("failed to select mask %q: %w", mask, err)
}
} else if len(strings.Split(mask, ",")) > 1 {
return fmt.Errorf("cannot select mask from %q when IP address is unknown", mask)
selectedMask, err := SelectSubnet(configurator.IP(), mask)
if err != nil {
return fmt.Errorf("failed to select mask %q: %w", mask, err)
}
m, err := strconv.Atoi(selectedMask)
@@ -883,18 +767,16 @@ func (configurator *network) SetMask(mask string) error {
return err
}
size := DefaultMaskIPv4
family := utils.IPv4Family
size := 32
family := "IPv4"
if configurator.IP() != "" {
if utils.IsIPv6(configurator.IP()) {
size = DefaultMaskIPv6
family = utils.IPv6Family
}
if IsIPv6(configurator.IP()) {
size = 128
family = "IPv6"
}
if m > size {
return fmt.Errorf("provided CIDR mask '%d' is greater than the highest mask value for the %s family (%d)", m, family, size)
}
if m > size {
return fmt.Errorf("provided CIDR mask '%d' is greater than the highest mask value for the %s family (%d)", m, family, size)
}
toSet := net.CIDRMask(m, size)
@@ -923,14 +805,6 @@ func (configurator *network) ARPName() string {
return fmt.Sprintf("%s-%s", configurator.CIDR(), configurator.Interface())
}
func (configurator *network) GetPossibleSubnets() string {
return configurator.possibleSubnets
}
func (configurator *network) DHCPFamily() string {
return configurator.dhcpFamily
}
// SelectSubnet formats an IP address with the appropriate CIDR based on the input.
// The input SubnetMasks can be "32,128" (dual-stack), "32", "128" (SingleStack).
func SelectSubnet(rawIP string, subnetMasks string) (string, error) {

View File

@@ -2,8 +2,7 @@ package vip
import (
"context"
"fmt"
"sync"
"net"
"time"
log "log/slog"
@@ -15,43 +14,42 @@ import (
// for the dDNSHostName
// will return the IP allocated
type DDNSManager interface {
Start(ctx context.Context, wg *sync.WaitGroup) (string, error)
Start() (string, error)
}
type ddnsManager struct {
network Network
backoffAttempts uint
ctx context.Context
network Network
}
// NewDDNSManager returns a newly created Dynamic DNS manager
func NewDDNSManager(network Network, backoffAttempts uint) DDNSManager {
func NewDDNSManager(ctx context.Context, network Network) DDNSManager {
return &ddnsManager{
network: network,
backoffAttempts: backoffAttempts,
ctx: ctx,
network: network,
}
}
// Start will start the dhcpclient routine to keep the lease
// and return the IP it got from DHCP
func (ddns *ddnsManager) Start(ctx context.Context, wg *sync.WaitGroup) (string, error) {
client, err := NewDHCPClient(ddns.network, ddns.backoffAttempts)
func (ddns *ddnsManager) Start() (string, error) {
interfaceName := ddns.network.Interface()
iface, err := net.InterfaceByName(interfaceName)
if err != nil {
return "", fmt.Errorf("unable to create DHCP client: %w", err)
return "", err
}
client := NewDHCPClient(iface, false, "")
client.WithHostName(ddns.network.DDNSHostName())
wg.Go(func() {
if err := client.Start(ctx); err != nil {
log.Error("[ddns] DHCP client error: %w")
}
})
go client.Start()
log.Info("waiting for ip from dhcp")
ip, timeout := "", time.After(1*time.Minute)
select {
case <-ctx.Done():
case <-ddns.ctx.Done():
client.Stop()
return "", errors.New("context cancelled")
case <-timeout:
@@ -69,7 +67,7 @@ func (ddns *ddnsManager) Start(ctx context.Context, wg *sync.WaitGroup) (string,
// start a go routine to stop dhclient when lose leader election
// also to keep read the ip from channel
// so onbound function is unblocked to send the ip
wg.Go(func() {
go func(ctx context.Context) {
for {
select {
case <-ctx.Done():
@@ -80,7 +78,7 @@ func (ddns *ddnsManager) Start(ctx context.Context, wg *sync.WaitGroup) (string,
log.Info("got address from dhcp", "ip", ip)
}
}
})
}(ddns.ctx)
return ip, nil
}

View File

@@ -1,39 +1,289 @@
package vip
// DHCP client implementation that refers to https://www.rfc-editor.org/rfc/rfc2131.html
import (
"context"
"fmt"
"net"
"strings"
"time"
"github.com/kube-vip/kube-vip/pkg/utils"
log "log/slog"
"github.com/insomniacslk/dhcp/dhcpv4"
"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
"github.com/jpillora/backoff"
)
type DHCPClient interface {
ErrorChannel() chan error
IPChannel() chan string
Start(ctx context.Context) error
Stop()
WithHostName(hostname string) DHCPClient
const dhcpClientPort = "68"
const defaultDHCPRenew = time.Hour
const maxBackoffAttempts = 3
// DHCPClient is responsible for maintaining ipv4 lease for one specified interface
type DHCPClient struct {
iface *net.Interface
ddnsHostName string
lease *nclient4.Lease
initRebootFlag bool
requestedIP net.IP
stopChan chan struct{} // used as a signal to release the IP and stop the dhcp client daemon
releasedChan chan struct{} // indicate that the IP has been released
errorChan chan error // indicates there was an error on the IP request
ipChan chan string
}
func NewDHCPClient(network Network, backoffAttempts uint) (DHCPClient, error) {
interfaceName := network.Interface()
iface, err := net.InterfaceByName(interfaceName)
if err != nil {
return nil, fmt.Errorf("failed to get interface %q: %w", interfaceName, err)
// NewDHCPClient returns a new DHCP Client.
func NewDHCPClient(iface *net.Interface, initRebootFlag bool, requestedIP string) *DHCPClient {
return &DHCPClient{
iface: iface,
stopChan: make(chan struct{}),
releasedChan: make(chan struct{}),
errorChan: make(chan error),
initRebootFlag: initRebootFlag,
requestedIP: net.ParseIP(requestedIP),
ipChan: make(chan string),
}
}
var client DHCPClient
func (c *DHCPClient) WithHostName(hostname string) *DHCPClient {
c.ddnsHostName = hostname
return c
}
if strings.EqualFold(network.DHCPFamily(), utils.IPv6Family) {
client, err = NewDHCPv6Client(iface, nil, false, "", backoffAttempts)
if err != nil {
return nil, fmt.Errorf("failed to create DHCP client: %w", err)
// Stop state-transition process and close dhcp client
func (c *DHCPClient) Stop() {
close(c.ipChan)
close(c.stopChan)
<-c.releasedChan
}
// Gets the IPChannel for consumption
func (c *DHCPClient) IPChannel() chan string {
return c.ipChan
}
// Gets the ErrorChannel for consumption
func (c *DHCPClient) ErrorChannel() chan error {
return c.errorChan
}
// Start state-transition process of dhcp client
//
// -------- -------
//
// | | +-------------------------->| |<-------------------+
// | INIT- | | +-------------------->| INIT | |
// | REBOOT |DHCPNAK/ +---------->| |<---+ |
// | |Restart| | ------- | |
//
// -------- | DHCPNAK/ | | |
// | Discard offer | -/Send DHCPDISCOVER |
//
// -/Send DHCPREQUEST | | |
//
// | | | DHCPACK v | |
// ----------- | (not accept.)/ ----------- | |
//
// | | | Send DHCPDECLINE | | |
// | REBOOTING | | | | SELECTING |<----+ |
// | | | / | | |DHCPOFFER/ |
//
// ----------- | / ----------- | |Collect |
// | | / | | | replies |
//
// DHCPACK/ | / +----------------+ +-------+ |
// Record lease, set| | v Select offer/ |
// timers T1, T2 ------------ send DHCPREQUEST | |
//
// | +----->| | DHCPNAK, Lease expired/ |
// | | | REQUESTING | Halt network |
// DHCPOFFER/ | | | |
// Discard ------------ | |
// | | | | ----------- |
// | +--------+ DHCPACK/ | | |
// | Record lease, set -----| REBINDING | |
// | timers T1, T2 / | | |
// | | DHCPACK/ ----------- |
// | v Record lease, set ^ |
// +----------------> ------- /timers T1,T2 | |
// +----->| |<---+ | |
// | | BOUND |<---+ | |
// DHCPOFFER, DHCPACK, | | | T2 expires/ DHCPNAK/
// DHCPNAK/Discard ------- | Broadcast Halt network
// | | | | DHCPREQUEST |
// +-------+ | DHCPACK/ | |
// T1 expires/ Record lease, set | |
// Send DHCPREQUEST timers T1, T2 | |
// to leasing server | | |
// | ---------- | |
// | | |------------+ |
// +->| RENEWING | |
// | |----------------------------+
// ----------
// Figure: State-transition diagram for DHCP clients
func (c *DHCPClient) Start() {
lease := c.requestWithBackoff()
c.initRebootFlag = false
c.lease = lease
// Set up two ticker to renew/rebind regularly
t1Timeout := c.lease.ACK.IPAddressLeaseTime(defaultDHCPRenew) / 2
t2Timeout := (c.lease.ACK.IPAddressLeaseTime(defaultDHCPRenew) / 8) * 7
log.Debug("dhcp timeouts", "timeout1", t1Timeout, "timeoute2", t2Timeout)
t1, t2 := time.NewTicker(t1Timeout), time.NewTicker(t2Timeout)
for {
select {
case <-t1.C:
// renew is a unicast request of the IP renewal
// A point on renew is: the library does not return the right message (NAK)
// on renew error due to IP Change, but instead it returns a different error
// This way there's not much to do other than log and continue, as the renew error
// may be an offline server, or may be an incorrect package match
lease, err := c.renew()
if err == nil {
c.lease = lease
log.Info("renew", "lease", lease)
t2.Reset(t2Timeout)
} else {
log.Error("renew failed", "err", err)
}
case <-t2.C:
// rebind is just like a request, but forcing to provide a new IP address
lease, err := c.request(true)
if err == nil {
c.lease = lease
log.Info("rebind", "lease", lease)
} else {
if _, ok := err.(*nclient4.ErrNak); !ok {
t1.Stop()
t2.Stop()
log.Error("rebind failed", "err", err)
return
}
log.Warn("ip may have changed", "ip", c.lease.ACK.YourIPAddr, "err", err)
c.initRebootFlag = false
c.lease = c.requestWithBackoff()
}
t1.Reset(t1Timeout)
t2.Reset(t2Timeout)
case <-c.stopChan:
// release is a unicast request of the IP release.
if err := c.release(); err != nil {
log.Error("release lease failed", "lease", lease, "err", err)
} else {
log.Info("release", "lease", lease)
}
t1.Stop()
t2.Stop()
close(c.releasedChan)
return
}
} else {
client = NewDHCPv4Client(iface, false, "", backoffAttempts)
}
}
// --------------------------------------------------------
// | |INIT-REBOOT | RENEWING |REBINDING |
// --------------------------------------------------------
// |broad/unicast |broadcast | unicast |broadcast |
// |server-ip |MUST NOT | MUST NOT |MUST NOT |
// |requested-ip |MUST | MUST NOT |MUST NOT |
// |ciaddr |zero | IP address |IP address|
// --------------------------------------------------------
func (c *DHCPClient) requestWithBackoff() *nclient4.Lease {
backoff := backoff.Backoff{
Factor: 2,
Jitter: true,
Min: 10 * time.Second,
Max: 1 * time.Minute,
}
return client, nil
var lease *nclient4.Lease
var err error
for {
log.Debug("trying to get a new IP", "attempt", backoff.Attempt())
lease, err = c.request(false)
if err != nil {
dur := backoff.Duration()
if backoff.Attempt() > maxBackoffAttempts-1 {
errMsg := fmt.Errorf("failed to get an IP address after %d attempts, error %s, giving up", maxBackoffAttempts, err.Error())
log.Error(errMsg.Error())
c.errorChan <- errMsg
c.Stop()
return nil
}
log.Error("request failed", "err", err.Error(), "waiting", dur)
time.Sleep(dur)
continue
}
backoff.Reset()
break
}
if c.ipChan != nil {
log.Debug("using channel")
c.ipChan <- lease.ACK.YourIPAddr.String()
}
return lease
}
func (c *DHCPClient) request(rebind bool) (*nclient4.Lease, error) {
dhclient, err := nclient4.New(c.iface.Name)
if err != nil {
return nil, fmt.Errorf("create a client for iface %s failed, error: %w", c.iface.Name, err)
}
defer dhclient.Close()
modifiers := make([]dhcpv4.Modifier, 0)
if c.ddnsHostName != "" {
modifiers = append(modifiers,
dhcpv4.WithOption(dhcpv4.OptHostName(c.ddnsHostName)),
dhcpv4.WithOption(dhcpv4.OptClientIdentifier([]byte(c.ddnsHostName))),
)
}
// if initRebootFlag is set, this means we have an IP already set on c.requestedIP that should be used
if c.initRebootFlag {
log.Debug("init-reboot", "ip", c.requestedIP)
modifiers = append(modifiers, dhcpv4.WithOption(dhcpv4.OptRequestedIPAddress(c.requestedIP)))
}
// if this is a rebind, then the IP we should set is the one that already exists in lease
if rebind {
log.Debug("rebinding", "ip", c.lease.ACK.YourIPAddr)
modifiers = append(modifiers, dhcpv4.WithOption(dhcpv4.OptRequestedIPAddress(c.lease.ACK.YourIPAddr)))
}
return dhclient.Request(context.TODO(), modifiers...)
}
func (c *DHCPClient) release() error {
dhclient, err := nclient4.New(c.iface.Name)
if err != nil {
return fmt.Errorf("create release client failed, error: %w, iface: %s, server ip: %v", err, c.iface.Name, c.lease.ACK.ServerIPAddr)
}
defer dhclient.Close()
// TODO modify lease
return dhclient.Release(c.lease)
}
func (c *DHCPClient) renew() (*nclient4.Lease, error) {
// renew needs a unicast client. This is due to some servers (like dnsmasq) require the exact request coming from the vip interface
dhclient, err := nclient4.New(c.iface.Name,
nclient4.WithUnicast(&net.UDPAddr{IP: c.lease.ACK.YourIPAddr, Port: nclient4.ClientPort}))
if err != nil {
return nil, fmt.Errorf("create renew client failed, error: %w, server ip: %v", err, c.lease.ACK.ServerIPAddr)
}
defer dhclient.Close()
return dhclient.Renew(context.TODO(), c.lease)
}

Some files were not shown because too many files have changed in this diff Show More