Compare commits

..

1 Commits

Author SHA1 Message Date
Dan Finneran
296c9c1580 Unintentionally created duplicate functions that can't be used
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-08-12 12:17:20 +00:00
225 changed files with 6609 additions and 31864 deletions

View File

@@ -1,13 +0,0 @@
I'll help you add a suggestion. Unfortunately, I can't directly add a suggestion to an existing comment through the API. However, here's what I recommend:
**Option 1: Reply with a suggestion**
Create a new comment with a suggested fix:
```suggestion
failed to get an IPv6 address after %d attempt(s), giving up, error: %s
```
**Option 2: Edit your existing comment**
Update your comment to include the suggestion details pointing out that line 284 in the error message says "IPv4" but should say "IPv6" since this is the DHCPv6Client.
Would you like me to create a new reply comment with the suggestion instead?

View File

@@ -22,10 +22,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v4
with:
ref: ${{ github.ref_name }}
- name: Anchore SBOM Action
uses: anchore/sbom-action@v0.24.2
uses: anchore/sbom-action@v0.20.4
with:
format: cyclonedx-json

View File

@@ -1,88 +1,51 @@
name: For each PR
on:
pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
e2e-tests:
runs-on: ubuntu-latest
name: E2E tests
timeout-minutes: 120
env:
GINKGO_PROCS: ${{ matrix.ginkgo-procs }}
strategy:
fail-fast: false
max-parallel: 3
matrix:
include:
- mode: arp
ginkgo-procs: 4
- mode: rt
ginkgo-procs: 4
- mode: bgp
ginkgo-procs: 4
mode: ["arp", "rt", "bgp"]
fail-fast: true
max-parallel: 1
steps:
- name: Get current date
id: date
run: echo "date=$(date +'%Y-%m-%d-%H-%M')" >> "$GITHUB_OUTPUT"
- name: Ensure fs wont cause issues
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v7
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image locally
run: make dockerx86Local
- name: Run Manifest generation tests
run: make manifest-test
if: matrix.mode == 'arp'
- name: Run ARP mode tests
run: DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true GINKGO_ARGS="--output-dir=/tmp --json-report=kube-vip-test-report-arp.json" make e2e-tests-arp
if: matrix.mode == 'arp'
- name: Run RT mode tests
run: DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true GINKGO_ARGS="--output-dir=/tmp --json-report=kube-vip-test-report-rt.json" make e2e-tests-rt
if: matrix.mode == 'rt'
- name: Run ARP mode tests v1.29.0 onwards
run: make e2e-tests129-arp
if: matrix.mode== 'arp'
- name: Run RT mode tests v1.29.0 onwards
run: make e2e-tests129-rt
if: matrix.mode== 'rt'
- name: Get GoBGP binaries
run: make get-gobgp
if: matrix.mode == 'bgp'
- name: Run BGP mode tests
run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true GINKGO_ARGS="--output-dir=/tmp --json-report=kube-vip-test-report-bgp.json" make e2e-tests-bgp
if: matrix.mode == 'bgp'
- name: Change log directory permissions
run: sudo chmod -R 755 /tmp/kube-vip-test*
if: matrix.mode == 'bgp' && always()
- name: Save logs
uses: actions/upload-artifact@v7
with:
name: e2e-test-logs-${{ matrix.mode }}-${{ steps.date.outputs.date }}
path: /tmp/kube-vip-test*
if: always()
if: matrix.mode== 'bgp'
- name: Run BGP mode tests v1.29.0 onwards
run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 make e2e-tests129-bgp
if: matrix.mode== 'bgp'
service-e2e-tests:
runs-on: ubuntu-latest
name: E2E service tests
timeout-minutes: 60
steps:
- name: Get current date
id: date
run: echo "date=$(date +'%Y-%m-%d-%H-%M')" >> "$GITHUB_OUTPUT"
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v7
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image with iptables
run: make dockerx86ActionIPTables
- name: Run tests
run: DOCKER_API_VERSION=1.48 DOCKERTAG=action E2E_KEEP_LOGS=true make service-tests
- name: Save logs
uses: actions/upload-artifact@v7
with:
name: services-test-logs-${{ steps.date.outputs.date }}
path: /tmp/kube-vip-services*
if: always()
run: DOCKERTAG=action make service-tests

View File

@@ -1,69 +1,44 @@
name: For each commit
on:
push:
branches: [main]
tags: ['v*']
pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
validation:
runs-on: ubuntu-latest
name: Checks and linters
timeout-minutes: 30
steps:
- name: Init
run: sudo apt-get update && sudo apt-get install -y build-essential && sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Install golangci-lint
uses: golangci/golangci-lint-action@v9
with:
version: v2.12
install-only: true
run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.64.8
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v7
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Verify gofmt
run: |
unformatted=$(gofmt -l .)
if [ -n "$unformatted" ]; then
echo "The following files are not gofmt-formatted:"
echo "$unformatted"
exit 1
fi
- name: All checks
run: make check
unit-tests:
runs-on: ubuntu-latest
name: Unit tests
timeout-minutes: 30
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v7
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Run tests
run: make unit-tests
- name: Run privileged network tests
run: |
sudo -E env PATH="$PATH" KUBE_VIP_REQUIRE_NETNS=1 go test -race ./pkg/services ./pkg/vip ./pkg/instance \
-run 'TestRecover|TestServiceAddressRetained|TestRetainControlPlaneVIPs|TestAddressProtocol|TestKubeVIPAddressProtocol|TestCleanupKubeVIPAddresses|TestMonitorDefaultInterfaceReturnsErrorWhenTestLinkIsSetDown|TestCleanupLinkAttachmentsOnlyDeletesOwnedVLAN'
integration-tests:
name: Integration tests
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- name: Checkout code
uses: actions/checkout@v7
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v7
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Run tests
@@ -71,12 +46,11 @@ jobs:
image-vul-check:
runs-on: ubuntu-latest
name: Image vulnerability scan
timeout-minutes: 30
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Build image
run: make dockerx86Action
uses: actions/checkout@v4
- name: Build image with iptables
run: make dockerx86ActionIPTables
- name: Run Trivy vulnerability scanner
uses: aquasecurity/trivy-action@master
with:

View File

@@ -38,16 +38,16 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v7
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v7
uses: actions/setup-go@v5
with:
go-version-file: go.mod
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v4
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@@ -58,7 +58,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v4
uses: github/codeql-action/autobuild@v3
# Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
@@ -72,4 +72,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v4
uses: github/codeql-action/analyze@v3

View File

@@ -11,25 +11,25 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to Github Packages
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build standard version
id: docker_build
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
@@ -39,7 +39,7 @@ jobs:
ghcr.io/kube-vip/kube-vip:${{ github.ref_name }}
- name: Build iptables version
id: docker_build_iptables
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: .
file: Dockerfile_iptables

View File

@@ -1,111 +0,0 @@
name: Nightly e2e
on:
schedule:
- cron: '30 2 * * *'
workflow_dispatch:
permissions:
contents: read
jobs:
unit-coverage:
runs-on: ubuntu-latest
name: Unit tests with coverage
timeout-minutes: 30
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Install Go
uses: actions/setup-go@v7
with:
go-version-file: go.mod
- name: Run tests
run: make unit-tests
- name: Summarize coverage
if: always()
run: |
if test -f coverage.out; then
echo "### Unit coverage" >> "$GITHUB_STEP_SUMMARY"
go tool cover -func=coverage.out | tail -1 >> "$GITHUB_STEP_SUMMARY"
else
echo "### Unit coverage: report missing" >> "$GITHUB_STEP_SUMMARY"
fi
- name: Upload coverage
uses: actions/upload-artifact@v7
with:
name: unit-coverage
path: coverage.out
if-no-files-found: error
if: always()
etcd-e2e:
runs-on: ubuntu-latest
name: Etcd E2E tests
timeout-minutes: 60
steps:
- name: Ensure fs wont cause issues
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Checkout code
uses: actions/checkout@v7
- name: Install Go
uses: actions/setup-go@v7
with:
go-version-file: go.mod
- name: Build image locally
run: make dockerx86Local
- name: Prepare Etcd artifacts
if: always()
run: |
mkdir -p /tmp/kube-vip-etcd-artifacts
: > /tmp/kube-vip-etcd-artifacts/suite.log
printf '[]\n' > /tmp/kube-vip-etcd-artifacts/report.json
- name: Run Etcd tests
id: etcd
# Scheduled failures are tolerated only during the initial stabilization window.
# The enforcement step below makes manual runs and later schedules blocking.
continue-on-error: true
shell: bash
run: |
set +e
set -o pipefail
DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true \
GINKGO_ARGS="--json-report=report.json --output-dir=/tmp/kube-vip-etcd-artifacts" \
make e2e-tests-etcd 2>&1 | tee /tmp/kube-vip-etcd-artifacts/suite.log
exit_code=${PIPESTATUS[0]}
echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT"
exit "$exit_code"
- name: Summarize Etcd suite
if: always()
env:
OUTCOME: ${{ steps.etcd.outcome }}
EXIT_CODE: ${{ steps.etcd.outputs.exit_code }}
run: |
echo "### Etcd E2E result: ${OUTCOME}" >> "$GITHUB_STEP_SUMMARY"
printf '{"outcome":"%s","exit_code":%s,"event":"%s","cutoff":"2026-10-01"}\n' \
"${OUTCOME:-skipped}" "${EXIT_CODE:-null}" "$GITHUB_EVENT_NAME" \
> /tmp/kube-vip-etcd-artifacts/result.json
- name: Save logs
uses: actions/upload-artifact@v7
continue-on-error: true
with:
name: etcd-e2e-logs
path: |
/tmp/kube-vip-etcd-artifacts
/tmp/kube-vip-test*
if-no-files-found: warn
if: always()
- name: Enforce Etcd result
if: always()
env:
EVENT_NAME: ${{ github.event_name }}
OUTCOME: ${{ steps.etcd.outcome }}
run: |
if test "$OUTCOME" = success; then
exit 0
fi
if test "$EVENT_NAME" = schedule && test "$(date -u +%Y-%m-%d)" \< 2026-10-01; then
echo "::warning::etcd e2e suite outcome was ${OUTCOME:-skipped} during stabilization through 2026-09-30"
exit 0
fi
echo "::error::etcd e2e suite outcome was ${OUTCOME:-skipped}; see the etcd-e2e-logs artifact"
exit 1

View File

@@ -11,31 +11,31 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to Github Packages
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Generate Metadata
uses: docker/metadata-action@v6.2.0
uses: docker/metadata-action@v5.7.0
id: metadata
with:
labels: |
org.opencontainers.image.documentation=https://kube-vip.io/docs/
- name: Build and push main branch
id: docker_build
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
@@ -48,7 +48,7 @@ jobs:
ghcr.io/kube-vip/kube-vip:latest
- name: Build iptables version and push main branch
id: docker_build_iptables
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: .
file: Dockerfile_iptables

2
.gitignore vendored
View File

@@ -3,7 +3,5 @@ kube-vip
.vscode
bin
testing/e2e/etcd/certs
coverage.out
pkg/etcd/etcd.pid
pkg/etcd/etcd-data
testing/e2e/e2e.test

View File

@@ -1,38 +1,13 @@
version: "2"
run:
timeout: 10m
linters:
enable:
- bodyclose
- gosec
- misspell
- unconvert
- unparam
settings:
misspell:
ignore-rules:
- creater
staticcheck:
checks:
- all
# Disable QF1008 to retain embedded fields for better readability.
- "-QF1008"
exclusions:
generated: lax
presets:
- comments
- common-false-positives
- legacy
- std-error-handling
paths:
- third_party$
- builtin$
- examples$
formatters:
enable:
- gofmt
- goimports
exclusions:
generated: lax
paths:
- third_party$
- builtin$
- examples$
- bodyclose
- gofmt
- goimports
- revive
- gosec
- misspell
- unconvert
- unparam

View File

@@ -1,62 +0,0 @@
# Changelog
All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
### Fixed
- Propagate `bgp_attach_ip_to_interface` into per-service config so it attaches BGP-mode Service VIPs to the interface as configured.
- Add a configurable kube-vip instance name and use it to isolate internal nftables egress tables, persist table ownership on Services, and migrate per-Service chains without affecting other deployments. Fixes #1634.
- Retry on 403 Forbidden and 401 Unauthorized in `ServicesWatcher` at startup with exponential backoff. Fixes #1464.
- Reintroduce BGP config via node annotations. Fixes #1488.
- Fail fast in runtime `manager` and `service` paths when legacy `vip_address` is used without `vip_subnet` in control-plane ARP, BGP, or Routing Table mode.
- Cancel the mode context on init or configuration failure before waiting on goroutines during shutdown.
### Added
- Configurable control-plane health check for BGP mode without leader election
- Polls a configurable HTTP(S) endpoint (e.g. `https://localhost:6443/livez`) to verify the exposed service is healthy (usually the local kube-apiserver)
- Withdraws the BGP route after a configurable number of consecutive failures, removing the unhealthy node from the ECMP set
- Re-announces the route automatically once the endpoint recovers
- Gracefully withdraws the route on shutdown (SIGTERM)
- Supports custom CA certificates for TLS verification
- Configuration via environment variables or CLI flags:
- `control_plane_health_check_address` / `--controlPlaneHealthCheckAddress`: URL to poll
- `control_plane_health_check_period_seconds` / `--controlPlaneHealthCheckPeriodSeconds`: interval between checks (default: 5)
- `control_plane_health_check_timeout_seconds` / `--controlPlaneHealthCheckTimeoutSeconds`: per-request timeout (default: 3)
- `control_plane_health_check_failure_threshold` / `--controlPlaneHealthCheckFailureThreshold`: consecutive failures before withdrawal (default: 3)
- `control_plane_health_check_ca_path` / `--controlPlaneHealthCheckCAPath`: CA cert for HTTPS verification
- SIGUSR1 signal handler for runtime configuration dumps (#1301)
- Send SIGUSR1 to kube-vip process to dump current configuration to stdout
- Configuration dump includes:
- Basic configuration (VIP, interface, port, namespace settings)
- BGP configuration (enabled status, AS number, router ID, peers)
- ARP/NDP configuration (enabled status, broadcast rate)
- Services configuration (enabled status, load balancer settings)
- Network interfaces status
- Leader election configuration (type, lease details)
- Runtime statistics (load balancer, Prometheus, health check settings)
- Output format: Human-readable plaintext via fmt.Printf()
- Thread-safe implementation using mutex protection
- Non-disruptive: Process continues running after configuration dump
- Added comprehensive unit tests for all dump methods
- Added E2E tests for signal handling
- Opt-in support for endpointless `LoadBalancer` services with `externalTrafficPolicy: Cluster`
- Annotation: `kube-vip.io/allow-reconcile-without-endpoints: "true"`
- Starts service handling path for opted-in endpointless Cluster services while preserving default endpoint-gated behavior for non-opt-in services and `Local` policy
- Added endpoint behavior tests and README usage documentation
- Added support in ipoib interfaces in ARP mode. Fixes #694
### Changed
- BGP mode now honours `enable_leader_election` for services: a single global services leader advertises the service VIPs instead of every node advertising them. Deployments that enabled `enable_leader_election` for the control plane and relied on ECMP/multipath for services must unset it (or switch to `enable_service_election`) to keep the previous datapath. kube-vip logs a warning on startup when this path is taken.
- Updated signal handlers in manager_arp.go, manager_bgp.go, manager_wireguard.go, and manager_table.go to use switch statement pattern for handling multiple signals (SIGUSR1, SIGINT, SIGTERM)
- wireguard.go now manages a complete wireguard interface on the current network namespace
- manager_wireguard.go uses the new wireguard.go implementation
## [v1.0.1] - Previous Release
### Previous changes
- See git history for changes prior to CHANGELOG.md introduction

View File

@@ -113,8 +113,8 @@ and *merged* sorts of commits.
To make it easier for reviewers to review your PR, consider the following:
1. Follow the golang [coding conventions](https://github.com/golang/go/wiki/CodeReviewComments).
2. Format your code with `make simplify` to automatically fix formatting issues.
2. Lint your code with `make check`; if the linters flag an issue that cannot be fixed automatically, an error message will be displayed so you can address the issue.
2. Format your code with `make golangci-fix`; if the [linters](ci/README.md) flag an issue that
cannot be fixed automatically, an error message will be displayed so you can address the issue.
3. Follow [git commit](https://chris.beams.io/posts/git-commit/) guidelines.
4. Follow [logging](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md) guidelines.

View File

@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:experimental
FROM golang:1.27.1-alpine3.23 as dev
FROM golang:1.24.5-alpine3.22 as dev
RUN apk add --no-cache git ca-certificates make
RUN adduser -D appuser
COPY . /src/

View File

@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:experimental
FROM golang:1.27.1-alpine3.23 as dev
FROM golang:1.24.5-alpine3.22 as dev
RUN apk add --no-cache git make
RUN adduser -D appuser
COPY . /src/
@@ -11,7 +11,7 @@ RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
--mount=type=cache,sharing=locked,id=goroot,target=/root/.cache/go-build \
CGO_ENABLED=0 GOOS=linux make build
FROM alpine:3.24.1
FROM alpine:3.22.1
# Update pkgs and add iptables
RUN apk upgrade && \
apk add --no-cache iptables iptables-legacy

View File

@@ -5,7 +5,7 @@ TARGET := kube-vip
.DEFAULT_GOAL := $(TARGET)
# These will be provided to the target
VERSION := v1.2.4
VERSION := v1.0.0
BUILD := `git rev-parse HEAD`
@@ -15,15 +15,9 @@ TARGETOS=linux
# Use linker flags to provide version/build settings to the target
LDFLAGS=-ldflags "-s -w -X=main.Version=$(VERSION) -X=main.Build=$(BUILD) -extldflags -static"
DOCKERTAG ?= $(VERSION)
REPOSITORY ?= docker.io/plndr
GO_VERSION := $(word 2,$(shell grep '^go ' go.mod))
K8S_VERSION ?= v1.35.0
GINKGO_ARGS ?=
GINKGO_PROCS ?=
GINKGO_PARALLEL := $(if $(GINKGO_PROCS),--procs=$(GINKGO_PROCS),-p)
BUILDX_CACHE_FLAGS ?=
REPOSITORY ?= plndr
.PHONY: all build clean install uninstall simplify check run e2e-tests unit-tests integration-tests unit-tests-docker integration-tests-docker e2e-tests-etcd
.PHONY: all build clean install uninstall simplify check run e2e-tests
all: check install
@@ -81,17 +75,17 @@ docker:
# This will build a local docker image (x86 only), use make dockerLocal for all architectures
dockerx86Local:
@-rm ./kube-vip
@docker buildx build --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) $(BUILDX_CACHE_FLAGS) .
@docker buildx build --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):$(DOCKERTAG) .
@echo New Multi Architecture Docker image created
dockerx86Action:
@-rm ./kube-vip
@docker buildx build --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):action $(BUILDX_CACHE_FLAGS) .
@docker buildx build --platform linux/amd64 --load -t $(REPOSITORY)/$(TARGET):action .
@echo New Multi Architecture Docker image created
dockerx86ActionIPTables:
@-rm ./kube-vip
@docker buildx build --platform linux/amd64 -f ./Dockerfile_iptables --load -t $(REPOSITORY)/$(TARGET):action $(BUILDX_CACHE_FLAGS) .
@docker buildx build --platform linux/amd64 -f ./Dockerfile_iptables --load -t $(REPOSITORY)/$(TARGET):action .
@echo New Multi Architecture Docker image created
dockerLocal:
@@ -115,49 +109,45 @@ run: install
manifests:
@make build
@mkdir -p ./docs/manifests/$(VERSION)/
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-arp.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-lb.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-bgp.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-arp.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-lb.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --bgp --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-bgp.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
@-rm ./kube-vip
manifest-test:
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --enableLoadBalancer
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --controlplane --services
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --enableLoadBalancer
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --bgp --controlplane --services
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster
unit-tests:
go test -race -coverprofile=coverage.out -covermode=atomic ./...
unit-tests-docker:
docker run --rm -w /kube-vip -v $$(pwd):/kube-vip -v kube-vip-gomod-cache:/go/pkg/mod -v kube-vip-gobuild-cache:/root/.cache/go-build golang:$(GO_VERSION) sh -c "make unit-tests; status=$$?; chmod 666 coverage.out 2>/dev/null || true; exit $$status"
go test ./...
integration-tests:
go test -tags=integration,e2e -v ./pkg/etcd
e2e-tests-arp: get-whoami
GOMAXPROCS=4 TEST_MODE=arp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e
e2e-tests:
GOMAXPROCS=4 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd
e2e-tests-rt: get-whoami
GOMAXPROCS=4 TEST_MODE=rt K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e
e2e-tests129-arp:
GOMAXPROCS=4 TEST_MODE=arp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests-bgp: get-whoami get-gobgp
GOMAXPROCS=4 TEST_MODE=bgp K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e
e2e-tests129-rt:
GOMAXPROCS=4 TEST_MODE=rt V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests-etcd: get-whoami
GOMAXPROCS=4 K8S_IMAGE_PATH=kindest/node:$(K8S_VERSION) E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v $(GINKGO_PARALLEL) $(GINKGO_ARGS) ./testing/e2e/etcd
e2e-tests129-bgp:
GOMAXPROCS=4 TEST_MODE=bgp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests: e2e-tests-arp e2e-tests-rt e2e-tests-bgp
e2e-tests129: e2e-tests129-arp e2e-tests129-rt e2e-tests129-bgp
service-tests:
$(MAKE) -C testing/e2e/e2e dockerLocal
E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/services -Services -simple -deployments -leaderActive -leaderFailover -localDeploy -electionFaults -egress -egressIPv6 -dualStack -egressInternal
E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/services -Services -simple -deployments -leaderActive -leaderFailover -localDeploy -egress -egressIPv6 -dualStack
trivy: dockerx86ActionIPTables
docker run -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy:0.47.0 \
@@ -176,16 +166,14 @@ kind-quick:
kubectl create configmap --namespace kube-system kubevip --from-literal range-global=172.18.100.10-172.18.100.30
kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
kind load docker-image --name kube-vip $(REPOSITORY)/$(TARGET):$(DOCKERTAG)
docker run --network host --rm $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --services --inCluster --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --servicesElection --interface eth0 | kubectl apply -f -
docker run --network host --rm $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --services --inCluster --arp --servicesElection --interface eth0 | kubectl apply -f -
kind-reload:
kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name services
kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name kube-vip
kubectl rollout restart -n kube-system daemonset/kube-vip-ds
get-gobgp:
mkdir -p bin
wget -nc --directory-prefix=bin https://github.com/osrg/gobgp/releases/download/v4.6.0/gobgp_4.6.0_linux_amd64.tar.gz
tar -xvzf bin/gobgp_4.6.0_linux_amd64.tar.gz -C bin
wget -nc --directory-prefix=bin https://github.com/osrg/gobgp/releases/download/v3.37.0/gobgp_3.37.0_linux_amd64.tar.gz
tar -xvzf bin/gobgp_3.37.0_linux_amd64.tar.gz -C bin
get-whoami:
docker pull ghcr.io/traefik/whoami:v1.11

View File

@@ -18,8 +18,6 @@ The idea behind `kube-vip` is a small self-contained Highly-Available option for
**NOTE:** All documentation of both usage and architecture are now available at [https://kube-vip.io](https://kube-vip.io).
For upgrading an existing install in place (static Pod or DaemonSet), see the [upgrade guide](https://kube-vip.io/docs/upgrade/).
## Features
Kube-Vip was originally created to provide a HA solution for the Kubernetes control plane, over time it has evolved to incorporate that same functionality into Kubernetes service type [load-balancers](https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer).
@@ -60,81 +58,14 @@ All of these would require a separate level of configuration and in some infrast
## Troubleshooting and Feedback
### SELinux and IPVS kernel modules
When using IPVS load balancing on nodes with SELinux enforcing, kube-vip may be
blocked from requesting kernel modules from inside the container. Symptoms can
include the kube-vip pod entering `Error` or `CrashLoopBackOff`, logs that show
`ensure IPVS kernel modules are loaded`, or audit denials for `module_request`
from `container_t`.
Load the required IPVS modules on every node that can run kube-vip before
deploying it:
```shell
sudo modprobe ip_vs
sudo modprobe ip_vs_rr
```
To persist this across reboots, add the modules to a file such as
`/etc/modules-load.d/kube-vip-ipvs.conf`:
```text
ip_vs
ip_vs_rr
```
Preloading only the required modules is preferred to enabling the SELinux
`domain_kernel_load_modules` boolean for containers.
### Gateway API `LoadBalancer` services with no endpoints
Some Gateway API controllers create `LoadBalancer` services that intentionally have no Endpoints/EndpointSlices backends.
If you want kube-vip to reconcile such a service, opt in with:
```yaml
metadata:
annotations:
kube-vip.io/allow-reconcile-without-endpoints: "true"
spec:
type: LoadBalancer
externalTrafficPolicy: Cluster
```
Scope:
- Works only with `externalTrafficPolicy: Cluster`
- No effect for `Local`
- Default endpoint-gated behavior remains unchanged for services without this annotation
Please raise issues on the GitHub repository and as mentioned check the documentation at [https://kube-vip.io](https://kube-vip.io/).
## Community Tools
- **[KubeStellar Console — Guided kube-vip Install](https://console.kubestellar.io/missions/install-kube-vip)** — A step-by-step guided installation experience for kube-vip with pre-flight checks, validation, troubleshooting, and rollback support.
## Contributing
Thanks for taking the time to join our community and start contributing! We welcome pull requests. Feel free to dig through the [issues](https://github.com/kube-vip/kube-vip/issues) and jump in.
:warning: This project has issue compiling on MacOS, please compile it on linux distribution
Additionally it is now relatively easy and quick to develop with [skaffold](https://skaffold.dev/), and the `skaffold.yaml` exists within the root folder of the gir repository.
### Set up a kind development environment
1. `kind create cluster --config ./testing/kind.yaml`
2. `kubectl apply -f https://kube-vip.io/manifests/rbac.yaml`
3. Create a load balancer range `configMap` from the kind cluster
4. Apply the CCM manifest
5. Start skaffold with `skaffold dev`
6. Start developing and see changes applied in real-time.
## Star History
[![Star History Chart](https://star-history.dera.page/svg?repos=kube-vip/kube-vip&type=Date)](https://star-history.dera.page/#kube-vip/kube-vip&type=date)
[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fkube-vip%2Fkube-vip.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fkube-vip%2Fkube-vip?ref=badge_shield)
## License
[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fkube-vip%2Fkube-vip.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fkube-vip%2Fkube-vip?ref=badge_large)
[![Star History Chart](https://api.star-history.com/svg?repos=kube-vip/kube-vip&type=Date)](https://star-history.com/#kube-vip/kube-vip&Date)

View File

@@ -23,45 +23,25 @@ func init() {
var kubeKubeadm = &cobra.Command{
Use: "kubeadm",
Short: "Kubeadm functions",
Long: `This command group provides utilities for generating static Pod manifests specifically tailored for the kubeadm bootstrapping process.
It contains two subcommands:
- init: Generates a manifest to be used during 'kubeadm init' on the first control-plane node.
- join: Generates a manifest to be used during 'kubeadm join' for additional control-plane nodes.
The generated YAML manifest should be saved to the kubeadm static Pod directory (typically /etc/kubernetes/manifests/) so that kubeadm launches the kube-vip static Pod automatically.`,
Run: func(cmd *cobra.Command, _ []string) {
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
_ = cmd.Help()
// TODO - A load of text detailing what's actually happening
},
}
var kubeKubeadmInit = &cobra.Command{
Use: "init",
Short: "kube-vip init",
Long: `The 'init' subcommand generates a Kubernetes Pod manifest that kubeadm will start as a static Pod during the cluster initialisation phase.
This manifest runs kube-vip on the first control-plane node to advertise the Virtual IP (VIP) for the API server. The VIP is typically configured using ARP (Layer 2) or BGP (dynamic routing).
Required flags for this command:
--interface : The network interface to bind the VIP to (e.g., eth0).
--vip or --address : The Virtual IP address or DNS name to use.
Example:
kube-vip kubeadm init --interface eth0 --vip 192.168.1.100 --controlplane
The output YAML should be written to the kubeadm manifests directory, e.g.:
kube-vip kubeadm init ... > /etc/kubernetes/manifests/kube-vip.yaml`,
Run: func(cmd *cobra.Command, _ []string) {
Long: "The \"init\" subcommand will generate the Kubernetes manifest that will be started by kubeadm through the kubeadm init process",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
// TODO - A load of text detailing what's actually happening
err := kubevip.ParseEnvironment(&initConfig)
if err != nil {
log.Error("parsing environment", "err", err)
return
}
if err := initConfig.Validate(); err != nil {
log.Error("validating configuration", "err", err)
return
}
// TODO - check for certain things VIP/interfaces
if initConfig.Interface == "" {
@@ -78,18 +58,14 @@ The output YAML should be written to the kubeadm manifests directory, e.g.:
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating VIPSubnet", "err", err)
return
}
}
cfg, err := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
if err != nil {
log.Error("unable to create manifest", "err", err)
return
}
cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
fmt.Println(cfg) // output manifest to stdout
},
}
@@ -97,30 +73,15 @@ The output YAML should be written to the kubeadm manifests directory, e.g.:
var kubeKubeadmJoin = &cobra.Command{
Use: "join",
Short: "kube-vip join",
Long: `The 'join' subcommand generates a Kubernetes Pod manifest for additional control-plane nodes joining an existing cluster via 'kubeadm join'.
It functions identically to the 'init' subcommand, but is intended for secondary control-plane nodes. It validates that the kubeconfig file (specified by --config, defaulting to /etc/kubernetes/admin.conf) exists on the node to ensure the node can authenticate with the cluster.
Required flags for this command:
--interface : The network interface to bind the VIP to.
--vip or --address : The Virtual IP address or DNS name (must match the VIP used during 'init').
Example:
kube-vip kubeadm join --interface eth0 --vip 192.168.1.100
The output YAML should be saved to the kubeadm manifests directory on the joining node.`,
Run: func(cmd *cobra.Command, _ []string) {
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
// TODO - A load of text detailing what's actually happening
err := kubevip.ParseEnvironment(&initConfig)
if err != nil {
log.Error("parsing environment", "err", err)
return
}
if err := initConfig.Validate(); err != nil {
log.Error("validating configuration", "err", err)
return
}
// TODO - check for certain things VIP/interfaces
if initConfig.Interface == "" {
@@ -142,18 +103,14 @@ The output YAML should be saved to the kubeadm manifests directory on the joinin
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating VIPSubnet", "err", err)
return
}
}
cfg, err := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
if err != nil {
log.Error("unable to create manifest", "err", err)
return
}
cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
fmt.Println(cfg) // output manifest to stdout
},
}

View File

@@ -19,7 +19,6 @@ var taint, role, rolebinding bool
func init() {
kubeManifest.PersistentFlags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
kubeManifest.PersistentFlags().StringVar(&image, "image", "ghcr.io/kube-vip/kube-vip", "Define a hardcoded image with or without tag for the manifest")
kubeManifestDaemon.PersistentFlags().BoolVar(&taint, "taint", false, "Taint the manifest for only running on control planes")
kubeManifestRbac.PersistentFlags().BoolVar(&role, "role", false, "Generate only a Role inside the serviceNamespace access")
kubeManifestRbac.PersistentFlags().BoolVar(&rolebinding, "rolebinding", false, "Generate only a RoleBinding for namespaced access")
@@ -32,71 +31,42 @@ func init() {
var kubeManifest = &cobra.Command{
Use: "manifest",
Short: "Manifest functions",
Long: `This command group provides flexible manifest generation for deploying kube-vip in various Kubernetes environments.
Unlike the 'kubeadm' subcommands, which are tightly coupled to kubeadm's static Pod requirements, these generators produce standard Kubernetes manifests (Pod, DaemonSet, RBAC) that can be used with any Kubernetes distribution (e.g., k3s, RKE, or vanilla Kubernetes).
Subcommands:
pod : Generates a standalone Pod manifest (similar to a static pod).
daemonset : Generates a DaemonSet manifest to run kube-vip on selected nodes.
rbac : Generates the necessary ServiceAccount, Role/ClusterRole, and Binding manifests.
All output is written to stdout as YAML, typically piped to 'kubectl apply -f -' or saved to a file.`,
Run: func(cmd *cobra.Command, _ []string) {
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
_ = cmd.Help()
// TODO - A load of text detailing what's actually happening
},
}
var kubeManifestPod = &cobra.Command{
Use: "pod",
Short: "Generate a Pod Manifest",
Long: `Generate a standalone Pod manifest for kube-vip.
This is ideal for environments that do not use DaemonSets or where you want to run kube-vip as a static Pod (similar to the 'kubeadm' subcommand, but without kubeadm-specific assumptions). It includes all the necessary container specifications, volumes, and environment variables derived from the provided flags.
Key flags:
--interface : Network interface for the VIP.
--vip or --address : The Virtual IP address or DNS name.
--image : Override the container image (default: ghcr.io/kube-vip/kube-vip).
The manifest is generated based on the current configuration flags set on the root command.
Example:
kube-vip manifest pod --interface eth0 --vip 10.0.0.100 --controlplane | kubectl apply -f -`,
Run: func(cmd *cobra.Command, _ []string) {
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
var err error
initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
// TODO - A load of text detailing what's actually happening
if err := kubevip.ParseEnvironment(&initConfig); err != nil {
log.Error("parsing environment", "err", err)
return
}
if err := initConfig.Validate(); err != nil {
log.Error("validating configuration", "err", err)
return
}
// The control plane has a requirement for a VIP being specified
if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
_ = cmd.Help()
log.Error("no address is specified for kube-vip to expose services on")
log.Error("No address is specified for kube-vip to expose services on")
return
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("config parse", "err", err)
return
}
}
cfg, err := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
if err != nil {
log.Error("unable to create manifest", "err", err)
return
}
cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
fmt.Println(cfg) // output manifest to stdout
},
}
@@ -104,50 +74,32 @@ Example:
var kubeManifestDaemon = &cobra.Command{
Use: "daemonset",
Short: "Generate a Daemonset Manifest",
Long: `Generate a DaemonSet manifest to run kube-vip across multiple nodes.
This is the recommended deployment method for production clusters running kube-vip as a service. It ensures that kube-vip runs on all control-plane nodes (or selected nodes via tolerations) and can handle both control-plane HA and service load-balancing.
Flags specific to this subcommand:
--taint : Adds a toleration to the DaemonSet so that pods are scheduled only on nodes with the control-plane taint (node-role.kubernetes.io/control-plane:NoSchedule). This is essential for control-plane-only deployments.
All other standard kube-vip flags (--interface, --vip, --enableARP, --enableBGP, etc.) are respected and embedded into the DaemonSet pod template.
Example:
kube-vip manifest daemonset --interface eth0 --vip 192.168.1.100 --controlplane --taint | kubectl apply -f -`,
Run: func(cmd *cobra.Command, _ []string) {
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
var err error
initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
// TODO - A load of text detailing what's actually happening
if err := kubevip.ParseEnvironment(&initConfig); err != nil {
log.Error("parsing environment", "err", err)
return
}
if err := initConfig.Validate(); err != nil {
log.Error("validating configuration", "err", err)
return
}
// The control plane has a requirement for a VIP being specified
if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
_ = cmd.Help()
log.Error("no address is specified for kube-vip to expose services on")
log.Error("No address is specified for kube-vip to expose services on")
return
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("config parse", "err", err)
return
}
}
cfg, err := kubevip.GenerateDaemonsetManifestFromConfig(&initConfig, image, Release.Version, inCluster, taint)
if err != nil {
log.Error("unable to create manifest", "err", err)
return
}
cfg := kubevip.GenerateDaemonsetManifestFromConfig(&initConfig, Release.Version, inCluster, taint)
fmt.Println(cfg) // output manifest to stdout
},
}
@@ -155,40 +107,26 @@ Example:
var kubeManifestRbac = &cobra.Command{
Use: "rbac",
Short: "Generate an RBAC Manifest",
Long: `Generate the RBAC (Role-Based Access Control) manifests required for kube-vip to interact with the Kubernetes API.
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
var err error
kube-vip needs permissions to watch services, endpoints, configmaps, and manage leader election leases. This command outputs the minimum required ServiceAccount, Role (or ClusterRole), and the corresponding binding.
Flags:
--role : If true, generates a namespaced Role instead of a ClusterRole. The namespace is taken from the root --namespace flag (default: kube-system).
--rolebinding : If true, generates a RoleBinding (if --role is also true). If --role is false, a ClusterRoleBinding is generated automatically.
The output is a multi-document YAML (separated by '---'). It is safe to apply directly:
kube-vip manifest rbac --role --rolebinding | kubectl apply -f -
Without --role, it generates a ClusterRole and ClusterRoleBinding, which is the default behaviour and suitable for most cluster-wide deployments.`,
Run: func(cmd *cobra.Command, _ []string) {
initConfig.LoadBalancers = append(initConfig.LoadBalancers, initLoadBalancer)
// TODO - A load of text detailing what's actually happening
if err := kubevip.ParseEnvironment(&initConfig); err != nil {
log.Error("parsing environment", "err", err)
return
}
if err := initConfig.Validate(); err != nil {
log.Error("validating configuration", "err", err)
return
}
// The control plane has a requirement for a VIP being specified
if initConfig.EnableControlPlane && (initConfig.VIP == "" && initConfig.Address == "" && !initConfig.DDNS) {
_ = cmd.Help()
log.Error("no address is specified for kube-vip to expose services on")
log.Error("No address is specified for kube-vip to expose services on")
return
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
var err error
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating VIPSubnet", "err", err)
return

View File

@@ -4,29 +4,24 @@ import (
"context"
"fmt"
"net"
"net/http"
"os"
"slices"
"strconv"
"strings"
"sync"
"time"
log "log/slog"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/spf13/cobra"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
"github.com/kube-vip/kube-vip/pkg/debouncer"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/manager"
"github.com/kube-vip/kube-vip/pkg/metrics"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
// Is an option to set the image
var image string
// Is kube-vip running within cluster
var inCluster bool
@@ -49,16 +44,14 @@ var (
)
var kubeVipCmd = &cobra.Command{
Use: "kube-vip",
Short: "This is a server for providing a Virtual IP and load-balancer for the Kubernetes control-plane",
SilenceErrors: true,
Use: "kube-vip",
Short: "This is a server for providing a Virtual IP and load-balancer for the Kubernetes control-plane",
}
func init() {
// Basic flags
kubeVipCmd.PersistentFlags().StringVar(&initConfig.Interface, "interface", "", "Name of the interface to bind to")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesInterface, "serviceInterface", "", "Name of the interface to bind to (for services)")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.AllowInterfaceNotUp, "allowInterfaceNotUp", false, "Allow kube-vip to start even if the interface is not up")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIP, "vip", "", "The Virtual IP address")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPSubnet, "vipSubnet", "", "The Virtual IP address subnet e.g. /32 /24 /8 etc.. (Default to 32 for IPv4 and 128 for IPv6)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.NodeName, "nodeName", "", "Name to be used for lease holder. Must be unique for each node/instance")
@@ -69,9 +62,6 @@ func init() {
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for VIP changes")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableWireguard, "wireguard", false, "Enable Wireguard for services VIPs")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableRoutingTable, "table", false, "Enable Routing Table for services VIPs")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.PreserveVIPOnLeadershipLoss, "preserveVipOnLeadershipLoss", false, "Preserve ARP VIP addresses on interface when leadership is lost (default: false for backward compatibility)")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoseLeadership, "loseLeadership", false, "Lose leadership when VIP interface goes down")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.LoseLeadershipTimeoutSeconds, "loseLeadershiptTimeoutSeconds", 30, "Timeout before re-electing a leader when the VIP interface is down")
// LoadBalancer flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "enableLoadBalancer", false, "enable loadbalancing on the VIP with IPVS")
@@ -84,13 +74,12 @@ func init() {
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLeaderElection, "leaderElection", false, "Use the Kubernetes leader election mechanism for clustering")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaderElectionType, "leaderElectionType", "kubernetes", "Defines the backend to run the leader election: kubernetes or etcd. Defaults to kubernetes.")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.LeaseName, "leaseName", "plndr-cp-lock", "Name of the lease that is used for leader election")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 15, "Length of time (in seconds) a Kubernetes leader lease can be held for")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 10, "Length of time (in seconds) a Kubernetes leader can attempt to renew its lease")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 2, "Length of time (in seconds) the LeaderElector clients should wait between tries of actions")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.LeaseDuration, "leaseDuration", 5, "Length of time (in seconds) a Kubernetes leader lease can be held for")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time (in seconds) a Kubernetes leader can attempt to renew its lease")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Length of time (in seconds) the LeaderElector clients should wait between tries of actions")
// BGP flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableBGP, "bgp", false, "This will enable BGP support within kube-vip")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPAttachIPToInterface, "bgpAttachIPToInterface", false, "Assign BGP service VIPs to the configured interface")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.RouterID, "bgpRouterID", "", "The routerID for the bgp server")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.SourceIF, "sourceIF", "", "The source interface for bgp peering (not to be used with sourceIP)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.SourceIP, "sourceIP", "", "The source address for bgp peering (not to be used with sourceIF)")
@@ -108,11 +97,6 @@ func init() {
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.URL, "zebraUrl", "unix:/var/run/frr/zserv.api", "Path to the unix domain socket for connecting to Zebra daemon")
kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.Zebra.Version, "zebraVersion", 6, "Zebra API Version")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.SoftwareName, "zebraSoftwareName", "frr8.3", "Software Name for Zebra")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ControlPlaneHealthCheck.Address, "controlPlaneHealthCheckAddress", "", "URL to poll for the control-plane health check when using BGP without leader election")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.ControlPlaneHealthCheck.PeriodSeconds, "controlPlaneHealthCheckPeriodSeconds", 5, "Seconds between control-plane health checks")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.ControlPlaneHealthCheck.TimeoutSeconds, "controlPlaneHealthCheckTimeoutSeconds", 3, "Timeout for each control-plane health check request")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.ControlPlaneHealthCheck.FailureThreshold, "controlPlaneHealthCheckFailureThreshold", 3, "Consecutive control-plane health check failures before withdrawing the BGP route")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ControlPlaneHealthCheck.CAPath, "controlPlaneHealthCheckCAPath", "", "Path to CA certificate for TLS verification when the control-plane health check URL is HTTPS")
// Namespace for kube-vip
kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The namespace for the configmap defined within the cluster")
@@ -136,22 +120,17 @@ func init() {
// Extended behaviour flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServicesElection, "servicesElection", false, "Enable leader election per kubernetes service")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassOnly, "lbClassOnly", false, fmt.Sprintf("Enable load balancing only for services with LoadBalancerClass %q", kubevip.LBClassName))
kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerClassName, "lbClassName", kubevip.LBClassName, fmt.Sprintf("Name of load balancer class for kube-VIP, defaults to %q", kubevip.LBClassName))
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassOnly, "lbClassOnly", false, "Enable load balancing only for services with LoadBalancerClass \"kube-vip.io/kube-vip-class\"")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerClassName, "lbClassName", "kube-vip.io/kube-vip-class", "Name of load balancer class for kube-VIP, defaults to \"kube-vip.io/kube-vip-class\"")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassLegacyHandling, "lbClassNameLegacyHandling", true, "Use legacy LoadBalancer class name handling (e.g. accepting services both with empty and non-empty class)")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServiceSecurity, "onlyAllowTrafficServicePorts", false, "Only allow traffic to service ports, others will be dropped, defaults to false")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableNodeLabeling, "enableNodeLabeling", false, fmt.Sprintf("Enable leader node labeling with %q, defaults to false", kubevip.HasIP))
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableNodeLabeling, "enableNodeLabeling", false, "Enable leader node labeling with \"kube-vip.io/has-ip=<VIP address>\", defaults to false")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesLeaseName, "servicesLeaseName", "plndr-svcs-lock", "Name of the lease that is used for leader election for services (in arp mode)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.DNSMode, "dnsMode", "first", "Name of the mode that DNS lookup will be performed (first, ipv4, ipv6, dual)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.DHCPMode, "dhcpMode", "", "Mode DHCP resolving will use to obtain IP addresses (ipv4, ipv6, dual)")
kubeVipCmd.PersistentFlags().UintVar(&initConfig.DHCPBackoffAttempts, "dhcpBackoffAttempts", kubevip.DefaultDHCPBackoffAttempts,
fmt.Sprintf("number of times DHCP client will try to obtain an IP address (defaults to: %d, 0 for unlimited retries)", kubevip.DefaultDHCPBackoffAttempts))
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DisableServiceUpdates, "disableServiceUpdates", false, "If true, kube-vip will process services as usual, but will not update service's Status.LoadBalancer.Ingress slice")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpoints, "enableEndpoints", false, "If enabled, kube-vip will only advertise services, but will use the (deprecated since v1.33) endpoints for IP addresses")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoInterfaceGlobalScope, "loInterfaceGlobalScope", false, "If true, kube-vip will set global scope when using the lo interface, otherwise a host scope will be used by default")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.HealthCheckPort, "healthCheckPort", 0, "If set to non-zero (> 1024), then this is the port that the healthcheck will listen on")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.DebounceTime, "debounceTime", debouncer.DefaultTime,
"Configures the time that the event debouncer will wait for the events arrival (default 0s - debouncer disabled, enable with min. 200ms)")
// Prometheus HTTP Server
kubeVipCmd.PersistentFlags().StringVar(&initConfig.PrometheusHTTPServer, "prometheusHTTPServer", ":2112", "Host and port used to expose Prometheus metrics via an HTTP server")
@@ -165,14 +144,6 @@ func init() {
// Kubernetes client specific flags
kubeVipCmd.PersistentFlags().StringVar(&initConfig.K8sConfigFile, "k8sConfigPath", "/etc/kubernetes/admin.conf", "Path to the configuration file used with the Kubernetes client")
// Configuration file flag
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ConfigFile, "config-file", "", "Path to a JSON/YAML configuration file to load settings from")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.InstanceName, "instanceName", "", "Unique name for this kube-vip instance (currently used to isolate nftables egress tables)")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EgressWithNftables, "egressWithNftables", true, "Use nftables-based egress implementation")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.PerServiceElectionOnDemand, "perServiceElectionOnDemand", false, "Allow kube-vip to use per-service election for annotated services")
kubeVipCmd.AddCommand(kubeKubeadm)
kubeVipCmd.AddCommand(kubeManifest)
kubeVipCmd.AddCommand(kubeVipManager)
@@ -185,16 +156,11 @@ func init() {
}
// Execute - starts the command parsing process
func Execute() int {
cmd, err := kubeVipCmd.ExecuteC()
if err != nil {
log.Error("command failed", "err", err)
if cmd == kubeVipCmd {
_ = cmd.Usage()
}
return 1
func Execute() {
if err := kubeVipCmd.Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
return 0
}
var kubeVipVersion = &cobra.Command{
@@ -218,24 +184,13 @@ var kubeVipSample = &cobra.Command{
var kubeVipService = &cobra.Command{
Use: "service",
Short: "Start the Virtual IP / Load balancer as a service within a Kubernetes cluster",
RunE: func(cmd *cobra.Command, args []string) error { //nolint TODO
cmd.SilenceUsage = true
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
// Load configuration from file if specified (lowest priority)
if initConfig.ConfigFile != "" {
err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
if err != nil {
return fmt.Errorf("loading config file: %w", err)
}
}
// parse environment variables, these will overwrite anything loaded from config file
// parse environment variables, these will overwrite anything loaded or flags
err := kubevip.ParseEnvironment(&initConfig)
if err != nil {
return fmt.Errorf("parsing environment: %w", err)
}
if err := initConfig.Validate(); err != nil {
return fmt.Errorf("validating configuration: %w", err)
log.Error("parsing env", "err", err)
return
}
// Change RTN_UNSPEC to default type
@@ -247,7 +202,8 @@ var kubeVipService = &cobra.Command{
log.SetLogLoggerLevel(log.Level(initConfig.Logging))
if err := initConfig.CheckInterface(); err != nil {
return fmt.Errorf("checking interface: %w", err)
log.Error("checking interface", "err", err)
return
}
// User Environment variables as an option to make manifest clearer
@@ -256,61 +212,40 @@ var kubeVipService = &cobra.Command{
configMap = envConfigMap
}
// Legacy vip_address requires vip_subnet for control-plane ARP, BGP, and Routing Table modes.
if initConfig.EnableControlPlane &&
(initConfig.EnableARP || initConfig.EnableBGP || initConfig.EnableRoutingTable) {
if err := initConfig.CheckSubnetExists(); err != nil {
return fmt.Errorf("checking subnet exists if vip_address defined: %w", err)
}
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
return fmt.Errorf("generating CIDR: %w", err)
log.Error("generating CIDR", "err", err)
return
}
}
ctx, cancel := context.WithCancel(cmd.Context())
defer cancel()
// Define the new service manager
mgr, err := manager.New(ctx, configMap, &initConfig)
mgr, err := manager.New(configMap, &initConfig)
if err != nil {
return fmt.Errorf("new manager: %w", err)
log.Error("new manager", "err", err)
return
}
// Start the service manager, this will watch the config Map and construct kube-vip services for it
err = mgr.Start(ctx)
err = mgr.Start()
if err != nil {
return fmt.Errorf("manager start: %w", err)
log.Error("manager start", "err", err)
return
}
return nil
},
}
var kubeVipManager = &cobra.Command{
Use: "manager",
Short: "Start the kube-vip manager",
RunE: func(cmd *cobra.Command, args []string) error { //nolint TODO
cmd.SilenceUsage = true
// Load configuration from file if specified (lowest priority)
if initConfig.ConfigFile != "" {
err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
if err != nil {
return fmt.Errorf("loading config file: %w", err)
}
}
// parse environment variables, these will overwrite anything loaded from config file
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
// parse environment variables, these will overwrite anything loaded or flags
err := kubevip.ParseEnvironment(&initConfig)
if err != nil {
return fmt.Errorf("parsing environment: %w", err)
}
if err := initConfig.Validate(); err != nil {
return fmt.Errorf("validating configuration: %w", err)
log.Error("parsing environment", "err", err)
return
}
// Change RTN_UNSPEC to default type
@@ -321,77 +256,41 @@ var kubeVipManager = &cobra.Command{
// Set the logging level for all subsequent functions
log.SetLogLoggerLevel(log.Level(initConfig.Logging))
// Legacy vip_address requires vip_subnet for control-plane ARP, BGP, and Routing Table modes.
if initConfig.EnableControlPlane &&
(initConfig.EnableARP || initConfig.EnableBGP || initConfig.EnableRoutingTable) {
if err := initConfig.CheckSubnetExists(); err != nil {
return fmt.Errorf("checking subnet exists if vip_address defined: %w", err)
}
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address)
if err != nil {
return fmt.Errorf("generating CIDR: %w", err)
log.Error("No interface is specified for kube-vip to bind to")
return
}
}
// Welome messages
log.Info("kube-vip.io", "version", Release.Version, "build", Release.Build)
wg := sync.WaitGroup{}
defer wg.Wait()
// create main manager context
ctx, cancel := context.WithCancel(cmd.Context())
defer cancel()
metrics.RegisterPrometheusMetrics()
// start prometheus server
if initConfig.PrometheusHTTPServer != "" {
wg.Go(func() {
if err := metrics.Serve(ctx, metrics.ServerConfig{
Addr: initConfig.PrometheusHTTPServer,
}); err != nil {
// Continue even if metrics server fails
log.Error("prometheus HTTP server", "err", err)
}
go servePrometheusHTTPServer(cmd.Context(), PrometheusHTTPServerConfig{
Addr: initConfig.PrometheusHTTPServer,
})
}
// Determine the kube-vip mode
var (
mode string
modesEnabled int
)
var mode string
if initConfig.EnableARP {
mode = "ARP"
modesEnabled++
}
if initConfig.EnableBGP {
mode = "BGP"
modesEnabled++
}
if initConfig.EnableWireguard {
mode = "Wireguard"
modesEnabled++
}
if initConfig.EnableRoutingTable {
mode = "Routing Table"
modesEnabled++
}
if mode == "" {
return fmt.Errorf("no valid kube-vip mode detected, ensure a supported mode is configured")
}
if modesEnabled > 1 {
return fmt.Errorf("multiple kube-vip modes detected, ensure only one mode is configured")
}
// Provide configuration to output/logging
@@ -399,15 +298,18 @@ var kubeVipManager = &cobra.Command{
// End if nothing is enabled
if !initConfig.EnableServices && !initConfig.EnableControlPlane {
return fmt.Errorf("no features are enabled")
log.Error("no features are enabled")
return
}
if !initConfig.EnableARP && strings.Contains(initConfig.VIPSubnet, kubevip.Auto) {
return fmt.Errorf("auto subnet discovery cannot be used outside ARP mode")
log.Error("auto subnet discovery cannot be used outside ARP mode")
return
}
if strings.Contains(initConfig.VIPSubnet, kubevip.Auto) && initConfig.Address != "" {
return fmt.Errorf("auto subnet discovery cannot be used if VIP address was provided")
log.Error("auto subnet discovery cannot be used if VIP address was provided")
return
}
// If we're using wireguard then all traffic goes through the wg0 interface
@@ -424,17 +326,20 @@ var kubeVipManager = &cobra.Command{
log.Warn("attempting to create wireguard interface", "interface not found", initConfig.Interface)
err = netlink.LinkAdd(&netlink.Wireguard{LinkAttrs: netlink.LinkAttrs{Name: initConfig.Interface}})
if err != nil {
return fmt.Errorf("adding link: %w", err)
log.Error("adding link", "err", err)
return
}
l, err = netlink.LinkByName(initConfig.Interface)
if err != nil {
return fmt.Errorf("finding link: %w", err)
log.Error("finding link", "err", err)
return
}
}
}
err = netlink.LinkSetUp(l)
if err != nil {
return fmt.Errorf("setting link UP: %w", err)
log.Error("setting link UP", "err", err)
return
}
} else { // if we're not using Wireguard then we'll need to use an actual interface
@@ -444,23 +349,25 @@ var kubeVipManager = &cobra.Command{
defaultIF, err := vip.GetDefaultGatewayInterface()
if err != nil {
_ = cmd.Help()
return fmt.Errorf("detecting interface: %w", err)
log.Error("detecting interface", "err", err)
return
}
initConfig.Interface = defaultIF.Name
log.Info("kube-vip bind", "interface", initConfig.Interface)
wg.Go(func() {
if err := vip.MonitorDefaultInterface(ctx, defaultIF); err != nil {
go func() {
if err := vip.MonitorDefaultInterface(context.TODO(), defaultIF); err != nil {
log.Error("interface monitor", "err", err)
return
}
})
}()
}
}
// Perform a check on the state of the interface
if err := initConfig.CheckInterface(); err != nil {
return fmt.Errorf("checking interface: %w", err)
log.Error("checking interface", "err", err)
return
}
// User Environment variables as an option to make manifest clearer
@@ -470,55 +377,97 @@ var kubeVipManager = &cobra.Command{
}
// Define the new service manager
mgr, err := manager.New(ctx, configMap, &initConfig)
mgr, err := manager.New(configMap, &initConfig)
if err != nil {
return fmt.Errorf("new manager: %w", err)
log.Error("new manager", "err", err)
return
}
// Label metrics after the call to manager.New, as it may modify the node name
// if it was not set in the configuration.
metrics.BuildInfo.WithLabelValues(Release.Version, Release.Build, initConfig.NodeName)
prometheus.MustRegister(mgr.PrometheusCollector()...)
// Start the service manager, this will watch the config Map and construct kube-vip services for it
err = mgr.Start(ctx)
err = mgr.Start()
if err != nil {
return fmt.Errorf("start manager: %w", err)
log.Error("start manager", "err", err)
return
}
return nil
},
}
func GenerateCidrRange(address string, dnsMode string) (string, error) {
// PrometheusHTTPServerConfig defines the Prometheus server configuration.
type PrometheusHTTPServerConfig struct {
// Addr sets the http server address used to expose the metric endpoint
Addr string
}
func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerConfig) {
var err error
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.Handler())
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { //nolint TODO
_, _ = w.Write([]byte(`<html>
<head><title>kube-vip</title></head>
<body>
<h1>kube-vip Metrics</h1>
<p><a href="` + "/metrics" + `">Metrics</a></p>
</body>
</html>`))
})
srv := &http.Server{
Addr: config.Addr,
Handler: mux,
ReadHeaderTimeout: 2 * time.Second,
}
go func() {
if err = srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Error("prometheus HTTP server", "err", err)
return
}
}()
log.Info("prometheus HTTP server started")
<-ctx.Done()
log.Info("prometheus HTTP server stopped")
ctxShutDown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer func() {
cancel()
}()
if err = srv.Shutdown(ctxShutDown); err != nil {
log.Error("shutting down prometheus HTTP server", "err", err)
return
}
if err == http.ErrServerClosed {
err = nil
}
}
func GenerateCidrRange(address string) (string, error) {
var cidrs []string
addresses := strings.Split(address, ",")
for _, a := range addresses {
ip := net.ParseIP(a)
if ip == nil {
// we probably are a DNS name
ips, err := utils.LookupHost(a, dnsMode, true)
ips, err := net.LookupIP(a)
if len(ips) == 0 || err != nil {
return "", fmt.Errorf("invalid IP address: %s from [%s], %v", a, address, err)
}
for _, addr := range ips {
ip = net.ParseIP(addr)
if ip.To4() != nil {
cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv4))
} else {
cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv6))
}
}
ip = ips[0]
}
if ip.To4() != nil {
cidrs = append(cidrs, "32")
} else {
if ip.To4() != nil {
cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv4))
} else {
cidrs = append(cidrs, strconv.Itoa(vip.DefaultMaskIPv6))
}
cidrs = append(cidrs, "128")
}
}
// compact as DNS could have a lot of addresses
slices.Sort(cidrs)
cidrs = slices.Compact(cidrs)
slices.Reverse(cidrs)
return strings.Join(cidrs, ","), nil
}

View File

@@ -19,7 +19,7 @@ func main() {
var errorOccurred bool
for {
p := make([]byte, 2048)
conn, err := net.Dial("udp", net.JoinHostPort(*address, fmt.Sprint(port)))
conn, err := net.Dial("udp", fmt.Sprintf("%s:%d", *address, *port))
if err != nil {
if !errorOccurred {
errorTime = time.Now()

74
example/deploy/0.1.2.yaml Normal file
View File

@@ -0,0 +1,74 @@
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
app: kube-vip-cluster
name: kube-vip-cluster
spec:
replicas: 3
selector:
matchLabels:
app: kube-vip-cluster
strategy: {}
template:
metadata:
creationTimestamp: null
labels:
app: kube-vip-cluster
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- kube-vip-cluster
topologyKey: "kubernetes.io/hostname"
containers:
- image: ghcr.io/kube-vip/kube-vip:0.3.7
imagePullPolicy: Always
name: kube-vip
command:
- /kube-vip
- service
- --configMap
- plndr-configmap
- --arp
- --interface
- ens192
- --log
- "5"
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
hostNetwork: true
status: {}
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: lease-access
rules:
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "create", "update", "list", "put"]
- apiGroups: [""]
resources: ["configMap"]
verbs: ["get"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: lease-access
subjects:
- kind: User
name: system:serviceaccount:default:default
roleRef:
kind: Role
name: lease-access
apiGroup: rbac.authorization.k8s.io

83
example/deploy/0.1.3.yaml Normal file
View File

@@ -0,0 +1,83 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: vip
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: vip-role
rules:
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "create", "update", "list", "put"]
- apiGroups: [""]
resources: ["configmaps", "endpoints"]
verbs: ["watch", "get"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: vip-role-bind
subjects:
- kind: ServiceAccount
name: vip
apiGroup: ""
roleRef:
kind: Role
name: vip-role
apiGroup: ""
---
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
app: kube-vip-cluster
name: kube-vip-cluster
spec:
replicas: 3
selector:
matchLabels:
app: kube-vip-cluster
strategy: {}
template:
metadata:
creationTimestamp: null
labels:
app: kube-vip-cluster
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- kube-vip-cluster
topologyKey: "kubernetes.io/hostname"
containers:
- image: ghcr.io/kube-vip/kube-vip:0.3.7
imagePullPolicy: Always
name: kube-vip
command:
- /kube-vip
- service
env:
- name: vip_interface
value: "ens192"
- name: vip_configmap
value: "plndr"
- name: vip_arp
value: "true"
- name: vip_loglevel
value: "5"
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
hostNetwork: true
serviceAccountName: vip
status: {}

83
example/deploy/0.1.4.yaml Normal file
View File

@@ -0,0 +1,83 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: vip
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: vip-role
rules:
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "create", "update", "list", "put"]
- apiGroups: [""]
resources: ["configmaps", "endpoints"]
verbs: ["watch", "get"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: vip-role-bind
subjects:
- kind: ServiceAccount
name: vip
apiGroup: ""
roleRef:
kind: Role
name: vip-role
apiGroup: ""
---
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
app: kube-vip-cluster
name: kube-vip-cluster
spec:
replicas: 3
selector:
matchLabels:
app: kube-vip-cluster
strategy: {}
template:
metadata:
creationTimestamp: null
labels:
app: kube-vip-cluster
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- kube-vip-cluster
topologyKey: "kubernetes.io/hostname"
containers:
- image: plndr/kube-vip:0.1.4
imagePullPolicy: Always
name: kube-vip
command:
- /kube-vip
- service
env:
- name: vip_interface
value: "ens192"
- name: vip_configmap
value: "plndr"
- name: vip_arp
value: "true"
- name: vip_loglevel
value: "5"
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
hostNetwork: true
serviceAccountName: vip
status: {}

55
example/deploy/0.3.5.yaml Normal file
View File

@@ -0,0 +1,55 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
creationTimestamp: null
name: kube-vip-ds
namespace: kube-system
spec:
selector:
matchLabels:
name: kube-vip-ds
template:
metadata:
creationTimestamp: null
labels:
name: kube-vip-ds
spec:
containers:
- args:
- manager
env:
- name: vip_arp
value: "true"
- name: vip_interface
value: eth0
- name: port
value: "6443"
- name: vip_cidr
value: "32"
- name: svc_enable
value: "true"
- name: vip_startleader
value: "false"
- name: vip_addpeerstolb
value: "true"
- name: vip_localpeer
value: ip-172-20-40-207:172.20.40.207:10000
- name: vip_address
image: plndr/kube-vip:v0.3.5
imagePullPolicy: Always
name: kube-vip
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
- SYS_TIME
hostNetwork: true
serviceAccountName: kube-vip
updateStrategy: {}
status:
currentNumberScheduled: 0
desiredNumberScheduled: 0
numberMisscheduled: 0
numberReady: 0

179
go.mod
View File

@@ -1,154 +1,165 @@
module github.com/kube-vip/kube-vip
go 1.26.4
go 1.24.4
require (
github.com/cloudflare/ipvs v0.12.0
github.com/containernetworking/plugins v1.9.1
github.com/docker/docker v28.5.2+incompatible
github.com/florianl/go-conntrack v0.7.0
github.com/cloudflare/ipvs v0.11.0
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/docker/docker v28.3.3+incompatible
github.com/florianl/go-conntrack v0.4.0
github.com/google/go-cmp v0.7.0
github.com/google/go-containerregistry v0.22.1
github.com/google/nftables v0.3.0
github.com/gookit/slog v0.7.1
github.com/gookit/slog v0.5.8
github.com/huin/goupnp v1.3.0
github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82
github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d
github.com/jpillora/backoff v1.0.0
github.com/mdlayher/ndp v1.1.0
github.com/onsi/ginkgo/v2 v2.32.2
github.com/onsi/gomega v1.43.0
github.com/osrg/gobgp/v4 v4.9.0
github.com/onsi/ginkgo/v2 v2.23.4
github.com/onsi/gomega v1.37.0
github.com/osrg/gobgp/v3 v3.37.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.24.1
github.com/spf13/cobra v1.10.2
github.com/stretchr/testify v1.12.1
github.com/vishvananda/netlink v1.3.2-0.20260830232854-cf01b55a4a4b
github.com/vishvananda/netns v0.0.5
go.etcd.io/etcd/api/v3 v3.7.1
go.etcd.io/etcd/client/pkg/v3 v3.7.1
go.etcd.io/etcd/client/v3 v3.7.1
go.uber.org/zap v1.28.0
github.com/prometheus/client_golang v1.22.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.9.1
github.com/stretchr/testify v1.10.0
github.com/vishvananda/netlink v1.3.1
go.etcd.io/etcd/api/v3 v3.6.1
go.etcd.io/etcd/client/pkg/v3 v3.6.1
go.etcd.io/etcd/client/v3 v3.6.1
go.uber.org/zap v1.27.0
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
golang.org/x/sync v0.23.0
golang.org/x/sys v0.48.0
golang.org/x/sync v0.15.0
golang.org/x/sys v0.33.0
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
google.golang.org/grpc v1.83.2
gopkg.in/yaml.v3 v3.0.1
k8s.io/api v0.36.4
k8s.io/apimachinery v0.36.4
k8s.io/client-go v0.36.4
k8s.io/klog/v2 v2.140.0
sigs.k8s.io/kind v0.33.0
sigs.k8s.io/yaml v1.6.0
google.golang.org/grpc v1.73.0
google.golang.org/protobuf v1.36.6
k8s.io/api v0.33.3
k8s.io/apimachinery v0.33.3
k8s.io/client-go v0.33.3
k8s.io/klog/v2 v2.130.1
sigs.k8s.io/kind v0.29.0
sigs.k8s.io/yaml v1.4.0
)
require (
al.essio.dev/pkg/shellescape v1.5.1 // indirect
github.com/BurntSushi/toml v1.5.0 // indirect
github.com/Masterminds/semver/v3 v3.4.0 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/BurntSushi/toml v1.4.0 // indirect
github.com/Microsoft/go-winio v0.4.14 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/containerd/errdefs v1.0.0 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/coreos/go-semver v0.3.1 // indirect
github.com/coreos/go-systemd/v22 v22.7.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/go-connections v0.7.0 // indirect
github.com/dlclark/regexp2 v1.11.4 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/eapache/channels v1.1.0 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/emicklei/go-restful/v3 v3.13.0 // indirect
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
github.com/gaissmai/bart v0.26.1 // indirect
github.com/fsnotify/fsnotify v1.8.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.21.0 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.7.0 // indirect
github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect
github.com/google/gnostic-models v0.6.9 // indirect
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gookit/color v1.6.1 // indirect
github.com/gookit/goutil v0.7.6 // indirect
github.com/gookit/gsr v0.1.1 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
github.com/gookit/color v1.5.4 // indirect
github.com/gookit/goutil v0.6.18 // indirect
github.com/gookit/gsr v0.1.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/josharian/native v1.1.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/k-sone/critbitgo v1.4.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/magiconair/properties v1.8.9 // indirect
github.com/mailru/easyjson v0.9.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mdlayher/genetlink v1.3.2 // indirect
github.com/mdlayher/netlink v1.8.0 // indirect
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect
github.com/mdlayher/packet v1.1.2 // indirect
github.com/mdlayher/socket v0.5.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/sys/atomicwriter v0.1.0 // indirect
github.com/moby/term v0.5.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
github.com/morikuni/aec v1.1.0 // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/pierrec/lz4/v4 v4.1.22 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect
github.com/sagikazarmark/locafero v0.7.0 // indirect
github.com/segmentio/fasthash v1.0.3 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.62.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/sagikazarmark/locafero v0.6.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.12.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.7.1 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/spf13/viper v1.20.1 // indirect
github.com/spf13/pflag v1.0.6 // indirect
github.com/spf13/viper v1.19.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tj/go-spin v1.1.0 // indirect
github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/vishvananda/netns v0.0.5 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xlab/c-for-go v1.3.0 // indirect
github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
go.opentelemetry.io/otel v1.44.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 // indirect
go.opentelemetry.io/otel/metric v1.44.0 // indirect
go.opentelemetry.io/otel/trace v1.44.0 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.37.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 // indirect
go.opentelemetry.io/otel/metric v1.37.0 // indirect
go.opentelemetry.io/otel/sdk v1.37.0 // indirect
go.opentelemetry.io/otel/trace v1.37.0 // indirect
go.uber.org/automaxprocs v1.6.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/crypto v0.55.0 // indirect
golang.org/x/mod v0.39.0 // indirect
golang.org/x/net v0.58.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/term v0.45.0 // indirect
golang.org/x/text v0.41.0 // indirect
golang.org/x/time v0.14.0 // indirect
golang.org/x/tools v0.49.0 // indirect
golang.org/x/crypto v0.39.0 // indirect
golang.org/x/mod v0.25.0 // indirect
golang.org/x/net v0.41.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/term v0.32.0 // indirect
golang.org/x/text v0.26.0 // indirect
golang.org/x/time v0.9.0 // indirect
golang.org/x/tools v0.33.0 // indirect
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect
k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect
modernc.org/cc/v4 v4.24.4 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/opt v0.1.4 // indirect
modernc.org/sortutil v1.2.1 // indirect
modernc.org/strutil v1.2.1 // indirect
modernc.org/token v1.1.0 // indirect
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
sigs.k8s.io/randfill v1.0.0 // indirect
sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
)

555
go.sum
View File

@@ -2,34 +2,32 @@ al.essio.dev/pkg/shellescape v1.5.1 h1:86HrALUujYS/h+GtqoB26SBEdkWfmMI6FubjXlsXy
al.essio.dev/pkg/shellescape v1.5.1/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8=
github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cloudflare/ipvs v0.12.0 h1:UeKRM4q82F+XQjw1sGRwsH3IOy6keBYUwlqTUqJhy7Y=
github.com/cloudflare/ipvs v0.12.0/go.mod h1:SvzXyDX2E33X/Cokbgts7rFXueiSqp6NlEMGDT7Bu24=
github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
github.com/cloudflare/ipvs v0.11.0 h1:niLcbqfv8+RSYk+yI+jhiCIHobEdEhO++mcwCX0Znsw=
github.com/cloudflare/ipvs v0.11.0/go.mod h1:XsbuKcQpqb3rpjYPtsFsM8BNZlgLw+Z0iC6RhocqDd8=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
github.com/containernetworking/cni v1.3.0 h1:v6EpN8RznAZj9765HhXQrtXgX+ECGebEYEmnuFjskwo=
github.com/containernetworking/cni v1.3.0/go.mod h1:Bs8glZjjFfGPHMw6hQu82RUgEPNGEaBb9KS5KtNMnJ4=
github.com/containernetworking/plugins v1.9.1 h1:8oU6WsIsU3bpnNZuvHp74a6cE1MJwbj2P7s4/yTUNlA=
github.com/containernetworking/plugins v1.9.1/go.mod h1:fj7kS55qg3o/RgS+WGsF3+ZxwIImMPusQZKzBpcSr4c=
github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA=
github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w=
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -39,38 +37,35 @@ github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa5
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c=
github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k=
github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0=
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes=
github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU=
github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/florianl/go-conntrack v0.7.0 h1:kWbRhLRUqBlmi2ncASmygN5yI281wlpnTUJs69tcs7g=
github.com/florianl/go-conntrack v0.7.0/go.mod h1:7kzfTjtQCRsAVkuornpI8KiwFfa79QmcZXZQex1IXhU=
github.com/florianl/go-conntrack v0.4.0 h1:TlYkxytdwgVayfU0cKwkHurQA0Rd1ZSEBRckRYDUu18=
github.com/florianl/go-conntrack v0.4.0/go.mod h1:iPDx4oIats2T7X7Jm3PFyRCJM1GfZhJaSHOWROYOrE8=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
github.com/gaissmai/bart v0.26.1 h1:+w4rnLGNlA2GDVn382Tfe3jOsK5vOr5n4KmigJ9lbTo=
github.com/gaissmai/bart v0.26.1/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs=
github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo=
github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M=
github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk=
github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE=
github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc=
github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -84,96 +79,125 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr
github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw=
github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-containerregistry v0.22.1 h1:RZuuSYhTvlDvtsK+NkutoCZ//C0X2ebLK8X8l3ULs84=
github.com/google/go-containerregistry v0.22.1/go.mod h1:bJR35SK8XgisYmhg/FMQ/5RK0S/XrOAqLBV5/LR2XE0=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg=
github.com/google/nftables v0.3.0/go.mod h1:BCp9FsrbF1Fn/Yu6CLUc9GGZFw/+hsxfluNXXmxBfRM=
github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg=
github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gookit/assert v0.1.1 h1:lh3GcawXe/p+cU7ESTZ5Ui3Sm/x8JWpIis4/1aF0mY0=
github.com/gookit/assert v0.1.1/go.mod h1:jS5bmIVQZTIwk42uXl4lyj4iaaxx32tqH16CFj0VX2E=
github.com/gookit/color v1.6.1 h1:KoTnDxJPRgrL0SoX0f8rCFg2zI0t4E3GZZBMo2nN8LU=
github.com/gookit/color v1.6.1/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs=
github.com/gookit/goutil v0.7.6 h1:700ZP6QPWhw5ms7X13JH9fUs4LTyYMmncFFMGpK73ns=
github.com/gookit/goutil v0.7.6/go.mod h1:vJS9HXctYTCLtCsZot5L5xF+O1oR17cDYO9R0HxBmnU=
github.com/gookit/gsr v0.1.1 h1:TaHD3M7qa6lcAf9D2J4mGNg+QjgDtD1bw7uctF8RXOM=
github.com/gookit/gsr v0.1.1/go.mod h1:7wv4Y4WCnil8+DlDYHBjidzrEzfHhXEoFjEA0pPPWpI=
github.com/gookit/rotatefile v0.3.0 h1:9MtCRBM79/Chcqp6ySHHmeDGpJE09WvyRFHo7yCQ+is=
github.com/gookit/rotatefile v0.3.0/go.mod h1:MUaLyw2tEKNe8nta7o2qMfCGST30kzJqybG4KUreIu4=
github.com/gookit/slog v0.7.1 h1:/q4YtsaJfdtzK+Q1g3QtNoO8cuEF6EjjU56W5U069i8=
github.com/gookit/slog v0.7.1/go.mod h1:aJ4SGHlMR5YdfeQcICQBEn5bnF0Rpnuh+a5FEzQqXpE=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
github.com/gookit/goutil v0.6.18 h1:MUVj0G16flubWT8zYVicIuisUiHdgirPAkmnfD2kKgw=
github.com/gookit/goutil v0.6.18/go.mod h1:AY/5sAwKe7Xck+mEbuxj0n/bc3qwrGNe3Oeulln7zBA=
github.com/gookit/gsr v0.1.0 h1:0gadWaYGU4phMs0bma38t+Do5OZowRMEVlHv31p0Zig=
github.com/gookit/gsr v0.1.0/go.mod h1:7wv4Y4WCnil8+DlDYHBjidzrEzfHhXEoFjEA0pPPWpI=
github.com/gookit/slog v0.5.8 h1:XZCeHLQvvOZWcSUDZcqxXITsL9+d1ESsKZoASBmK1lI=
github.com/gookit/slog v0.5.8/go.mod h1:s0ViFOY/IgUuT4MDPF0l9x5/npcciy8pL4xwWZadnoc=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8CrSJVmaolDVOxTfS9kc36uB6H40kdbQq8=
github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Goc3h8EklBH5mspfHFxBnEoURQCGzQQH1ga9Myjvis=
github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc=
github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82 h1:y5aU8Uvl7eyM5WNgdQvRxbMJb+zo7pD+S72/Yo4pvnQ=
github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82/go.mod h1:qfvBmyDNp+/liLEYWRvqny/PEz9hGe2Dz833eXILSmo=
github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d h1:VkCNWh6tuQLgDBc6KrUOz/L1mCUQGnR1Ujj8uTgpwwk=
github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d/go.mod h1:VvGYjkZoJyKqlmT1yzakUs4mfKMNB0XdODP0+rdml6k=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE=
github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung=
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ=
github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok=
github.com/jsimonetti/rtnetlink v0.0.0-20201216134343-bde56ed16391/go.mod h1:cR77jAZG3Y3bsb8hF6fHJbFoyFukLFOkQ98S0pQz3xw=
github.com/jsimonetti/rtnetlink v0.0.0-20201220180245-69540ac93943/go.mod h1:z4c53zj6Eex712ROyh8WI0ihysb5j2ROyV42iNogmAs=
github.com/jsimonetti/rtnetlink v0.0.0-20210122163228-8d122574c736/go.mod h1:ZXpIyOK59ZnN7J0BV99cZUPmsqDRZ3eq5X+st7u/oSA=
github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b/go.mod h1:8w9Rh8m+aHZIG69YPGGem1i5VzoyRC8nw2kA8B+ik5U=
github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190/go.mod h1:NmKSdU4VGSiv1bMsdqNALI4RSvvjtz65tTMCnD05qLo=
github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/k-sone/critbitgo v1.4.0 h1:l71cTyBGeh6X5ATh6Fibgw3+rtNT80BA0uNNWgkPrbE=
github.com/k-sone/critbitgo v1.4.0/go.mod h1:7E6pyoyADnFxlUBEKcnfS49b7SUAQGMK+OAp/UQvo0s=
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a8RkpQM=
github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo=
github.com/mdlayher/ethtool v0.0.0-20211028163843-288d040e9d60/go.mod h1:aYbhishWc4Ai3I2U4Gaa2n3kHWSwzme6EsG/46HRQbE=
github.com/mdlayher/genetlink v1.0.0/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc=
github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw=
github.com/mdlayher/genetlink v1.3.2/go.mod h1:tcC3pkCrPUGIKKsCsp0B3AdaaKuHtaxoJRz3cc+528o=
github.com/mdlayher/ndp v1.1.0 h1:QylGKGVtH60sKZUE88+IW5ila1Z/M9/OXhWdsVKuscs=
github.com/mdlayher/ndp v1.1.0/go.mod h1:FmgESgemgjl38vuOIyAHWUUL6vQKA/pQNkvXdWsdQFM=
github.com/mdlayher/netlink v1.6.2/go.mod h1:O1HXX2sIWSMJ3Qn1BYZk1yZM+7iMki/uYGGiwGyq/iU=
github.com/mdlayher/netlink v1.8.0 h1:e7XNIYJKD7hUct3Px04RuIGJbBxy1/c4nX7D5YyvvlM=
github.com/mdlayher/netlink v1.8.0/go.mod h1:UhgKXUlDQhzb09DrCl2GuRNEglHmhYoWAHid9HK3594=
github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA=
github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M=
github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY=
github.com/mdlayher/netlink v1.1.1/go.mod h1:WTYpFb/WTvlRJAyKhZL5/uy69TDDpHHu2VZmb2XgV7o=
github.com/mdlayher/netlink v1.2.0/go.mod h1:kwVW1io0AZy9A1E2YYgaD4Cj+C+GPkU6klXCMzIJ9p8=
github.com/mdlayher/netlink v1.2.1/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU=
github.com/mdlayher/netlink v1.2.2-0.20210123213345-5cc92139ae3e/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU=
github.com/mdlayher/netlink v1.3.0/go.mod h1:xK/BssKuwcRXHrtN04UBkwQ6dY9VviGGuriDdoPSWys=
github.com/mdlayher/netlink v1.4.0/go.mod h1:dRJi5IABcZpBD2A3D0Mv/AiX8I9uDEu5oGkAVrekmf8=
github.com/mdlayher/netlink v1.4.1/go.mod h1:e4/KuJ+s8UhfUpO9z00/fDZZmhSrs+oxyqAS9cNgn6Q=
github.com/mdlayher/netlink v1.5.0/go.mod h1:1Kr8BBFxGyUyNmztC9WLOayqYVAd2wsgOZm18nqGuzQ=
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 h1:A1Cq6Ysb0GM0tpKMbdCXCIfBclan4oHk1Jb+Hrejirg=
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42/go.mod h1:BB4YCPDOzfy7FniQ/lxuYQ3dgmM2cZumHbK8RpTjN2o=
github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY=
github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4=
github.com/mdlayher/socket v0.2.3/go.mod h1:bz12/FozYNH/VbvC3q7TRIK/Y6dH1kCKsXaUeXi/FmY=
github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00/go.mod h1:GAFlyu4/XV68LkQKYzKhIo/WW7j3Zi0YRAz/BOoanUc=
github.com/mdlayher/socket v0.0.0-20211007213009-516dcbdf0267/go.mod h1:nFZ1EtZYK8Gi/k6QNu7z7CgO20i/4ExeQswwWuPmG/g=
github.com/mdlayher/socket v0.1.0/go.mod h1:mYV5YIZAfHh4dzDVzI8x8tWLWCliuX8Mon5Awbj+qDs=
github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos=
github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ=
github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE=
github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A=
github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws=
github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw=
@@ -185,214 +209,301 @@ github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFL
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/morikuni/aec v1.1.0 h1:vBBl0pUnvi/Je71dsRrhMBtreIqNMYErSAbEeb8jrXQ=
github.com/morikuni/aec v1.1.0/go.mod h1:xDRgiq/iw5l+zkao76YTKzKttOp2cwPEne25HDkJnBw=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/onsi/ginkgo/v2 v2.32.2 h1:2o6vyFvR6snrJWgRVztC+OwuqqPEMI1UzYl2s2iU7Cg=
github.com/onsi/ginkgo/v2 v2.32.2/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
github.com/onsi/gomega v1.43.0 h1:VlG/1FxqNxhSO+lq/OHBNaaqwiBK/mO8JbVkX9Y+FeU=
github.com/onsi/gomega v1.43.0/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus=
github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8=
github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y=
github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c=
github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM=
github.com/osrg/gobgp/v4 v4.9.0 h1:pKOw914kwQ4I/lWNVTfEDosEN3FuqPGytMEInXxpTyQ=
github.com/osrg/gobgp/v4 v4.9.0/go.mod h1:bJbFm7T2nRANggShfl3I9h0UpPCzu4uAY5J/6dTdRvs=
github.com/osrg/gobgp/v3 v3.37.0 h1:+ObuOdvj7G7nxrT0fKFta+EAupdWf/q1WzbXydr8IOY=
github.com/osrg/gobgp/v3 v3.37.0/go.mod h1:kVHVFy1/fyZHJ8P32+ctvPeJogn9qKwa1YCeMRXXrP0=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM=
github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY=
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk=
github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs=
github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4=
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds=
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 h1:pyC9PaHYZFgEKFdlp3G8RaCKgVpHZnecvArXvPXcFkM=
github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/vishvananda/netlink v1.3.2-0.20260830232854-cf01b55a4a4b h1:XtEhFJO3IqjQWHJZ3bbNm7LtbDehriJK65KW+6lnw+Q=
github.com/vishvananda/netlink v1.3.2-0.20260830232854-cf01b55a4a4b/go.mod h1:lEui7SPMd9fgxzHVGRAvTxsBGCF6PRH81o2kLWLWHgw=
github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/xlab/c-for-go v1.3.0 h1:WDc+/E59g1OcZ4dYB4K9V4Doh4NnVBMYxaZFhNi77Lg=
github.com/xlab/c-for-go v1.3.0/go.mod h1:0qVnGIfoNg7c5gABptM38Y7nnXgAMo5KYWVQPycu63Y=
github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 h1:Sw125DKxZhPUI4JLlWugkzsrlB50jR9v2khiD9FxuSo=
github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245/go.mod h1:C+diUUz7pxhNY6KAoLgrTYARGWnt82zWTylZlxT92vk=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
go.etcd.io/etcd/api/v3 v3.7.1 h1:KJG0/DcWGfe3Y1otDf/fsBf0TSSgpxZ5RO/L8SFt73E=
go.etcd.io/etcd/api/v3 v3.7.1/go.mod h1:8bXIpCMeV7E3/XL0Ix123ATn3dB+0V7d9zklHbB0m78=
go.etcd.io/etcd/client/pkg/v3 v3.7.1 h1:rKYsj3pRkR0eK3yjT3XOgrhqfmIfj9pzNgxjh7mfFv4=
go.etcd.io/etcd/client/pkg/v3 v3.7.1/go.mod h1:cnzZGIUzSfjEwLC6UBVsSXlEK1eepS/JUD7wE6PLRT0=
go.etcd.io/etcd/client/v3 v3.7.1 h1:0PEMMC0KuZmVIN+RAbdqfkZ45pYTgKVtmBEbRCvZFUg=
go.etcd.io/etcd/client/v3 v3.7.1/go.mod h1:ffNqALa8tRCYhYo1F9oR489y23K39Gz+BSR3ApAGYq0=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak=
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g=
go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.etcd.io/etcd/api/v3 v3.6.1 h1:yJ9WlDih9HT457QPuHt/TH/XtsdN2tubyxyQHSHPsEo=
go.etcd.io/etcd/api/v3 v3.6.1/go.mod h1:lnfuqoGsXMlZdTJlact3IB56o3bWp1DIlXPIGKRArto=
go.etcd.io/etcd/client/pkg/v3 v3.6.1 h1:CxDVv8ggphmamrXM4Of8aCC8QHzDM4tGcVr9p2BSoGk=
go.etcd.io/etcd/client/pkg/v3 v3.6.1/go.mod h1:aTkCp+6ixcVTZmrJGa7/Mc5nMNs59PEgBbq+HCmWyMc=
go.etcd.io/etcd/client/v3 v3.6.1 h1:KelkcizJGsskUXlsxjVrSmINvMMga0VWwFF0tSPGEP0=
go.etcd.io/etcd/client/v3 v3.6.1/go.mod h1:fCbPUdjWNLfx1A6ATo9syUmFVxqHH9bCnPLBZmnLmMY=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 h1:bDMKF3RUSxshZ5OjOTi8rsHGaPKsAt76FaqgvIUySLc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0/go.mod h1:dDT67G/IkA46Mr2l9Uj7HsQVwsjASyV9SjGofsiUZDA=
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo=
go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329 h1:9kj3STMvgqy3YA4VQXBrN7925ICMxD5wzMRcgA30588=
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c=
golang.org/x/mod v0.39.0 h1:UF5zwQdCRRUpHfyPwr7d4UrGiVeldIsogtzWVnczL74=
golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20220923203811-8be639271d50/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210110051926-789bb1bd4061/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210123111255-9b0068b26619/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uIfPMv78iAJGcPKDeqAFnaLBropIC4=
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA=
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10 h1:3GDAcqdIg1ozBNLgPy4SLT84nfcBjr6rhGtXYtrkWLU=
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10/go.mod h1:T97yPqesLiNrOYxkwmhMI0ZIlJDm+p0PMR8eRVeR5tQ=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8=
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY=
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok=
google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc=
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo=
gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4=
gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
k8s.io/api v0.36.4 h1:RxrvqCL6vgH5/+UnTeu1IIFqYmGfy0hnyrod1rn35Oo=
k8s.io/api v0.36.4/go.mod h1:S2B3orCFBDhrgyWbLeuKcT2QdHIpQesBkCYSlWtwUOw=
k8s.io/apimachinery v0.36.4 h1:PT2UzkupGuAx/+xT5XjiMJ1WGpY3fn9/hdAvjweRet4=
k8s.io/apimachinery v0.36.4/go.mod h1:p2I2dipt7JHG+quVwQ1d02d28O4GdDi77RByQ13MTpk=
k8s.io/client-go v0.36.4 h1:MDvfDNvMSt0Br94SK8neviVlwL9qifw9B26hJCpD1K0=
k8s.io/client-go v0.36.4/go.mod h1:pNK4WKELbwlEDvtbE8l22lEZL5THYF61H5EealokZmA=
k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc=
k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0=
k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg=
k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0=
k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU=
k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk=
honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8=
k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE=
k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA=
k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM=
k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA=
k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg=
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4=
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8=
k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0=
k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
modernc.org/cc/v4 v4.24.4 h1:TFkx1s6dCkQpd6dKurBNmpo+G8Zl4Sq/ztJ+2+DEsh0=
modernc.org/cc/v4 v4.24.4/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
modernc.org/ccorpus2 v1.5.2 h1:Ui+4tc58mf/W+2arcYCJR903y3zl3ecsI7Fpaaqozyw=
modernc.org/ccorpus2 v1.5.2/go.mod h1:Wifvo4Q/qS/h1aRoC2TffcHsnxwTikmi1AuLANuucJQ=
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
sigs.k8s.io/kind v0.33.0 h1:AjvDv3vOygb/VKLVQW87lfktIBzkxR8Ump9DjxC8+Lk=
sigs.k8s.io/kind v0.33.0/go.mod h1:FSqriGaoTPruiXWfRnUXNykF8r2t+fHtK0P0m1AbGF8=
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
sigs.k8s.io/kind v0.29.0 h1:3TpCsyh908IkXXpcSnsMjWdwdWjIl7o9IMZImZCWFnI=
sigs.k8s.io/kind v0.29.0/go.mod h1:ldWQisw2NYyM6k64o/tkZng/1qQW7OlzcN5a8geJX3o=
sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw=
sigs.k8s.io/structured-merge-diff/v6 v6.3.3/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc=
sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps=
sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

View File

@@ -1,8 +1,6 @@
package main
import (
"os"
"github.com/kube-vip/kube-vip/cmd"
)
@@ -16,5 +14,5 @@ func main() {
cmd.Release.Version = Version
cmd.Release.Build = Build
os.Exit(cmd.Execute())
cmd.Execute()
}

View File

@@ -2,21 +2,15 @@ package arp
import (
"context"
"fmt"
log "log/slog"
"sync"
"time"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
)
const linkSubscriptionBuffer = 64
type Manager struct {
mu sync.Mutex
instances map[string]*Instance
config *kubevip.Config
}
@@ -24,14 +18,11 @@ type Manager struct {
type Instance struct {
network vip.Network
ndp *vip.NdpResponder
mu sync.Mutex
counter int
}
func NewManager(config *kubevip.Config) *Manager {
if config.ArpBroadcastRate < 500 {
log.Warn("[ARP manager] arp broadcast rate is too low", "rate (ms)", config.ArpBroadcastRate, "setting to (ms)", "3000")
config.ArpBroadcastRate = 3000
}
return &Manager{
instances: make(map[string]*Instance),
config: config,
@@ -51,122 +42,57 @@ func (i *Instance) Name() string {
}
func (m *Manager) Insert(instance *Instance) {
m.mu.Lock()
defer m.mu.Unlock()
existing := m.instances[instance.Name()]
if existing == nil {
i, ok := m.instances[instance.Name()]
if !ok {
log.Info("inserting ARP/NDP instance", "name", instance.Name())
m.instances[instance.Name()] = instance
log.Info("[ARP manager] inserting ARP/NDP instance", "name", instance.Name())
return
} else {
i.mu.Lock()
defer i.mu.Unlock()
i.counter++
}
existing.counter++
}
func (m *Manager) Remove(instance *Instance) {
m.RemoveWithIPDelete(instance, true)
}
// RemoveOnLeadershipLoss removes an ARP instance when leadership is lost
func (m *Manager) RemoveOnLeadershipLoss(instance *Instance) {
// Use the inverse of PreserveVIPOnLeadershipLoss to decide whether to delete the IP
// If preserve is true, don't delete IP (deleteIP = false)
// If preserve is false, delete IP (deleteIP = true), This is the legacy behavior
deleteIP := !m.config.PreserveVIPOnLeadershipLoss
m.RemoveWithIPDelete(instance, deleteIP)
}
func (m *Manager) RemoveWithIPDelete(instance *Instance, deleteIP bool) {
m.mu.Lock()
defer m.mu.Unlock()
i := m.instances[instance.Name()]
if i != nil {
i.counter--
if i.counter == 0 {
log.Info("[ARP manager] removing ARP/NDP instance", "name", instance.Name())
if deleteIP {
if _, err := instance.network.DeleteIP(); err != nil {
log.Error("failed to delete IP", "address", instance.network.IP(), "err", err)
}
}
if i, ok := m.instances[instance.Name()]; ok {
i.mu.Lock()
defer i.mu.Unlock()
if i.counter > 1 {
i.counter--
} else {
log.Info("removing ARP/NDP instance", "name", instance.Name())
delete(m.instances, instance.Name())
}
} else {
log.Warn("[ARP manager] unable to remove the instance - instance not found", "name", instance.Name())
}
}
func (m *Manager) Count(name string) int {
m.mu.Lock()
defer m.mu.Unlock()
i := m.instances[name]
if i != nil {
if i, ok := m.instances[name]; ok {
i.mu.Lock()
defer i.mu.Unlock()
return i.counter
}
return 0
}
func (m *Manager) StartAdvertisement(ctx context.Context, killFunc func()) {
if m.config.LoseLeadership {
var wg sync.WaitGroup
defer wg.Wait()
log.Info("[ARP manager] starting watching network device", "interface", m.config.Interface)
duration := time.Duration(m.config.LoseLeadershipTimeoutSeconds) * time.Second
timeout := time.NewTimer(duration)
timeout.Stop()
wg.Go(func() {
select {
case <-timeout.C:
killFunc()
case <-ctx.Done():
return
}
})
wg.Go(func() {
if err := watch(ctx, m.config.Interface, func(s netlink.LinkOperState) {
if isUp(s) {
timeout.Stop()
return
}
timeout.Reset(duration)
}); err != nil {
log.Error("[ARP manager] stopped watching interface", "err", err)
killFunc()
}
})
}
log.Info("[ARP manager] starting ARP/NDP advertisement")
ticker := time.NewTicker(time.Duration(m.config.ArpBroadcastRate) * time.Millisecond)
defer ticker.Stop()
func (m *Manager) StartAdvertisement(ctx context.Context) {
log.Info("Starting ARP/NDP advertisement")
for {
select {
case <-ctx.Done(): // if cancel() execute
return
case <-ticker.C: // send gratuitous ARP/NDP on each tick
m.advertiseAll()
default:
for _, instance := range m.instances {
if instance.counter > 0 {
ensureIPAndSendGratuitous(instance)
}
}
}
}
}
func (m *Manager) advertiseAll() {
m.mu.Lock()
defer m.mu.Unlock()
for _, instance := range m.instances {
if instance.counter > 0 {
ensureIPAndSendGratuitous(instance)
} else if _, err := instance.network.DeleteIP(); err != nil {
log.Error("[ARP manager] failed to delete IP", "address", instance.network.IP(), "err", err)
if m.config.ArpBroadcastRate < 500 {
log.Error("arp broadcast rate is too low", "rate (ms)", m.config.ArpBroadcastRate, "setting to (ms)", "3000")
m.config.ArpBroadcastRate = 3000
}
time.Sleep(time.Duration(m.config.ArpBroadcastRate) * time.Millisecond)
}
}
@@ -185,27 +111,23 @@ func ensureIPAndSendGratuitous(instance *Instance) {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted and recreating address with NODAD flag to skip DAD", "IP", ipString, "interface", iface)
// Re-add immediately without DAD check since we're recovering from DADFAILED
// The AddIP function will set IFA_F_NODAD flag for IPv6 addresses when skipDAD=true
if _, err := instance.network.AddIP(false, true); err != nil {
log.Error("failed to recreate address after DADFAILED", "IP", ipString, "interface", iface, "err", err)
} else {
log.Info("successfully recreated address after DADFAILED recovery", "IP", ipString, "interface", iface)
}
log.Info("deleted and recreating address", "IP", ipString, "interface", iface)
// if _, err := instance.network.AddIP(false); err != nil {
// log.Error("failed to recreate address", "IP", ipString, "interface", iface)
// }
}
// Return early after DADFAILED recovery to avoid double IP addition
return
}
// Normal case: add IP with precheck and normal DAD process
if added, err := instance.network.AddIP(true, false); err != nil {
// Ensure the address exists on the interface before attempting to ARP
// if instance.network.HasEndpoints() {
if added, err := instance.network.AddIP(true); err != nil {
log.Warn(err.Error())
} else if added {
log.Warn("Re-applied the VIP configuration", "ip", ipString, "interface", iface)
}
// }
if utils.IsIPv6(ipString) {
if vip.IsIPv6(ipString) {
// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
if instance.ndp == nil {
log.Error("NDP responder was not created")
@@ -215,6 +137,7 @@ func ensureIPAndSendGratuitous(instance *Instance) {
log.Warn(err.Error())
}
}
} else {
// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
err := vip.ARPSendGratuitous(ipString, iface)
@@ -222,75 +145,5 @@ func ensureIPAndSendGratuitous(instance *Instance) {
log.Warn(err.Error())
}
}
}
// watch subscribing to the network interface events and calls handler
func watch(ctx context.Context, interfaceName string, operStateHandler func(netlink.LinkOperState)) error {
ifname, err := netlink.LinkByName(interfaceName)
if err != nil {
return fmt.Errorf("failed to watch interface %q: %w", interfaceName, err)
}
// verify if this interface is physical device
if _, ok := ifname.(*netlink.Device); !ok {
return fmt.Errorf("interface %s is not physical, ignoring", interfaceName)
}
// The subscription is buffered and drained on exit: netlink parks its reader
// goroutine on an unread send, which closing done alone does not release.
events := make(chan netlink.LinkUpdate, linkSubscriptionBuffer)
done := make(chan struct{})
if err := netlink.LinkSubscribe(events, done); err != nil {
return fmt.Errorf("failed to subscribe to the interface events: %w", err)
}
defer func() {
close(done)
drainLinkUpdates(events)
}()
// handle initial state
operStateHandler(ifname.Attrs().OperState)
for {
select {
case <-ctx.Done():
return ctx.Err()
case event, ok := <-events:
if !ok {
return fmt.Errorf("interface events channel closed")
}
attrs := event.Attrs()
// LinkSubscribe captures events for all network devices found
// so we only care about vip interface
if ifname.Attrs().Name != attrs.Name {
continue
}
log.Debug("handling device change", "state", attrs.OperState)
operStateHandler(attrs.OperState)
}
}
}
func isUp(operState netlink.LinkOperState) bool {
return operState == netlink.OperUp
}
// drainLinkUpdates releases a netlink sender that is parked on an unread update
// so its goroutine can observe the closed subscription and exit.
func drainLinkUpdates(events <-chan netlink.LinkUpdate) {
timer := time.NewTimer(100 * time.Millisecond)
defer timer.Stop()
for {
select {
case _, ok := <-events:
if !ok {
return
}
case <-timer.C:
return
}
}
}

View File

@@ -1,119 +0,0 @@
package arp
import (
"sync"
"testing"
"time"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
)
func TestDrainLinkUpdatesReleasesParkedSender(t *testing.T) {
events := make(chan netlink.LinkUpdate)
sent := make(chan struct{})
go func() {
events <- netlink.LinkUpdate{}
close(sent)
}()
drainLinkUpdates(events)
select {
case <-sent:
case <-time.After(time.Second):
t.Fatal("netlink sender is still parked on an unread link update")
}
}
// stubNetwork is a minimal vip.Network implementation; only ARPName matters here.
type stubNetwork struct {
name string
deleteStarted chan struct{}
releaseDelete chan struct{}
}
func (s *stubNetwork) AddIP(bool, bool, ...int) (bool, error) { return false, nil }
func (s *stubNetwork) AddRoute(bool) (bool, error) { return false, nil }
func (s *stubNetwork) ReplaceRoute() error { return nil }
func (s *stubNetwork) DeleteIP() (bool, error) {
if s.deleteStarted != nil {
close(s.deleteStarted)
<-s.releaseDelete
}
return true, nil
}
func (s *stubNetwork) DeleteRoute() error { return nil }
func (s *stubNetwork) UpdateRoutes() (bool, error) { return false, nil }
func (s *stubNetwork) IsSet() (*netlink.Addr, error) { return nil, nil }
func (s *stubNetwork) IP() string { return "" }
func (s *stubNetwork) CIDR() string { return "" }
func (s *stubNetwork) IPisLinkLocal() bool { return false }
func (s *stubNetwork) PrepareRoute() *netlink.Route { return nil }
func (s *stubNetwork) RouteHash() string { return "" }
func (s *stubNetwork) SetIP(string) error { return nil }
func (s *stubNetwork) SetServicePorts(*v1.Service) {}
func (s *stubNetwork) Interface() string { return "eth0" }
func (s *stubNetwork) IsDADFAIL() bool { return false }
func (s *stubNetwork) IsDNS() bool { return false }
func (s *stubNetwork) IsDDNS() bool { return false }
func (s *stubNetwork) DDNSHostName() string { return "" }
func (s *stubNetwork) DNSName() string { return "" }
func (s *stubNetwork) SetMask(string) error { return nil }
func (s *stubNetwork) SetHasEndpoints(bool) {}
func (s *stubNetwork) HasEndpoints() bool { return false }
func (s *stubNetwork) ARPName() string { return s.name }
func (s *stubNetwork) GetPossibleSubnets() string { return "" }
func (s *stubNetwork) DHCPFamily() string { return "" }
func (s *stubNetwork) IPVSMark() uint32 { return 0 }
// TestManagerInsertConcurrentFirstRegistrationsDoNotLoseClaims guards the
// get-then-store race: two never-before-seen instances for the same ARP name
// registering concurrently must both be counted, not just the last writer.
func TestManagerInsertConcurrentFirstRegistrationsDoNotLoseClaims(t *testing.T) {
m := NewManager(&kubevip.Config{ArpBroadcastRate: 3000})
const concurrent = 8
var wg sync.WaitGroup
for range concurrent {
wg.Add(1)
go func() {
defer wg.Done()
m.Insert(NewInstance(&stubNetwork{name: "shared"}, nil))
}()
}
wg.Wait()
if got := m.Count("shared"); got != concurrent {
t.Fatalf("Count() = %d, want %d claims registered", got, concurrent)
}
}
func TestManagerInsertDoesNotJoinEntryBeingRemoved(t *testing.T) {
m := NewManager(&kubevip.Config{ArpBroadcastRate: 3000})
deleteStarted := make(chan struct{})
releaseDelete := make(chan struct{})
first := NewInstance(&stubNetwork{name: "shared", deleteStarted: deleteStarted, releaseDelete: releaseDelete}, nil)
m.Insert(first)
removeDone := make(chan struct{})
go func() {
m.Remove(first)
close(removeDone)
}()
<-deleteStarted
insertDone := make(chan struct{})
go func() {
m.Insert(NewInstance(&stubNetwork{name: "shared"}, nil))
close(insertDone)
}()
close(releaseDelete)
<-removeDone
<-insertDone
if got := m.Count("shared"); got != 1 {
t.Fatalf("Count() = %d, want replacement claim registered", got)
}
}

View File

@@ -1,15 +1,14 @@
package backend
import (
"context"
"fmt"
"sync"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
@@ -22,21 +21,6 @@ type Entry struct {
type Map map[Entry]bool
// kubeConfigPath is an explicitly configured kubeconfig used by Check when
// set; static pod deployments configure it since neither admin.conf nor
// in-cluster config are available there.
var (
kubeConfigPath string
pathMtx sync.Mutex
)
// SetKubeConfigPath configures the kubeconfig used by backend health checks.
func SetKubeConfigPath(path string) {
pathMtx.Lock()
defer pathMtx.Unlock()
kubeConfigPath = path
}
func (e *Entry) Check() bool {
var client *kubernetes.Clientset
var err error
@@ -47,19 +31,13 @@ func (e *Entry) Check() bool {
// homeConfigPath := filepath.Join(os.Getenv("HOME"), ".kube", "config")
var k8sAddr string
if utils.IsIPv6(e.Addr) {
k8sAddr = fmt.Sprintf("[%s]:%v", e.Addr, e.Port)
} else {
if vip.IsIPv4(e.Addr) {
k8sAddr = fmt.Sprintf("%s:%v", e.Addr, e.Port)
} else {
k8sAddr = fmt.Sprintf("[%s]:%v", e.Addr, e.Port)
}
switch {
case kubeConfigPath != "" && utils.FileExists(kubeConfigPath):
config, err = k8s.NewRestConfig(kubeConfigPath, false, k8sAddr)
if err != nil {
log.Error("create k8s REST config", "path", kubeConfigPath, "err", err)
return false
}
case utils.FileExists(adminConfigPath):
config, err = k8s.NewRestConfig(adminConfigPath, false, k8sAddr)
if err != nil {
@@ -88,7 +66,7 @@ func (e *Entry) Check() bool {
return true
}
func Watch(ctx context.Context, interval int, tickAction func()) {
func Watch(tickAction func(), interval int, stop chan struct{}) {
if interval <= 0 {
interval = 5
}
@@ -98,10 +76,13 @@ func Watch(ctx context.Context, interval int, tickAction func()) {
for {
select {
case <-ctx.Done():
case <-stop:
ticker.Stop()
return
case <-ticker.C:
ticker.Stop()
tickAction()
ticker.Reset(time.Second * time.Duration(interval))
}
}
}

View File

@@ -3,80 +3,46 @@ package bgp
import (
"context"
"fmt"
log "log/slog"
"net"
"github.com/osrg/gobgp/v4/pkg/apiutil"
api "github.com/osrg/gobgp/v3/api"
)
// AddHost will update peers of a host
func (b *Server) AddHost(ctx context.Context, addr string, object string) error {
b.mtx.Lock()
defer b.mtx.Unlock()
objects, exists := b.tracker[addr]
if !exists {
b.tracker[addr] = make(map[string]bool)
objects = b.tracker[addr]
ip, _, err := net.ParseCIDR(addr)
if err != nil {
return err
}
p := b.getPath(ip)
if p == nil {
return fmt.Errorf("failed to get path for %v", ip)
}
if _, err := b.s.AddPath(apiutil.AddPathRequest{
Paths: []*apiutil.Path{p},
}); err != nil {
return err
}
log.Debug("[BGP] added host", "addr", addr, "cnt", len(objects)+1, "object", object)
}
objects[object] = true
return nil
}
// DelHost will inform peers to remove a host
func (b *Server) DelHost(ctx context.Context, addr string, object string) error {
b.mtx.Lock()
defer b.mtx.Unlock()
objects, exists := b.tracker[addr]
if !exists {
log.Debug("[BGP] deleting host - nothing to delete", "addr", addr, "object", object)
return nil
}
func (b *Server) AddHost(addr string) (err error) {
ip, _, err := net.ParseCIDR(addr)
if err != nil {
return err
}
delete(objects, object)
if len(objects) == 0 {
p := b.getPath(ip)
if p == nil {
return nil
}
if err := b.s.DeletePath(apiutil.DeletePathRequest{
Paths: []*apiutil.Path{p},
}); err != nil {
return err
}
delete(b.tracker, addr)
log.Debug("[BGP] deleted host", "addr", addr, "cnt", len(objects), "object", object)
} else {
log.Debug("[BGP] deleting from tracker only", "addr", addr, "object", object)
p := b.getPath(ip)
if p == nil {
return fmt.Errorf("failed to get path for %v", ip)
}
return nil
_, err = b.s.AddPath(context.Background(), &api.AddPathRequest{
Path: p,
})
if err != nil {
return err
}
return
}
// DelHost will inform peers to remove a host
func (b *Server) DelHost(addr string) (err error) {
ip, _, err := net.ParseCIDR(addr)
if err != nil {
return err
}
p := b.getPath(ip)
if p == nil {
return
}
return b.s.DeletePath(context.Background(), &api.DeletePathRequest{
Path: p,
})
}

View File

@@ -3,39 +3,24 @@ package bgp
import (
"context"
"fmt"
log "log/slog"
"net"
"net/netip"
"strconv"
"time"
"strings"
"github.com/jpillora/backoff"
"github.com/kube-vip/kube-vip/pkg/kubevip"
//nolint
"github.com/kube-vip/kube-vip/pkg/vip"
api "github.com/osrg/gobgp/v4/api"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/osrg/gobgp/v4/pkg/apiutil"
"github.com/osrg/gobgp/v4/pkg/config/oc"
bgp "github.com/osrg/gobgp/v4/pkg/packet/bgp"
"github.com/osrg/gobgp/v4/pkg/server"
api "github.com/osrg/gobgp/v3/api"
"github.com/osrg/gobgp/v3/pkg/server"
"google.golang.org/protobuf/types/known/anypb"
)
const defaultBGPPort uint32 = 179
// AddPeer will add peers to the BGP configuration
func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error) {
remotePort := defaultBGPPort
if peer.Port != 0 {
remotePort = uint32(peer.Port)
}
func (b *Server) AddPeer(peer Peer) (err error) {
p := &api.Peer{
Conf: &api.PeerConf{
NeighborAddress: peer.Address,
PeerAsn: peer.AS,
NeighborInterface: peer.Interface,
AuthPassword: peer.Password,
NeighborAddress: peer.Address,
PeerAsn: peer.AS,
AuthPassword: peer.Password,
},
Timers: &api.Timers{
@@ -55,165 +40,236 @@ func (b *Server) AddPeer(ctx context.Context, peer kubevip.BGPPeer) (err error)
Transport: &api.Transport{
MtuDiscovery: true,
RemoteAddress: peer.Address,
RemotePort: remotePort,
RemotePort: uint32(179),
},
}
if peer.BFDEnabled {
p.Bfd = &api.BfdPeerConfig{
Enabled: true,
DesiredMinimumTxInterval: peer.BFDTransmitInterval,
RequiredMinimumReceive: peer.BFDReceiveInterval,
DetectionMultiplier: peer.BFDDetectMultiplier,
Port: 3784, // TODO: Should this be configurable??
}
}
if peer.Interface != "" {
neighborAddress, err := getIPv6LinkLocalNeighborAddress(ctx, peer.Interface)
if err != nil {
return fmt.Errorf("failed to get link-local address of interface %s: %w", peer.Interface, err)
}
p.State = &api.PeerState{
NeighborAddress: neighborAddress,
}
}
mpBGP := b.c.MpbgpNexthop
if peer.MpbgpNexthop != "" {
mpBGP = peer.MpbgpNexthop
}
if mpBGP != "" {
ipv4Address, ipv6Address, err := peer.FindMpbgpAddresses(p, b.c)
if err != nil {
log.Error("failed to get MP-BGP addresses, will not us MP-BGP for this host", "error", err)
b.setPeerSource(p)
} else {
p.AfiSafis = []*api.AfiSafi{
{
Config: &api.AfiSafiConfig{
Family: &api.Family{
Afi: api.Family_AFI_IP,
Safi: api.Family_SAFI_UNICAST,
},
Enabled: true,
if b.c.MpbgpNexthop != "" {
p.AfiSafis = []*api.AfiSafi{
{
Config: &api.AfiSafiConfig{
Family: &api.Family{
Afi: api.Family_AFI_IP,
Safi: api.Family_SAFI_UNICAST,
},
Enabled: true,
},
{
Config: &api.AfiSafiConfig{
Family: &api.Family{
Afi: api.Family_AFI_IP6,
Safi: api.Family_SAFI_UNICAST,
},
Enabled: true,
},
{
Config: &api.AfiSafiConfig{
Family: &api.Family{
Afi: api.Family_AFI_IP6,
Safi: api.Family_SAFI_UNICAST,
},
Enabled: true,
},
}
},
}
peer.SetMpbgpOptions(b.c)
peer.setMpbgpOptions(b.c)
mask := strconv.Itoa(vip.DefaultMaskIPv6)
address := ipv4Address
family := api.Family_AFI_IP
if utils.IsIPv4(p.Conf.NeighborAddress) {
mask = strconv.Itoa(vip.DefaultMaskIPv4)
address = ipv6Address
family = api.Family_AFI_IP6
}
ipv4Address, ipv6Address, err := peer.findMpbgpAddresses(p, b.c)
if err != nil {
return fmt.Errorf("failed to get MP-BGP addresses: %w", err)
}
err = b.s.AddDefinedSet(ctx, &api.AddDefinedSetRequest{
DefinedSet: &api.DefinedSet{
DefinedType: api.DefinedType_DEFINED_TYPE_NEIGHBOR,
Name: fmt.Sprintf("peer-%s", p.Conf.NeighborAddress),
List: []string{fmt.Sprintf("%s/%s", p.Conf.NeighborAddress, mask)},
},
})
if err != nil {
return fmt.Errorf("failed to add defined set: %v", err)
}
mask := "128"
address := ipv4Address
family := api.Family_AFI_IP
if vip.IsIPv4(p.Conf.NeighborAddress) {
mask = "32"
address = ipv6Address
family = api.Family_AFI_IP6
}
if address != "" {
if err := insertPolicy(ctx, b.s, address, p, family); err != nil {
return fmt.Errorf("failed to add policy: %w", err)
}
err = b.s.AddDefinedSet(context.Background(), &api.AddDefinedSetRequest{
DefinedSet: &api.DefinedSet{
DefinedType: api.DefinedType_NEIGHBOR,
Name: fmt.Sprintf("peer-%s", p.Conf.NeighborAddress),
List: []string{fmt.Sprintf("%s/%s", p.Conf.NeighborAddress, mask)},
},
})
if err != nil {
return fmt.Errorf("failed to add defined set: %v", err)
}
if address != "" {
if err := insertPolicy(b.s, address, p, family); err != nil {
return fmt.Errorf("failed to add policy: %w", err)
}
}
} else {
b.setPeerSource(p)
if b.c.SourceIP != "" {
p.Transport.LocalAddress = b.c.SourceIP
}
if b.c.SourceIF != "" {
p.Transport.BindInterface = b.c.SourceIF
}
}
if err := b.s.AddPeer(ctx, &api.AddPeerRequest{Peer: p}); err != nil {
if err := b.s.AddPeer(context.Background(), &api.AddPeerRequest{Peer: p}); err != nil {
return fmt.Errorf("failed to add peer: %v", err)
}
log.Info("[BGP]", "peer", p.Conf.NeighborAddress, "AS", p.Conf.PeerAsn, "BFD", p.Bfd)
return nil
}
func (b *Server) setPeerSource(p *api.Peer) {
if b.c.SourceIP != "" {
p.Transport.LocalAddress = b.c.SourceIP
}
if b.c.SourceIF != "" {
p.Transport.BindInterface = b.c.SourceIF
}
}
func (b *Server) getPath(ip net.IP) *apiutil.Path {
func (b *Server) getPath(ip net.IP) (path *api.Path) {
isV6 := ip.To4() == nil
//nolint
originAttr, _ := anypb.New(&api.OriginAttribute{
Origin: 0,
})
if !isV6 {
prefix, err := bgp.NewIPAddrPrefix(netip.MustParsePrefix(
fmt.Sprintf("%s/%d", ip.String(), vip.DefaultMaskIPv4),
))
if err != nil {
return nil
}
//nolint
nlri, _ := anypb.New(&api.IPAddressPrefix{
Prefix: ip.String(),
PrefixLen: 32,
})
nh, err := bgp.NewPathAttributeNextHop(netip.MustParseAddr("0.0.0.0"))
if err != nil {
return nil
}
//nolint
nhAttr, _ := anypb.New(&api.NextHopAttribute{
NextHop: "0.0.0.0", // gobgp will fill this
})
return &apiutil.Path{
Family: bgp.RF_IPv4_UC,
Nlri: prefix,
Attrs: []bgp.PathAttributeInterface{
bgp.NewPathAttributeOrigin(0),
nh,
path = &api.Path{
Family: &api.Family{
Afi: api.Family_AFI_IP,
Safi: api.Family_SAFI_UNICAST,
},
Nlri: nlri,
Pattrs: []*anypb.Any{originAttr, nhAttr},
}
} else {
//nolint
nlri, _ := anypb.New(&api.IPAddressPrefix{
Prefix: ip.String(),
PrefixLen: 128,
})
v6Family := &api.Family{
Afi: api.Family_AFI_IP6,
Safi: api.Family_SAFI_UNICAST,
}
//nolint
mpAttr, _ := anypb.New(&api.MpReachNLRIAttribute{
Family: v6Family,
NextHops: []string{"::"}, // gobgp will fill this
Nlris: []*anypb.Any{nlri},
})
path = &api.Path{
Family: v6Family,
Nlri: nlri,
Pattrs: []*anypb.Any{originAttr, mpAttr},
}
}
prefix, err := bgp.NewIPAddrPrefix(netip.MustParsePrefix(
fmt.Sprintf("%s/%d", ip.String(), vip.DefaultMaskIPv6),
))
if err != nil {
return nil
}
mpReach, err := bgp.NewPathAttributeMpReachNLRI(
bgp.RF_IPv6_UC,
[]bgp.PathNLRI{{NLRI: prefix}},
netip.MustParseAddr("::"),
)
if err != nil {
return nil
}
return &apiutil.Path{
Family: bgp.RF_IPv6_UC,
Nlri: prefix,
Attrs: []bgp.PathAttributeInterface{
bgp.NewPathAttributeOrigin(0),
mpReach,
},
}
return
}
func insertPolicy(ctx context.Context, s *server.BgpServer, address string, p *api.Peer, family api.Family_Afi) error {
// ParseBGPPeerConfig - take a string and parses it into an array of peers
func ParseBGPPeerConfig(config string) (bgpPeers []Peer, err error) {
peers := strings.Split(config, ",")
if len(peers) == 0 {
return nil, fmt.Errorf("no BGP Peer configurations found")
}
for x := range peers {
peerStr := peers[x]
config := strings.Split(peerStr, "/")
peerStr = config[0]
if peerStr == "" {
continue
}
isV6Peer := peerStr[0] == '['
address := ""
if isV6Peer {
addressEndPos := strings.IndexByte(peerStr, ']')
if addressEndPos == -1 {
return nil, fmt.Errorf("no matching ] found for IPv6 BGP Peer")
}
address = peerStr[1:addressEndPos]
peerStr = peerStr[addressEndPos+1:]
}
peer := strings.Split(peerStr, ":")
if len(peer) < 2 {
return nil, fmt.Errorf("mandatory peering params <host>:<AS> incomplete")
}
if !isV6Peer {
address = peer[0]
}
ASNumber, err := strconv.ParseUint(peer[1], 10, 32)
if err != nil {
return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
}
password := ""
if len(peer) >= 3 {
password = peer[2]
}
multiHop := false
if len(peer) >= 4 {
multiHop, err = strconv.ParseBool(peer[3])
if err != nil {
return nil, fmt.Errorf("BGP MultiHop format error (true/false) [%s]", peer[1])
}
}
var port uint64
if len(peer) >= 5 {
port, err = strconv.ParseUint(peer[4], 10, 16)
if err != nil {
return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
}
} else {
port = 179
}
var mpbgpNexthop, mpbgpIPv4, mpbgpIPv6 string
if len(config) > 1 {
configData := strings.Split(config[1], ";")
for _, cfg := range configData {
c := strings.Split(cfg, "=")
switch c[0] {
case "mpbgp_nexthop":
mpbgpNexthop = c[1]
case "mpbgp_ipv4":
mpbgpIPv4 = c[1]
case "mpbgp_ipv6":
mpbgpIPv6 = c[1]
default:
return nil, fmt.Errorf("peer configuration parameter '%s' is not supported", c[0])
}
}
}
peerConfig := Peer{
Address: address,
AS: uint32(ASNumber),
Port: uint16(port),
Password: password,
MultiHop: multiHop,
MpbgpNexthop: mpbgpNexthop,
MpbgpIPv4: mpbgpIPv4,
MpbgpIPv6: mpbgpIPv6,
}
bgpPeers = append(bgpPeers, peerConfig)
}
return
}
func insertPolicy(s *server.BgpServer, address string, p *api.Peer, family api.Family_Afi) error {
familyType := "v4"
if family == api.Family_AFI_IP6 {
familyType = "v6"
@@ -234,12 +290,12 @@ func insertPolicy(ctx context.Context, s *server.BgpServer, address string, p *a
},
},
NeighborSet: &api.MatchSet{
Type: api.MatchSet_TYPE_ANY,
Type: api.MatchSet_ANY,
Name: setName,
},
},
Actions: &api.Actions{
RouteAction: api.RouteAction_ROUTE_ACTION_ACCEPT,
RouteAction: api.RouteAction_ACCEPT,
Nexthop: &api.NexthopAction{
Address: address,
},
@@ -248,28 +304,28 @@ func insertPolicy(ctx context.Context, s *server.BgpServer, address string, p *a
{
Conditions: &api.Conditions{
NeighborSet: &api.MatchSet{
Type: api.MatchSet_TYPE_ANY,
Type: api.MatchSet_ANY,
Name: setName,
},
},
Actions: &api.Actions{
RouteAction: api.RouteAction_ROUTE_ACTION_ACCEPT,
RouteAction: api.RouteAction_ACCEPT,
},
},
},
}
err := s.AddPolicy(ctx, &api.AddPolicyRequest{
err := s.AddPolicy(context.Background(), &api.AddPolicyRequest{
Policy: policy,
})
if err != nil {
return fmt.Errorf("failed to add policy: %w", err)
}
err = s.AddPolicyAssignment(ctx, &api.AddPolicyAssignmentRequest{
err = s.AddPolicyAssignment(context.Background(), &api.AddPolicyAssignmentRequest{
Assignment: &api.PolicyAssignment{
Name: "global",
Direction: api.PolicyDirection_POLICY_DIRECTION_EXPORT,
Direction: api.PolicyDirection_EXPORT,
Policies: []*api.Policy{
{
Name: policy.Name,
@@ -283,45 +339,3 @@ func insertPolicy(ctx context.Context, s *server.BgpServer, address string, p *a
return nil
}
func getIPv6LinkLocalNeighborAddress(ctx context.Context, peerInterface string) (string, error) {
neighCtx, neighCancel := context.WithTimeout(ctx, time.Minute)
defer neighCancel()
bo := backoff.Backoff{
Factor: 2,
Jitter: true,
Min: 1 * time.Second,
Max: 5 * time.Second,
}
maxAttempts := 20.0
var err error
for {
select {
case <-neighCtx.Done():
if err != nil {
return "", fmt.Errorf("failed to get link-local address of interface %s: %w", peerInterface, err)
}
return "", fmt.Errorf("failed to get link-local address of interface %s: %w", peerInterface, neighCtx.Err())
default:
dur := bo.Duration()
var neighborAddress string
neighborAddress, err = oc.GetIPv6LinkLocalNeighborAddress(peerInterface)
if err != nil && bo.Attempt() >= maxAttempts {
return "", fmt.Errorf("failed to get link-local address of interface %s: %w", peerInterface, err)
}
if neighborAddress != "" {
return neighborAddress, nil
}
t := time.NewTimer(dur)
select {
case <-neighCtx.Done():
t.Stop()
case <-t.C:
}
}
}
}

View File

@@ -1,146 +0,0 @@
package bgp
import (
"context"
"sync"
"testing"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
api "github.com/osrg/gobgp/v4/api"
gobgp "github.com/osrg/gobgp/v4/pkg/server"
)
func TestAddPeerConfiguresTransportOptions(t *testing.T) {
tests := []struct {
name string
newServer func(*testing.T) *Server
peer kubevip.BGPPeer
wantPort uint32
wantLocalAddr string
wantInterface string
}{
{
name: "configured remote port",
newServer: func(t *testing.T) *Server {
return newStartedTestBGPServer(t, kubevip.BGPConfig{
AS: 65000,
RouterID: "192.0.2.1",
Peers: []kubevip.BGPPeer{{Address: "192.0.2.10", AS: 65001}},
})
},
peer: kubevip.BGPPeer{Address: "192.0.2.10", AS: 65001, Port: 180},
wantPort: 180,
},
{
name: "configured source interface after MP-BGP fallback",
newServer: func(t *testing.T) *Server {
return newPeerTestServer(t, kubevip.BGPConfig{
AS: 65000,
RouterID: "192.0.2.1",
SourceIF: "lo",
MpbgpNexthop: "fixed",
Peers: []kubevip.BGPPeer{{Address: "192.0.2.20", AS: 65001}},
MpbgpIPv4: "",
MpbgpIPv6: "",
})
},
peer: kubevip.BGPPeer{Address: "192.0.2.20", AS: 65001},
wantInterface: "lo",
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
server := tt.newServer(t)
if err := server.AddPeer(context.Background(), tt.peer); err != nil {
t.Fatalf("AddPeer() error = %v", err)
}
peer := listTestPeer(t, server, tt.peer.Address)
if peer.GetTransport() == nil {
t.Fatal("configured peer has no transport")
}
if tt.wantPort != 0 && peer.GetTransport().GetRemotePort() != tt.wantPort {
t.Fatalf("remote port = %d, want %d", peer.GetTransport().GetRemotePort(), tt.wantPort)
}
if tt.wantLocalAddr != "" && peer.GetTransport().GetLocalAddress() != tt.wantLocalAddr {
t.Fatalf("local address = %q, want %q", peer.GetTransport().GetLocalAddress(), tt.wantLocalAddr)
}
if tt.wantInterface != "" && peer.GetTransport().GetBindInterface() != tt.wantInterface {
t.Fatalf("bind interface = %q, want %q", peer.GetTransport().GetBindInterface(), tt.wantInterface)
}
})
}
}
func newStartedTestBGPServer(t *testing.T, config kubevip.BGPConfig) *Server {
t.Helper()
server, err := NewBGPServer(config, log.LevelError)
if err != nil {
t.Fatalf("NewBGPServer() error = %v", err)
}
go server.s.Serve()
if err := server.s.StartBgp(context.Background(), &api.StartBgpRequest{
Global: &api.Global{
Asn: config.AS,
RouterId: config.RouterID,
ListenPort: -1,
},
}); err != nil {
server.s.Stop()
t.Fatalf("StartBgp() error = %v", err)
}
t.Cleanup(server.s.Stop)
return server
}
func listTestPeer(t *testing.T, server *Server, address string) *api.Peer {
t.Helper()
var got *api.Peer
if err := server.s.ListPeer(context.Background(), &api.ListPeerRequest{Address: address}, func(peer *api.Peer) {
got = peer
}); err != nil {
t.Fatalf("ListPeer() error = %v", err)
}
if got == nil {
t.Fatalf("ListPeer() returned no peer for %s", address)
}
return got
}
func newPeerTestServer(t *testing.T, cfg kubevip.BGPConfig) *Server {
t.Helper()
raw := startEmbeddedRawBGP(t)
return &Server{s: raw, c: &cfg, tracker: make(map[string]map[string]bool)}
}
func startEmbeddedRawBGP(t *testing.T) *gobgp.BgpServer {
t.Helper()
raw := gobgp.NewBgpServer()
go raw.Serve()
if err := raw.StartBgp(context.Background(), &api.StartBgpRequest{
Global: &api.Global{
Asn: 65000,
RouterId: "192.0.2.1",
ListenPort: -1,
},
}); err != nil {
t.Fatalf("starting embedded BGP server: %v", err)
}
var stopOnce sync.Once
t.Cleanup(func() {
stopOnce.Do(func() {
if err := raw.StopBgp(context.Background(), &api.StopBgpRequest{}); err != nil {
t.Logf("stopping embedded BGP server: %v", err)
}
})
})
return raw
}

54
pkg/bgp/peers_test.go Normal file
View File

@@ -0,0 +1,54 @@
package bgp
import (
"reflect"
"testing"
)
func TestParseBGPPeerConfig(t *testing.T) {
type args struct {
config string
}
tests := []struct {
name string
args args
wantBgpPeers []Peer
wantErr bool
}{
{
name: "IPv4, default port",
args: args{config: "192.168.0.10:65000::false,192.168.0.11:65000::false"},
wantBgpPeers: []Peer{
{Address: "192.168.0.10", Port: 179, AS: 65000, MultiHop: false},
{Address: "192.168.0.11", Port: 179, AS: 65000, MultiHop: false},
},
},
{
name: "IPv4, different port",
args: args{config: "192.168.0.10:65000::false:180,192.168.0.11:65000::false:190"},
wantBgpPeers: []Peer{
{Address: "192.168.0.10", Port: 180, AS: 65000, MultiHop: false},
{Address: "192.168.0.11", Port: 190, AS: 65000, MultiHop: false},
},
},
{
name: "IPv6, multi-protocol",
args: args{config: "[fd00:1111:2222:3333:c7d9:7235:6bf7:5d52]:65501::false/mpbgp_nexthop=auto_sourceif"},
wantBgpPeers: []Peer{
{Address: "fd00:1111:2222:3333:c7d9:7235:6bf7:5d52", Port: 179, AS: 65501, MultiHop: false, MpbgpNexthop: "auto_sourceif"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotBgpPeers, err := ParseBGPPeerConfig(tt.args.config)
if (err != nil) != tt.wantErr {
t.Errorf("ParseBGPPeerConfig() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(gotBgpPeers, tt.wantBgpPeers) {
t.Errorf("ParseBGPPeerConfig() = %v, want %v", gotBgpPeers, tt.wantBgpPeers)
}
})
}
}

View File

@@ -3,33 +3,17 @@ package bgp
import (
"context"
"fmt"
"sync"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
api "github.com/osrg/gobgp/v4/api"
"github.com/osrg/gobgp/v4/pkg/apiutil"
bgp "github.com/osrg/gobgp/v4/pkg/packet/bgp"
gobgp "github.com/osrg/gobgp/v4/pkg/server"
api "github.com/osrg/gobgp/v3/api"
gobgp "github.com/osrg/gobgp/v3/pkg/server"
"github.com/prometheus/client_golang/prometheus"
)
type BGPManager interface {
AddHost(ctx context.Context, addr string, object string) error
DelHost(ctx context.Context, addr string, object string) error
}
// Server manages a server object
type Server struct {
s *gobgp.BgpServer
c *kubevip.BGPConfig
mtx sync.Mutex
tracker map[string]map[string]bool
}
// NewBGPServer takes a configuration and returns a running BGP server instance
func NewBGPServer(c kubevip.BGPConfig, logLevel log.Level) (b *Server, err error) {
func NewBGPServer(c *Config) (b *Server, err error) {
if c.AS == 0 {
return nil, fmt.Errorf("you need to provide AS")
}
@@ -41,23 +25,26 @@ func NewBGPServer(c kubevip.BGPConfig, logLevel log.Level) (b *Server, err error
if len(c.Peers) == 0 {
return nil, fmt.Errorf("you need to provide at least one peer")
}
bgpLogger := log.Default()
lvl := &log.LevelVar{}
lvl.Set(logLevel)
b = &Server{
s: gobgp.NewBgpServer(gobgp.LoggerOption(bgpLogger, lvl)),
c: &c,
tracker: make(map[string]map[string]bool),
s: gobgp.NewBgpServer(),
c: c,
BGPSessionInfoGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "kube_vip",
Subsystem: "manager",
Name: "bgp_session_info",
Help: "Display state of session by setting metric for label value with current state to 1",
}, []string{"state", "peer"}),
}
return
}
// Start starts the BGP server
func (b *Server) Start(ctx context.Context, peerStateChangeCallback func(*apiutil.WatchEventMessage_PeerEvent)) (err error) {
func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (err error) {
go b.s.Serve()
if err = b.s.StartBgp(ctx, &api.StartBgpRequest{
if err = b.s.StartBgp(context.Background(), &api.StartBgpRequest{
Global: &api.Global{
Asn: b.c.AS,
RouterId: b.c.RouterID,
@@ -67,25 +54,25 @@ func (b *Server) Start(ctx context.Context, peerStateChangeCallback func(*apiuti
return
}
if err = b.s.WatchEvent(ctx, gobgp.WatchEventMessageCallbacks{
OnPeerUpdate: func(p *apiutil.WatchEventMessage_PeerEvent, _ time.Time) {
log.Info("[BGP]", "peer", fmt.Sprintf("%+v", p))
if err = b.s.WatchEvent(context.Background(), &api.WatchEventRequest{Peer: &api.WatchEventRequest_Peer{}}, func(r *api.WatchEventResponse) {
if p := r.GetPeer(); p != nil && p.Type == api.WatchEventResponse_PeerEvent_STATE {
log.Info("[BGP]", "peer", p.String())
if peerStateChangeCallback != nil {
peerStateChangeCallback(p)
}
},
}, gobgp.WatchPeer()); err != nil {
}
}); err != nil {
return
}
for _, p := range b.c.Peers {
if err = b.AddPeer(ctx, p); err != nil {
if err = b.AddPeer(p); err != nil {
return
}
}
if b.c.Zebra.Enabled {
if err = b.s.EnableZebra(ctx, &api.EnableZebraRequest{
if err = b.s.EnableZebra(context.Background(), &api.EnableZebraRequest{
Url: b.c.Zebra.URL,
Version: b.c.Zebra.Version,
SoftwareName: b.c.Zebra.SoftwareName,
@@ -100,51 +87,7 @@ func (b *Server) Start(ctx context.Context, peerStateChangeCallback func(*apiuti
// Close will stop a running BGP Server
func (b *Server) Close() error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
ctx, cf := context.WithTimeout(context.Background(), 5*time.Second)
defer cf()
return b.s.StopBgp(ctx, &api.StopBgpRequest{})
}
// ListAdvertisedRoutes retrieves all active routes inside GoBGP's local RIB.
// It queries the GLOBAL table type to find routes that kube-vip has requested GoBGP to advertise.
func (b *Server) ListAdvertisedRoutes(ctx context.Context, isIPv6 bool) ([]*api.Destination, error) {
afi := bgp.AFI_IP
if isIPv6 {
afi = bgp.AFI_IP6
}
family := bgp.NewFamily(uint16(afi), bgp.SAFI_UNICAST)
var destinations []*api.Destination
req := apiutil.ListPathRequest{
TableType: api.TableType_TABLE_TYPE_GLOBAL,
Family: family,
}
// GoBGP's embedded server API uses a callback function to stream results
// locally without requiring a gRPC client stream setup.
err := b.s.ListPath(req, func(prefix bgp.NLRI, paths []*apiutil.Path) {
var newPaths []*api.Path
for _, p := range paths {
np, err := apiutil.NewPath(p.Family, p.Nlri, p.Withdrawal, p.Attrs, time.Unix(p.Age, 0))
if err != nil {
log.Error("failed to create BGP path details", "err", err)
continue
}
newPaths = append(newPaths, np)
}
d := &api.Destination{
Prefix: prefix.String(),
Paths: newPaths,
}
destinations = append(destinations, d)
})
if err != nil {
return nil, fmt.Errorf("failed to extract local RIB: %w", err)
}
return destinations, nil
}

145
pkg/bgp/types.go Normal file
View File

@@ -0,0 +1,145 @@
package bgp
import (
"fmt"
"net"
"github.com/kube-vip/kube-vip/pkg/vip"
api "github.com/osrg/gobgp/v3/api"
gobgp "github.com/osrg/gobgp/v3/pkg/server"
"github.com/prometheus/client_golang/prometheus"
"github.com/vishvananda/netlink"
)
// Peer defines a BGP Peer
type Peer struct {
Address string
Port uint16
AS uint32
Password string
MultiHop bool
MpbgpNexthop string
MpbgpIPv4 string
MpbgpIPv6 string
}
func (p *Peer) setMpbgpOptions(server *Config) {
if p.MpbgpNexthop == "" {
p.MpbgpNexthop = server.MpbgpNexthop
}
if p.MpbgpIPv4 == "" {
p.MpbgpIPv4 = server.MpbgpIPv4
}
if p.MpbgpIPv6 == "" {
p.MpbgpIPv6 = server.MpbgpIPv6
}
}
func (p *Peer) findMpbgpAddresses(ap *api.Peer, server *Config) (string, string, error) {
var ipv4Address, ipv6Address string
switch p.MpbgpNexthop {
case "fixed":
ap.Transport.LocalAddress = server.SourceIP
if p.MpbgpIPv4 == "" && p.MpbgpIPv6 == "" {
return "", "", fmt.Errorf("to use MP-BGP with fixed address at least one IPv4 or IPv6 address has to be provided [current - IPv4: %s, IPv6: %s]",
p.MpbgpIPv4, p.MpbgpIPv6)
}
if p.MpbgpIPv4 != "" {
if net.ParseIP(p.MpbgpIPv4) == nil {
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv4 address", p.MpbgpIPv4)
}
}
if p.MpbgpIPv6 != "" {
if net.ParseIP(p.MpbgpIPv6) == nil {
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv6 address", p.MpbgpIPv6)
}
}
ipv4Address = p.MpbgpIPv4
ipv6Address = p.MpbgpIPv6
case "auto_sourceip":
ap.Transport.LocalAddress = server.SourceIP
// Resolve the local interface by SourceIP
iface, err := vip.GetInterfaceByIP(server.SourceIP)
if err != nil {
return "", "", fmt.Errorf("failed to get interface by IP: %v", err)
}
if vip.IsIPv4(server.SourceIP) {
// Get the non link-local IPv6 address on that interface
ipv6Address, err = vip.GetNonLinkLocalIP(iface, netlink.FAMILY_V6)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
}
} else {
// Get the non link-local IPv4 address on that interface
ipv4Address, err = vip.GetNonLinkLocalIP(iface, netlink.FAMILY_V4)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
}
}
case "auto_sourceif":
ap.Transport.BindInterface = server.SourceIF
iface, err := netlink.LinkByName(server.SourceIF)
if err != nil {
return "", "", fmt.Errorf("failed to get interface by name: %v", err)
}
// Get the non link-local IPv4 address on that interface
ipv4Address, err = vip.GetNonLinkLocalIP(&iface, netlink.FAMILY_V4)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
}
// Get the non link-local IPv6 address on that interface
ipv6Address, err = vip.GetNonLinkLocalIP(&iface, netlink.FAMILY_V6)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
}
default:
return "", "", fmt.Errorf("option %s for MP-BPG nexthop is not supported", server.MpbgpNexthop)
}
return ipv4Address, ipv6Address, nil
}
// Config defines the BGP server configuration
type Config struct {
AS uint32
RouterID string
SourceIP string
SourceIF string
MpbgpNexthop string
MpbgpIPv4 string
MpbgpIPv6 string
HoldTime uint64
KeepaliveInterval uint64
Peers []Peer
Zebra ZebraConfig
}
// Defines Zebra connection configuration. More on the topic - https://github.com/osrg/gobgp/blob/master/docs/sources/zebra.md#configuration
type ZebraConfig struct {
Enabled bool
URL string
Version uint32
SoftwareName string
}
// Server manages a server object
type Server struct {
s *gobgp.BgpServer
c *Config
// This is a prometheus gauge indicating the state of the sessions.
// 1 means "ESTABLISHED", 0 means "NOT ESTABLISHED"
BGPSessionInfoGauge *prometheus.GaugeVec
}

View File

@@ -1,42 +1,28 @@
package cluster
import (
"crypto/tls"
"crypto/x509"
"fmt"
"net/http"
"os"
"sync"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/node"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
// Cluster - The Cluster object manages the state of the cluster for a particular node
type Cluster struct {
stop chan bool
stopMutex sync.Mutex
Network []vip.Network
arpMgr *arp.Manager
routeMgr *route.Manager
nodeLabelMgr node.Labeler
labelAdded bool
healthCheckHTTPClient *http.Client
stop chan bool
completed chan bool
once sync.Once
Network []vip.Network
arpMgr *arp.Manager
}
// InitCluster - Will attempt to initialise all of the required settings for the cluster
func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.Manager, arpMgr *arp.Manager,
routeMgr *route.Manager, nodeLabelMgr node.Labeler) (*Cluster, error) {
func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Cluster, error) {
var networks []vip.Network
var healthCheckHTTPClient *http.Client
var err error
if !disableVIP {
@@ -46,22 +32,10 @@ func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.M
return nil, err
}
}
if c.ControlPlaneHealthCheck.Address != "" {
healthCheckHTTPClient, err = newHealthCheckHTTPClient(c)
if err != nil {
return nil, fmt.Errorf("initializing BGP health check client: %w", err)
}
}
// Initialise the Cluster structure
newCluster := &Cluster{
Network: networks,
arpMgr: arpMgr,
stop: make(chan bool),
routeMgr: routeMgr,
nodeLabelMgr: nodeLabelMgr,
healthCheckHTTPClient: healthCheckHTTPClient,
Network: networks,
arpMgr: arpMgr,
}
log.Debug("service security", "enabled", c.EnableServiceSecurity)
@@ -80,10 +54,9 @@ func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vi
networks := []vip.Network{}
for _, addr := range addresses {
network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.DHCPMode,
c.RequireDualStack, c.IsDualStack, c.RoutingTableID, c.RoutingTableType, c.RoutingProtocol, c.DNSMode,
c.LoadBalancerForwardingMethod, c.IptablesBackend, c.EnableLoadBalancer, c.LoadBalancerPort,
c.EnableServiceSecurity, intfMgr, c.EgressWithNftables, c.SkipDAD)
network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.RoutingTableID,
c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend,
c.EnableLoadBalancer, c.EnableServiceSecurity, intfMgr)
if err != nil {
return nil, err
}
@@ -95,83 +68,13 @@ func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vi
// Stop - Will stop the Cluster and release VIP if needed
func (cluster *Cluster) Stop() {
cluster.stopMutex.Lock()
defer cluster.stopMutex.Unlock()
// Close the stop channel, which will shut down the VIP (if needed)
if cluster.stop != nil {
close(cluster.stop)
cluster.stop = make(chan bool) // recreate channel for future use
}
}
func newHealthCheckHTTPClient(c *kubevip.Config) (*http.Client, error) {
defaultTransport, ok := http.DefaultTransport.(*http.Transport)
if !ok {
return nil, fmt.Errorf("unexpected default HTTP transport type %T", http.DefaultTransport)
}
transport := defaultTransport.Clone()
if c.ControlPlaneHealthCheck.CAPath != "" {
caCert, err := os.ReadFile(c.ControlPlaneHealthCheck.CAPath)
if err != nil {
return nil, fmt.Errorf("reading health check CA cert %q: %w", c.ControlPlaneHealthCheck.CAPath, err)
}
rootCAs, err := x509.SystemCertPool()
if err != nil || rootCAs == nil {
rootCAs = x509.NewCertPool()
}
if !rootCAs.AppendCertsFromPEM(caCert) {
return nil, fmt.Errorf("health check CA cert %q contains no valid certificates", c.ControlPlaneHealthCheck.CAPath)
}
tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12}
if transport.TLSClientConfig != nil {
tlsConfig = transport.TLSClientConfig.Clone()
}
tlsConfig.RootCAs = rootCAs
transport.TLSClientConfig = tlsConfig
}
return &http.Client{
Timeout: time.Duration(c.ControlPlaneHealthCheck.TimeoutSeconds) * time.Second,
Transport: transport,
}, nil
}
// cleanupVIPs handles VIP removal based on the PreserveVIPOnLeadershipLoss configuration.
// When preservation is enabled, IPv6 VIPs are always removed immediately to prevent DAD
// failures on the new leader, while IPv4 VIPs are intentionally left in place.
// When preservation is disabled (legacy behavior), all VIPs are removed.
func (cluster *Cluster) cleanupVIPs(c *kubevip.Config) {
for i := range cluster.Network {
if c.EnableARP && cluster.arpMgr.Count(cluster.Network[i].ARPName()) > 1 {
continue
}
if c.PreserveVIPOnLeadershipLoss {
if utils.IsIPv6(cluster.Network[i].IP()) {
log.Info("[VIP] Removing IPv6 VIP immediately (required to prevent DAD failures on new leader)", "ip", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
} else {
log.Info("[VIP] Preserving IPv4 VIP address on interface, only stopped ARP broadcasting", "ip", cluster.Network[i].IP())
}
} else {
log.Info("[VIP] Deleting VIP", "ip", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
}
cluster.once.Do(func() { // Ensure that the close channel can only ever be called once
close(cluster.stop)
})
}
// Wait until the completed channel is closed, signallign all shutdown tasks completed
<-cluster.completed
}

View File

@@ -2,7 +2,6 @@ package cluster
import (
"context"
"sync"
"github.com/kube-vip/kube-vip/pkg/vip"
)
@@ -13,14 +12,16 @@ import (
// during runtime if IP changes, startDDNS don't have to do reconfigure because
// dnsUpdater already have the functionality to keep trying resolve the IP
// and update the VIP configuration if it changes
func (cluster *Cluster) StartDDNS(ctx context.Context, network vip.Network, backoffAttempts uint, wg *sync.WaitGroup) error {
ddnsMgr := vip.NewDDNSManager(network, backoffAttempts)
ip, err := ddnsMgr.Start(ctx, wg)
if err != nil {
return err
}
if err = network.SetIP(ip); err != nil {
return err
func (cluster *Cluster) StartDDNS(ctx context.Context) error {
for i := range cluster.Network {
ddnsMgr := vip.NewDDNSManager(ctx, cluster.Network[i])
ip, err := ddnsMgr.Start()
if err != nil {
return err
}
if err = cluster.Network[i].SetIP(ip); err != nil {
return err
}
}
return nil

View File

@@ -3,59 +3,131 @@ package cluster
import (
"context"
"fmt"
"sync"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/etcd"
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/loadbalancer"
log "log/slog"
clientv3 "go.etcd.io/etcd/client/v3"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
watchtools "k8s.io/client-go/tools/watch"
)
// StartCluster - Begins a running instance of the Leader Election cluster
func (cluster *Cluster) StartCluster(ctx context.Context, c *kubevip.Config,
em *election.Manager, bgpServer *bgp.Server, leaseMgr *lease.Manager, killFunc func()) error {
// Manager degines the manager of the load-balancing services
type Manager struct {
KubernetesClient *kubernetes.Clientset
RetryWatcherClient *kubernetes.Clientset
// This channel is used to signal a shutdown
SignalChan chan os.Signal
ns, leaseName := lease.NamespaceName(c.LeaseName, c)
EtcdClient *clientv3.Client
}
leaseID := lease.NewID(c.LeaderElectionType, ns, leaseName)
// NewManager will create a new managing object
func NewManager(path string, inCluster bool, port int) (*Manager, error) {
var hostname string
log.Info("cluster membership", "namespace", leaseID.Namespace(), "lock", leaseID.Name(), "id", c.NodeName)
// If inCluster is set then it will likely have started as a static pod or won't have the
// VIP up before trying to connect to the API server, we set the API endpoint to this machine to
// ensure connectivity. Else if the path passed is empty and not running in the cluster,
// attempt to look for a kubeconfig in the default HOME dir.
objectName := lease.ObjectName(leaseID, "cp")
objLease := leaseMgr.Add(ctx, leaseID)
isNew := objLease.Add(objectName)
hostname = fmt.Sprintf("kubernetes:%v", port)
wg := sync.WaitGroup{}
defer wg.Wait()
if len(path) == 0 && !inCluster {
path = filepath.Join(os.Getenv("HOME"), ".kube", "config")
// Start a goroutine that will delete the lease when the service context is cancelled.
// This is important for proper cleanup when a service is deleted - it ensures that
// the lease context (svcLease.Ctx) gets cancelled, which causes RunOrDie to return.
// Without this, RunOrDie would continue running until leadership is naturally lost.
wg.Go(func() {
<-objLease.Ctx.Done()
leaseMgr.Delete(leaseID, objectName, objLease)
})
if !isNew {
log.Debug("this election was already done, waiting for it to finish", "lease", leaseName)
<-objLease.Ctx.Done()
return nil
}
wg.Go(func() {
select {
case <-cluster.stop:
case <-ctx.Done():
// We modify the config so that we can always speak to the correct host
id, err := os.Hostname()
if err != nil {
return nil, err
}
hostname = fmt.Sprintf("%s:%v", id, port)
}
config, err := k8s.NewRestConfig(path, inCluster, hostname)
if err != nil {
return nil, fmt.Errorf("failed to create k8s REST config: %w", err)
}
clientset, err := k8s.NewClientset(config)
if err != nil {
return nil, fmt.Errorf("error creating a new k8s clientset: %v", err)
}
rwConfig, err := k8s.NewRestConfig(path, inCluster, hostname)
if err != nil {
return nil, fmt.Errorf("failed to create k8s REST config for retryClientSet: %w", err)
}
rwConfig.Timeout = 0 // empty value to disable the timeout
rwClientSet, err := k8s.NewClientset(rwConfig)
if err != nil {
return nil, fmt.Errorf("failed to create k8s client for retry watcher: %w", err)
}
return &Manager{
KubernetesClient: clientset,
RetryWatcherClient: rwClientSet,
}, nil
}
// StartCluster - Begins a running instance of the Leader Election cluster
func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *bgp.Server) error {
var err error
log.Info("cluster membership", "namespace", c.Namespace, "lock", c.LeaseName, "id", c.NodeName)
// use a Go context so we can tell the leaderelection code when we
// want to step down
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// use a Go context so we can tell the arp loop code when we
// want to step down
ctxArp, cancelArp := context.WithCancel(context.Background())
defer cancelArp()
// use a Go context so we can tell the dns loop code when we
// want to step down
ctxDNS, cancelDNS := context.WithCancel(context.Background())
defer cancelDNS()
// listen for interrupts or the Linux SIGTERM signal and cancel
// our context, which the leader election code will observe and
// step down
signalChan := make(chan os.Signal, 1)
// Add Notification for Userland interrupt
signal.Notify(signalChan, syscall.SIGINT)
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(signalChan, syscall.SIGTERM)
go func() {
<-signalChan
log.Info("Received termination, signaling cluster shutdown")
// Cancel the leader context, which will in turn cancel the leadership
objLease.Cancel()
})
// Cancel the context, which will in turn cancel the leadership
cancel()
// Cancel the arp context, which will in turn stop any broadcasts
}()
// (attempt to) Remove the virtual IP, in case it already exists
@@ -69,147 +141,250 @@ func (cluster *Cluster) StartCluster(ctx context.Context, c *kubevip.Config,
}
}
objLease.Lock()
// Defer a function to check if the bgpServer has been created and if so attempt to close it
defer func() {
objLease.Unlock()
if bgpServer != nil {
bgpServer.Close()
}
}()
// this object is sharing lease with another object
if objLease.Elected.Load() {
log.Debug("this election was already done, shared lease", "lease", leaseName)
// wait for leader election to start or context to be done
select {
case <-objLease.Started:
case <-objLease.Ctx.Done():
// Lease was cancelled (e.g., leader election ended), return immediately
// This allows the restart loop to create a fresh lease
log.Debug("lease context cancelled before leader election started", "lease", leaseName)
return fmt.Errorf("lease %q context cancelled before leader election started", leaseName)
if c.EnableBGP && bgpServer == nil {
// Lets start BGP
log.Info("Starting the BGP server to advertise VIP routes to VGP peers")
bgpServer, err = bgp.NewBGPServer(&c.BGPConfig)
if err != nil {
log.Error("new BGP server", "err", err)
}
if err := bgpServer.Start(nil); err != nil {
log.Error("starting BGP server", "err", err)
}
cluster.OnStartedLeading(c, objLease, em, bgpServer, killFunc, true)
log.Debug("cluster waiting for leader context done", "lease", leaseName)
// wait for leaderelection to be finished
<-objLease.Ctx.Done()
cluster.OnStoppedLeading(c, objLease, bgpServer)
return nil
}
run := &election.RunConfig{
Config: c,
LeaseID: leaseID,
LeaseAnnotations: c.LeaseAnnotations,
Mgr: em,
OnStartedLeading: func(context.Context) { //nolint TODO: potential clean code
cluster.OnStartedLeading(c, objLease, em, bgpServer, killFunc, false)
run := &runConfig{
config: c,
leaseID: c.NodeName,
sm: sm,
onStartedLeading: func(ctx context.Context) { //nolint TODO: potential clean code
// As we're leading lets start the vip service
err := cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, cancel)
if err != nil {
log.Error("starting VIP service on leader", "err", err)
}
},
OnStoppedLeading: func() {
objLease.Elected.Store(false)
cluster.OnStoppedLeading(c, objLease, bgpServer)
onStoppedLeading: func() {
// we can do cleanup here
log.Info("This node is becoming a follower within the cluster")
// Stop the dns context
cancelDNS()
// Stop the Arp context if it is running
cancelArp()
// Stop the BGP server
if bgpServer != nil {
err := bgpServer.Close()
if err != nil {
log.Warn("close BGP server", "err", err)
}
}
for i := range cluster.Network {
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("delete VIP", "err", err)
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
}
log.Error("lost leadership, restarting kube-vip")
panic("") // TODO - we could also return here
},
OnNewLeader: func(identity string) {
cluster.OnNewLeader(identity, c)
onNewLeader: func(identity string) {
// we're notified when new leader elected
log.Info("New leader", "leader", identity)
},
}
if err := election.RunOrDie(objLease.Ctx, run, c); err != nil {
cluster.Stop()
return fmt.Errorf("leaderelection failed: %w", err)
switch c.LeaderElectionType {
case "kubernetes", "":
cluster.runKubernetesLeaderElectionOrDie(ctx, run)
case "etcd":
cluster.runEtcdLeaderElectionOrDie(ctx, run)
default:
log.Info(fmt.Sprintf("LeaderElectionMode %s not supported, exiting", c.LeaderElectionType))
}
return nil
}
func (cluster *Cluster) OnStartedLeading(c *kubevip.Config, objLease *lease.Lease,
em *election.Manager, bgpServer *bgp.Server, killFunc func(), isShared bool) {
objLease.Elected.Store(true)
objLease.Unlock()
type runConfig struct {
config *kubevip.Config
leaseID string
sm *Manager
// When we become leader, ensure we can take over VIPs even if they're preserved on other nodes
if !isShared {
close(objLease.Started)
// onStartedLeading is called when this member starts leading.
onStartedLeading func(context.Context)
// onStoppedLeading is called when this member stops leading.
onStoppedLeading func()
// onNewLeader is called when the client observes a leader that is
// not the previously observed leader. This includes the first observed
// leader when the client starts.
onNewLeader func(identity string)
}
func (cluster *Cluster) runKubernetesLeaderElectionOrDie(ctx context.Context, run *runConfig) {
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: run.config.LeaseName,
Namespace: run.config.Namespace,
Annotations: run.config.LeaseAnnotations,
},
Client: run.sm.KubernetesClient.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: run.leaseID,
},
}
labels := generateLabelsFromConfig(c.Address, kubevip.HasIP)
if err := cluster.nodeLabelMgr.AddLabel(labels); err != nil {
log.Error("error adding label to node", "err", err)
}
cluster.labelAdded = true
// start the leader election code loop
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(run.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(run.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(run.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: run.onStartedLeading,
OnStoppedLeading: run.onStoppedLeading,
OnNewLeader: run.onNewLeader,
},
})
}
if c.PreserveVIPOnLeadershipLoss {
log.Info("Becoming leader with VIP preservation enabled - ensuring VIP takeover")
// Force add the VIPs (this will work even if they exist due to the precheck logic)
for i := range cluster.Network {
added, err := cluster.Network[i].AddIP(true, false)
if err != nil {
log.Error("failed to ensure VIP on leader takeover", "vip", cluster.Network[i].IP(), "err", err)
} else if added {
log.Info("took over VIP as new leader", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
} else {
log.Info("VIP already configured on interface", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
}
func (cluster *Cluster) runEtcdLeaderElectionOrDie(ctx context.Context, run *runConfig) {
etcd.RunElectionOrDie(ctx, &etcd.LeaderElectionConfig{
EtcdConfig: etcd.ClientConfig{Client: run.sm.EtcdClient},
Name: run.config.LeaseName,
MemberID: run.leaseID,
LeaseDurationSeconds: int64(run.config.LeaseDuration),
Callbacks: etcd.LeaderCallbacks{
OnStartedLeading: run.onStartedLeading,
OnStoppedLeading: run.onStoppedLeading,
OnNewLeader: run.onNewLeader,
},
})
}
func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
log.Info("Kube-Vip is watching nodes for control-plane labels")
listOptions := metav1.ListOptions{
LabelSelector: "node-role.kubernetes.io/control-plane",
}
// As we're leading lets start the vip service
err := cluster.StartVipService(objLease.Ctx, c, em, bgpServer, killFunc)
rw, err := watchtools.NewRetryWatcherWithContext(ctxArp, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.RetryWatcherClient.CoreV1().Nodes().Watch(context.Background(), listOptions)
},
})
if err != nil {
log.Error("starting VIP service on leader", "err", err)
killFunc()
}
}
func (cluster *Cluster) OnStoppedLeading(c *kubevip.Config, objLease *lease.Lease,
bgpServer *bgp.Server) {
// we can do cleanup here
log.Info("This node is becoming a follower within the cluster")
if cluster.labelAdded {
labels := generateLabelsFromConfig(c.Address, kubevip.HasIP)
if err := cluster.nodeLabelMgr.RemoveLabel(labels); err != nil {
log.Error("error removing label from node", "err", err)
}
cluster.labelAdded = false
return fmt.Errorf("error creating label watcher: %s", err.Error())
}
// Stop the cluster context if it is running
objLease.Cancel()
go func() {
<-sm.SignalChan
log.Info("Received termination, signaling shutdown")
// Cancel the context
rw.Stop()
}()
cluster.cleanupVIPs(c)
ch := rw.ResultChan()
// defer rw.Stop()
log.Error("lost leadership, restarting kube-vip")
}
func (cluster *Cluster) OnNewLeader(identity string, c *kubevip.Config) {
// we're notified when new leader elected
log.Info("New leader", "leader", identity)
// If we're not the new leader and we have VIPs preserved from previous leadership,
// we need to clean them up to avoid conflicts.
if identity != c.NodeName && c.PreserveVIPOnLeadershipLoss {
log.Info("Cleaning up preserved VIPs as another node became leader", "new_leader", identity)
for i := range cluster.Network {
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("failed to cleanup preserved VIP", "vip", cluster.Network[i].IP(), "err", err)
for event := range ch {
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
node, ok := event.Object.(*v1.Node)
if !ok {
return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
}
if deleted {
log.Info("cleaned up preserved VIP to avoid conflict", "IP", cluster.Network[i].IP(),
"interface", cluster.Network[i].Interface(), "new_leader", identity)
} else {
log.Debug("VIP was not present on this node", "IP", cluster.Network[i].IP(),
"interface", cluster.Network[i].Interface())
// Find the node IP address (this isn't foolproof)
for x := range node.Status.Addresses {
if node.Status.Addresses[x].Type == v1.NodeInternalIP {
if checkIfNodeIsReady(node) {
err = lb.AddBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("add IPVS backend", "err", err)
}
} else {
err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("remove IPVS backend", "err", err)
}
}
}
}
case watch.Deleted:
node, ok := event.Object.(*v1.Node)
if !ok {
return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
}
// Find the node IP address (this isn't foolproof)
for x := range node.Status.Addresses {
if node.Status.Addresses[x].Type == v1.NodeInternalIP {
err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("Del IPVS backend", "err", err)
}
}
}
log.Info("Node deleted", "name", node.Name)
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes Nodes")
// This round trip allows us to handle unstructured status
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Error("watcher", "status", status)
default:
}
}
log.Info("Exiting Node watcher")
return nil
}
func checkIfNodeIsReady(node *v1.Node) bool {
if node == nil {
return false
}
for _, condition := range node.Status.Conditions {
if condition.Type == v1.NodeReady {
if condition.Status == v1.ConditionTrue {
return true
}
}
}
}
func generateLabelsFromConfig(addr, labelKey string) map[string]string {
return map[string]string{
labelKey: utils.SanitizeIPForLabel(addr),
}
return false
}

View File

@@ -1,32 +0,0 @@
package cluster
import (
"sync"
"sync/atomic"
"testing"
)
func TestStopConcurrentDoesNotRaceOrPanic(t *testing.T) {
c := &Cluster{stop: make(chan bool)}
start := make(chan struct{})
var wg sync.WaitGroup
var panics atomic.Int64
for range 128 {
wg.Go(func() {
<-start
defer func() {
if recover() != nil {
panics.Add(1)
}
}()
c.Stop()
})
}
close(start)
wg.Wait()
if got := panics.Load(); got != 0 {
t.Fatalf("concurrent Stop panicked %d time(s)", got)
}
}

View File

@@ -1,125 +0,0 @@
package cluster_test
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"encoding/pem"
"math/big"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/kubevip"
)
func TestInitCluster_HealthCheckClientNoCA(t *testing.T) {
t.Parallel()
cfg := &kubevip.Config{
EnableBGP: true,
ControlPlaneHealthCheck: kubevip.HealthCheck{
Address: "http://localhost:6443/livez",
TimeoutSeconds: 5,
},
}
_, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
func TestInitCluster_HealthCheckClientValidCA(t *testing.T) {
t.Parallel()
caPEM := generateTestCACert(t)
caFile := filepath.Join(t.TempDir(), "ca.crt")
if err := os.WriteFile(caFile, caPEM, 0o600); err != nil {
t.Fatal(err)
}
cfg := &kubevip.Config{
EnableBGP: true,
ControlPlaneHealthCheck: kubevip.HealthCheck{
Address: "https://localhost:6443/livez",
TimeoutSeconds: 3,
CAPath: caFile,
},
}
_, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
func TestInitCluster_HealthCheckClientInvalidCAPath(t *testing.T) {
t.Parallel()
cfg := &kubevip.Config{
EnableBGP: true,
ControlPlaneHealthCheck: kubevip.HealthCheck{
Address: "https://localhost:6443/livez",
CAPath: "/nonexistent/ca.crt",
},
}
_, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
if err == nil {
t.Fatal("expected error for invalid CA path")
}
if !strings.Contains(err.Error(), "reading health check CA cert") {
t.Errorf("expected error about reading CA cert, got: %v", err)
}
}
func TestInitCluster_HealthCheckClientInvalidCAContent(t *testing.T) {
t.Parallel()
caFile := filepath.Join(t.TempDir(), "bad-ca.crt")
if err := os.WriteFile(caFile, []byte("not a certificate"), 0o600); err != nil {
t.Fatal(err)
}
cfg := &kubevip.Config{
EnableBGP: true,
ControlPlaneHealthCheck: kubevip.HealthCheck{
Address: "https://localhost:6443/livez",
CAPath: caFile,
},
}
_, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
if err == nil {
t.Fatal("expected error for invalid CA content")
}
if !strings.Contains(err.Error(), "contains no valid certificates") {
t.Errorf("expected error about invalid certificates, got: %v", err)
}
}
// generateTestCACert creates a self-signed CA certificate in PEM format for testing.
func generateTestCACert(t *testing.T) []byte {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
template := &x509.Certificate{
SerialNumber: big.NewInt(1),
NotBefore: time.Now(),
NotAfter: time.Now().Add(time.Hour),
IsCA: true,
BasicConstraintsValid: true,
}
certDER, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
}

View File

@@ -5,129 +5,103 @@ import (
"errors"
"fmt"
"io/fs"
"net/http"
"net/url"
"strconv"
"strings"
"net"
"os"
"os/signal"
"sync"
"syscall"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/backend"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/loadbalancer"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config, em *election.Manager, bgpServer bgp.BGPManager, killFunc func()) error {
func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Config, sm *Manager, bgpServer *bgp.Server, cancelLeaderElection context.CancelFunc) error {
var err error
var wg sync.WaitGroup
defer wg.Wait()
// listen for interrupts or the Linux SIGTERM signal and cancel
// our context, which the leader election code will observe and
// step down
signalChan := make(chan os.Signal, 1)
// Add Notification for Userland interrupt
signal.Notify(signalChan, syscall.SIGINT)
wg.Go(func() {
<-ctx.Done()
killFunc()
})
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(signalChan, syscall.SIGTERM)
loadbalancers := []*loadbalancer.IPVSLoadBalancer{}
var arpWG sync.WaitGroup
for i := range cluster.Network {
network := cluster.Network[i]
if network.IsDDNS() {
if err := cluster.StartDDNS(ctx, cluster.Network[i], c.DHCPBackoffAttempts, &wg); err != nil {
log.Error("failed to start DDNS", "err", err)
}
}
if err := network.SetMask(c.VIPSubnet); err != nil {
killFunc()
return fmt.Errorf("failed to set mask for subnet %q: %w", c.VIPSubnet, err)
log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
panic("")
}
if network.IsDDNS() {
if err := cluster.StartDDNS(ctxDNS); err != nil {
log.Error(err.Error())
}
}
// start the dns updater if address is dns
if network.IsDNS() {
log.Info("starting the DNS updater", "address", network.DNSName())
ipUpdater := vip.NewIPUpdater(network)
wg.Go(func() {
ipUpdater.Run(ctx)
})
ipUpdater.Run(ctxDNS)
}
if !c.EnableRoutingTable {
// Normal VIP addition, use skipDAD=false for normal DAD process
if _, err = network.AddIP(false, false); err != nil {
log.Error("failed to add IP", "address", network.IP(), "error", err)
if _, err = network.AddIP(false); err != nil {
log.Error(err.Error())
}
}
if c.EnableBGP {
if c.ControlPlaneHealthCheck.Address != "" {
// The health check loop owns route advertisement/withdrawal when configured.
wg.Go(func() {
cluster.bgpHealthCheckLoop(ctx, c, bgpServer, network.CIDR())
})
} else {
// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation.
log.Debug("Attempting to advertise over BGP", "address", network.CIDR())
err = bgpServer.AddHost(ctx, network.CIDR(), c.NodeName)
if err != nil {
log.Error(err.Error())
}
// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
log.Debug("Attempting to advertise over BGP", "address", network.CIDR())
err = bgpServer.AddHost(network.CIDR())
if err != nil {
log.Error(err.Error())
}
}
if c.EnableLoadBalancer {
lb, err := loadbalancer.NewIPVSLB(ctx, network, c.LoadBalancerPort, c.LoadBalancerForwardingMethod,
c.BackendHealthCheckInterval, c.EgressWithNftables, killFunc, &wg)
lb, err := loadbalancer.NewIPVSLB(network.IP(), c.LoadBalancerPort, c.LoadBalancerForwardingMethod, c.BackendHealthCheckInterval, c.Interface, cancelLeaderElection, signalChan)
if err != nil {
killFunc()
return fmt.Errorf("creating IPVS LoadBalancer: %w", err)
log.Error("Error creating IPVS LoadBalancer", "err", err)
}
wg.Go(func() {
for {
select {
case <-ctx.Done():
return
default:
err = em.NodeWatcher(ctx, lb, c.Port)
if err != nil {
log.Error("Error watching node labels", "err", err)
if utils.IsPanicError(err) {
killFunc()
return
}
}
}
go func() {
err = sm.NodeWatcher(ctxArp, lb, c.Port) //TODO: We're using the ctxARP as the context this will change when rkatz finishes his change
if err != nil {
log.Error("Error watching node labels", "err", err)
}
})
}()
loadbalancers = append(loadbalancers, lb)
}
if c.EnableARP {
wg.Go(func() {
cluster.layer2Update(ctx, network, c)
})
arpWG.Add(1)
go cluster.layer2Update(ctxArp, network, c, &arpWG)
}
}
if c.EnableLoadBalancer {
// Shutdown function that will wait on this signal, unless we call it ourselves
<-ctx.Done()
<-signalChan
for _, lb := range loadbalancers {
err = lb.RemoveIPVSLB()
if err != nil {
@@ -141,51 +115,64 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
backendMapV6 := backend.Map{}
// only check localhost
// An explicitly configured Kubernetes API address (static-pod
// deployments point it at the local API server, whose loopback
// listener is often the only certificate-valid local endpoint)
// takes precedence over the Node object's addresses: the check
// answers "is the local API server healthy" for every VIP family,
// regardless of the transport family of the override itself.
if entry := kubernetesAddrBackendEntry(c.KubernetesAddr, c.Port); entry != nil {
log.Info("using configured Kubernetes address for backend health checks", "address", c.KubernetesAddr)
backendMapV4[*entry] = false
backendMapV6[*entry] = false
nodename := ""
if c.NodeName != "" {
nodename = c.NodeName
} else {
ips := []string{}
if c.NodeName != "" {
if ips, err = getNodeIPs(ctx, c.NodeName, em.KubernetesClient); err != nil && !apierrors.IsNotFound(err) {
log.Error("failed to get IP of control-plane node", "err", err)
}
}
nodename = os.Getenv("HOSTNAME")
}
if len(ips) == 0 {
if !utils.IsIPv6(cluster.Network[0].IP()) {
ips = append(ips, "127.0.0.1")
} else {
ips = append(ips, "::1")
}
log.Info("no IP address found for node - will fallback to use localhost address", "addresses", ips)
}
for _, ip := range ips {
entry := backend.Entry{Addr: ip, Port: c.Port}
if !utils.IsIPv6(ip) {
backendMapV4[entry] = false
} else {
backendMapV6[entry] = false
}
ips := []string{}
if nodename != "" {
if ips, err = getNodeIPs(ctxArp, nodename, sm.KubernetesClient); err != nil && !apierrors.IsNotFound(err) {
log.Error("failed to get IP of control-plane nod", "err", err)
}
}
backend.SetKubeConfigPath(c.K8sConfigFile)
backend.Watch(ctx, c.BackendHealthCheckInterval, func() {
if len(ips) == 0 {
isV6, err := isV6(cluster.Network[0].IP())
if err != nil {
return fmt.Errorf("failed to parse IP '%s'", cluster.Network[0].IP())
}
if !isV6 {
ips = append(ips, "127.0.0.1")
} else {
ips = append(ips, "::1")
}
log.Info("no IP address found for node - will fallback to use localhost address", "addresses", ips)
}
for _, ip := range ips {
entry := backend.Entry{Addr: ip, Port: c.Port}
ipv6, err := isV6(ip)
if err != nil {
log.Error("failed to check IP type", "IP", ip, "error", err)
}
if !ipv6 {
backendMapV4[entry] = false
} else {
backendMapV6[entry] = false
}
}
stop := make(chan struct{})
// will wait for system interrupt and will send stop signal to backend watch
go func() {
<-signalChan
stop <- struct{}{}
}()
backend.Watch(func() {
for i := range cluster.Network {
network := cluster.Network[i]
networkIP := network.IP()
isNetworkV6 := utils.IsIPv6(networkIP)
log.Debug("current ip to process", "ip", networkIP)
isNetworkV6, err := isV6(networkIP)
if err != nil {
log.Error("failed to check IP type", "IP", networkIP, "error", err)
continue
}
backendMap := &backendMapV4
if isNetworkV6 {
@@ -193,28 +180,8 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
}
for entry := range *backendMap {
log.Debug("entry.Check() for entry", "entry", entry)
var healthy bool
if c.ControlPlaneHealthCheck.Address != "" {
req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, c.ControlPlaneHealthCheck.Address, nil)
if reqErr != nil {
log.Error("create health check request", "err", reqErr)
} else if resp, doErr := cluster.healthCheckHTTPClient.Do(req); doErr != nil {
log.Error("health check request failed", "url", c.ControlPlaneHealthCheck.Address, "err", doErr)
} else {
resp.Body.Close()
healthy = resp.StatusCode == http.StatusOK
if !healthy {
log.Warn("health check returned non-200 status", "url", c.ControlPlaneHealthCheck.Address, "status", resp.StatusCode)
}
}
} else {
healthy = entry.Check()
}
if healthy {
log.Debug("entry.Check() true")
// Normal VIP addition with precheck, use skipDAD=false for normal DAD process
_, err = network.AddIP(true, false)
if entry.Check() {
_, err = network.AddIP(true)
if err != nil {
log.Error("error adding address", "err", err)
}
@@ -222,21 +189,11 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
log.Info("added backend", "ip", network.IP())
}
err = cluster.routeMgr.Add(c.NodeName, network, true, false)
err = network.AddRoute(true)
if err != nil && !errors.Is(err, fs.ErrExist) && !errors.Is(err, syscall.ESRCH) {
log.Warn(err.Error())
} else if err == nil && !(*backendMap)[entry] {
log.Info("added route", "route", network.PrepareRoute())
} else if err == nil || errors.Is(err, fs.ErrExist) {
// Re-assert the route on every healthy cycle: routing daemons
// (e.g. zebra) can miss the single netlink event for the route,
// leaving it unadvertised even though it exists in the kernel.
// RouteReplace is idempotent and regenerates that event.
if replaceErr := network.ReplaceRoute(); replaceErr != nil {
log.Warn("re-asserting route", "err", replaceErr)
} else {
log.Debug("re-asserted route", "route", network.PrepareRoute())
}
log.Info("added route", "route", network.PrepareRoute().String())
}
(*backendMap)[entry] = true
@@ -254,128 +211,35 @@ func (cluster *Cluster) StartVipService(ctx context.Context, c *kubevip.Config,
}
if deleteAddress {
err = cluster.routeMgr.Delete(c.NodeName, network)
if err != nil {
err = network.DeleteRoute()
if err != nil && !errors.Is(err, fs.ErrNotExist) && !errors.Is(err, syscall.ESRCH) {
log.Warn("deleting route", "err", err)
} else if err == nil {
log.Info("deleted route", "route", network.PrepareRoute().String())
}
deleted, err := network.DeleteIP()
if err != nil {
log.Error("error deleting IP", "err", err)
killFunc()
return
panic("")
}
if deleted {
log.Info("deleted address", "IP", network.IP(), "interface", network.Interface())
}
}
}
})
}
if c.EnableBGP {
<-ctx.Done()
}, c.BackendHealthCheckInterval, stop)
}
return nil
}
func (cluster *Cluster) bgpHealthCheck(ctx context.Context, c *kubevip.Config) (bool, error) {
statusCode := 0
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.ControlPlaneHealthCheck.Address, nil)
if err != nil {
return false, fmt.Errorf("building request %v: %w", req, err)
} else {
resp, err := cluster.healthCheckHTTPClient.Do(req)
if err != nil {
return false, fmt.Errorf("checking control-plane: %w", err)
}
defer resp.Body.Close()
statusCode = resp.StatusCode
func isV6(ip string) (bool, error) {
ipaddr := net.ParseIP(ip)
if ipaddr == nil {
return false, fmt.Errorf("failed to parse IP '%s'", ip)
}
healthy := statusCode == http.StatusOK
if !healthy {
return healthy, fmt.Errorf("wrong status code: %d", statusCode)
}
return healthy, nil
}
func (cluster *Cluster) bgpHealthCheckLoop(ctx context.Context, c *kubevip.Config, bgpServer bgp.BGPManager, vipCIDR string) {
period := time.Duration(c.ControlPlaneHealthCheck.PeriodSeconds) * time.Second
consecutiveFailures := 0
routeAnnounced := false
ticker := time.NewTicker(period)
defer ticker.Stop()
log.Info("Starting BGP health check",
"address", c.ControlPlaneHealthCheck.Address,
"cidr", vipCIDR,
"period", period,
"timeout", cluster.healthCheckHTTPClient.Timeout,
"threshold", c.ControlPlaneHealthCheck.FailureThreshold,
)
for {
healthy, healthErr := cluster.bgpHealthCheck(ctx, c)
if healthy {
consecutiveFailures = 0
if !routeAnnounced {
log.Info("BGP health check passed, announcing route", "cidr", vipCIDR)
if err := bgpServer.AddHost(ctx, vipCIDR, c.NodeName); err != nil {
log.Error("BGP health check: failed to announce route", "cidr", vipCIDR, "err", err)
} else {
routeAnnounced = true
}
}
} else {
consecutiveFailures++
if healthErr != nil {
log.Warn("BGP health check failed", "address", c.ControlPlaneHealthCheck.Address, "consecutive", consecutiveFailures, "err", healthErr)
}
if consecutiveFailures >= c.ControlPlaneHealthCheck.FailureThreshold && routeAnnounced {
log.Warn("BGP health check threshold reached, withdrawing route", "failureThreshold", c.ControlPlaneHealthCheck.FailureThreshold, "cidr", vipCIDR)
if err := bgpServer.DelHost(ctx, vipCIDR, c.NodeName); err != nil {
log.Error("BGP health check: failed to withdraw route", "cidr", vipCIDR, "err", err)
} else {
routeAnnounced = false
}
}
}
select {
case <-ctx.Done():
if routeAnnounced {
if err := bgpServer.DelHost(ctx, vipCIDR, c.NodeName); err != nil {
log.Error("BGP health check: failed to withdraw route", "cidr", vipCIDR, "err", err)
}
}
return
case <-ticker.C:
}
}
}
// kubernetesAddrBackendEntry converts an explicitly configured Kubernetes
// API address override (config.KubernetesAddr, e.g. "https://127.0.0.1:6443"
// on static-pod deployments) into a backend health-check entry. Returns nil
// when no usable override is configured.
func kubernetesAddrBackendEntry(kubernetesAddr string, defaultPort uint16) *backend.Entry {
if kubernetesAddr == "" {
return nil
}
u, err := url.Parse(kubernetesAddr)
if err != nil || u.Hostname() == "" {
return nil
}
port := defaultPort
if p := u.Port(); p != "" {
if parsed, err := strconv.ParseUint(p, 10, 16); err == nil {
port = uint16(parsed)
}
}
return &backend.Entry{Addr: u.Hostname(), Port: port}
return ipaddr.To4() == nil, nil
}
func getNodeIPs(ctx context.Context, nodename string, client *kubernetes.Clientset) ([]string, error) {
@@ -393,136 +257,109 @@ func getNodeIPs(ctx context.Context, nodename string, client *kubernetes.Clients
}
// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp bgp.BGPManager, name string, wg *sync.WaitGroup) error {
func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp *bgp.Server, name string, CountRouteReferences func(*netlink.Route) int) {
// use a Go context so we can tell the arp loop code when we
// want to step down
//nolint
lbCtx, lbCancel := context.WithCancel(ctx)
ctxArp, cancelArp := context.WithCancel(ctx)
var lbWg sync.WaitGroup
cluster.stop = make(chan bool, 1)
cluster.completed = make(chan bool, 1)
var arpWG sync.WaitGroup
for i := range cluster.Network {
network := cluster.Network[i]
if network.IsDDNS() {
ddnsReady := make(chan struct{})
lbWg.Go(func() {
// start the DDNS if requested
log.Debug("(svcs) start DDNS", "name", network.DNSName())
if err := cluster.StartDDNS(lbCtx, cluster.Network[i], c.DHCPBackoffAttempts, &lbWg); err != nil {
log.Error("failed to start DDNS", "err", err)
}
close(ddnsReady)
<-lbCtx.Done()
})
<-ddnsReady
}
log.Debug("current ip to process", "ip", network.IP(), "mask", c.VIPSubnet)
if err := network.SetMask(c.VIPSubnet); err != nil {
log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
lbCancel()
return utils.WrapPanicError(err, "failed to set mask for subnet %q", c.VIPSubnet)
panic("")
}
_, err := network.DeleteIP()
if err != nil {
log.Warn("attempted to clean existing VIP", "err", err)
}
log.Debug("config flags", "enable_routing_table", c.EnableRoutingTable, "enable_leader_election", c.EnableLeaderElection, "enable_services_election", c.EnableServicesElection)
if c.EnableRoutingTable && (c.EnableLeaderElection || c.EnableServicesElection) {
err = cluster.routeMgr.Add(name, network, false, false)
err = network.AddRoute(false)
if err != nil {
log.Warn(err.Error())
} else {
log.Info("successful add Route")
}
}
if shouldAddServiceIP(c) {
// Normal VIP addition, use skipDAD=false for normal DAD process
// Note: When WireGuard is enabled, the VIP is added to the tunnel interface
// instead of lo, so we skip adding it here.
if _, err = network.AddIP(false, false); err != nil {
} else if !c.EnableRoutingTable {
if _, err = network.AddIP(false); err != nil {
log.Warn(err.Error())
} else {
log.Info("successful add IP", "address", network.IP())
}
}
if c.EnableARP {
lbWg.Go(func() {
cluster.layer2Update(lbCtx, network, c)
})
arpWG.Add(1)
go cluster.layer2Update(ctxArp, network, c, &arpWG)
}
if c.EnableBGP && (c.EnableLeaderElection || c.EnableServicesElection) {
// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
log.Debug("(svcs) attempting to advertise over BGP", "address", network.CIDR())
err = bgp.AddHost(lbCtx, network.CIDR(), name)
err = bgp.AddHost(network.CIDR())
if err != nil {
log.Error(err.Error())
}
}
}
wg.Go(func() {
for i := range cluster.Network {
network := cluster.Network[i]
go func() {
<-cluster.stop
// Stop the Arp context if it is running
cancelArp()
// start the dns updater if address is dns
if network.IsDNS() {
log.Info("(svcs) starting the DNS updater", "address", network.DNSName(), "ip", network.IP())
ipUpdater := vip.NewIPUpdater(network)
wg.Go(func() {
ipUpdater.Run(lbCtx)
})
}
}
select {
case <-cluster.stop:
case <-ctx.Done():
}
// Stop the loadbalancer context if it is running
lbCancel()
lbWg.Wait() // wait for all cluster ARP/NDP to be finished
arpWG.Wait() // wait for all cluster ARP/NDP to be finished
log.Info("[LOADBALANCER] Stopping load balancers", "name", name)
if c.EnableRoutingTable {
for i := range cluster.Network {
if err := cluster.routeMgr.Delete(name, cluster.Network[i]); err != nil {
log.Warn(err.Error())
// chek if route is not referenced by another service
r := cluster.Network[i].PrepareRoute()
if CountRouteReferences(r) < 1 {
log.Info("[VIP] Deleting Route for VIP", "IP", cluster.Network[i].IP())
if err := cluster.Network[i].DeleteRoute(); err != nil {
log.Warn(err.Error())
}
}
}
close(cluster.completed)
return
}
for i := range cluster.Network {
if c.EnableARP && cluster.arpMgr.Count(cluster.Network[i].ARPName()) > 0 {
continue
}
log.Info("[VIP] Deleting VIP", "ip", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
}
cluster.cleanupVIPs(c)
})
return nil
}
func shouldAddServiceIP(c *kubevip.Config) bool {
return !c.EnableRoutingTable && (!c.EnableBGP || c.BGPAttachIPToInterface) && !c.EnableWireguard
close(cluster.completed)
}()
}
// Layer2Update, handles the creation of the
func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config) {
func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config, arpWG *sync.WaitGroup) {
defer arpWG.Done()
log.Info("layer 2 broadcaster starting")
var ndp *vip.NdpResponder
var err error
ipString := network.IP()
if utils.IsIPv6(ipString) {
if vip.IsIPv6(ipString) {
if network.IPisLinkLocal() {
log.Error("layer2 is link-local can't use NDP", "address", ipString)
} else {
ndp, err = waitNDPResponder(ctx, network.Interface())
ndp, err = vip.NewNDPResponder(network.Interface())
if err != nil {
log.Error("failed to create new NDP Responder", "error", err)
} else {
@@ -533,7 +370,6 @@ func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c
}
}
log.Info("layer 2 broadcaster starting", "IP", network.IP(), "device", network.Interface())
log.Debug("layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
arpInstance := arp.NewInstance(network, ndp)
@@ -541,32 +377,5 @@ func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c
<-ctx.Done() // if cancel() execute
log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
cluster.arpMgr.RemoveOnLeadershipLoss(arpInstance)
}
func waitNDPResponder(ctx context.Context, ifaceName string) (*vip.NdpResponder, error) {
ndp, err := vip.NewNDPResponder(ifaceName)
if err != nil && strings.Contains(err.Error(), "no such device") {
log.Warn("unable to create NDP responder at first try", "interface", ifaceName, "err", err)
ndpCreateCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
ticker := time.NewTicker(time.Second)
for {
select {
case <-ndpCreateCtx.Done():
return nil, fmt.Errorf("failed to create NDP responder for interface %q: %w", ifaceName, ndpCreateCtx.Err())
case <-ticker.C:
ndp, err = vip.NewNDPResponder(ifaceName)
if err != nil {
log.Warn("unable to create NDP responder on retry", "interface", ifaceName, "err", err)
} else {
return ndp, nil
}
}
}
} else if err != nil {
return nil, fmt.Errorf("unable to create NDP responder for interface %q: %w", ifaceName, err)
}
return ndp, nil
cluster.arpMgr.Remove(arpInstance)
}

View File

@@ -1,55 +0,0 @@
package cluster
import (
"testing"
"github.com/kube-vip/kube-vip/pkg/kubevip"
)
func TestShouldAddServiceIP(t *testing.T) {
tests := []struct {
name string
config *kubevip.Config
want bool
}{
{
name: "BGP default does not attach IP",
config: &kubevip.Config{EnableBGP: true},
want: false,
},
{
name: "BGP opt-in attaches IP",
config: &kubevip.Config{
EnableBGP: true,
BGPAttachIPToInterface: true,
},
want: true,
},
{
name: "routing table takes precedence",
config: &kubevip.Config{
EnableBGP: true,
BGPAttachIPToInterface: true,
EnableRoutingTable: true,
},
want: false,
},
{
name: "WireGuard takes precedence",
config: &kubevip.Config{
EnableBGP: true,
BGPAttachIPToInterface: true,
EnableWireguard: true,
},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := shouldAddServiceIP(tt.config); got != tt.want {
t.Fatalf("shouldAddServiceIP() = %t, want %t", got, tt.want)
}
})
}
}

View File

@@ -1,61 +0,0 @@
package cluster
import (
"testing"
)
func TestKubernetesAddrBackendEntry(t *testing.T) {
cases := []struct {
name string
addr string
port uint16
wantAddr string
wantPort uint16
wantNil bool
}{
{
name: "explicit v4 loopback with port",
addr: "https://127.0.0.1:6443",
port: 9999,
wantAddr: "127.0.0.1",
wantPort: 6443,
},
{
name: "hostname without port falls back to config port",
addr: "https://localhost",
port: 6443,
wantAddr: "localhost",
wantPort: 6443,
},
{
name: "empty override",
addr: "",
port: 6443,
wantNil: true,
},
{
name: "garbage override",
addr: "://not-a-url",
port: 6443,
wantNil: true,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
entry := kubernetesAddrBackendEntry(tc.addr, tc.port)
if tc.wantNil {
if entry != nil {
t.Fatalf("expected nil entry, got %+v", entry)
}
return
}
if entry == nil {
t.Fatal("expected an entry, got nil")
}
if entry.Addr != tc.wantAddr || entry.Port != tc.wantPort {
t.Fatalf("got %+v, want addr %q port %d", entry, tc.wantAddr, tc.wantPort)
}
})
}
}

View File

@@ -1,430 +0,0 @@
package cluster_test
import (
"context"
"encoding/pem"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
corev1 "k8s.io/api/core/v1"
)
const testCIDR = "10.0.0.34/32"
func TestBGPHealthCheckLoop_AnnouncesOnHealthy(t *testing.T) {
t.Parallel()
healthcheck := newTestHealthServer(t, http.StatusOK)
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be announced")
}
func TestBGPHealthCheckLoop_NoAnnouncementUntilHealthy(t *testing.T) {
t.Parallel()
healthcheck := newTestHealthServer(t, http.StatusInternalServerError)
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
expectConsistently(t, func() bool { return !bgpManager.isAnnounced() },
2*time.Second, "route should not be announced while unhealthy")
healthcheck.setStatus(http.StatusOK)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be announced after recovery")
}
func TestBGPHealthCheckLoop_WithdrawsAfterThreshold(t *testing.T) {
t.Parallel()
healthcheck := newTestHealthServer(t, http.StatusOK)
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
cfg := newBGPConfig(healthcheck.server.URL, healthcheck.caPath)
cfg.ControlPlaneHealthCheck.FailureThreshold = 3
startVipService(t, cfg, bgpManager)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be announced")
healthcheck.setStatus(http.StatusServiceUnavailable)
expectConsistently(t, func() bool { return bgpManager.isAnnounced() },
1500*time.Millisecond, "route should stay announced before threshold is reached")
expectEventually(t, func() bool { return !bgpManager.isAnnounced() },
"route should be withdrawn after threshold")
}
func TestBGPHealthCheckLoop_ReAnnouncesOnRecovery(t *testing.T) {
t.Parallel()
healthcheck := newTestHealthServer(t, http.StatusOK)
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
cfg := newBGPConfig(healthcheck.server.URL, healthcheck.caPath)
cfg.ControlPlaneHealthCheck.FailureThreshold = 1
startVipService(t, cfg, bgpManager)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be announced")
healthcheck.setStatus(http.StatusServiceUnavailable)
expectEventually(t, func() bool { return !bgpManager.isAnnounced() },
"route should be withdrawn")
healthcheck.setStatus(http.StatusOK)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be re-announced")
}
func TestBGPHealthCheckLoop_StopsOnContextCancel(t *testing.T) {
t.Parallel()
healthcheck := newTestHealthServer(t, http.StatusOK)
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
cancelContext, vipServiceDone := startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be announced")
cancelContext()
select {
case <-vipServiceDone:
case <-time.After(5 * time.Second):
t.Fatal("vipService did not stop after context cancellation")
}
}
func TestBGPHealthCheckLoop_RetriesAddHostOnFailure(t *testing.T) {
t.Parallel()
healthcheck := newTestHealthServer(t, http.StatusOK)
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
bgpManager.setAddErr(errTestAddHost)
startVipService(t, newBGPConfig(healthcheck.server.URL, healthcheck.caPath), bgpManager)
expectConsistently(t, func() bool { return !bgpManager.isAnnounced() },
2*time.Second, "route should not be announced while AddHost errors")
bgpManager.setAddErr(nil)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be announced after clearing AddHost error")
}
func TestBGPHealthCheckLoop_RetriesDelHostOnFailure(t *testing.T) {
t.Parallel()
healthcheck := newTestHealthServer(t, http.StatusOK)
t.Cleanup(healthcheck.server.Close)
bgpManager := newMockBGPRouteManager()
cfg := newBGPConfig(healthcheck.server.URL, healthcheck.caPath)
cfg.ControlPlaneHealthCheck.FailureThreshold = 1
startVipService(t, cfg, bgpManager)
expectEventually(t, func() bool { return bgpManager.isAnnounced() },
"route should be announced")
bgpManager.setDelErr(errTestDelHost)
healthcheck.setStatus(http.StatusServiceUnavailable)
expectConsistently(t, func() bool { return bgpManager.isAnnounced() },
1500*time.Millisecond, "route should stay announced while DelHost errors")
bgpManager.setDelErr(nil)
expectEventually(t, func() bool { return !bgpManager.isAnnounced() },
"route should be withdrawn after clearing DelHost error")
}
func TestRoutingTableHealthCheck_AddsVIPWhenHealthy(t *testing.T) {
t.Parallel()
healthcheck := newTestHealthServer(t, http.StatusOK)
t.Cleanup(healthcheck.server.Close)
network := &mockNetwork{ip: "10.0.0.1", cidr: testCIDR}
startRoutingTableVipService(t, newRoutingTableConfig(healthcheck.server.URL, healthcheck.caPath), network)
expectEventually(t, network.isPresent,
"VIP should be added while health check is healthy")
}
func TestRoutingTableHealthCheck_RemovesVIPWhenUnhealthy(t *testing.T) {
t.Parallel()
healthcheck := newTestHealthServer(t, http.StatusOK)
t.Cleanup(healthcheck.server.Close)
network := &mockNetwork{ip: "10.0.0.1", cidr: testCIDR}
startRoutingTableVipService(t, newRoutingTableConfig(healthcheck.server.URL, healthcheck.caPath), network)
expectEventually(t, network.isPresent,
"VIP should be added while health check is healthy")
healthcheck.setStatus(http.StatusServiceUnavailable)
expectEventually(t, func() bool { return !network.isPresent() },
"VIP should be removed once health check becomes unhealthy")
}
var (
errTestAddHost = &testError{msg: "mock AddHost error"}
errTestDelHost = &testError{msg: "mock DelHost error"}
)
type testError struct{ msg string }
func (e *testError) Error() string { return e.msg }
// startVipService launches vipService in a goroutine with a mock network and
// registers a cleanup to cancel the context and wait for it to finish.
// Uses InitCluster so the real code parses certs for the BGP health check client.
func startVipService(t *testing.T, cfg *kubevip.Config, bgpServer bgp.BGPManager) (context.CancelFunc, <-chan struct{}) {
t.Helper()
c, err := cluster.InitCluster(cfg, true, nil, nil, nil, nil)
if err != nil {
t.Fatalf("InitCluster: %v", err)
}
c.Network = []vip.Network{&mockNetwork{ip: "10.0.0.1", cidr: testCIDR}}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
_ = c.StartVipService(ctx, cfg, nil, bgpServer, func() {})
close(done)
}()
t.Cleanup(func() {
cancel()
<-done
})
return cancel, done
}
// startRoutingTableVipService launches vipService in routing-table mode with a
// mock network and a real route.Manager (which only drives the mock network's
// route methods, so no netlink calls happen). Registers cleanup to stop it.
func startRoutingTableVipService(t *testing.T, cfg *kubevip.Config, network *mockNetwork) {
t.Helper()
c, err := cluster.InitCluster(cfg, true, nil, nil, route.NewManager(), nil)
if err != nil {
t.Fatalf("InitCluster: %v", err)
}
c.Network = []vip.Network{network}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
_ = c.StartVipService(ctx, cfg, nil, nil, func() {})
close(done)
}()
t.Cleanup(func() {
cancel()
<-done
})
}
func newRoutingTableConfig(url, caPath string) *kubevip.Config {
cfg := newBGPConfig(url, caPath)
cfg.EnableBGP = false
cfg.EnableRoutingTable = true
cfg.BackendHealthCheckInterval = 1
return cfg
}
func newBGPConfig(url, caPath string) *kubevip.Config {
return &kubevip.Config{
EnableBGP: true,
ControlPlaneHealthCheck: kubevip.HealthCheck{
Address: url,
CAPath: caPath,
PeriodSeconds: 1,
TimeoutSeconds: 2,
FailureThreshold: 1,
},
}
}
// mockBGPRouteManager tracks announced addresses as a set.
// AddHost adds, DelHost removes. Errors prevent state changes.
type mockBGPRouteManager struct {
mu sync.Mutex
announced map[string]bool
addErr error
delErr error
}
func newMockBGPRouteManager() *mockBGPRouteManager {
return &mockBGPRouteManager{announced: make(map[string]bool)}
}
func (m *mockBGPRouteManager) AddHost(_ context.Context, addr string, _ string) error {
m.mu.Lock()
defer m.mu.Unlock()
if m.addErr != nil {
return m.addErr
}
m.announced[addr] = true
return nil
}
func (m *mockBGPRouteManager) DelHost(_ context.Context, addr string, _ string) error {
m.mu.Lock()
defer m.mu.Unlock()
if m.delErr != nil {
return m.delErr
}
delete(m.announced, addr)
return nil
}
func (m *mockBGPRouteManager) isAnnounced() bool {
m.mu.Lock()
defer m.mu.Unlock()
return m.announced[testCIDR]
}
func (m *mockBGPRouteManager) setAddErr(err error) {
m.mu.Lock()
m.addErr = err
m.mu.Unlock()
}
func (m *mockBGPRouteManager) setDelErr(err error) {
m.mu.Lock()
m.delErr = err
m.mu.Unlock()
}
// mockNetwork implements vip.Network with no-op operations.
type mockNetwork struct {
ip string
cidr string
mu sync.Mutex
present bool
}
func (m *mockNetwork) AddIP(bool, bool, ...int) (bool, error) {
m.mu.Lock()
defer m.mu.Unlock()
m.present = true
return true, nil
}
func (m *mockNetwork) DeleteIP() (bool, error) {
m.mu.Lock()
defer m.mu.Unlock()
m.present = false
return false, nil
}
func (m *mockNetwork) isPresent() bool {
m.mu.Lock()
defer m.mu.Unlock()
return m.present
}
func (m *mockNetwork) AddRoute(bool) (bool, error) { return false, nil }
func (m *mockNetwork) ReplaceRoute() error { return nil }
func (m *mockNetwork) DeleteRoute() error { return nil }
func (m *mockNetwork) UpdateRoutes() (bool, error) { return false, nil }
func (m *mockNetwork) IsSet() (*netlink.Addr, error) { return nil, nil }
func (m *mockNetwork) IP() string { return m.ip }
func (m *mockNetwork) CIDR() string { return m.cidr }
func (m *mockNetwork) IPisLinkLocal() bool { return false }
func (m *mockNetwork) PrepareRoute() *netlink.Route { return nil }
func (m *mockNetwork) RouteHash() string { return "" }
func (m *mockNetwork) SetIP(string) error { return nil }
func (m *mockNetwork) SetServicePorts(*corev1.Service) {}
func (m *mockNetwork) Interface() string { return "eth0" }
func (m *mockNetwork) IsDADFAIL() bool { return false }
func (m *mockNetwork) IsDNS() bool { return false }
func (m *mockNetwork) IsDDNS() bool { return false }
func (m *mockNetwork) DDNSHostName() string { return "" }
func (m *mockNetwork) DNSName() string { return "" }
func (m *mockNetwork) SetMask(string) error { return nil }
func (m *mockNetwork) SetHasEndpoints(bool) {}
func (m *mockNetwork) HasEndpoints() bool { return false }
func (m *mockNetwork) ARPName() string { return "" }
func (m *mockNetwork) GetPossibleSubnets() string { return "" }
func (m *mockNetwork) DHCPFamily() string { return "" }
func (m *mockNetwork) IPVSMark() uint32 { return 0 }
// testHealthServer wraps an HTTPS httptest.Server with an atomic status code.
// caPath is the path to the server's CA cert for client verification.
type testHealthServer struct {
server *httptest.Server
statusCode atomic.Int64
caPath string
}
func newTestHealthServer(t *testing.T, status int) *testHealthServer {
t.Helper()
healthcheck := &testHealthServer{}
healthcheck.statusCode.Store(int64(status))
healthcheck.server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(int(healthcheck.statusCode.Load()))
}))
cert := healthcheck.server.Certificate()
if cert == nil {
t.Fatal("TLS server has no certificate")
}
caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})
caFile := filepath.Join(t.TempDir(), "ca.crt")
if err := os.WriteFile(caFile, caPEM, 0o600); err != nil {
t.Fatal(err)
}
healthcheck.caPath = caFile
return healthcheck
}
func (ths *testHealthServer) setStatus(code int) {
ths.statusCode.Store(int64(code))
}
// expectConsistently continuously checks that condition remains true for the given duration.
// Fails immediately if the condition becomes false at any point.
func expectConsistently(t *testing.T, condition func() bool, duration time.Duration, msg string) {
t.Helper()
deadline := time.Now().Add(duration)
for time.Now().Before(deadline) {
if !condition() {
t.Fatalf("condition violated: %s", msg)
}
time.Sleep(50 * time.Millisecond)
}
}
// expectEventually polls condition until it returns true or 5s timeout is reached.
func expectEventually(t *testing.T, condition func() bool, msg string) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if condition() {
return
}
time.Sleep(50 * time.Millisecond)
}
t.Fatalf("timeout: %s", msg)
}

84
pkg/cluster/singleNode.go Normal file
View File

@@ -0,0 +1,84 @@
package cluster
import (
"context"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/vip"
)
// StartSingleNode will start a single node cluster
func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) error {
// Start kube-vip as a single node server
// TODO - Split all this code out as a separate function
log.Info("Starting kube-vip as a single node cluster")
log.Info("This node is assuming leadership of the cluster")
cluster.stop = make(chan bool, 1)
cluster.completed = make(chan bool, 1)
for i := range cluster.Network {
if !disableVIP {
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("Attempted to clean existing VIP", "err", err)
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
_, err = cluster.Network[i].AddIP(false)
if err != nil {
log.Warn(err.Error())
}
}
if c.EnableARP {
// Gratuitous ARP, will broadcast to new MAC <-> IP
err := vip.ARPSendGratuitous(cluster.Network[i].IP(), c.Interface)
if err != nil {
log.Warn(err.Error())
}
}
}
go func() {
<-cluster.stop
if !disableVIP {
for i := range cluster.Network {
log.Info("[VIP] Releasing the VIP", "address", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
}
}
close(cluster.completed)
}()
log.Info("Started Load Balancer and Virtual IP")
return nil
}
func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp.Server) error {
// use a Go context so we can tell the arp loop code when we
// want to step down
ctxArp, cancelArp := context.WithCancel(context.Background())
defer cancelArp()
// use a Go context so we can tell the dns loop code when we
// want to step down
ctxDNS, cancelDNS := context.WithCancel(context.Background())
defer cancelDNS()
return cluster.vipService(ctxArp, ctxDNS, c, sm, bgp, nil)
}

View File

@@ -1,277 +0,0 @@
package debouncer
import (
"context"
"fmt"
log "log/slog"
"sync"
"sync/atomic"
"time"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
"k8s.io/apimachinery/pkg/watch"
)
const (
DefaultTime = "0s"
minimalTime = time.Millisecond * 200
)
type debouncer struct {
input <-chan watch.Event
output chan watch.Event
stopChan chan any
stopOnce sync.Once
// events holds event per namespace
namespaces sync.Map
debounceTime time.Duration
}
type ns struct {
sync.Map
cnt atomic.Int64
}
func (n *ns) get(name string) (*object, bool) {
value, exists := n.Load(name)
if !exists {
return nil, false
}
i, ok := value.(*object)
if !ok {
return nil, false
}
return i, true
}
func (n *ns) add(name string, output chan<- watch.Event) *object {
i := newObject(output)
n.Store(name, i)
n.cnt.Add(1)
return i
}
func (n *ns) del(name string, object *object) {
if n.CompareAndDelete(name, object) {
n.cnt.Add(-1)
}
}
func New(input <-chan watch.Event, debounceTime string) (*debouncer, error) {
dt, err := time.ParseDuration(debounceTime)
if err != nil {
// debouncer was configured with invalid unparsable value, return error
return nil, fmt.Errorf("failed to parse debounce time configuration: %w", err)
}
if dt < minimalTime {
if dt > 0 {
log.Warn("configured debounce time is less than the minimal threshold of 200ms, debouncer will remain disabled", "config value", dt.String())
}
return nil, nil
}
return &debouncer{
input: input,
output: make(chan watch.Event),
stopChan: make(chan any),
debounceTime: dt,
}, nil
}
func (d *debouncer) Start(ctx context.Context) error {
wg := sync.WaitGroup{}
debouncerCtx, cancel := context.WithCancel(ctx)
defer func() {
cancel()
wg.Wait()
close(d.output)
}()
for {
select {
case <-debouncerCtx.Done():
// return if debouncer context was cancelled
return nil
case <-d.stopChan:
// return if Stop() was called
return nil
case tmp := <-d.input:
// event has no type, probably error
if tmp.Type == "" {
return fmt.Errorf("get undefined object (input channel probably closed)")
}
var namespace, name string
// type switch event object
switch v := tmp.Object.(type) {
case *discoveryv1.EndpointSlice:
namespace = v.Namespace
name = v.Name
case *v1.Endpoints: //nolint:staticcheck
namespace = v.Namespace
name = v.Name
case *v1.Service:
namespace = v.Namespace
name = v.Name
default:
return fmt.Errorf("objects of type %T are not supported", v)
}
processEvent:
for {
eventNs, exists := d.getNs(namespace)
if !exists {
// if not, create new map for the namespace
eventNs = d.addNs(namespace)
}
// check if the object was previously reconciled
eventObject, exists := eventNs.get(name)
// if not and the event is not of type 'Deleted', create new object
if !exists && tmp.Type != watch.Deleted {
eventObject = eventNs.add(name, d.output)
workerObject := eventObject
workerNs := eventNs
workerName := name
workerNamespace := namespace
workerObject.onStop = func() {
// Remove the object before its worker can become receiver-less.
workerNs.del(workerName, workerObject)
}
wg.Go(func() {
// start deboucing events for this object
workerObject.start(debouncerCtx, d.debounceTime)
// if debouncer for the object ended - e.g. object was deleted - clean the map of objects
workerNs.del(workerName, workerObject)
// if namespace is empty, delete the namespace map
if workerNs.cnt.Load() == 0 {
d.delNs(workerNamespace, workerNs)
}
})
}
if eventObject == nil {
break processEvent
}
// pass the watch event to the debouncer object
select {
case eventObject.input <- tmp:
break processEvent
case <-eventObject.stopChan:
// The object stopped after the map lookup. Retry the event
// against the newly-created object instead of dropping it.
continue processEvent
case <-debouncerCtx.Done():
return nil
}
}
}
}
}
func (d *debouncer) Stop() {
d.stopOnce.Do(func() {
close(d.stopChan)
})
}
func (d *debouncer) Output() chan watch.Event {
return d.output
}
func (d *debouncer) getNs(namespace string) (*ns, bool) {
value, exists := d.namespaces.Load(namespace)
if !exists {
return nil, false
}
n, ok := value.(*ns)
if !ok {
return nil, false
}
return n, true
}
func (d *debouncer) addNs(namespace string) *ns {
n := ns{}
d.namespaces.Store(namespace, &n)
return &n
}
func (d *debouncer) delNs(namespace string, ns *ns) {
d.namespaces.CompareAndDelete(namespace, ns)
}
type object struct {
input chan watch.Event
output chan<- watch.Event
stopChan chan any
stopOnce sync.Once
onStop func()
}
func newObject(output chan<- watch.Event) *object {
return &object{
input: make(chan watch.Event),
output: output,
stopChan: make(chan any),
}
}
func (o *object) start(ctx context.Context, debounceTime time.Duration) {
t := time.NewTicker(debounceTime)
var last *watch.Event
defer func() {
if last != nil {
o.output <- *last
last = nil
}
}()
for {
select {
case <-ctx.Done():
// if context is done, return
return
case <-o.stopChan:
// return if Stop() was called
return
case tmp := <-o.input:
// if last event is known, but an event of another type arrived,
// send out the previous event
if last != nil && last.Type != tmp.Type {
o.output <- *last
}
// save current event as the last event
last = &tmp
// reset the ticker to wait for more events
t.Reset(debounceTime)
case <-t.C:
if last != nil {
// on tick, if we have an event, send it out
o.output <- *last
// if the event is of type 'Deleted', stop the debouncer for the object
if last.Type == watch.Deleted {
o.stop()
}
// reset last known event, so it won't be send out twice
last = nil
}
}
}
}
func (o *object) stop() {
o.stopOnce.Do(func() {
if o.onStop != nil {
o.onStop()
}
close(o.stopChan)
})
}

View File

@@ -1,162 +0,0 @@
package debouncer
import (
"context"
"runtime"
"testing"
"time"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
)
func TestStartReturnsWhenCancellationInterruptsObjectForwarding(t *testing.T) {
input := make(chan watch.Event)
d := &debouncer{
input: input,
output: make(chan watch.Event),
stopChan: make(chan any),
debounceTime: 200 * time.Millisecond,
}
// Leave the object without a receiver. This is the state reached when its
// worker exits on context cancellation just before Start forwards an event.
ns := d.addNs("default")
ns.add("example", d.output)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := make(chan error, 1)
go func() { done <- d.Start(ctx) }()
event := watch.Event{
Type: watch.Modified,
Object: &v1.Service{ObjectMeta: metav1.ObjectMeta{
Name: "example", Namespace: "default",
}},
}
sent := make(chan struct{})
go func() {
input <- event
close(sent)
}()
select {
case <-sent:
case <-time.After(250 * time.Millisecond):
cancel()
t.Fatal("debouncer did not receive the test event")
}
cancel()
// Nothing ever receives from the object, so Start can only return by
// abandoning the blocked forward when the context is cancelled. Receiving
// here instead would make the forward succeed and the assertion racy.
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("debouncer remained blocked forwarding an event after context cancellation")
}
}
func TestStartRecreatesObjectAfterDeletionWithoutCancellation(t *testing.T) {
previousProcs := runtime.GOMAXPROCS(1)
t.Cleanup(func() { runtime.GOMAXPROCS(previousProcs) })
input := make(chan watch.Event)
d, err := New(input, "200ms")
if err != nil {
t.Fatalf("failed to create debouncer: %s", err)
}
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
done := make(chan error, 1)
go func() { done <- d.Start(ctx) }()
service := func(eventType watch.EventType, resourceVersion string) watch.Event {
return watch.Event{
Type: eventType,
Object: &v1.Service{ObjectMeta: metav1.ObjectMeta{
Name: "example",
Namespace: "default",
ResourceVersion: resourceVersion,
}},
}
}
send := func(event watch.Event) {
t.Helper()
select {
case input <- event:
case <-time.After(time.Second):
t.Fatal("debouncer did not receive the test event")
}
}
send(service(watch.Added, "initial"))
select {
case event := <-d.output:
if event.Type != watch.Added {
t.Fatalf("expected initial Added event, got %s", event.Type)
}
case <-time.After(time.Second):
t.Fatal("debouncer did not emit the initial event")
}
eventNs, exists := d.getNs("default")
if !exists {
t.Fatal("debouncer did not create the namespace map")
}
oldObject, exists := eventNs.get("example")
if !exists {
t.Fatal("debouncer did not create the object")
}
send(service(watch.Deleted, "deleted"))
select {
case event := <-d.output:
if event.Type != watch.Deleted {
t.Fatalf("expected Deleted event, got %s", event.Type)
}
case <-time.After(time.Second):
t.Fatal("debouncer did not emit the Deleted event")
}
select {
case <-oldObject.stopChan:
case <-time.After(time.Second):
t.Fatal("object did not self-terminate")
}
if _, exists := eventNs.get("example"); exists {
t.Fatal("self-terminated object remained in the namespace map")
}
send(service(watch.Modified, "fresh"))
select {
case event := <-d.output:
if event.Type != watch.Modified {
t.Fatalf("expected fresh Modified event, got %s", event.Type)
}
service, ok := event.Object.(*v1.Service)
if !ok {
t.Fatalf("expected a Service event, got %T", event.Object)
}
if service.ResourceVersion != "fresh" {
t.Fatalf("expected the fresh event, got resource version %q", service.ResourceVersion)
}
case <-time.After(time.Second):
t.Fatal("debouncer did not process the fresh event after object deletion")
}
cancel()
select {
case err := <-done:
if err != nil {
t.Fatalf("debouncer returned an error: %s", err)
}
case <-time.After(time.Second):
t.Fatal("debouncer did not stop")
}
}

View File

@@ -1,394 +0,0 @@
package debouncer
import (
"context"
"fmt"
"strconv"
"sync"
"testing"
"time"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
)
func TestTimeSetting(t *testing.T) {
tcs := []struct {
name string
configured string
expected string
}{
{
name: "configured proper value 10s",
configured: "10s",
expected: "10s",
},
{
name: "configured value less than 200ms",
configured: "0s",
expected: "disabled",
},
{
name: "configured proper value 1s",
configured: "1s",
expected: "1s",
},
{
name: "configured proper value 1500ms",
configured: "1500ms",
expected: "1.5s",
},
{
name: "configured to value greater than 0s but lower than 200ms",
configured: "150ms",
expected: "disabled",
},
{
name: "configured invalid value that cannot be parsed",
configured: "invalid",
expected: "error",
},
{
name: "configured negative value",
configured: "-1s",
expected: "disabled",
},
}
input := make(chan watch.Event)
defer close(input)
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
d, err := New(input, tc.configured)
switch tc.expected {
case "disabled":
if err != nil {
t.Fatalf("failed to create debouncer with debounce time %q", tc.configured)
}
if d != nil {
t.Fatalf("debouncer was created but should be disabled for value %q", tc.configured)
}
case "error":
if err == nil {
t.Fatalf("debouncer was created but should error for value %q", tc.configured)
}
default:
if d == nil {
t.Fatalf("debouncer was not created for value %q", tc.configured)
}
if d.debounceTime.String() != tc.expected {
t.Fatalf("invalid debounce time %q was configured instead of expected %q", d.debounceTime.String(), tc.expected)
}
}
})
}
}
func TestStartStop(t *testing.T) {
t.Run("Run and stop the debouncer without issues", func(t *testing.T) {
input := make(chan watch.Event)
defer close(input)
expected := "200ms"
d, err := New(input, expected)
if err != nil {
t.Fatalf("failed to create debouncer with debounce time %q", expected)
}
if d.debounceTime.String() != expected {
t.Fatalf("invalid debounce time %q was configured instead of expected %q", d.debounceTime.String(), expected)
}
ctx, cancel := context.WithCancel(context.Background())
wg := sync.WaitGroup{}
wg.Go(func() {
if err := d.Start(ctx); err != nil {
t.Fatalf("debouncer error: %s", err.Error())
}
})
cancel()
timedOut := waitTimeout(&wg, time.Second*3)
if timedOut {
t.Fatal("debouncer was not closed before timeout")
}
})
}
func TestDebouncing(t *testing.T) {
tcs := []string{"endpointslices", "endpoints", "services"}
for _, tc := range tcs {
t.Run(fmt.Sprintf("Get the newest event as the only one when using %s", tc), func(t *testing.T) {
expected := "500ms"
fw := watch.NewFake()
defer fw.Stop()
d, err := New(fw.ResultChan(), expected)
if err != nil {
t.Fatalf("failed to create debouncer with debounce time %q", expected)
}
if d.debounceTime.String() != expected {
t.Fatalf("invalid debounce time %q was configured instead of expected %q", d.debounceTime.String(), expected)
}
ctx, cancel := context.WithCancel(context.Background())
wg := sync.WaitGroup{}
wg.Go(func() {
if err := d.Start(ctx); err != nil {
t.Fatalf("debouncer error: %s", err.Error())
}
})
numOfUpdates := 100
switch tc {
case "endpointslices":
epslice := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "test",
},
Endpoints: make([]discoveryv1.Endpoint, 1),
}
addrEpslices := []string{}
for i := range numOfUpdates {
addrEpslices = append(addrEpslices, strconv.Itoa(i))
epslice.Endpoints[0].Addresses = addrEpslices
fw.Add(epslice)
}
case "endpoints":
ep := &v1.Endpoints{ //nolint:staticcheck
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "test",
},
Subsets: make([]v1.EndpointSubset, 1), //nolint:staticcheck
}
addrEp := []v1.EndpointAddress{}
for i := range numOfUpdates {
addrEp = append(addrEp, v1.EndpointAddress{IP: strconv.Itoa(i)})
ep.Subsets[0].Addresses = addrEp
fw.Add(ep)
}
case "services":
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "test",
},
}
svcPorts := []v1.ServicePort{}
for i := range numOfUpdates {
svcPorts = append(svcPorts, v1.ServicePort{Port: int32(i)})
svc.Spec.Ports = svcPorts
fw.Add(svc)
}
default:
t.Fatal("unknown test", "type", tc)
}
out := <-d.output
switch tc {
case "endpointslices":
outEps, ok := out.Object.(*discoveryv1.EndpointSlice)
if !ok {
t.Fatal("got different type of object than EndpointSlice, failed to cast")
}
if len(outEps.Endpoints[0].Addresses) != numOfUpdates {
t.Fatalf("expected to aggregate %d events, but got %d", numOfUpdates, len(outEps.Endpoints[0].Addresses))
}
case "endpoints":
outEps, ok := out.Object.(*v1.Endpoints) //nolint:staticcheck
if !ok {
t.Fatal("got different type of object than EndpointSlice, failed to cast")
}
if len(outEps.Subsets[0].Addresses) != numOfUpdates {
t.Fatalf("expected to aggregate %d events, but got %d", numOfUpdates, len(outEps.Subsets[0].Addresses))
}
case "services":
outSvc, ok := out.Object.(*v1.Service) //nolint:staticcheck
if !ok {
t.Fatal("got different type of object than EndpointSlice, failed to cast")
}
if len(outSvc.Spec.Ports) != numOfUpdates {
t.Fatalf("expected to aggregate %d events, but got %d", numOfUpdates, len(outSvc.Spec.Ports))
}
default:
t.Fatal("unknown test", "type", tc)
}
cancel()
timedOut := waitTimeout(&wg, time.Second*3)
if timedOut {
t.Fatal("debouncer was not closed before timeout")
}
})
}
}
func TestTypeChange(t *testing.T) {
tcs := []string{"endpointslices", "endpoints", "services"}
for _, tc := range tcs {
t.Run(fmt.Sprintf("Get the newest event as the only one when using %s", tc), func(t *testing.T) {
expected := "500ms"
fw := watch.NewFake()
defer fw.Stop()
d, err := New(fw.ResultChan(), expected)
if err != nil {
t.Fatalf("failed to create debouncer with debounce time %q", expected)
}
if d.debounceTime.String() != expected {
t.Fatalf("invalid debounce time %q was configured instead of expected %q", d.debounceTime.String(), expected)
}
ctx, cancel := context.WithCancel(context.Background())
wg := sync.WaitGroup{}
wg.Go(func() {
if err := d.Start(ctx); err != nil {
t.Fatalf("debouncer error: %s", err.Error())
}
})
numOfUpdates := 100
switch tc {
case "endpointslices":
epslice := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "test",
},
Endpoints: make([]discoveryv1.Endpoint, 1),
}
addrEpslices := []string{}
for i := range numOfUpdates {
addrEpslices = append(addrEpslices, strconv.Itoa(i))
epslice.Endpoints[0].Addresses = addrEpslices
if i < numOfUpdates-1 {
fw.Add(epslice)
} else {
fw.Delete(epslice)
}
}
case "endpoints":
ep := &v1.Endpoints{ //nolint:staticcheck
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "test",
},
Subsets: make([]v1.EndpointSubset, 1), //nolint:staticcheck
}
addrEp := []v1.EndpointAddress{}
for i := range numOfUpdates {
addrEp = append(addrEp, v1.EndpointAddress{IP: strconv.Itoa(i)})
ep.Subsets[0].Addresses = addrEp
if i < numOfUpdates-1 {
fw.Add(ep)
} else {
fw.Delete(ep)
}
}
case "services":
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "test",
},
}
svcPorts := []v1.ServicePort{}
for i := range numOfUpdates {
svcPorts = append(svcPorts, v1.ServicePort{Port: int32(i)})
svc.Spec.Ports = svcPorts
if i < numOfUpdates-1 {
fw.Add(svc)
} else {
fw.Delete(svc)
}
}
default:
t.Fatal("unknown test", "type", tc)
}
out := <-d.output
if out.Type != watch.Added {
t.Fatalf("expected to get add event, but got %s event", out.Type)
}
out = <-d.output
if out.Type != watch.Deleted {
t.Fatalf("expected to get delete event, but got %s event", out.Type)
}
cancel()
timedOut := waitTimeout(&wg, time.Second*3)
if timedOut {
t.Fatal("debouncer was not closed before timeout")
}
})
}
}
// waitTimeout waits for the waitgroup for the specified max timeout.
// Returns true if waiting timed out.
func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
c := make(chan struct{})
go func() {
defer close(c)
wg.Wait()
}()
select {
case <-c:
return false // completed normally
case <-time.After(timeout):
return true // timed out
}
}

View File

@@ -27,7 +27,7 @@ func FindIPAddress(addrName string) (string, string, error) {
// If we're not searching for a specific adapter return the first one
if addrName == "" {
return iface.Name, address, nil
}
} else
// If this is the correct adapter return the details
if iface.Name == addrName {
return iface.Name, address, nil
@@ -37,5 +37,5 @@ func FindIPAddress(addrName string) (string, string, error) {
}
}
return "", "", fmt.Errorf("unknown interface [%s]", addrName)
return "", "", fmt.Errorf("Unknown interface [%s]", addrName)
}

View File

@@ -7,7 +7,6 @@ import (
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
@@ -19,13 +18,15 @@ func Teardown(podIP, vipIP, namespace, serviceUUID string, annotations map[strin
internalEgress := annotations[kubevip.EgressInternal]
protocol := iptables.ProtocolIPv4
if utils.IsIPv6(podIP) {
IPv6 := false
if vip.IsIPv6(podIP) {
protocol = iptables.ProtocolIPv6
IPv6 = true
}
// Use the internal egress implementation
if internalEgress != "" || useNftables {
return nftables.DeleteSNATFromAllTables(serviceUUID)
if internalEgress != "" {
return nftables.DeleteSNAT(IPv6, serviceUUID)
}
i, err := vip.CreateIptablesClient(useNftables, namespace, protocol)

View File

@@ -1,268 +0,0 @@
package election
import (
"context"
"fmt"
"sync"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/etcd"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/loadbalancer"
"github.com/kube-vip/kube-vip/pkg/utils"
clientv3 "go.etcd.io/etcd/client/v3"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
watchtools "k8s.io/client-go/tools/watch"
)
type Manager struct {
KubernetesClient *kubernetes.Clientset
RetryWatcherClient *kubernetes.Clientset
// This channel is used to signal a shutdown
EtcdClient *clientv3.Client
}
// NewManager will create a new managing object
func NewManager(config *kubevip.Config, k8sClientset, rwClientset *kubernetes.Clientset) (*Manager, error) {
m := &Manager{}
switch config.LeaderElectionType {
case "kubernetes", "":
if k8sClientset == nil || rwClientset == nil {
return nil, fmt.Errorf("provided nil clientset")
}
m.KubernetesClient = k8sClientset
m.RetryWatcherClient = rwClientset
case "etcd":
client, err := etcd.NewClient(config)
if err != nil {
return nil, err
}
m.EtcdClient = client
default:
return nil, fmt.Errorf("invalid LeaderElectionMode %s not supported", config.LeaderElectionType)
}
return m, nil
}
func RunOrDie(ctx context.Context, run *RunConfig, c *kubevip.Config) error {
switch c.LeaderElectionType {
case "kubernetes", "":
return runKubernetesLeaderElectionOrDie(ctx, run)
case "etcd":
if err := runEtcdLeaderElectionOrDie(ctx, run); err != nil {
return err
}
default:
log.Info("LeaderElectionMode not supported, exiting", "mode", c.LeaderElectionType)
}
return nil
}
func runKubernetesLeaderElectionOrDie(ctx context.Context, run *RunConfig) error {
annotations, err := kubevip.WithLeaseVIPs(run.LeaseAnnotations, run.Config.InstanceName, run.Config.RoutingProtocol, run.VIPs)
if err != nil {
return err
}
leaseClient := run.Mgr.KubernetesClient.CoordinationV1().Leases(run.LeaseID.Namespace())
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
baseLock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: run.LeaseID.Name(),
Namespace: run.LeaseID.Namespace(),
},
Client: run.Mgr.KubernetesClient.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: run.Config.NodeName,
},
}
lock := newAnnotatedLeaseLock(baseLock, leaseClient, run.LeaseID.Name(), annotations)
// start the leader election code loop
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(run.Config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(run.Config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(run.Config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: run.OnStartedLeading,
OnStoppedLeading: run.OnStoppedLeading,
OnNewLeader: run.OnNewLeader,
},
})
return nil
}
func runEtcdLeaderElectionOrDie(ctx context.Context, run *RunConfig) error {
if err := etcd.RunElectionOrDie(ctx, &etcd.LeaderElectionConfig{
EtcdConfig: etcd.ClientConfig{Client: run.Mgr.EtcdClient},
Name: run.LeaseID.NamespacedName(),
MemberID: run.Config.NodeName,
LeaseDurationSeconds: int64(run.Config.LeaseDuration),
Callbacks: etcd.LeaderCallbacks{
OnStartedLeading: run.OnStartedLeading,
OnStoppedLeading: run.OnStoppedLeading,
OnNewLeader: run.OnNewLeader,
},
}); err != nil {
return fmt.Errorf("etcd leaderelection: %w", err)
}
return nil
}
type Actions interface {
OnStartedLeading(ctx context.Context)
OnStoppedLeading()
OnNewLeader(identity string)
}
type RunConfig struct {
Config *kubevip.Config
LeaseID lease.ID
Mgr *Manager
LeaseAnnotations map[string]string
VIPs []string
// onStartedLeading is called when this member starts leading.
OnStartedLeading func(context.Context)
// onStoppedLeading is called when this member stops leading.
OnStoppedLeading func()
// onNewLeader is called when the client observes a leader that is
// not the previously observed leader. This includes the first observed
// leader when the client starts.
OnNewLeader func(identity string)
}
func (em *Manager) NodeWatcher(ctx context.Context, lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
log.Info("Kube-Vip is watching nodes for control-plane labels")
listOptions := metav1.ListOptions{
LabelSelector: "node-role.kubernetes.io/control-plane",
}
wg := sync.WaitGroup{}
defer wg.Wait()
watchCtx, watchCancel := context.WithCancel(ctx)
defer watchCancel()
rw, err := watchtools.NewRetryWatcherWithContext(watchCtx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return utils.WatchWithAuthRetry(ctx, func(ctx context.Context) (watch.Interface, error) {
return em.RetryWatcherClient.CoreV1().Nodes().Watch(watchCtx, listOptions)
})
},
})
if err != nil {
return fmt.Errorf("error creating label watcher: %w", err)
}
wg.Go(func() {
<-watchCtx.Done()
log.Info("Node watcher context cancelled, stopping")
// Stop the retrywatcher
rw.Stop()
})
ch := rw.ResultChan()
var watchErr error
for event := range ch {
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
node, ok := event.Object.(*v1.Node)
if !ok {
return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
}
// Find the node IP address (this isn't foolproof)
for x := range node.Status.Addresses {
if node.Status.Addresses[x].Type == v1.NodeInternalIP {
if checkIfNodeIsReady(node) {
err = lb.AddBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("adding node to load balancer", "node", node.Name, "ip", node.Status.Addresses[x].Address, "err", err)
if utils.IsPanicError(err) {
return fmt.Errorf("add IPVS backend: %w", err)
}
}
} else {
err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("removing node from load balancer", "node", node.Name, "ip", node.Status.Addresses[x].Address, "err", err)
}
}
}
}
case watch.Deleted:
node, ok := event.Object.(*v1.Node)
if !ok {
return fmt.Errorf("unable to parse Kubernetes Node from Annotation watcher")
}
// Find the node IP address (this isn't foolproof)
for x := range node.Status.Addresses {
if node.Status.Addresses[x].Type == v1.NodeInternalIP {
err = lb.RemoveBackend(node.Status.Addresses[x].Address, port)
if err != nil {
log.Error("removing node from load balancer", "node", node.Name, "ip", node.Status.Addresses[x].Address, "err", err)
}
}
}
log.Info("Node deleted", "name", node.Name)
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes Nodes")
watchErr = fmt.Errorf("node watcher error: %w", utils.WatchError(event.Object))
log.Error("watcher", "err", watchErr)
default:
}
}
log.Info("Exiting Node watcher")
if watchErr != nil {
return watchErr
}
if ctx.Err() != nil {
return nil
}
return utils.NewPanicError("node watcher channel closed unexpectedly")
}
func checkIfNodeIsReady(node *v1.Node) bool {
if node == nil {
return false
}
for _, condition := range node.Status.Conditions {
if condition.Type == v1.NodeReady {
if condition.Status == v1.ConditionTrue {
return true
}
}
}
return false
}

View File

@@ -1,92 +0,0 @@
package election
import (
"context"
log "log/slog"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
coordinationv1client "k8s.io/client-go/kubernetes/typed/coordination/v1"
"k8s.io/client-go/tools/leaderelection/resourcelock"
"k8s.io/client-go/util/retry"
)
type annotatedLeaseLock struct {
resourcelock.Interface
leases coordinationv1client.LeaseInterface
name string
annotations map[string]string
}
func newAnnotatedLeaseLock(lock resourcelock.Interface, leases coordinationv1client.LeaseInterface,
name string, annotations map[string]string) resourcelock.Interface {
return &annotatedLeaseLock{Interface: lock, leases: leases, name: name, annotations: annotations}
}
func (lock *annotatedLeaseLock) Create(ctx context.Context, record resourcelock.LeaderElectionRecord) error {
if err := lock.Interface.Create(ctx, record); err != nil {
return err
}
lock.ensure(ctx, record)
return nil
}
func (lock *annotatedLeaseLock) Update(ctx context.Context, record resourcelock.LeaderElectionRecord) error {
if err := lock.Interface.Update(ctx, record); err != nil {
return err
}
lock.ensure(ctx, record)
return nil
}
// ensure applies the configured annotations once this process holds the lease. Failures are
// logged rather than returned: the lease write already succeeded, so reporting an error would
// make the elector stand down while it still holds the lease.
func (lock *annotatedLeaseLock) ensure(ctx context.Context, record resourcelock.LeaderElectionRecord) {
if record.HolderIdentity != lock.Identity() {
return
}
changed, err := lock.ensureAnnotations(ctx)
if err != nil {
log.Warn("failed to annotate lease", "lease", lock.name, "err", err)
return
}
if !changed {
return
}
// Annotating out of band bumps the resourceVersion, so refresh the wrapped lock's
// cached lease or its next optimistic Update conflicts.
if _, _, err := lock.Interface.Get(ctx); err != nil {
log.Warn("failed to refresh lease after annotating", "lease", lock.name, "err", err)
}
}
func (lock *annotatedLeaseLock) ensureAnnotations(ctx context.Context) (bool, error) {
changed := false
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
resource, err := lock.leases.Get(ctx, lock.name, metav1.GetOptions{})
if err != nil {
return err
}
if resource.Annotations == nil {
resource.Annotations = make(map[string]string, len(lock.annotations))
}
resourceChanged := false
for key, value := range lock.annotations {
if resource.Annotations[key] == value {
continue
}
resource.Annotations[key] = value
resourceChanged = true
}
if !resourceChanged {
return nil
}
_, err = lock.leases.Update(ctx, resource, metav1.UpdateOptions{})
if err == nil {
changed = true
}
return err
})
return changed, err
}

View File

@@ -1,177 +0,0 @@
package election
import (
"context"
"fmt"
"testing"
"github.com/kube-vip/kube-vip/pkg/kubevip"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/fake"
k8stesting "k8s.io/client-go/testing"
"k8s.io/client-go/tools/leaderelection/resourcelock"
)
func TestAnnotatedLeaseLockPersistsAnnotationsOnCreateAndUpdate(t *testing.T) {
client := fake.NewSimpleClientset()
leaseClient := client.CoordinationV1().Leases("default")
base := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{Name: "lease", Namespace: "default"},
Client: client.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: "node-a",
},
}
annotations, err := kubevip.WithLeaseVIPs(map[string]string{"example.test/preserved": "true"},
"release_a", 248, []string{"192.0.2.10"})
if err != nil {
t.Fatalf("WithLeaseVIPs() error = %v", err)
}
lock := newAnnotatedLeaseLock(base, leaseClient, "lease", annotations)
record := resourcelock.LeaderElectionRecord{HolderIdentity: "node-a"}
if err := lock.Create(context.Background(), record); err != nil {
t.Fatalf("Create() error = %v", err)
}
if err := lock.Update(context.Background(), record); err != nil {
t.Fatalf("Update() error = %v", err)
}
resource, err := leaseClient.Get(context.Background(), "lease", metav1.GetOptions{})
if err != nil {
t.Fatalf("get Lease: %v", err)
}
value, err := kubevip.ParseLeaseVIPs(resource.Annotations[kubevip.LeaseVIPs])
if err != nil {
t.Fatalf("ParseLeaseVIPs() error = %v", err)
}
if value.InstanceName != "release_a" || value.IFAProto != 248 || len(value.VIPs) != 1 ||
value.VIPs[0] != (kubevip.LeaseVIP{Index: 0, Value: "192.0.2.10", Kind: kubevip.LeaseVIPKindAddress}) {
t.Fatalf("Lease VIP metadata = %+v", value)
}
if resource.Annotations["example.test/preserved"] != "true" {
t.Fatal("Lease update dropped a configured annotation")
}
}
// A failed annotation write must not be reported to the leader elector: the lease itself
// was already written, and an error makes the elector stand down while it still holds it.
func TestAnnotatedLeaseLockAnnotationFailureDoesNotSurfaceToElector(t *testing.T) {
client := fake.NewSimpleClientset()
base := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{Name: "lease", Namespace: "default"},
Client: client.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{Identity: "node-a"},
}
failing := fake.NewSimpleClientset()
failing.PrependReactor("get", "leases", func(k8stesting.Action) (bool, runtime.Object, error) {
return true, nil, fmt.Errorf("annotation backend unavailable")
})
annotations, err := kubevip.WithLeaseVIPs(nil, "release_a", 248, []string{"192.0.2.10"})
if err != nil {
t.Fatal(err)
}
lock := newAnnotatedLeaseLock(base, failing.CoordinationV1().Leases("default"), "lease", annotations)
record := resourcelock.LeaderElectionRecord{HolderIdentity: "node-a"}
if err := lock.Create(context.Background(), record); err != nil {
t.Fatalf("Create() error = %v, want nil so the elector keeps the lease", err)
}
if err := lock.Update(context.Background(), record); err != nil {
t.Fatalf("Update() error = %v, want nil so the elector keeps the lease", err)
}
if _, err := client.CoordinationV1().Leases("default").Get(context.Background(), "lease",
metav1.GetOptions{}); err != nil {
t.Fatalf("wrapped lock did not write the lease: %v", err)
}
}
func TestAnnotatedLeaseLockFollowerDoesNotOverwriteAnnotations(t *testing.T) {
client := fake.NewSimpleClientset()
leaseClient := client.CoordinationV1().Leases("default")
newBase := func(identity string) *resourcelock.LeaseLock {
return &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{Name: "lease", Namespace: "default"},
Client: client.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{Identity: identity},
}
}
ownerBase := newBase("node-a")
followerBase := newBase("node-b")
active, err := kubevip.WithLeaseVIPs(nil, "release_a", 248, []string{"192.0.2.10"})
if err != nil {
t.Fatal(err)
}
creator := newAnnotatedLeaseLock(ownerBase, leaseClient, "lease", active)
if err := creator.Create(context.Background(), resourcelock.LeaderElectionRecord{HolderIdentity: "node-a"}); err != nil {
t.Fatalf("Create() error = %v", err)
}
follower, err := kubevip.WithLeaseVIPs(nil, "release_b", 249, []string{"192.0.2.20"})
if err != nil {
t.Fatal(err)
}
observer := newAnnotatedLeaseLock(followerBase, leaseClient, "lease", follower)
if _, _, err := observer.Get(context.Background()); err != nil {
t.Fatalf("Get() error = %v", err)
}
resource, err := leaseClient.Get(context.Background(), "lease", metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
metadata, err := kubevip.ParseLeaseVIPs(resource.Annotations[kubevip.LeaseVIPs])
if err != nil {
t.Fatal(err)
}
if metadata.InstanceName != "release_a" || metadata.IFAProto != 248 {
t.Fatalf("follower overwrote active metadata: %+v", metadata)
}
}
func TestAnnotatedLeaseLockReleaseDoesNotOverwriteSuccessorMetadata(t *testing.T) {
client := fake.NewSimpleClientset()
leaseClient := client.CoordinationV1().Leases("default")
newLock := func(identity, instanceName string, protocol int, vip string) resourcelock.Interface {
base := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{Name: "lease", Namespace: "default"},
Client: client.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{Identity: identity},
}
annotations, err := kubevip.WithLeaseVIPs(nil, instanceName, protocol, []string{vip})
if err != nil {
t.Fatal(err)
}
return newAnnotatedLeaseLock(base, leaseClient, "lease", annotations)
}
first := newLock("node-a", "release_a", 248, "192.0.2.10")
if err := first.Create(context.Background(), resourcelock.LeaderElectionRecord{HolderIdentity: "node-a"}); err != nil {
t.Fatal(err)
}
if err := first.Update(context.Background(), resourcelock.LeaderElectionRecord{}); err != nil {
t.Fatal(err)
}
second := newLock("node-b", "release_b", 249, "192.0.2.20")
if _, _, err := second.Get(context.Background()); err != nil {
t.Fatal(err)
}
if err := second.Update(context.Background(), resourcelock.LeaderElectionRecord{HolderIdentity: "node-b"}); err != nil {
t.Fatal(err)
}
resource, err := leaseClient.Get(context.Background(), "lease", metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
metadata, err := kubevip.ParseLeaseVIPs(resource.Annotations[kubevip.LeaseVIPs])
if err != nil {
t.Fatal(err)
}
if metadata.InstanceName != "release_b" || metadata.IFAProto != 249 || metadata.VIPs[0].Value != "192.0.2.20" {
t.Fatalf("successor metadata = %+v", metadata)
}
}

View File

@@ -4,9 +4,6 @@ import (
"context"
"fmt"
"net"
"strings"
"sync"
"time"
log "log/slog"
@@ -14,15 +11,10 @@ import (
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/metrics"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/wireguard"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
)
type Processor struct {
@@ -31,47 +23,38 @@ type Processor struct {
bgpServer *bgp.Server
worker endpointWorker
instances *[]*instance.Instance
leaseMgr *lease.Manager
}
func NewEndpointProcessor(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
instances *[]*instance.Instance, leaseMgr *lease.Manager, tunnelMgr *wireguard.TunnelManager, routeMgr *route.Manager) *Processor {
instances *[]*instance.Instance) *Processor {
return &Processor{
config: config,
provider: provider,
bgpServer: bgpServer,
instances: instances,
leaseMgr: leaseMgr,
worker: newEndpointWorker(config, provider, bgpServer, instances, leaseMgr, tunnelMgr, routeMgr),
worker: newEndpointWorker(config, provider, bgpServer, instances),
}
}
// Reconcile applies a watch event to the provider and reconciles the service
// against the endpoints that remain afterwards. A deleted object is only one of
// potentially several backing the service, so deletions are recomputed rather
// than assumed to empty it. It reports whether the caller should skip this event
// and wait for the next one.
func (p *Processor) Reconcile(svcCtx *servicecontext.Context, event watch.Event,
lastKnownGoodEndpoint *string, service *v1.Service, id string,
serviceFunc func(*servicecontext.Context, *v1.Service, *sync.WaitGroup, bool) error, wg *sync.WaitGroup,
clientSet *kubernetes.Clientset,
egressUpdateFunc func(context.Context, *v1.Service) error) (bool, error) {
func (p *Processor) AddOrModify(ctx *servicecontext.Context, event watch.Event,
lastKnownGoodEndpoint *string, service *v1.Service, id string, leaderElectionActive *bool,
serviceFunc func(context.Context, *v1.Service) error,
leaderCtx *context.Context, cancel *context.CancelFunc) (bool, error) {
if err := p.applyEvent(svcCtx, event); err != nil {
return false, err
var err error
if err = p.provider.LoadObject(event.Object, *cancel); err != nil {
return false, fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
}
endpoints, err := p.worker.getEndpoints(service, id)
if err != nil {
return false, fmt.Errorf("[%s] error getting endpoints: %w", p.provider.GetLabel(), err)
return false, err
}
if err := p.worker.setInstanceEndpointsStatus(svcCtx.Ctx, service, endpoints); err != nil {
if err := p.worker.setInstanceEndpointsStatus(service, endpoints); err != nil {
log.Error("updating instance", "err", err)
}
allowReconcileWithoutEndpoints := shouldAllowReconcileWithoutEndpoints(service)
// Find out if we have any local endpoints
// if out endpoint is empty then populate it
// if not, go through the endpoints and see if ours still exists
@@ -81,97 +64,50 @@ func (p *Processor) Reconcile(svcCtx *servicecontext.Context, event watch.Event,
// Check that we have local endpoints
if len(endpoints) != 0 {
// Ignore IPv4
if service.Annotations[kubevip.EgressIPv6] == "true" && !hasV6(endpoints) {
if service.Annotations[kubevip.EgressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
return true, nil
}
p.updateLastKnownGoodEndpoint(lastKnownGoodEndpoint, endpoints, service)
if err := p.startServiceHandlingIfNeeded(svcCtx, service, serviceFunc, wg); err != nil {
return true, err
p.updateLastKnownGoodEndpoint(lastKnownGoodEndpoint, endpoints, service, leaderElectionActive, *cancel)
// start leader election if it's enabled and not already started
if !*leaderElectionActive && p.config.EnableServicesElection {
go func() {
*leaderCtx, *cancel = context.WithCancel(ctx.Ctx)
startLeaderElection(*leaderCtx, leaderElectionActive, service, serviceFunc)
}()
}
svcCtx.SignalReadiness()
if p.shouldProcessInstance() {
if err := p.worker.processInstance(svcCtx, service); err != nil {
// There are local endpoints available on the node
if !p.config.EnableServicesElection && !p.config.EnableLeaderElection {
if err := p.worker.processInstance(ctx, service, leaderElectionActive); err != nil {
return false, fmt.Errorf("failed to process non-empty instance: %w", err)
}
}
} else {
if allowReconcileWithoutEndpoints {
// Explicit opt-in for controllers that create LoadBalancer services without endpoints
if err := p.startServiceHandlingIfNeeded(svcCtx, service, serviceFunc, wg); err != nil {
return true, err
}
svcCtx.SignalReadiness()
if p.shouldProcessInstance() {
if err := p.worker.processInstance(svcCtx, service); err != nil {
return false, fmt.Errorf("failed to process endpointless instance: %w", err)
}
}
} else if svcCtx.Signalled.Load() {
p.handleNoEndpoints(svcCtx, service, lastKnownGoodEndpoint)
}
// There are no local endpoints
p.worker.clear(ctx, lastKnownGoodEndpoint, service, *cancel, leaderElectionActive)
}
// Set the service accordingly
p.updateAnnotations(service, lastKnownGoodEndpoint, clientSet, egressUpdateFunc)
p.updateAnnotations(service, lastKnownGoodEndpoint)
log.Debug("watcher", "provider",
p.provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", *lastKnownGoodEndpoint)
p.provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", *lastKnownGoodEndpoint, "active leader election", *leaderElectionActive)
return false, nil
}
// applyEvent updates the provider's view of the objects backing this service.
func (p *Processor) applyEvent(svcCtx *servicecontext.Context, event watch.Event) error {
if event.Type == watch.Deleted {
if err := p.provider.DeleteObject(event.Object); err != nil {
return fmt.Errorf("[%s] error deleting k8s object: %w", p.provider.GetLabel(), err)
}
return nil
}
if err := p.provider.LoadObject(event.Object, svcCtx.Cancel); err != nil {
return fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
func (p *Processor) Delete(service *v1.Service, id string) error {
if err := p.worker.delete(service, id); err != nil {
return fmt.Errorf("[%s] error deleting service: %w", p.provider.GetLabel(), err)
}
return nil
}
// shouldProcessInstance reports whether this node has to program the datapath
// itself, rather than waiting to be told to by a leader election callback.
// WireGuard always reprograms, because its DNAT rules are per-endpoint.
func (p *Processor) shouldProcessInstance() bool {
return (!p.config.EnableServicesElection && !p.config.EnableLeaderElection) || p.config.EnableWireguard
}
// handleNoEndpoints tears down everything backing a service that no longer has
// any usable endpoints.
func (p *Processor) handleNoEndpoints(svcCtx *servicecontext.Context, service *v1.Service, lastKnownGoodEndpoint *string) {
svcCtx.ResetReadiness()
p.worker.clear(svcCtx, lastKnownGoodEndpoint, service)
if p.config.EnableARP && !p.config.EnableServicesElection && p.instances != nil {
if i := instance.FindServiceInstance(service, *p.instances); i != nil {
for _, c := range i.Clusters {
c.Stop()
}
}
}
}
func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service) {
func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service, leaderElectionActive *bool, cancel context.CancelFunc) {
// if we haven't populated one, then do so
family := utils.IPv4Family
if service.Annotations[kubevip.EgressIPv6] == "true" {
family = utils.IPv6Family
}
ep := getEndpoint(endpoints, family)
if *lastKnownGoodEndpoint == "" {
*lastKnownGoodEndpoint = ep
*lastKnownGoodEndpoint = endpoints[0]
return
}
@@ -185,197 +121,45 @@ func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, e
}
// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
if !stillExists {
ip := net.ParseIP(*lastKnownGoodEndpoint)
if (ip.To4() != nil && service.Annotations[kubevip.Egress] == "true") ||
(ip.To4() == nil && service.Annotations[kubevip.EgressIPv6] == "true") {
p.worker.removeEgress(service, lastKnownGoodEndpoint)
p.worker.removeEgress(service, lastKnownGoodEndpoint)
if *leaderElectionActive && (p.config.EnableServicesElection || p.config.EnableLeaderElection) {
log.Warn("existing endpoint has been removed, restarting leaderElection", "provider", p.provider.GetLabel(), "endpoint", *lastKnownGoodEndpoint)
// Stop the existing leaderElection
cancel()
// disable last leaderElection flag
*leaderElectionActive = false
}
// Set our active endpoint to an existing one
*lastKnownGoodEndpoint = ep
*lastKnownGoodEndpoint = endpoints[0]
}
}
func (p *Processor) updateAnnotations(service *v1.Service, lastKnownGoodEndpoint *string,
clientSet *kubernetes.Clientset,
egressUpdateFunc func(context.Context, *v1.Service) error) {
func (p *Processor) updateAnnotations(service *v1.Service, lastKnownGoodEndpoint *string) {
// Set the service accordingly
if service.Annotations[kubevip.Egress] == "true" {
if *lastKnownGoodEndpoint != "" {
ip := net.ParseIP(*lastKnownGoodEndpoint)
expectIPv6 := service.Annotations[kubevip.EgressIPv6] == "true"
if ip == nil || (ip.To4() == nil) != expectIPv6 {
log.Warn("ignoring active endpoint with unexpected address family",
"service", service.Name,
"namespace", service.Namespace,
"endpoint", *lastKnownGoodEndpoint,
"expected_ipv6", expectIPv6)
return
}
}
// Store old values from ServiceSnapshot to detect if annotation actually changed
// We use the ServiceSnapshot instead of the service parameter because the service parameter
// may have stale annotations if the last update failed
var oldEndpoint, oldEndpointIPv6 string
snapshotFound := false
if p.instances != nil {
serviceInstance := instance.FindServiceInstance(service, *p.instances)
if serviceInstance != nil && serviceInstance.ServiceSnapshot != nil {
snapshotFound = true
oldEndpoint = serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint]
oldEndpointIPv6 = serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpointIPv6]
}
}
// Empty annotations in an existing snapshot are meaningful after a zero-endpoint transition.
if !snapshotFound {
oldEndpoint = service.Annotations[kubevip.ActiveEndpoint]
oldEndpointIPv6 = service.Annotations[kubevip.ActiveEndpointIPv6]
}
// Determine which annotation to update based on IP version
var endpoint, endpointIPv6 string
if service.Annotations[kubevip.EgressIPv6] == "true" && !p.config.EnableEndpoints {
// IPv6
endpointIPv6 = *lastKnownGoodEndpoint
endpoint = oldEndpoint // Preserve existing IPv4 if any
} else {
// IPv4
endpoint = *lastKnownGoodEndpoint
endpointIPv6 = oldEndpointIPv6 // Preserve existing IPv6 if any
}
// Check if annotation actually changed
annotationChanged := (oldEndpoint != endpoint) || (oldEndpointIPv6 != endpointIPv6)
if !annotationChanged {
return // Nothing to do
}
// Persist to Kubernetes
ctx := context.Background()
if err := p.provider.UpdateServiceAnnotation(ctx, endpoint, endpointIPv6, service, clientSet); err != nil {
log.Warn("failed to update service annotation", "service", service.Name, "namespace", service.Namespace, "err", err)
return
}
log.Debug("updated active endpoint annotation", "service", service.Name, "namespace", service.Namespace, "endpoint", *lastKnownGoodEndpoint)
// Trigger egress reconfiguration
// For services with leader election, the service watcher doesn't process Modified events
// after initial setup, so we need to directly call the update function
if egressUpdateFunc != nil {
// Create a copy of service with updated annotations
svcCopy := service.DeepCopy()
svcCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
svcCopy.Annotations[kubevip.ActiveEndpointIPv6] = endpointIPv6
if err := egressUpdateFunc(ctx, svcCopy); err != nil {
log.Error("failed to reconfigure egress", "service", service.Name, "namespace", service.Namespace, "err", err)
}
activeEndpointAnnotation := kubevip.ActiveEndpoint
if !p.config.EnableEndpoints && p.provider.GetProtocol() == string(discoveryv1.AddressTypeIPv6) {
activeEndpointAnnotation = kubevip.ActiveEndpointIPv6
}
service.Annotations[activeEndpointAnnotation] = *lastKnownGoodEndpoint
}
}
func (p *Processor) startServiceHandlingIfNeeded(svcCtx *servicecontext.Context, service *v1.Service,
serviceFunc func(*servicecontext.Context, *v1.Service, *sync.WaitGroup, bool) error, wg *sync.WaitGroup) error {
if p.config.EnableServicesElection {
// startLeaderElection restarts itself until the service context is cancelled,
// so start it only once instead of on every endpoint event.
svcCtx.StartLeaderElectionOnce(func() {
wg.Go(func() {
p.startLeaderElection(svcCtx, service, serviceFunc, wg)
})
})
return nil
}
if p.config.EnableARP || (p.config.EnableRoutingTable && p.config.EnableLeaderElection) {
if !svcCtx.Signalled.Load() {
inst := instance.FindServiceInstance(service, *p.instances)
if inst == nil {
return fmt.Errorf("[%s] failed to find an instance for service %s/%s", p.provider.GetLabel(), service.Namespace, service.Name)
}
for x := range inst.VIPConfigs {
log.Debug("starting loadbalancer for service", "provider", p.provider.GetLabel(), "name", service.Name, "namespace", service.Namespace, "uid", service.UID)
if err := inst.Clusters[x].StartLoadBalancerService(svcCtx.Ctx, inst.VIPConfigs[x], p.bgpServer, lease.ServiceNamespacedName(service), wg); err != nil {
return fmt.Errorf("failed to start lb: %w", err)
}
}
}
}
return nil
}
func (p *Processor) startLeaderElection(svcCtx *servicecontext.Context, service *v1.Service, serviceFunc func(*servicecontext.Context, *v1.Service, *sync.WaitGroup, bool) error, wg *sync.WaitGroup) {
// Track this loop for the lifetime of the goroutine. There has to be at most
// one per service, so a value above 1 means loops leaked.
loops := metrics.ServiceElectionLoops.WithLabelValues(service.Namespace, service.Name)
loops.Inc()
defer loops.Dec()
attempts := metrics.ServiceElectionAttemptsTotal.WithLabelValues(service.Namespace, service.Name)
func startLeaderElection(ctx context.Context, leaderElectionActive *bool, service *v1.Service, serviceFunc func(context.Context, *v1.Service) error) {
// This is a blocking function, that will restart (in the event of failure)
for {
select {
case <-svcCtx.Ctx.Done():
return
default:
leaseNamespace, serviceLease := lease.ServiceName(service)
id := lease.NewID(p.config.LeaderElectionType, leaseNamespace, serviceLease)
// The lease is retired once its last service is gone, so an absent one means
// this loop has nothing left to elect for.
l := p.leaseMgr.Get(id)
if l == nil {
return
}
l.Lock()
if !l.Elected.Load() {
l.Unlock()
attempts.Inc()
err := serviceFunc(svcCtx, service, wg, true)
if err != nil {
log.Error(err.Error())
}
} else {
l.Unlock()
time.Sleep(time.Millisecond * 200)
// if the context isn't cancelled restart
if ctx.Err() != context.Canceled {
*leaderElectionActive = true
err := serviceFunc(ctx, service)
if err != nil {
log.Error(err.Error())
}
*leaderElectionActive = false
} else {
*leaderElectionActive = false
break
}
}
}
func shouldAllowReconcileWithoutEndpoints(service *v1.Service) bool {
if service == nil || service.Spec.ExternalTrafficPolicy != v1.ServiceExternalTrafficPolicyTypeCluster {
return false
}
return strings.EqualFold(service.Annotations[kubevip.AllowReconcileWithoutEndpoints], "true")
}
func hasV6(endpoints []string) bool {
for _, e := range endpoints {
ip := net.ParseIP(e)
if ip != nil {
if ip.To4() == nil {
return true
}
}
}
return false
}
func getEndpoint(endpoints []string, family string) string {
for _, e := range endpoints {
ip := net.ParseIP(e)
if family == utils.IPv4Family && ip.To4() != nil {
return e
}
if family == utils.IPv6Family && ip.To4() == nil {
return e
}
}
return ""
}

View File

@@ -2,11 +2,11 @@ package endpoints
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
)
@@ -23,19 +23,20 @@ func newBGP(generic generic, bgpServer *bgp.Server) endpointWorker {
}
}
func (b *BGP) processInstance(svcCtx *servicecontext.Context, service *v1.Service) error {
func (b *BGP) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
if !svcCtx.IsNetworkConfigured(cluster.Network[i].IP()) {
if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) {
log.Debug("attempting to advertise BGP service", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP())
err := b.bgpServer.AddHost(svcCtx.Ctx, cluster.Network[i].CIDR(), lease.ServiceNamespacedName(service))
err := b.bgpServer.AddHost(cluster.Network[i].CIDR())
if err != nil {
log.Error("error adding BGP host", "provider", b.provider.GetLabel(), "err", err)
} else {
log.Info("added BGP host", "provider",
b.provider.GetLabel(), "ip", cluster.Network[i].CIDR(), "service name", service.Name, "namespace", service.Namespace)
svcCtx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
*leaderElectionActive = true
}
}
}
@@ -44,19 +45,20 @@ func (b *BGP) processInstance(svcCtx *servicecontext.Context, service *v1.Servic
return nil
}
func (b *BGP) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service) {
func (b *BGP) clear(ctx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
// If BGP mode is enabled - routes should be deleted
if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
err := b.bgpServer.DelHost(svcCtx.Ctx, cluster.Network[i].CIDR(), lease.ServiceNamespacedName(service))
err := b.bgpServer.DelHost(cluster.Network[i].CIDR())
if err != nil {
log.Error("deleting BGP host", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "err", err)
} else {
log.Info("deleted BGP host", "provider",
b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace)
svcCtx.ConfiguredNetworks.Delete(cluster.Network[i].IP())
ctx.ConfiguredNetworks.Delete(cluster.Network[i])
*leaderElectionActive = false
}
}
}
@@ -64,34 +66,54 @@ func (b *BGP) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *strin
}
}
b.clearEgress(lastKnownGoodEndpoint, service)
svcCtx.CallLeaderCancel()
b.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (b *BGP) getEndpoints(service *v1.Service, id string) ([]string, error) {
return b.getAllEndpoints(service, id)
}
func (b *BGP) setInstanceEndpointsStatus(_ context.Context, _ *v1.Service, _ []string) error {
func (b *BGP) delete(service *v1.Service, id string) error {
// When no-leader-elecition mode
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
// find all existing local endpoints
endpoints, err := b.getEndpoints(service, id)
if err != nil {
return fmt.Errorf("[%s] error getting endpoints: %w", b.provider.GetLabel(), err)
}
// If there were local endpoints deleted
if len(endpoints) > 0 {
b.deleteAction(service)
}
}
return nil
}
func ClearBGPHosts(ctx context.Context, service *v1.Service, instances *[]*instance.Instance, bgpServer *bgp.Server) {
func (b *BGP) deleteAction(service *v1.Service) {
b.clearBGPHosts(service)
}
func (b *BGP) clearBGPHosts(service *v1.Service) {
ClearBGPHosts(service, b.instances, b.bgpServer)
}
func (b *BGP) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
return nil
}
func ClearBGPHosts(service *v1.Service, instances *[]*instance.Instance, bgpServer *bgp.Server) {
if instance := instance.FindServiceInstance(service, *instances); instance != nil {
ClearBGPHostsByInstance(ctx, instance, bgpServer)
ClearBGPHostsByInstance(instance, bgpServer)
}
}
func ClearBGPHostsByInstance(ctx context.Context, instance *instance.Instance, bgpServer *bgp.Server) {
if instance == nil {
log.Error("failed to clear BGP host for nil instance")
return
}
func ClearBGPHostsByInstance(instance *instance.Instance, bgpServer *bgp.Server) {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
network := cluster.Network[i]
err := bgpServer.DelHost(ctx, network.CIDR(), lease.ServiceNamespacedName(instance.ServiceSnapshot))
err := bgpServer.DelHost(network.CIDR())
if err != nil {
log.Error("[endpoint] error deleting BGP host", "err", err)
} else {

View File

@@ -10,30 +10,24 @@ import (
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/wireguard"
v1 "k8s.io/api/core/v1"
)
type endpointWorker interface {
processInstance(svcCtx *servicecontext.Context, service *v1.Service) error
clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service)
processInstance(svcCtx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error
clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool)
getEndpoints(service *v1.Service, id string) ([]string, error)
removeEgress(service *v1.Service, lastKnownGoodEndpoint *string)
setInstanceEndpointsStatus(ctx context.Context, service *v1.Service, endpoints []string) error
delete(service *v1.Service, id string) error
setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error
}
func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server, instances *[]*instance.Instance,
leaseMgr *lease.Manager, tunnelMgr *wireguard.TunnelManager, routeMgr *route.Manager) endpointWorker {
generic := newGeneric(config, provider, instances, leaseMgr)
func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server, instances *[]*instance.Instance) endpointWorker {
generic := newGeneric(config, provider, instances)
if config.EnableWireguard {
return newWireguardWorker(config, provider, bgpServer, instances, leaseMgr, tunnelMgr)
}
if config.EnableRoutingTable {
return newRoutingTable(generic, routeMgr)
return newRoutingTable(generic)
}
if config.EnableBGP {
return newBGP(generic, bgpServer)
@@ -46,47 +40,60 @@ type generic struct {
config *kubevip.Config
provider providers.Provider
instances *[]*instance.Instance
leaseMgr *lease.Manager
}
func newGeneric(config *kubevip.Config, provider providers.Provider, instances *[]*instance.Instance, leaseMgr *lease.Manager) generic {
func newGeneric(config *kubevip.Config, provider providers.Provider, instances *[]*instance.Instance) generic {
return generic{
config: config,
provider: provider,
instances: instances,
leaseMgr: leaseMgr,
}
}
func (g *generic) processInstance(_ *servicecontext.Context, _ *v1.Service) error {
func (g *generic) processInstance(_ *servicecontext.Context, _ *v1.Service, _ *bool) error {
return nil
}
func (g *generic) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service) {
g.clearEgress(lastKnownGoodEndpoint, service)
svcCtx.CallLeaderCancel()
func (g *generic) clear(_ *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
g.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service) {
func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if *lastKnownGoodEndpoint != "" {
log.Warn("existing endpoint has been removed, no remaining endpoints for leaderElection", "provider", g.provider.GetLabel(), "endpoint", lastKnownGoodEndpoint)
log.Warn("existing endpoint has been removed, no remaining endpoints for leaderElection", "provider", g.provider.GetLabel(), "endpoint", lastKnownGoodEndpoint)
if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP, service.Namespace, string(service.UID), service.Annotations, g.config.EgressWithNftables); err != nil {
log.Error("error removing redundant egress rules", "err", err)
}
*lastKnownGoodEndpoint = "" // reset endpoint
if g.config.EnableServicesElection || g.config.EnableLeaderElection {
cancel() // stop services watcher
}
*leaderElectionActive = false
}
}
func (g *generic) getEndpoints(service *v1.Service, id string) ([]string, error) {
return g.getAllEndpoints(service, id)
func (g *generic) getEndpoints(_ *v1.Service, id string) ([]string, error) {
return g.getLocalEndpoints(id)
}
func (g *generic) getLocalEndpoints(id string) ([]string, error) {
// Build endpoints
var endpoints []string
var err error
if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
}
return endpoints, nil
}
func (g *generic) getAllEndpoints(service *v1.Service, id string) ([]string, error) {
// Build endpoints
var err error
var endpoints []string
if service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
if !g.config.EnableLeaderElection && !g.config.EnableServicesElection &&
service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
if endpoints, err = g.provider.GetAllEndpoints(); err != nil {
return nil, fmt.Errorf("[%s] error getting all endpoints: %w", g.provider.GetLabel(), err)
}
@@ -102,6 +109,10 @@ func (g *generic) getAllEndpoints(service *v1.Service, id string) ([]string, err
func (g *generic) removeEgress(_ *v1.Service, _ *string) {
}
func (g *generic) setInstanceEndpointsStatus(_ context.Context, _ *v1.Service, _ []string) error {
func (g *generic) delete(_ *v1.Service, _ string) error {
return nil
}
func (g *generic) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
return nil
}

View File

@@ -2,46 +2,64 @@ package endpoints
import (
"context"
"errors"
"fmt"
"net"
"sync"
"syscall"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/egress"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
)
type RoutingTable struct {
generic
mtx sync.Mutex
routeMgr *route.Manager
}
func newRoutingTable(generic generic, routeMgr *route.Manager) endpointWorker {
func newRoutingTable(generic generic) endpointWorker {
return &RoutingTable{
generic: generic,
routeMgr: routeMgr,
generic: generic,
}
}
func (rt *RoutingTable) processInstance(svcCtx *servicecontext.Context, service *v1.Service) error {
inst := instance.FindServiceInstance(service, *rt.instances)
if inst != nil {
for _, cluster := range inst.Clusters {
func (rt *RoutingTable) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
instance := instance.FindServiceInstance(service, *rt.instances)
if instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
if !svcCtx.IsNetworkConfigured(cluster.Network[i].IP()) && cluster.Network[i].HasEndpoints() {
if err := rt.routeMgr.Add(lease.ServiceNamespacedName(service), cluster.Network[i], false, true); err != nil {
return fmt.Errorf("[%s] error adding route: %s", rt.provider.GetLabel(), err.Error())
if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) && cluster.Network[i].HasEndpoints() {
err := cluster.Network[i].AddRoute(false)
if err != nil {
if errors.Is(err, syscall.EEXIST) {
// If route exists, but protocol is not set (e.g. the route was created by the older version
// of kube-vip) try to update it if necessary
isUpdated, err := cluster.Network[i].UpdateRoutes()
if err != nil {
return fmt.Errorf("[%s] error updating existing routes: %w", rt.provider.GetLabel(), err)
}
if isUpdated {
log.Info("updated route", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
} else {
log.Info("route already present", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
}
} else {
// If other error occurs, return error
return fmt.Errorf("[%s] error adding route: %s", rt.provider.GetLabel(), err.Error())
}
} else {
log.Info("added route", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
svcCtx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
*leaderElectionActive = true
}
}
}
@@ -51,11 +69,9 @@ func (rt *RoutingTable) processInstance(svcCtx *servicecontext.Context, service
return nil
}
func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service) {
rt.mtx.Lock()
defer rt.mtx.Unlock()
if !rt.config.EnableServicesElection {
if errs := ClearRoutes(service, rt.instances, rt.routeMgr); len(errs) == 0 {
func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
if errs := ClearRoutes(service, rt.instances); len(errs) == 0 {
svcCtx.ConfiguredNetworks.Clear()
} else {
for _, err := range errs {
@@ -64,9 +80,7 @@ func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpo
}
}
rt.clearEgress(lastKnownGoodEndpoint, service)
svcCtx.CallLeaderCancel()
rt.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (rt *RoutingTable) getEndpoints(service *v1.Service, id string) ([]string, error) {
@@ -80,24 +94,42 @@ func (rt *RoutingTable) removeEgress(service *v1.Service, lastKnownGoodEndpoint
}
}
func (rt *RoutingTable) setInstanceEndpointsStatus(ctx context.Context, service *v1.Service, endpoints []string) error {
inst := instance.FindServiceInstance(service, *rt.instances)
if inst == nil {
log.Error("failed to find the instance", "namespace", service.Namespace, "name", service.Name, "uid", service.UID, "provider", rt.provider.GetLabel())
func (rt *RoutingTable) delete(service *v1.Service, id string) error {
// When no-leader-elecition mode
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
// find all existing local endpoints
endpoints, err := rt.getEndpoints(service, id)
if err != nil {
return fmt.Errorf("[%s] error getting endpoints: %w", rt.provider.GetLabel(), err)
}
// If there were local endpoints deleted
if len(endpoints) > 0 {
rt.deleteAction(service)
}
}
return nil
}
func (rt *RoutingTable) deleteAction(service *v1.Service) {
ClearRoutes(service, rt.instances)
}
func (rt *RoutingTable) setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error {
instance := instance.FindServiceInstance(service, *rt.instances)
if instance == nil {
log.Error("failed to find the instance", "service", service.UID, "provider", rt.provider.GetLabel())
} else {
for _, c := range inst.Clusters {
for _, c := range instance.Clusters {
for n := range c.Network {
// if there are no endpoints set HasEndpoints false just in case
if len(endpoints) < 1 {
c.Network[n].SetHasEndpoints(false)
} else {
// check if endpoint are available and are of same IP family as service
for _, ep := range endpoints {
if (net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(ep).To4() == nil) {
c.Network[n].SetHasEndpoints(true)
break
}
}
}
// check if endpoint are available and are of same IP family as service
if len(endpoints) > 0 && ((net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(endpoints[0]).To4() == nil)) {
c.Network[n].SetHasEndpoints(true)
}
}
}
@@ -106,32 +138,41 @@ func (rt *RoutingTable) setInstanceEndpointsStatus(ctx context.Context, service
return nil
}
func ClearRoutes(service *v1.Service, instances *[]*instance.Instance, routeMgr *route.Manager) []error {
func ClearRoutes(service *v1.Service, instances *[]*instance.Instance) []error {
errs := []error{}
if svcInst := instance.FindServiceInstance(service, *instances); svcInst != nil {
clearErrs := ClearRoutesByInstance(service, svcInst, instances, routeMgr)
errs = append(errs, clearErrs...)
}
return errs
}
func ClearRoutesByInstance(service *v1.Service, svcInst *instance.Instance, instances *[]*instance.Instance, routeMgr *route.Manager) []error {
if svcInst == nil {
return []error{fmt.Errorf("failed to remove routes for nil instance of service %s/%s, uid: %s", service.Namespace, service.Name, service.UID)}
}
errs := []error{}
for _, cluster := range svcInst.Clusters {
for i := range cluster.Network {
err := routeMgr.Delete(lease.ServiceNamespacedName(service), cluster.Network[i])
if err != nil {
log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
errs = append(errs, err)
if instance := instance.FindServiceInstance(service, *instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
route := cluster.Network[i].PrepareRoute()
// check if route we are about to delete is not referenced by more than one service
if CountRouteReferences(route, instances) <= 1 {
err := cluster.Network[i].DeleteRoute()
if err != nil && !errors.Is(err, syscall.ESRCH) {
log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
errs = append(errs, err)
}
log.Debug("deleted route", "ip",
cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface())
}
}
log.Debug("deleted route", "ip",
cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface())
}
}
return errs
}
func CountRouteReferences(route *netlink.Route, instances *[]*instance.Instance) int {
cnt := 0
for _, instance := range *instances {
for _, cluster := range instance.Clusters {
for n := range cluster.Network {
if cluster.Network[n].HasEndpoints() {
r := cluster.Network[n].PrepareRoute()
if r.Dst.String() == route.Dst.String() {
cnt++
}
}
}
}
}
return cnt
}

View File

@@ -1,492 +0,0 @@
package endpoints
import (
"context"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/metrics"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/prometheus/client_golang/prometheus/testutil"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
)
func TestShouldAllowReconcileWithoutEndpoints(t *testing.T) {
if shouldAllowReconcileWithoutEndpoints(nil) {
t.Fatal("nil service should not be allowed")
}
clusterOptIn := &v1.Service{
Spec: v1.ServiceSpec{ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeCluster},
ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{kubevip.AllowReconcileWithoutEndpoints: "true"}},
}
if !shouldAllowReconcileWithoutEndpoints(clusterOptIn) {
t.Fatal("cluster service with opt-in annotation should be allowed")
}
localOptIn := &v1.Service{
Spec: v1.ServiceSpec{ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal},
ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{kubevip.AllowReconcileWithoutEndpoints: "true"}},
}
if shouldAllowReconcileWithoutEndpoints(localOptIn) {
t.Fatal("local service should not be allowed")
}
}
type fakeWorker struct {
endpoints []string
clearCalled bool
processCalled bool
}
type annotationUpdate struct {
endpoint string
endpointIPv6 string
}
type recordingProvider struct {
providers.Provider
updates []annotationUpdate
}
func (p *recordingProvider) UpdateServiceAnnotation(_ context.Context, endpoint, endpointIPv6 string,
_ *v1.Service, _ *kubernetes.Clientset) error {
p.updates = append(p.updates, annotationUpdate{endpoint: endpoint, endpointIPv6: endpointIPv6})
return nil
}
func TestUpdateAnnotationsZeroEndpointsThenSameEndpoint(t *testing.T) {
for _, enableEndpoints := range []bool{true, false} {
providerName := "EndpointSlices"
provider := providers.NewEndpointslices()
if enableEndpoints {
providerName = "Endpoints"
provider = providers.NewEndpoints()
}
for _, family := range []struct {
name string
endpoint string
other string
egressIPv6 bool
}{
{name: "IPv4", endpoint: "10.0.0.1", other: "fd00::1"},
{name: "IPv6", endpoint: "fd00::1", other: "10.0.0.1", egressIPv6: true},
} {
t.Run(providerName+"/"+family.name, func(t *testing.T) {
annotations := map[string]string{kubevip.Egress: "true"}
if family.egressIPv6 {
annotations[kubevip.EgressIPv6] = "true"
}
if !enableEndpoints {
if family.egressIPv6 {
annotations[kubevip.ActiveEndpoint] = family.other
annotations[kubevip.ActiveEndpointIPv6] = family.endpoint
} else {
annotations[kubevip.ActiveEndpoint] = family.endpoint
annotations[kubevip.ActiveEndpointIPv6] = family.other
}
} else {
annotations[kubevip.ActiveEndpoint] = family.endpoint
}
service := &v1.Service{ObjectMeta: metav1.ObjectMeta{
Name: "test-service", Namespace: "default", UID: "test-uid", Annotations: annotations,
}}
serviceInstance := &instance.Instance{ServiceUID: service.UID, ServiceSnapshot: service.DeepCopy()}
instances := []*instance.Instance{serviceInstance}
recorder := &recordingProvider{Provider: provider}
processor := &Processor{
config: &kubevip.Config{EnableEndpoints: enableEndpoints},
provider: recorder,
instances: &instances,
}
updateSnapshot := func(_ context.Context, updated *v1.Service) error {
serviceInstance.ServiceSnapshot = updated
return nil
}
noEndpoint := ""
processor.updateAnnotations(service, &noEndpoint, nil, updateSnapshot)
repopulatedEndpoint := family.endpoint
processor.updateAnnotations(service, &repopulatedEndpoint, nil, updateSnapshot)
cleared := annotationUpdate{}
repopulated := annotationUpdate{endpoint: family.endpoint}
if !enableEndpoints {
if family.egressIPv6 {
cleared = annotationUpdate{endpoint: family.other}
repopulated = annotationUpdate{endpoint: family.other, endpointIPv6: family.endpoint}
} else {
cleared = annotationUpdate{endpointIPv6: family.other}
repopulated = annotationUpdate{endpoint: family.endpoint, endpointIPv6: family.other}
}
}
want := []annotationUpdate{cleared, repopulated}
if len(recorder.updates) != len(want) {
t.Fatalf("annotation updates = %+v, want %+v", recorder.updates, want)
}
for index := range want {
if recorder.updates[index] != want[index] {
t.Errorf("annotation update %d = %+v, want %+v", index, recorder.updates[index], want[index])
}
}
})
}
}
}
func TestUpdateAnnotationsEndpointSlicesClearsConfiguredFamily(t *testing.T) {
for _, test := range []struct {
name string
egressIPv6 bool
want annotationUpdate
}{
{name: "IPv4", want: annotationUpdate{endpointIPv6: "fd00::1"}},
{name: "IPv6", egressIPv6: true, want: annotationUpdate{endpoint: "10.0.0.1"}},
} {
t.Run(test.name, func(t *testing.T) {
annotations := map[string]string{
kubevip.Egress: "true",
kubevip.ActiveEndpoint: "10.0.0.1",
kubevip.ActiveEndpointIPv6: "fd00::1",
}
if test.egressIPv6 {
annotations[kubevip.EgressIPv6] = "true"
}
service := &v1.Service{ObjectMeta: metav1.ObjectMeta{
Name: "test-service", Namespace: "default", UID: "test-uid", Annotations: annotations,
}}
instances := []*instance.Instance{{ServiceUID: service.UID, ServiceSnapshot: service.DeepCopy()}}
recorder := &recordingProvider{Provider: providers.NewEndpointslices()}
processor := &Processor{
config: &kubevip.Config{EnableEndpoints: false},
provider: recorder,
instances: &instances,
}
noEndpoint := ""
processor.updateAnnotations(service, &noEndpoint, nil, func(context.Context, *v1.Service) error { return nil })
if len(recorder.updates) != 1 || recorder.updates[0] != test.want {
t.Fatalf("annotation updates = %+v, want [%+v]", recorder.updates, test.want)
}
})
}
}
func TestUpdateAnnotationsValidatesEndpointFamily(t *testing.T) {
tests := []struct {
name string
endpoint string
egressIPv6 bool
want annotationUpdate
wantUpdate bool
}{
{name: "invalid address", endpoint: "not-an-ip"},
{name: "IPv6 endpoint for IPv4 egress", endpoint: "fd00::1"},
{name: "IPv4 endpoint for IPv6 egress", endpoint: "10.0.0.1", egressIPv6: true},
{name: "IPv4 endpoint", endpoint: "10.0.0.2", want: annotationUpdate{endpoint: "10.0.0.2", endpointIPv6: "fd00::1"}, wantUpdate: true},
{name: "IPv6 endpoint", endpoint: "fd00::2", egressIPv6: true, want: annotationUpdate{endpoint: "10.0.0.1", endpointIPv6: "fd00::2"}, wantUpdate: true},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
annotations := map[string]string{
kubevip.Egress: "true",
kubevip.ActiveEndpoint: "10.0.0.1",
kubevip.ActiveEndpointIPv6: "fd00::1",
}
if test.egressIPv6 {
annotations[kubevip.EgressIPv6] = "true"
}
service := &v1.Service{ObjectMeta: metav1.ObjectMeta{
Name: "test-service", Namespace: "default", Annotations: annotations,
}}
recorder := &recordingProvider{Provider: providers.NewEndpointslices()}
processor := &Processor{
config: &kubevip.Config{EnableEndpoints: false},
provider: recorder,
}
processor.updateAnnotations(service, &test.endpoint, nil, nil)
if !test.wantUpdate {
if len(recorder.updates) != 0 {
t.Fatalf("annotation updates = %+v, want none", recorder.updates)
}
return
}
if len(recorder.updates) != 1 || recorder.updates[0] != test.want {
t.Fatalf("annotation updates = %+v, want [%+v]", recorder.updates, test.want)
}
})
}
}
func (f *fakeWorker) processInstance(_ *servicecontext.Context, _ *v1.Service) error {
f.processCalled = true
return nil
}
func (f *fakeWorker) clear(_ *servicecontext.Context, _ *string, _ *v1.Service) {
f.clearCalled = true
}
func (f *fakeWorker) getEndpoints(_ *v1.Service, _ string) ([]string, error) { return f.endpoints, nil }
func (f *fakeWorker) removeEgress(_ *v1.Service, _ *string) {}
func (f *fakeWorker) setInstanceEndpointsStatus(_ context.Context, _ *v1.Service, _ []string) error {
return nil
}
// TestReconcile_RecomputesRemainingEndpoints asserts that deleting one EndpointSlice
// reconciles against the endpoints that remain, instead of assuming the service
// lost all of them.
func TestReconcile_RecomputesRemainingEndpoints(t *testing.T) {
t.Parallel()
tests := []struct {
name string
remaining []string
lastKnown string
expectReady bool
expectClear bool
expectProcess bool
expectedLastKnown string
}{
{
name: "remaining endpoints keep the service up",
remaining: []string{"10.0.0.2"},
lastKnown: "10.0.0.2",
expectReady: true,
expectProcess: true,
expectedLastKnown: "10.0.0.2",
},
{
name: "stale last known endpoint moves to a survivor",
remaining: []string{"10.0.0.2"},
lastKnown: "10.0.0.1",
expectReady: true,
expectProcess: true,
expectedLastKnown: "10.0.0.2",
},
{
name: "last endpoint removed tears the service down",
remaining: nil,
lastKnown: "10.0.0.1",
expectReady: false,
expectClear: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
worker := &fakeWorker{endpoints: test.remaining}
p := &Processor{
config: &kubevip.Config{},
provider: providers.NewEndpointslices(),
worker: worker,
}
svcCtx := servicecontext.New(context.Background())
svcCtx.SignalReadiness()
lastKnown := test.lastKnown
restart, err := p.Reconcile(
svcCtx,
watch.Event{
Type: watch.Deleted,
Object: &discoveryv1.EndpointSlice{ObjectMeta: metav1.ObjectMeta{Name: "slice-1"}},
},
&lastKnown,
&v1.Service{Spec: v1.ServiceSpec{ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal}},
"node-1",
func(*servicecontext.Context, *v1.Service, *sync.WaitGroup, bool) error { return nil },
&sync.WaitGroup{},
nil,
nil,
)
if err != nil {
t.Fatalf("Reconcile returned error: %v", err)
}
if restart {
t.Fatal("Reconcile unexpectedly requested restart")
}
if ready := svcCtx.Signalled.Load(); ready != test.expectReady {
t.Fatalf("readiness mismatch: expected %v, got %v", test.expectReady, ready)
}
if worker.clearCalled != test.expectClear {
t.Fatalf("clearCalled mismatch: expected %v, got %v", test.expectClear, worker.clearCalled)
}
if worker.processCalled != test.expectProcess {
t.Fatalf("processCalled mismatch: expected %v, got %v", test.expectProcess, worker.processCalled)
}
if test.expectedLastKnown != "" && lastKnown != test.expectedLastKnown {
t.Fatalf("lastKnownGoodEndpoint mismatch: expected %q, got %q", test.expectedLastKnown, lastKnown)
}
})
}
}
func TestReconcile_ZeroEndpointsBehavior(t *testing.T) {
t.Parallel()
run := func(t *testing.T, service *v1.Service, presetSignalled bool, expectReady bool, expectClear bool, expectProcess bool) {
t.Helper()
worker := &fakeWorker{endpoints: []string{}}
p := &Processor{
config: &kubevip.Config{},
provider: providers.NewEndpointslices(),
worker: worker,
}
svcCtx := servicecontext.New(context.Background())
if presetSignalled {
svcCtx.SignalReadiness()
}
restart, err := p.Reconcile(
svcCtx,
watch.Event{Type: watch.Modified, Object: &discoveryv1.EndpointSlice{}},
new(string),
service,
"node-1",
func(*servicecontext.Context, *v1.Service, *sync.WaitGroup, bool) error { return nil },
&sync.WaitGroup{},
nil,
nil,
)
if err != nil {
t.Fatalf("Reconcile returned error: %v", err)
}
if restart {
t.Fatal("Reconcile unexpectedly requested restart")
}
if ready := svcCtx.Signalled.Load(); ready != expectReady {
t.Fatalf("readiness mismatch: expected %v, got %v", expectReady, ready)
}
if worker.clearCalled != expectClear {
t.Fatalf("clearCalled mismatch: expected %v, got %v", expectClear, worker.clearCalled)
}
if worker.processCalled != expectProcess {
t.Fatalf("processCalled mismatch: expected %v, got %v", expectProcess, worker.processCalled)
}
}
t.Run("cluster opt-in keeps readiness and skips clear", func(t *testing.T) {
service := &v1.Service{
ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{kubevip.AllowReconcileWithoutEndpoints: "true"}},
Spec: v1.ServiceSpec{ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeCluster},
}
run(t, service, false, true, false, true)
})
t.Run("cluster without opt-in resets and clears when pre-signalled", func(t *testing.T) {
service := &v1.Service{
ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}},
Spec: v1.ServiceSpec{ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeCluster},
}
run(t, service, true, false, true, false)
})
t.Run("local opt-in still resets and clears when pre-signalled", func(t *testing.T) {
service := &v1.Service{
ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{kubevip.AllowReconcileWithoutEndpoints: "true"}},
Spec: v1.ServiceSpec{ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal},
}
run(t, service, true, false, true, false)
})
}
// TestReconcile_ServicesElectionStartsOnce asserts that repeated endpoint events
// for the same service start the leader-election restart loop exactly once.
//
// Reconcile runs on every EndpointSlice add/modify/resync event, and the loop it
// starts only returns once the service context is cancelled. Starting it per event
// therefore accumulates duplicate goroutines that all contend on the same lease.
//
// See https://github.com/kube-vip/kube-vip/issues/1665.
func TestReconcile_ServicesElectionStartsOnce(t *testing.T) {
config := &kubevip.Config{
EnableServicesElection: true,
LeaderElectionType: "kubernetes",
}
service := &v1.Service{
ObjectMeta: metav1.ObjectMeta{Name: "test-svc", Namespace: "default", UID: "test-uid"},
Spec: v1.ServiceSpec{Type: v1.ServiceTypeLoadBalancer},
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
leaseMgr := lease.NewManager()
leaseNamespace, serviceLease := lease.ServiceName(service)
svcLease := leaseMgr.Add(ctx, lease.NewID(config.LeaderElectionType, leaseNamespace, serviceLease))
svcCtx := servicecontext.New(svcLease.Ctx)
// The started loops only return once the service context is cancelled, so it has
// to be cancelled before waiting on them.
wg := &sync.WaitGroup{}
defer wg.Wait()
defer svcCtx.Cancel()
p := &Processor{
config: config,
provider: providers.NewEndpointslices(),
worker: &fakeWorker{endpoints: []string{"10.0.0.1"}},
leaseMgr: leaseMgr,
}
// starts counts the restart loops. The real StartServicesLeaderElection blocks
// until the service context is cancelled, so each loop parks in a single call.
var starts atomic.Int64
serviceFunc := func(svcCtx *servicecontext.Context, _ *v1.Service, _ *sync.WaitGroup, _ bool) error {
starts.Add(1)
<-svcCtx.Ctx.Done()
return nil
}
// Three endpoint events, as a flapping backend pod would produce.
for range 3 {
restart, err := p.Reconcile(svcCtx, watch.Event{Type: watch.Modified, Object: &discoveryv1.EndpointSlice{}},
new(string), service, "node-1", serviceFunc, wg, nil, nil)
if err != nil {
t.Fatalf("Reconcile returned error: %v", err)
}
if restart {
t.Fatal("Reconcile unexpectedly requested restart")
}
}
// Give every loop that is going to start a chance to reach serviceFunc.
for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); {
if starts.Load() > 1 {
break
}
time.Sleep(10 * time.Millisecond)
}
if got := starts.Load(); got != 1 {
t.Errorf("leader election started %d times, want 1", got)
}
// The gauge the e2e fault tests assert on has to agree with the call count.
if got := testutil.ToFloat64(metrics.ServiceElectionLoops.WithLabelValues(service.Namespace, service.Name)); got != 1 {
t.Errorf("kube_vip_service_election_loops is %v, want 1", got)
}
}

View File

@@ -1,278 +0,0 @@
package endpoints
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/wireguard"
v1 "k8s.io/api/core/v1"
)
// wireguardWorker handles endpoint changes for WireGuard-based services
type wireguardWorker struct {
config *kubevip.Config
provider providers.Provider
bgpServer *bgp.Server
instances *[]*instance.Instance
leaseMgr *lease.Manager
tunnelMgr *wireguard.TunnelManager
}
func newWireguardWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
instances *[]*instance.Instance, leaseMgr *lease.Manager, tunnelMgr *wireguard.TunnelManager) *wireguardWorker {
return &wireguardWorker{
config: config,
provider: provider,
bgpServer: bgpServer,
instances: instances,
leaseMgr: leaseMgr,
tunnelMgr: tunnelMgr,
}
}
// processInstance updates nftables DNAT rules when endpoints change
// This is called by the endpoint watcher when endpoints are added/modified
func (w *wireguardWorker) processInstance(svcCtx *servicecontext.Context, service *v1.Service) error {
log.Debug("[wireguard] processing instance for endpoint change", "service", service.Name, "namespace", service.Namespace)
// Get the target endpoint for this service
// For ExternalTrafficPolicy=Local, only use local endpoints
// For ExternalTrafficPolicy=Cluster, use all endpoints
var endpoints []string
var err error
if service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
endpoints, err = w.provider.GetLocalEndpoints(w.config.NodeName, w.config)
} else {
endpoints, err = w.provider.GetAllEndpoints()
}
if err != nil {
return fmt.Errorf("failed to get endpoints: %w", err)
}
if len(endpoints) == 0 {
log.Debug("[wireguard] no endpoints available", "service", service.Name)
w.clear(svcCtx, nil, service)
return nil
}
// Find the service processor to call updateServiceWireguardEndpoints
// Note: This requires access to the service processor which we don't have here
// So we'll recreate the DNAT rules directly
// First, clear existing rules
w.clear(svcCtx, nil, service)
// Get service VIPs
serviceIPs, err := utils.FetchServiceIPs(service)
if err != nil {
return fmt.Errorf("failed to get service IPs: %w", err)
}
log.Info("[wireguard] updating DNAT rules for endpoint change",
"service", service.Name,
"namespace", service.Namespace,
"endpoints", endpoints,
"vips", serviceIPs)
// Update DNAT rules for each port
for _, port := range service.Spec.Ports {
// Determine target port (resolve named ports if necessary)
targetPort := w.provider.ResolvePort(port)
log.Info("[wireguard] resolved port", "service", service.Name, "servicePort", port.Port, "targetPort", targetPort, "targetPortName", port.TargetPort.StrVal)
// Build targets list from all endpoints
targets := make([]nftables.DNATTarget, len(endpoints))
for i, ep := range endpoints {
targets[i] = nftables.DNATTarget{
IP: ep,
Port: uint16(targetPort), //nolint:gosec // Port range validated by Kubernetes
}
}
for _, vip := range serviceIPs {
// Strip CIDR notation if present
vipAddr := utils.StripCIDR(vip)
// Get WireGuard interface name from TunnelManager for this VIP
if w.tunnelMgr == nil {
log.Error("[wireguard] TunnelManager not configured; cannot update DNAT rules",
"service", service.Name,
"namespace", service.Namespace)
return fmt.Errorf("TunnelManager not configured")
}
tunnelConfig := w.tunnelMgr.GetConfigForVIP(vipAddr)
if tunnelConfig == nil {
log.Error("[wireguard] WireGuard interface name not configured; cannot update DNAT rules",
"service", service.Name,
"namespace", service.Namespace,
"vip", vipAddr)
return fmt.Errorf("wireguard interface name not configured for VIP %s", vipAddr)
}
wgInterface := tunnelConfig.InterfaceName
portServiceID, _ := wireguard.ServicePortIDs(service.Namespace, service.Name, port)
log.Info("[wireguard] applying DNAT rule with load balancing",
"service", service.Name,
"vip", vipAddr,
"interface", wgInterface,
"sourcePort", port.Port,
"targets", targets,
"chainID", portServiceID)
// Apply the DNAT rule with load balancing across all endpoints
// localEndpoint=true when using ExternalTrafficPolicy=Local, which preserves client source IP
isLocalEndpoint := service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal
err := nftables.ApplyDNAT(
wgInterface,
vipAddr,
uint16(port.Port), //nolint:gosec // Port range validated by Kubernetes
targets,
portServiceID,
port.Protocol,
isLocalEndpoint,
tunnelConfig.ListenPort,
)
if err != nil {
log.Error("[wireguard] failed to update DNAT rule",
"service", service.Name,
"vip", vipAddr,
"port", port.Port,
"err", err)
continue
}
log.Debug("[wireguard] DNAT rule updated successfully",
"service", service.Name,
"vip", vipAddr,
"port", port.Port,
"targetCount", len(targets))
}
}
return nil
}
// clear removes DNAT rules when no endpoints are available
func (w *wireguardWorker) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service) {
log.Info("[wireguard] clearing DNAT rules (no endpoints)", "service", service.Name, "namespace", service.Namespace)
// Get service IPs to determine IPv4 vs IPv6
serviceIPs, _ := utils.FetchServiceIPs(service)
// Delete DNAT chains for each port
for _, port := range service.Spec.Ports {
if port.Protocol != v1.ProtocolTCP && port.Protocol != v1.ProtocolUDP {
continue
}
// Determine if we have IPv4 or IPv6
hasIPv4, hasIPv6 := false, false
for _, vip := range serviceIPs {
if isIPv6Address(vip) {
hasIPv6 = true
} else {
hasIPv4 = true
}
}
for _, portServiceID := range wireguard.ServicePortIDSet(service.Namespace, service.Name, port) {
if hasIPv4 {
if err := nftables.DeleteIngressChains(false, portServiceID); err != nil {
log.Warn("[wireguard] failed to delete IPv4 DNAT chains",
"service", service.Name,
"port", port.Port,
"id", portServiceID,
"err", err)
}
}
if hasIPv6 {
if err := nftables.DeleteIngressChains(true, portServiceID); err != nil {
log.Warn("[wireguard] failed to delete IPv6 DNAT chains",
"service", service.Name,
"port", port.Port,
"id", portServiceID,
"err", err)
}
}
}
}
if svcCtx != nil {
svcCtx.CallLeaderCancel()
}
}
// getEndpoints retrieves the list of endpoints for a service
// For ExternalTrafficPolicy=Local, only local endpoints are returned
// For ExternalTrafficPolicy=Cluster, all endpoints are returned
func (w *wireguardWorker) getEndpoints(service *v1.Service, id string) ([]string, error) {
var endpoints []string
var err error
if service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
endpoints, err = w.provider.GetLocalEndpoints(id, w.config)
} else {
endpoints, err = w.provider.GetAllEndpoints()
}
if err != nil {
return nil, fmt.Errorf("[wireguard] failed to get endpoints: %w", err)
}
log.Debug("[wireguard] retrieved endpoints", "service", service.Name, "count", len(endpoints), "endpoints", endpoints)
return endpoints, nil
}
// removeEgress is a no-op for WireGuard since egress is handled separately
func (w *wireguardWorker) removeEgress(service *v1.Service, lastKnownGoodEndpoint *string) {
// WireGuard doesn't use egress in the same way as other modes
log.Debug("[wireguard] removeEgress called (no-op)", "service", service.Name)
}
// setInstanceEndpointsStatus updates the endpoint status on the service instance
func (w *wireguardWorker) setInstanceEndpointsStatus(_ context.Context, service *v1.Service, endpoints []string) error {
hasEndpoints := len(endpoints) > 0
log.Debug("[wireguard] setting instance endpoint status",
"service", service.Name,
"hasEndpoints", hasEndpoints,
"endpointCount", len(endpoints))
// Find the service instance
for _, inst := range *w.instances {
if inst.ServiceSnapshot == nil {
continue
}
if inst.ServiceSnapshot.UID == service.UID {
// Update the network status for all clusters
for _, cluster := range inst.Clusters {
for i := range cluster.Network {
cluster.Network[i].SetHasEndpoints(hasEndpoints)
}
}
log.Debug("[wireguard] updated instance endpoint status",
"service", service.Name,
"hasEndpoints", hasEndpoints)
return nil
}
}
log.Debug("[wireguard] instance not found for endpoint status update", "service", service.Name)
return nil
}
func isIPv6Address(ip string) bool {
// Strip CIDR notation if present before checking
addr := utils.StripCIDR(ip)
return utils.IsIPv6(addr)
}

View File

@@ -1,14 +0,0 @@
package endpoints
import (
"testing"
v1 "k8s.io/api/core/v1"
)
func TestWireguardClearDoesNotDereferenceNilServiceContext(t *testing.T) {
worker := &wireguardWorker{}
service := &v1.Service{}
worker.clear(nil, nil, service)
}

View File

@@ -61,18 +61,6 @@ func (ep *Endpoints) LoadObject(endpoints runtime.Object, cancel context.CancelF
return nil
}
// DeleteObject drops the tracked object. A service is backed by exactly one
// v1.Endpoints object, so there is nothing to match on and the cache is reset.
func (ep *Endpoints) DeleteObject(endpoints runtime.Object) error {
//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
if _, ok := endpoints.(*v1.Endpoints); !ok {
return fmt.Errorf("[%s] unable to parse Kubernetes object", ep.GetLabel())
}
//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
ep.endpoints = &v1.Endpoints{}
return nil
}
func (ep *Endpoints) GetAllEndpoints() ([]string, error) {
result := []string{}
for subset := range ep.endpoints.Subsets {
@@ -99,7 +87,7 @@ func (ep *Endpoints) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string,
continue
}
// 2. Compare the Hostname (only useful if address.NodeName is not available)
if address.NodeName == nil && id == address.Hostname {
if id == address.Hostname {
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname)
localEndpoints = append(localEndpoints, address.IP)
continue
@@ -109,11 +97,11 @@ func (ep *Endpoints) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string,
return localEndpoints, nil
}
func (ep *Endpoints) UpdateServiceAnnotation(ctx context.Context, endpoint string, _ string, service *v1.Service, clientSet *kubernetes.Clientset) error {
func (ep *Endpoints) UpdateServiceAnnotation(endpoint string, _ string, service *v1.Service, clientSet *kubernetes.Clientset) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(ctx, service.Name, metav1.GetOptions{})
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
@@ -125,7 +113,7 @@ func (ep *Endpoints) UpdateServiceAnnotation(ctx context.Context, endpoint strin
currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(ctx, currentServiceCopy, metav1.UpdateOptions{})
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("error updating Service Spec", "label", ep.GetLabel(), "name", currentServiceCopy.Name, "err", err)
return err
@@ -144,15 +132,6 @@ func (ep *Endpoints) GetLabel() string {
return ep.label
}
func (ep *Endpoints) ResolvePort(servicePort v1.ServicePort) int32 {
return ResolvePortWithLookup(servicePort, func(name string) int32 {
for _, subset := range ep.endpoints.Subsets {
for _, p := range subset.Ports {
if p.Name == name {
return p.Port
}
}
}
return 0
})
func (ep *Endpoints) GetProtocol() string {
return ""
}

View File

@@ -20,14 +20,13 @@ import (
)
type Endpointslices struct {
label string
slices map[string]*discoveryv1.EndpointSlice
label string
endpoints *discoveryv1.EndpointSlice
}
func NewEndpointslices() Provider {
return &Endpointslices{
label: "endpointslices",
slices: make(map[string]*discoveryv1.EndpointSlice),
label: "endpointslices",
}
}
@@ -57,83 +56,53 @@ func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.Ca
cancel()
return fmt.Errorf("[%s] error casting endpoints to v1.Endpoints struct", ep.label)
}
if ep.slices == nil {
ep.slices = make(map[string]*discoveryv1.EndpointSlice)
}
ep.slices[eps.Name] = eps.DeepCopy()
ep.endpoints = eps
return nil
}
func (ep *Endpointslices) DeleteObject(endpoints runtime.Object) error {
eps, ok := endpoints.(*discoveryv1.EndpointSlice)
if !ok {
return fmt.Errorf("[%s] unable to parse Kubernetes object", ep.GetLabel())
}
delete(ep.slices, eps.Name)
return nil
}
// isServing reports whether an endpoint should receive traffic. Per the
// EndpointConditions godoc a nil Serving defers to Ready, and a nil Ready is an
// unknown state that consumers should interpret as ready.
func isServing(conditions discoveryv1.EndpointConditions) bool {
serving := conditions.Serving
if serving == nil {
serving = conditions.Ready
}
return serving == nil || *serving
}
func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
result := []string{}
for _, eps := range ep.slices {
for _, e := range eps.Endpoints {
if !isServing(e.Conditions) {
continue
}
result = append(result, e.Addresses...)
}
for _, ep := range ep.endpoints.Endpoints {
result = append(result, ep.Addresses...)
}
return result, nil
}
func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
var localEndpoints []string
for _, eps := range ep.slices {
for _, endpoint := range eps.Endpoints {
if !isServing(endpoint.Conditions) {
for _, endpoint := range ep.endpoints.Endpoints {
if !*endpoint.Conditions.Serving {
continue
}
for _, address := range endpoint.Addresses {
log.Debug("processing endpoint", "provider", ep.label, "ip", address)
// 1. Compare the Nodename
if endpoint.NodeName != nil && id == *endpoint.NodeName {
if endpoint.Hostname != nil {
log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname, "nodename", *endpoint.NodeName)
} else {
log.Debug("found endpoint", "provider", ep.label, "ip", address, "nodename", *endpoint.NodeName)
}
localEndpoints = append(localEndpoints, address)
continue
}
for _, address := range endpoint.Addresses {
// 1. Compare the Nodename
if endpoint.NodeName != nil && id == *endpoint.NodeName {
if endpoint.Hostname != nil {
log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname, "nodename", *endpoint.NodeName)
} else {
log.Debug("found endpoint", "provider", ep.label, "ip", address, "nodename", *endpoint.NodeName)
}
localEndpoints = append(localEndpoints, address)
continue
}
// 2. Compare the Hostname (only useful if endpoint.NodeName is not available)
if endpoint.NodeName == nil && endpoint.Hostname != nil && id == *endpoint.Hostname {
log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname)
localEndpoints = append(localEndpoints, address)
}
// 2. Compare the Hostname (only useful if endpoint.NodeName is not available)
if endpoint.Hostname != nil && id == *endpoint.Hostname {
log.Debug("found endpoint", "provider", ep.label, "ip", address, "hostname", *endpoint.Hostname)
localEndpoints = append(localEndpoints, address)
}
}
}
return localEndpoints, nil
}
func (ep *Endpointslices) UpdateServiceAnnotation(ctx context.Context, endpoint, endpointIPv6 string, service *v1.Service, clientSet *kubernetes.Clientset) error {
func (ep *Endpointslices) UpdateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, clientSet *kubernetes.Clientset) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(ctx, service.Name, metav1.GetOptions{})
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
@@ -146,7 +115,7 @@ func (ep *Endpointslices) UpdateServiceAnnotation(ctx context.Context, endpoint,
currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
currentServiceCopy.Annotations[kubevip.ActiveEndpointIPv6] = endpointIPv6
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(ctx, currentServiceCopy, metav1.UpdateOptions{})
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("error updating Service Spec", "provider", ep.label, "service name", currentServiceCopy.Name, "err", err)
return err
@@ -165,15 +134,6 @@ func (ep *Endpointslices) GetLabel() string {
return ep.label
}
func (ep *Endpointslices) ResolvePort(servicePort v1.ServicePort) int32 {
return ResolvePortWithLookup(servicePort, func(name string) int32 {
for _, eps := range ep.slices {
for _, p := range eps.Ports {
if p.Name != nil && *p.Name == name && p.Port != nil {
return *p.Port
}
}
}
return 0
})
func (ep *Endpointslices) GetProtocol() string {
return string(ep.endpoints.AddressType)
}

View File

@@ -1,149 +0,0 @@
package providers
import (
"context"
"testing"
"github.com/kube-vip/kube-vip/pkg/kubevip"
discoveryv1 "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestEndpointslicesTracksAndDeletesSlices(t *testing.T) {
provider := NewEndpointslices().(*Endpointslices)
serving := true
nodeName := "node-1"
slice1 := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{Name: "slice-1"},
AddressType: discoveryv1.AddressTypeIPv4,
Endpoints: []discoveryv1.Endpoint{{
Addresses: []string{"10.0.0.1"},
Conditions: discoveryv1.EndpointConditions{Serving: &serving},
NodeName: &nodeName,
}},
}
slice2 := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{Name: "slice-2"},
AddressType: discoveryv1.AddressTypeIPv4,
Endpoints: []discoveryv1.Endpoint{{
Addresses: []string{"10.0.0.2"},
Conditions: discoveryv1.EndpointConditions{Serving: &serving},
NodeName: &nodeName,
}},
}
for _, slice := range []*discoveryv1.EndpointSlice{slice1, slice2} {
if err := provider.LoadObject(slice, func() {}); err != nil {
t.Fatalf("LoadObject returned error: %v", err)
}
}
assertEndpoints(t, provider, []string{"10.0.0.1", "10.0.0.2"})
assertLocalEndpoints(t, provider, nodeName, []string{"10.0.0.1", "10.0.0.2"})
if err := provider.DeleteObject(slice1); err != nil {
t.Fatalf("DeleteObject returned error: %v", err)
}
assertEndpoints(t, provider, []string{"10.0.0.2"})
assertLocalEndpoints(t, provider, nodeName, []string{"10.0.0.2"})
if err := provider.DeleteObject(slice2); err != nil {
t.Fatalf("DeleteObject returned error: %v", err)
}
assertEndpoints(t, provider, nil)
assertLocalEndpoints(t, provider, nodeName, nil)
}
func TestEndpointslicesReplacingSliceUpdatesState(t *testing.T) {
provider := NewEndpointslices().(*Endpointslices)
first := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{Name: "slice-1"},
AddressType: discoveryv1.AddressTypeIPv4,
Endpoints: []discoveryv1.Endpoint{{Addresses: []string{"10.0.0.1"}}},
}
replacement := first.DeepCopy()
replacement.Endpoints[0].Addresses = []string{"10.0.0.2"}
if err := provider.LoadObject(first, context.CancelFunc(func() {})); err != nil {
t.Fatalf("LoadObject returned error: %v", err)
}
if err := provider.LoadObject(replacement, context.CancelFunc(func() {})); err != nil {
t.Fatalf("LoadObject returned error: %v", err)
}
assertEndpoints(t, provider, []string{"10.0.0.2"})
}
func TestEndpointslicesEndpointConditions(t *testing.T) {
yes, no := true, false
nodeName := "node-1"
tests := []struct {
name string
conditions discoveryv1.EndpointConditions
want []string
}{
{"serving true", discoveryv1.EndpointConditions{Serving: &yes}, []string{"10.0.0.1"}},
{"serving false", discoveryv1.EndpointConditions{Serving: &no}, nil},
{"serving false overrides ready true", discoveryv1.EndpointConditions{Serving: &no, Ready: &yes}, nil},
{"nil serving defers to ready true", discoveryv1.EndpointConditions{Ready: &yes}, []string{"10.0.0.1"}},
{"nil serving defers to ready false", discoveryv1.EndpointConditions{Ready: &no}, nil},
{"both nil is treated as ready", discoveryv1.EndpointConditions{}, []string{"10.0.0.1"}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
provider := NewEndpointslices().(*Endpointslices)
slice := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{Name: "slice-1"},
AddressType: discoveryv1.AddressTypeIPv4,
Endpoints: []discoveryv1.Endpoint{{
Addresses: []string{"10.0.0.1"},
Conditions: test.conditions,
NodeName: &nodeName,
}},
}
if err := provider.LoadObject(slice, func() {}); err != nil {
t.Fatalf("LoadObject returned error: %v", err)
}
// Cluster and Local policy have to agree on which endpoints are usable.
assertEndpoints(t, provider, test.want)
assertLocalEndpoints(t, provider, nodeName, test.want)
})
}
}
func assertEndpoints(t *testing.T, provider *Endpointslices, want []string) {
t.Helper()
got, err := provider.GetAllEndpoints()
if err != nil {
t.Fatalf("GetAllEndpoints returned error: %v", err)
}
assertStringSet(t, got, want)
}
func assertLocalEndpoints(t *testing.T, provider *Endpointslices, nodeName string, want []string) {
t.Helper()
got, err := provider.GetLocalEndpoints(nodeName, &kubevip.Config{})
if err != nil {
t.Fatalf("GetLocalEndpoints returned error: %v", err)
}
assertStringSet(t, got, want)
}
func assertStringSet(t *testing.T, got, want []string) {
t.Helper()
counts := map[string]int{}
for _, value := range got {
counts[value]++
}
for _, value := range want {
counts[value]--
}
for value, count := range counts {
if count != 0 {
t.Fatalf("endpoint set mismatch for %q: got %v, want %v", value, got, want)
}
}
}

View File

@@ -16,25 +16,7 @@ type Provider interface {
GetAllEndpoints() ([]string, error)
GetLocalEndpoints(string, *kubevip.Config) ([]string, error)
GetLabel() string
UpdateServiceAnnotation(context.Context, string, string, *v1.Service, *kubernetes.Clientset) error
UpdateServiceAnnotation(string, string, *v1.Service, *kubernetes.Clientset) error
LoadObject(runtime.Object, context.CancelFunc) error
DeleteObject(runtime.Object) error
// ResolvePort resolves a service port to the actual target port.
// For named ports, it looks up the port number from the endpoint.
// For numeric ports, it returns the port as-is.
ResolvePort(servicePort v1.ServicePort) int32
}
// ResolvePortWithLookup is a helper that resolves a service port using a lookup function
// for named ports. This consolidates the common resolution logic.
func ResolvePortWithLookup(servicePort v1.ServicePort, lookupNamedPort func(string) int32) int32 {
if servicePort.TargetPort.IntVal != 0 {
return servicePort.TargetPort.IntVal
}
if servicePort.TargetPort.StrVal != "" {
if port := lookupNamedPort(servicePort.TargetPort.StrVal); port != 0 {
return port
}
}
return servicePort.Port
GetProtocol() string
}

View File

@@ -1,319 +0,0 @@
package providers
import (
"net"
"reflect"
"testing"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/kubernetes/fake"
)
func TestEndpointProvidersParityForLocalAndAllEndpoints(t *testing.T) {
t.Parallel()
nodeA := "node-a"
nodeB := "node-b"
serving := true
v4Addresses := []discoveryv1.Endpoint{
{Addresses: []string{"10.0.0.1"}, NodeName: &nodeA, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
{Addresses: []string{"10.0.0.2"}, NodeName: &nodeB, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
}
v6Addresses := []discoveryv1.Endpoint{
{Addresses: []string{"2001:db8::1"}, NodeName: &nodeA, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
{Addresses: []string{"2001:db8::2"}, NodeName: &nodeB, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
}
legacy := NewEndpoints()
//nolint:staticcheck // this test covers the deprecated legacy Endpoints provider on purpose
if err := legacy.LoadObject(&v1.Endpoints{
ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "default"},
//nolint:staticcheck // deprecated legacy Endpoints API is the subject under test
Subsets: []v1.EndpointSubset{{
Addresses: []v1.EndpointAddress{
{IP: "10.0.0.1", NodeName: &nodeA},
{IP: "10.0.0.2", NodeName: &nodeB},
{IP: "2001:db8::1", NodeName: &nodeA},
{IP: "2001:db8::2", NodeName: &nodeB},
},
}},
}, func() {}); err != nil {
t.Fatalf("loading legacy Endpoints: %v", err)
}
slices := NewEndpointslices()
if err := slices.LoadObject(&discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{Name: "service-v4", Namespace: "default"},
AddressType: discoveryv1.AddressTypeIPv4,
Endpoints: v4Addresses,
}, func() {}); err != nil {
t.Fatalf("loading IPv4 EndpointSlice: %v", err)
}
if err := slices.LoadObject(&discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{Name: "service-v6", Namespace: "default"},
AddressType: discoveryv1.AddressTypeIPv6,
Endpoints: v6Addresses,
}, func() {}); err != nil {
t.Fatalf("loading IPv6 EndpointSlice: %v", err)
}
legacyAll, err := legacy.GetAllEndpoints()
if err != nil {
t.Fatalf("legacy GetAllEndpoints() error = %v", err)
}
sliceAll, err := slices.GetAllEndpoints()
if err != nil {
t.Fatalf("EndpointSlice GetAllEndpoints() error = %v", err)
}
wantAll := endpointSet([]string{"10.0.0.1", "10.0.0.2", "2001:db8::1", "2001:db8::2"})
if got := endpointSet(legacyAll); !reflect.DeepEqual(got, wantAll) {
t.Errorf("legacy all endpoints = %v, want %v", got, wantAll)
}
if got := endpointSet(sliceAll); !reflect.DeepEqual(got, wantAll) {
t.Errorf("EndpointSlice all endpoints = %v, want %v", got, wantAll)
}
legacyLocal, err := legacy.GetLocalEndpoints(nodeA, &kubevip.Config{})
if err != nil {
t.Fatalf("legacy GetLocalEndpoints() error = %v", err)
}
sliceLocal, err := slices.GetLocalEndpoints(nodeA, &kubevip.Config{})
if err != nil {
t.Fatalf("EndpointSlice GetLocalEndpoints() error = %v", err)
}
wantLocal := endpointSet([]string{"10.0.0.1", "2001:db8::1"})
if got := endpointSet(legacyLocal); !reflect.DeepEqual(got, wantLocal) {
t.Errorf("legacy local endpoints = %v, want %v", got, wantLocal)
}
if got := endpointSet(sliceLocal); !reflect.DeepEqual(got, wantLocal) {
t.Errorf("EndpointSlice local endpoints = %v, want %v", got, wantLocal)
}
assertEndpointFamilies(t, legacyAll, 2, 2)
assertEndpointFamilies(t, sliceAll, 2, 2)
}
func TestEndpointSlicesLocalFilteringRequiresServingEndpoint(t *testing.T) {
t.Parallel()
node := "node-a"
serving := true
notServing := false
provider := NewEndpointslices()
if err := provider.LoadObject(&discoveryv1.EndpointSlice{
AddressType: discoveryv1.AddressTypeIPv4,
Endpoints: []discoveryv1.Endpoint{
{Addresses: []string{"10.0.0.1"}, NodeName: &node, Conditions: discoveryv1.EndpointConditions{Serving: &serving}},
{Addresses: []string{"10.0.0.2"}, NodeName: &node, Conditions: discoveryv1.EndpointConditions{Serving: &notServing}},
},
}, func() {}); err != nil {
t.Fatalf("loading EndpointSlice: %v", err)
}
local, err := provider.GetLocalEndpoints(node, &kubevip.Config{})
if err != nil {
t.Fatalf("GetLocalEndpoints() error = %v", err)
}
if got, want := endpointSet(local), endpointSet([]string{"10.0.0.1"}); !reflect.DeepEqual(got, want) {
t.Errorf("local endpoints = %v, want serving endpoints %v", got, want)
}
}
func TestResolvePortFromFakeClientObjects(t *testing.T) {
t.Parallel()
//nolint:staticcheck // deprecated legacy Endpoints API is the subject under test
legacyObject := &v1.Endpoints{
ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "default"},
//nolint:staticcheck // deprecated legacy Endpoints API is the subject under test
Subsets: []v1.EndpointSubset{{
Ports: []v1.EndpointPort{{Name: "web", Port: 8080}},
}},
}
legacyClient := fake.NewSimpleClientset(legacyObject)
legacyLoaded, err := legacyClient.CoreV1().Endpoints("default").Get(t.Context(), "service", metav1.GetOptions{})
if err != nil {
t.Fatalf("getting fake legacy Endpoints: %v", err)
}
legacy := NewEndpoints()
if err := legacy.LoadObject(legacyLoaded, func() {}); err != nil {
t.Fatalf("loading fake legacy Endpoints: %v", err)
}
portName := "web"
port := int32(8081)
sliceObject := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{Name: "service-slice", Namespace: "default"},
AddressType: discoveryv1.AddressTypeIPv4,
Ports: []discoveryv1.EndpointPort{{Name: &portName, Port: &port}},
}
sliceClient := fake.NewSimpleClientset(sliceObject)
sliceLoaded, err := sliceClient.DiscoveryV1().EndpointSlices("default").Get(t.Context(), "service-slice", metav1.GetOptions{})
if err != nil {
t.Fatalf("getting fake EndpointSlice: %v", err)
}
slices := NewEndpointslices()
if err := slices.LoadObject(sliceLoaded, func() {}); err != nil {
t.Fatalf("loading fake EndpointSlice: %v", err)
}
namedPort := v1.ServicePort{Port: 80, TargetPort: intstr.FromString("web")}
if got := legacy.ResolvePort(namedPort); got != 8080 {
t.Errorf("legacy ResolvePort() = %d, want 8080", got)
}
if got := slices.ResolvePort(namedPort); got != 8081 {
t.Errorf("EndpointSlice ResolvePort() = %d, want 8081", got)
}
}
func TestResolvePortWithLookup(t *testing.T) {
t.Parallel()
tests := []struct {
name string
port v1.ServicePort
lookup func(string) int32
want int32
}{
{
name: "numeric target port wins",
port: v1.ServicePort{Port: 80, TargetPort: intstr.FromInt(8080)},
lookup: func(string) int32 { return 9090 },
want: 8080,
},
{
name: "named target port is looked up",
port: v1.ServicePort{Port: 80, TargetPort: intstr.FromString("web")},
lookup: func(name string) int32 {
if name == "web" {
return 8081
}
return 0
},
want: 8081,
},
{
name: "missing named target falls back to service port",
port: v1.ServicePort{Port: 80, TargetPort: intstr.FromString("missing")},
lookup: func(string) int32 { return 0 },
want: 80,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := ResolvePortWithLookup(tt.port, tt.lookup); got != tt.want {
t.Errorf("ResolvePortWithLookup() = %d, want %d", got, tt.want)
}
})
}
}
func endpointSet(endpoints []string) map[string]struct{} {
result := make(map[string]struct{}, len(endpoints))
for _, endpoint := range endpoints {
result[endpoint] = struct{}{}
}
return result
}
func assertEndpointFamilies(t *testing.T, endpoints []string, wantIPv4, wantIPv6 int) {
t.Helper()
ipv4, ipv6 := 0, 0
for _, endpoint := range endpoints {
ip := net.ParseIP(endpoint)
if ip == nil {
t.Errorf("endpoint %q is not an IP address", endpoint)
continue
}
if ip.To4() != nil {
ipv4++
} else {
ipv6++
}
}
if ipv4 != wantIPv4 || ipv6 != wantIPv6 {
t.Errorf("endpoint families = IPv4 %d, IPv6 %d; want IPv4 %d, IPv6 %d", ipv4, ipv6, wantIPv4, wantIPv6)
}
}
func TestEndpointProvidersPreferNodeNameOverHostname(t *testing.T) {
t.Parallel()
nodeA := "node-a"
nodeB := "node-b"
serving := true
tests := []struct {
name string
load func(Provider) error
}{
{
name: "legacy Endpoints",
load: func(provider Provider) error {
//nolint:staticcheck // the legacy provider is deliberately under test
return provider.LoadObject(&v1.Endpoints{
Subsets: []v1.EndpointSubset{{
Addresses: []v1.EndpointAddress{{
IP: "10.0.0.1",
NodeName: &nodeB,
Hostname: nodeA,
}},
}},
}, func() {})
},
},
{
name: "EndpointSlice",
load: func(provider Provider) error {
hostname := nodeA
return provider.LoadObject(&discoveryv1.EndpointSlice{
AddressType: discoveryv1.AddressTypeIPv4,
Endpoints: []discoveryv1.Endpoint{{
Addresses: []string{"10.0.0.1"},
NodeName: &nodeB,
Hostname: &hostname,
Conditions: discoveryv1.EndpointConditions{Serving: &serving},
}},
}, func() {})
},
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
var provider Provider
if tt.name == "legacy Endpoints" {
provider = NewEndpoints()
} else {
provider = NewEndpointslices()
}
if err := tt.load(provider); err != nil {
t.Fatalf("LoadObject() error = %v", err)
}
local, err := provider.GetLocalEndpoints(nodeA, &kubevip.Config{})
if err != nil {
t.Fatalf("GetLocalEndpoints(%q) error = %v", nodeA, err)
}
if len(local) != 0 {
t.Fatalf("GetLocalEndpoints(%q) = %v, want no endpoints", nodeA, local)
}
local, err = provider.GetLocalEndpoints(nodeB, &kubevip.Config{})
if err != nil {
t.Fatalf("GetLocalEndpoints(%q) error = %v", nodeB, err)
}
if got, want := endpointSet(local), endpointSet([]string{"10.0.0.1"}); !reflect.DeepEqual(got, want) {
t.Fatalf("GetLocalEndpoints(%q) = %v, want %v", nodeB, got, want)
}
})
}
}

View File

@@ -2,9 +2,7 @@ package etcd
import (
"context"
"fmt"
"hash/fnv"
"sync"
"time"
log "log/slog"
@@ -66,11 +64,10 @@ type ClientConfig struct {
}
// RunElectionOrDie behaves the same way as RunElection but panics if there is an error.
func RunElectionOrDie(ctx context.Context, config *LeaderElectionConfig) error {
func RunElectionOrDie(ctx context.Context, config *LeaderElectionConfig) {
if err := RunElection(ctx, config); err != nil {
return fmt.Errorf("leaderelection error: %w", err)
panic(err)
}
return nil
}
// RunElection starts a client with the provided config or panics.
@@ -119,12 +116,7 @@ func RunElection(ctx context.Context, config *LeaderElectionConfig) error {
leaseTTL: lease.TTL,
}
wg := sync.WaitGroup{}
defer wg.Wait()
wg.Go(func() {
m.tryToBeLeader(ctx, &wg)
})
go m.tryToBeLeader(ctx)
m.watchLeaderChanges(ctx)
return nil
@@ -193,7 +185,7 @@ watcher:
log.Debug("Exiting watcher", "id", m.memberID)
}
func (m *member) tryToBeLeader(ctx context.Context, wg *sync.WaitGroup) {
func (m *member) tryToBeLeader(ctx context.Context) {
if err := m.election.Campaign(ctx, m.memberID); err != nil {
log.Error("Failed trying to become the leader", "err", err)
// Resign just in case we acquired leadership just before failing
@@ -212,9 +204,7 @@ func (m *member) tryToBeLeader(ctx context.Context, wg *sync.WaitGroup) {
m.weAreTheLeader <- struct{}{}
// Once we are the leader, start the routine to resign if context is canceled
wg.Go(func() {
m.resignOnCancel(ctx)
})
go m.resignOnCancel(ctx)
// After becoming the leader, we wait for at least a lease TTL to wait for
// the previous leader to detect the new leadership (if there was one) and
@@ -231,7 +221,7 @@ func (m *member) tryToBeLeader(ctx context.Context, wg *sync.WaitGroup) {
func (m *member) resignOnCancel(ctx context.Context) {
<-ctx.Done()
if err := m.election.Resign(m.client.Ctx()); err != nil && !errors.Is(err, context.Canceled) {
if err := m.election.Resign(m.client.Ctx()); err != nil {
log.Error("Failed to resign after the context was canceled", "err", err)
}
}

View File

@@ -20,26 +20,19 @@ import (
func TestRunElectionWithMemberIDCollision(t *testing.T) {
t.Parallel()
g := NewWithT(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ctx := context.Background()
cli := client(g)
defer cli.Close()
electionName := randomElectionNameForTest("memberIDConflict")
log.Printf("Election name %s\n", electionName)
memberCtx, cancelMember1 := context.WithCancel(ctx)
// Use a channel to signal when the first member has observed a new leader
// This ensures proper ordering without relying on sleep timing
firstMemberObservedLeader := make(chan struct{})
var firstMemberObservedOnce sync.Once
config := &etcd.LeaderElectionConfig{
EtcdConfig: etcd.ClientConfig{
Client: cli,
},
Name: electionName,
MemberID: randomElectionNameForTest("my-host"),
MemberID: "my-host",
LeaseDurationSeconds: 1,
Callbacks: etcd.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
@@ -49,11 +42,6 @@ func TestRunElectionWithMemberIDCollision(t *testing.T) {
},
OnNewLeader: func(identity string) {
log.Printf("New leader: %s\n", identity)
// Signal that the first member has observed a leader
// This means the lease has been created
firstMemberObservedOnce.Do(func() {
close(firstMemberObservedLeader)
})
},
OnStoppedLeading: func() {
log.Println("I'm not the leader anymore")
@@ -71,18 +59,8 @@ func TestRunElectionWithMemberIDCollision(t *testing.T) {
go func() {
defer wg.Done()
// Wait for the first member to observe a leader, which means the lease has been created
select {
case <-firstMemberObservedLeader:
// First member has created the lease, now try to create a conflicting one
case <-time.After(5 * time.Second):
t.Error("timeout waiting for first member to observe leader")
return
}
// Use a cancellable context to prevent hanging if this goroutine unexpectedly succeeds
member2Ctx, cancelMember2 := context.WithTimeout(ctx, 5*time.Second)
defer cancelMember2()
g.Expect(etcd.RunElection(member2Ctx, config)).Should(MatchError(ContainSubstring("creating lease")))
time.Sleep(time.Millisecond * 50) // make sure the first one becomes leader
g.Expect(etcd.RunElection(ctx, config)).Should(MatchError(ContainSubstring("creating lease")))
}()
wg.Wait()
@@ -91,8 +69,7 @@ func TestRunElectionWithMemberIDCollision(t *testing.T) {
func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
t.Parallel()
g := NewWithT(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ctx := context.Background()
cli := client(g)
defer cli.Close()
@@ -108,27 +85,24 @@ func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
LeaseDurationSeconds: 1,
}
member1Ctx, cancelMember1 := context.WithCancel(ctx)
member1Ctx, _ := context.WithCancel(ctx)
member2Ctx, cancelMember2 := context.WithCancel(ctx)
config1 := configBase
config1.EtcdConfig.Client = cliMember1
config1.MemberID = randomElectionNameForTest("my-host")
config1.MemberID = "my-host"
uniqueID := rand.Uint64()
config1.MemberUniqueID = &uniqueID
config1.Callbacks = baseCallbacksForName(config1.MemberID)
syncMembers := make(chan (any))
config1.Callbacks.OnStartedLeading = func(_ context.Context) {
log.Println("I'm my-host, the new leader!!!!")
close(syncMembers)
log.Println("Losing the leadership on purpose by stopping renewing the lease")
g.Expect(cliMember1.Lease.Close()).To(Succeed())
log.Println("Member1 leases closed")
cancelMember1()
}
config2 := configBase
config2.MemberID = randomElectionNameForTest("my-other-host")
config2.MemberID = "my-other-host"
config2.Callbacks = baseCallbacksForName(config2.MemberID)
config2.Callbacks.OnStartedLeading = func(_ context.Context) {
log.Println("I'm my-other-host, the new leader!!!!")
@@ -142,18 +116,17 @@ func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
go func() {
defer wg.Done()
g.Expect(etcd.RunElection(member1Ctx, &config1)).To(Succeed())
log.Printf("%s routine done\n", config1.MemberID)
log.Println("Member1 routine done")
}()
go func() {
defer wg.Done()
<-syncMembers
time.Sleep(time.Millisecond * 50) // Make sure member1 becomes leader
g.Expect(etcd.RunElection(member2Ctx, &config2)).To(Succeed())
log.Printf("%s routine done\n", config2.MemberID)
log.Println("Member2 routine done")
}()
wg.Wait()
}
func baseCallbacksForName(name string) etcd.LeaderCallbacks {

View File

@@ -15,6 +15,7 @@ import (
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
const (
@@ -27,8 +28,8 @@ const (
)
func TestMain(m *testing.M) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
logrus.SetLevel(logrus.DebugLevel)
ctx := context.Background()
expectSuccess(startEtcd(ctx), "starting etcd")
os.Exit(runTestsWithCleanup(m, func() {

View File

@@ -1,30 +1,22 @@
package instance
import (
"context"
"errors"
"fmt"
"net"
"slices"
"strconv"
"strings"
"sync"
"sync/atomic"
"log/slog"
log "log/slog"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/node"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/sysctl"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
@@ -37,82 +29,18 @@ type Instance struct {
Clusters []*cluster.Cluster
// Service uses DHCP
IsDHCPv4 bool
IsDHCPv6 bool
IsDHCP bool
DHCPInterface string
DHCPInterfaceHwaddr string
DHCPInterfaceIP string
DHCPInterfaceIPv4 string
DHCPInterfaceIPv6 string
DHCPHostname string
DHCPv4Client vip.DHCPClient
DHCPv6Client vip.DHCPClient
macvlanName string
dhcpBroadcast bool
dhcpInterfaceOwned atomic.Bool
// Service use Vlan
IsVLAN bool
VLANInterface string
vlanOwned atomic.Bool
DHCPClient *vip.DHCPClient
// External Gateway IP the service is forwarded from
UPNPGatewayIPs []string
// Kubernetes service mapping
ServiceUID types.UID
ServiceAddresses []string
ServiceSnapshot *v1.Service
cleanupInfo *ServiceCleanupInfo
// AddCalled determined that ActionAdd was already performed for the instance
AddCalled bool
// LabelAdded determined that node was labeled with
// service-provided.kube-vip.io label
LabelAdded bool
}
func (instance *Instance) UID() types.UID {
return instance.ServiceUID
}
func (instance *Instance) Addresses() []string {
if instance.ServiceAddresses != nil {
return append([]string(nil), instance.ServiceAddresses...)
}
addresses, _ := FetchServiceAddresses(instance.ServiceSnapshot)
return addresses
}
type ServiceCleanupInfo struct {
Namespace string
Name string
Lease string
ExternalTrafficPolicy v1.ServiceExternalTrafficPolicy
}
// CleanupInfo returns Service policy captured when the instance was created.
func (instance *Instance) CleanupInfo() (ServiceCleanupInfo, bool) {
if instance == nil {
return ServiceCleanupInfo{}, false
}
if instance.cleanupInfo != nil {
return *instance.cleanupInfo, true
}
if instance.ServiceSnapshot == nil {
return ServiceCleanupInfo{}, false
}
return serviceCleanupInfo(instance.ServiceSnapshot), true
}
func serviceCleanupInfo(service *v1.Service) ServiceCleanupInfo {
return ServiceCleanupInfo{
Namespace: service.Namespace,
Name: service.Name,
Lease: service.Annotations[kubevip.ServiceLease],
ExternalTrafficPolicy: service.Spec.ExternalTrafficPolicy,
}
ServiceSnapshot *v1.Service
}
type Port struct {
@@ -120,28 +48,10 @@ type Port struct {
Type string
}
func NewInstance(ctx context.Context, svc *v1.Service, config *kubevip.Config,
intfMgr *networkinterface.Manager, arpMgr *arp.Manager, routeMgr *route.Manager,
nodeLabelMgr node.Labeler, wg *sync.WaitGroup) (*Instance, error) {
instanceAddresses, instanceHostnames := FetchServiceAddresses(svc)
log.Info("new instance", "namespace", svc.Namespace, "service", svc.Name, "addresses", instanceAddresses, "hostnames", instanceHostnames)
func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Instance, error) {
instanceAddresses := FetchServiceAddresses(svc)
//instanceUID := string(svc.UID)
cleanupInfo := serviceCleanupInfo(svc)
instance := &Instance{
ServiceUID: svc.UID,
ServiceAddresses: append([]string(nil), instanceAddresses...),
ServiceSnapshot: svc,
cleanupInfo: &cleanupInfo,
}
if err := instance.initialize(ctx, svc, config, intfMgr, arpMgr, routeMgr, nodeLabelMgr, wg, instanceAddresses, instanceHostnames); err != nil {
return nil, errors.Join(err, instance.CleanupLinkAttachments())
}
return instance, nil
}
func (instance *Instance) initialize(ctx context.Context, svc *v1.Service, config *kubevip.Config,
intfMgr *networkinterface.Manager, arpMgr *arp.Manager, routeMgr *route.Manager,
nodeLabelMgr node.Labeler, wg *sync.WaitGroup, instanceAddresses, instanceHostnames []string) error {
var newVips []*kubevip.Config
var link netlink.Link
var err error
@@ -149,22 +59,7 @@ func (instance *Instance) initialize(ctx context.Context, svc *v1.Service, confi
for _, address := range instanceAddresses {
// Detect if we're using a specific interface for services
var svcInterface string
svcInterface = svc.Annotations[kubevip.ServiceVlan]
if svcInterface != "" {
parent, tag, err := utils.ParseVLANInterface(svcInterface)
if err != nil {
log.Error("failed to validate VLAN", "err", err)
}
if err := instance.addVLAN(parent, tag); err != nil {
log.Error("failed to create VLAN", "err", err)
}
} else {
// If no vlan defined use specific interface from annotation
svcInterface = svc.Annotations[kubevip.ServiceInterface]
}
svcInterface = svc.Annotations[kubevip.ServiceInterface] // If the service has a specific interface defined, then use it
if svcInterface == kubevip.Auto {
link, err = autoFindInterface(address)
if err != nil {
@@ -200,10 +95,10 @@ func (instance *Instance) initialize(ctx context.Context, svc *v1.Service, confi
if link == nil {
if link, err = netlink.LinkByName(svcInterface); err != nil {
return fmt.Errorf("failed to get interface %s: %w", svcInterface, err)
return nil, fmt.Errorf("failed to get interface %s: %w", svcInterface, err)
}
if link == nil {
return fmt.Errorf("failed to get interface %s", svcInterface)
return nil, fmt.Errorf("failed to get interface %s", svcInterface)
}
}
@@ -220,35 +115,35 @@ func (instance *Instance) initialize(ctx context.Context, svc *v1.Service, confi
}
if (config.Address != "" || config.VIP != "") && (ipv4AutoSubnet || ipv6AutoSubnet) {
return fmt.Errorf("auto subnet discovery cannot be used if VIP address was provided")
return nil, fmt.Errorf("auto subnet discovery cannot be used if VIP address was provided")
}
subnet := ""
var err error
if utils.IsIPv4(address) {
if vip.IsIPv4(address) {
if ipv4AutoSubnet {
subnet, err = autoFindSubnet(link, address)
if err != nil {
return fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
return nil, fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
}
} else {
if cidrs[0] != "" && cidrs[0] != kubevip.Auto {
subnet = cidrs[0]
} else {
subnet = strconv.Itoa(vip.DefaultMaskIPv4)
subnet = "32"
}
}
} else {
if ipv6AutoSubnet {
subnet, err = autoFindSubnet(link, address)
if err != nil {
return fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
return nil, fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
}
} else {
if len(cidrs) > 1 && cidrs[1] != "" && cidrs[1] != kubevip.Auto {
subnet = cidrs[1]
} else {
subnet = strconv.Itoa(vip.DefaultMaskIPv6)
subnet = "128"
}
}
}
@@ -260,85 +155,14 @@ func (instance *Instance) initialize(ctx context.Context, svc *v1.Service, confi
SingleNode: true,
EnableARP: config.EnableARP,
EnableBGP: config.EnableBGP,
BGPAttachIPToInterface: config.BGPAttachIPToInterface,
VIPSubnet: subnet,
EnableRoutingTable: config.EnableRoutingTable,
RoutingTableID: config.RoutingTableID,
RoutingTableType: config.RoutingTableType,
RoutingProtocol: config.RoutingProtocol,
SkipDAD: config.SkipDAD,
ArpBroadcastRate: config.ArpBroadcastRate,
EnableServiceSecurity: config.EnableServiceSecurity,
DNSMode: config.DNSMode,
DHCPMode: config.DHCPMode,
DHCPBackoffAttempts: config.DHCPBackoffAttempts,
DisableServiceUpdates: config.DisableServiceUpdates,
EnableServicesElection: config.EnableServicesElection,
// cleanupVIPs reads this from the per-VIP config, so Service VIPs need it too.
PreserveVIPOnLeadershipLoss: config.PreserveVIPOnLeadershipLoss,
KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
EnableLeaderElection: config.EnableLeaderElection,
},
})
}
for _, hostname := range instanceHostnames {
log.Info("hostname", "addr", hostname)
// Detect if we're using a specific interface for services
var svcInterface string
svcInterface = svc.Annotations[kubevip.ServiceVlan]
if svcInterface != "" {
parent, tag, err := utils.ParseVLANInterface(svcInterface)
if err != nil {
log.Error("failed to validate VLAN", "err", err)
}
if err := instance.addVLAN(parent, tag); err != nil {
log.Error("failed to create VLAN", "err", err)
}
} else {
// If no vlan defined use specific interface from annotation
svcInterface = svc.Annotations[kubevip.ServiceInterface]
}
// If it is still blank then use the
if svcInterface == "" {
switch config.ServicesInterface {
case "":
svcInterface = config.Interface
default:
svcInterface = config.ServicesInterface
}
}
if link == nil {
if link, err = netlink.LinkByName(svcInterface); err != nil {
return fmt.Errorf("failed to get interface %s: %w", svcInterface, err)
}
if link == nil {
return fmt.Errorf("failed to get interface %s", svcInterface)
}
}
// Generate new Virtual IP configuration
newVips = append(newVips, &kubevip.Config{
VIP: hostname,
Interface: svcInterface,
SingleNode: true,
EnableARP: config.EnableARP,
EnableBGP: config.EnableBGP,
BGPAttachIPToInterface: config.BGPAttachIPToInterface,
VIPSubnet: config.VIPSubnet,
EnableRoutingTable: config.EnableRoutingTable,
RoutingTableID: config.RoutingTableID,
RoutingTableType: config.RoutingTableType,
RoutingProtocol: config.RoutingProtocol,
SkipDAD: config.SkipDAD,
ArpBroadcastRate: config.ArpBroadcastRate,
EnableServiceSecurity: config.EnableServiceSecurity,
DNSMode: config.DNSMode,
DHCPMode: config.DHCPMode,
DisableServiceUpdates: config.DisableServiceUpdates,
EnableServicesElection: config.EnableServicesElection,
KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
@@ -347,26 +171,23 @@ func (instance *Instance) initialize(ctx context.Context, svc *v1.Service, confi
})
}
// Create new service
instance := &Instance{
//UID: instanceUID,
//VIPs: instanceAddresses,
ServiceSnapshot: svc,
}
// for _, port := range svc.Spec.Ports {
// instance.ExternalPorts = append(instance.ExternalPorts, Port{
// Port: uint16(port.Port), //nolint
// Type: string(port.Protocol),
// })
// }
if svc.Annotations != nil {
instance.DHCPInterfaceHwaddr = svc.Annotations[kubevip.HwAddrKey]
requestedIP := svc.Annotations[kubevip.RequestedIP]
if requestedIP != "" {
requestedIPs := strings.Split(requestedIP, ",")
if len(requestedIPs) > 2 {
return fmt.Errorf("annotation %q cannot request more than one IPv4 and one Ipv6 address", kubevip.RequestedIP)
}
for _, ip := range requestedIPs {
netip := net.ParseIP(ip)
if netip.To4() != nil {
instance.DHCPInterfaceIPv4 = ip
} else {
instance.DHCPInterfaceIPv6 = ip
}
}
}
instance.DHCPInterfaceIP = svc.Annotations[kubevip.RequestedIP]
instance.DHCPHostname = svc.Annotations[kubevip.LoadbalancerHostname]
instance.macvlanName = svc.Annotations[kubevip.MacvlanName]
instance.dhcpBroadcast = svc.Annotations[kubevip.DHCPBroadcast] == "true"
}
configPorts := make([]kubevip.Port, 0)
@@ -389,100 +210,41 @@ func (instance *Instance) initialize(ctx context.Context, svc *v1.Service, confi
// Create Add configuration to the new service
instance.VIPConfigs = newVips
// If this was purposely created with the address '0.0.0.0', or '::'
// If this was purposely created with the address 0.0.0.0,
// we will create a macvlan on the main interface and a DHCP client
if len(instanceAddresses) > 2 && (slices.Contains(instanceAddresses, "0.0.0.0") || slices.Contains(instanceAddresses, "::")) {
return fmt.Errorf("DHCP cannot be used if more than 2 addresses (one IPv4 and one IPv6) were specified")
}
for index := range instance.VIPConfigs {
if instance.VIPConfigs[index].VIP == "0.0.0.0" {
err := instance.startDHCP(ctx, index, config.DHCPBackoffAttempts, wg)
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
case err := <-instance.DHCPv4Client.ErrorChannel():
return fmt.Errorf("error starting DHCPv4 for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
case ip := <-instance.DHCPv4Client.IPChannel():
instance.VIPConfigs[index].Interface = instance.DHCPInterface
instance.VIPConfigs[index].VIP = ip
instance.DHCPInterfaceIPv4 = ip
}
}
if instance.VIPConfigs[index].VIP == "::" {
err := instance.startDHCP(ctx, index, config.DHCPBackoffAttempts, wg)
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
case err := <-instance.DHCPv6Client.ErrorChannel():
return fmt.Errorf("error starting DHCPv6 for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
case ip := <-instance.DHCPv6Client.IPChannel():
instance.VIPConfigs[index].Interface = instance.DHCPInterface
instance.VIPConfigs[index].VIP = ip
instance.DHCPInterfaceIPv6 = ip
}
}
ddnsAnnotation, exists := svc.Annotations[kubevip.ServiceDDNS]
if exists {
instance.VIPConfigs[index].DDNS, err = strconv.ParseBool(ddnsAnnotation)
if err != nil {
log.Error("Failed to add service", "err", err)
return err
}
}
if len(svc.Spec.IPFamilies) > 0 {
if len(svc.Spec.IPFamilies) > 1 {
instance.VIPConfigs[index].DHCPMode = utils.DualFamily
instance.VIPConfigs[index].DNSMode = utils.DualFamily
switch *svc.Spec.IPFamilyPolicy {
case v1.IPFamilyPolicyRequireDualStack:
instance.VIPConfigs[index].IsDualStack = true
instance.VIPConfigs[index].RequireDualStack = true
case v1.IPFamilyPolicyPreferDualStack:
instance.VIPConfigs[index].IsDualStack = true
instance.VIPConfigs[index].RequireDualStack = false
default:
instance.VIPConfigs[index].IsDualStack = false
instance.VIPConfigs[index].RequireDualStack = false
}
} else {
if strings.EqualFold(string(svc.Spec.IPFamilies[0]), utils.IPv4Family) {
instance.VIPConfigs[index].DHCPMode = strings.ToLower(utils.IPv4Family)
instance.VIPConfigs[index].DNSMode = strings.ToLower(utils.IPv4Family)
} else {
instance.VIPConfigs[index].DHCPMode = strings.ToLower(utils.IPv6Family)
instance.VIPConfigs[index].DNSMode = strings.ToLower(utils.IPv6Family)
}
}
}
instance.VIPConfigs[index].EgressWithNftables = config.EgressWithNftables
c, err := cluster.InitCluster(instance.VIPConfigs[index], false, intfMgr, arpMgr, routeMgr, nodeLabelMgr)
// TODO: Consider how best to handle DHCP with multiple addresses
if len(instanceAddresses) == 1 && instanceAddresses[0] == "0.0.0.0" {
err := instance.startDHCP()
if err != nil {
log.Error("failed to add service", "err", err)
return err
return nil, err
}
select {
case err := <-instance.DHCPClient.ErrorChannel():
return nil, fmt.Errorf("error starting DHCP for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
case ip := <-instance.DHCPClient.IPChannel():
instance.VIPConfigs[0].Interface = instance.DHCPInterface
instance.VIPConfigs[0].VIP = ip
instance.DHCPInterfaceIP = ip
}
}
for _, vipConfig := range instance.VIPConfigs {
c, err := cluster.InitCluster(vipConfig, false, intfMgr, arpMgr)
if err != nil {
log.Error("Failed to add Service %s/%s", svc.Namespace, svc.Name)
return nil, err
}
for networkIndex := range c.Network {
c.Network[networkIndex].SetServicePorts(svc)
for i := range c.Network {
c.Network[i].SetServicePorts(svc)
}
instance.Clusters = append(instance.Clusters, c)
log.Info("(svcs) adding VIP", "ip", instance.VIPConfigs[index].VIP, "interface", instance.VIPConfigs[index].Interface, "namespace", svc.Namespace, "name", svc.Name)
log.Info("(svcs) adding VIP", "ip", vipConfig.VIP, "interface", vipConfig.Interface, "namespace", svc.Namespace, "name", svc.Name)
}
return nil
return instance, nil
}
func autoFindInterface(ip string) (netlink.Link, error) {
@@ -542,152 +304,25 @@ func getAutoInterfaceName(link netlink.Link, defaultInterface string) string {
return link.Attrs().Name
}
func (instance *Instance) addVLAN(parentInterface string, tag int) error {
interfaceName := fmt.Sprintf("%s.%d", parentInterface, tag)
parent, err := netlink.LinkByName(parentInterface)
if err != nil {
return fmt.Errorf("finding VLAN parent interface %s: %w", parentInterface, err)
func (i *Instance) startDHCP() error {
if len(i.VIPConfigs) != 1 {
return fmt.Errorf("DHCP requires exactly 1 VIP config, got: %v", len(i.VIPConfigs))
}
iface, err := netlink.LinkByName(interfaceName)
if err != nil {
var notFound netlink.LinkNotFoundError
if !errors.As(err, &notFound) {
return fmt.Errorf("finding VLAN interface %s: %w", interfaceName, err)
}
log.Info("Creating new VLAN interface", "interface", interfaceName)
vlan := &netlink.Vlan{
LinkAttrs: netlink.LinkAttrs{
Name: interfaceName,
ParentIndex: parent.Attrs().Index,
},
VlanId: tag,
VlanProtocol: netlink.VLAN_PROTOCOL_8021Q,
}
err = netlink.LinkAdd(vlan)
if err != nil {
return fmt.Errorf("could not add VLAN %s: %v", interfaceName, err)
}
instance.vlanOwned.Store(true)
err = netlink.LinkSetUp(vlan)
if err != nil {
return fmt.Errorf("could not bring up VLAN interface [%s] : %v", interfaceName, err)
}
_, err = net.InterfaceByName(interfaceName)
if err != nil {
return fmt.Errorf("error finding new VLAN interface by name [%v]", err)
}
} else {
log.Info("Using existing VLAN interface", "interface", interfaceName)
if err := utils.ValidateVLANInterface(iface, parent, tag); err != nil {
return err
}
}
instance.VLANInterface = interfaceName
instance.IsVLAN = true
return nil
}
// CleanupLinkAttachments stops this instance's DHCP clients and removes only
// VLAN or macvlan links created by this instance that are not used by a
// remaining Service instance.
func (instance *Instance) CleanupLinkAttachments(remaining ...*Instance) error {
var errs []error
if instance.DHCPv4Client != nil {
instance.DHCPv4Client.Stop()
}
if instance.DHCPv6Client != nil {
instance.DHCPv6Client.Stop()
}
if instance.dhcpInterfaceOwned.Load() {
if transferLinkAttachmentOwnership(instance.DHCPInterface, remaining, false) {
instance.dhcpInterfaceOwned.Store(false)
} else if err := deleteOwnedLink(instance.DHCPInterface, "DHCP"); err != nil {
errs = append(errs, err)
} else {
instance.dhcpInterfaceOwned.Store(false)
}
}
if instance.vlanOwned.Load() {
if transferLinkAttachmentOwnership(instance.VLANInterface, remaining, true) {
instance.vlanOwned.Store(false)
} else if err := deleteOwnedLink(instance.VLANInterface, "VLAN"); err != nil {
errs = append(errs, err)
} else {
instance.vlanOwned.Store(false)
}
}
return errors.Join(errs...)
}
func transferLinkAttachmentOwnership(name string, instances []*Instance, vlan bool) bool {
if name == "" {
return false
}
for _, instance := range instances {
if instance == nil {
continue
}
if vlan && instance.IsVLAN && instance.VLANInterface == name {
instance.vlanOwned.Store(true)
return true
}
if !vlan && (instance.IsDHCPv4 || instance.IsDHCPv6) && instance.DHCPInterface == name {
instance.dhcpInterfaceOwned.Store(true)
return true
}
}
return false
}
func deleteOwnedLink(name, kind string) error {
if name == "" {
return nil
}
link, err := netlink.LinkByName(name)
if err != nil {
var notFound netlink.LinkNotFoundError
if errors.As(err, &notFound) {
return nil
}
return fmt.Errorf("find %s interface %q: %w", kind, name, err)
}
if err := netlink.LinkDel(link); err != nil {
return fmt.Errorf("delete %s interface %q: %w", kind, name, err)
}
return nil
}
func (instance *Instance) startDHCP(ctx context.Context, index int, backoffAttempts uint, wg *sync.WaitGroup) error {
if len(instance.VIPConfigs) > 2 {
return fmt.Errorf("DHCP can be used with 2 VIP config maximally, got: %v", len(instance.VIPConfigs))
}
parent, err := netlink.LinkByName(instance.VIPConfigs[index].Interface)
parent, err := netlink.LinkByName(i.VIPConfigs[0].Interface)
if err != nil {
return fmt.Errorf("error finding VIP Interface, for building DHCP Link : %v", err)
}
interfaceName := instance.macvlanName
if interfaceName == "" {
// Generate name from UID
interfaceName = fmt.Sprintf("vip-%s", instance.UID()[0:8])
}
// Generate name from UID
interfaceName := fmt.Sprintf("vip-%s", i.ServiceSnapshot.UID[0:8])
// Check if the interface doesn't exist first
iface, err := net.InterfaceByName(interfaceName)
if err != nil {
log.Info("creating new macvlan interface for DHCP", "interface", interfaceName)
hwaddr, err := net.ParseMAC(instance.DHCPInterfaceHwaddr)
if instance.DHCPInterfaceHwaddr != "" && err != nil {
hwaddr, err := net.ParseMAC(i.DHCPInterfaceHwaddr)
if i.DHCPInterfaceHwaddr != "" && err != nil {
return err
} else if hwaddr == nil {
hwaddr, err = net.ParseMAC(vip.GenerateMac())
@@ -710,7 +345,6 @@ func (instance *Instance) startDHCP(ctx context.Context, index int, backoffAttem
if err != nil {
return fmt.Errorf("could not add %s: %v", interfaceName, err)
}
instance.dhcpInterfaceOwned.Store(true)
err = netlink.LinkSetUp(mac)
if err != nil {
@@ -725,162 +359,124 @@ func (instance *Instance) startDHCP(ctx context.Context, index int, backoffAttem
log.Info("Using existing macvlan interface for DHCP", "interface", interfaceName)
}
var initRebootFlag bool
ip := net.ParseIP(instance.VIPConfigs[index].VIP)
// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
rpfilterSetting := "0"
var client vip.DHCPClient
if ip.To4() != nil {
// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
rpfilterSetting := "0"
// Check if we need to set an override rp_filter value for the interface
if instance.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
// Check the rp_filter value
rpFilter, err := strconv.Atoi(instance.ServiceSnapshot.Annotations[kubevip.RPFilter])
if err != nil {
log.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
// Check if we need to set an override rp_filter value for the interface
if i.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
// Check the rp_filter value
rpFilter, err := strconv.Atoi(i.ServiceSnapshot.Annotations[kubevip.RPFilter])
if err != nil {
slog.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
} else {
if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
rpfilterSetting = i.ServiceSnapshot.Annotations[kubevip.RPFilter]
} else {
if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
rpfilterSetting = instance.ServiceSnapshot.Annotations[kubevip.RPFilter]
} else {
log.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
}
slog.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
}
}
err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
if err != nil {
log.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
}
if instance.DHCPInterfaceIPv4 != "" {
initRebootFlag = true
}
client = vip.NewDHCPv4Client(iface, initRebootFlag, instance.DHCPInterfaceIPv4, backoffAttempts, instance.dhcpBroadcast)
// Add the client so that we can call it to stop function
instance.DHCPv4Client = client
// Set that DHCPv4 is enabled
instance.IsDHCPv4 = true
} else {
if instance.DHCPInterfaceIPv6 != "" {
initRebootFlag = true
}
client, err = vip.NewDHCPv6Client(iface, parent, initRebootFlag, instance.DHCPInterfaceIPv6, backoffAttempts)
if err != nil {
return fmt.Errorf("unable to create client: %w", err)
}
// Add the client so that we can call it to stop function
instance.DHCPv6Client = client
// Set that DHCPv6 is enabled
instance.IsDHCPv6 = true
}
err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
if err != nil {
slog.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
}
var initRebootFlag bool
if i.DHCPInterfaceIP != "" {
initRebootFlag = true
}
client := vip.NewDHCPClient(iface, initRebootFlag, i.DHCPInterfaceIP)
// Add hostname to dhcp client if annotated
if instance.DHCPHostname != "" {
log.Info("Hostname specified for dhcp lease", "interface", interfaceName, "hostname", instance.DHCPHostname)
client.WithHostName(instance.DHCPHostname)
if i.DHCPHostname != "" {
log.Info("Hostname specified for dhcp lease", "interface", interfaceName, "hostname", i.DHCPHostname)
client.WithHostName(i.DHCPHostname)
}
wg.Go(func() {
if err := client.Start(ctx); err != nil {
log.Error("[instance] DHCP client", "error", err)
client.Stop()
}
})
go client.Start()
// Set that DHCP is enabled
i.IsDHCP = true
// Set the name of the interface so that it can be removed on Service deletion
instance.DHCPInterface = interfaceName
instance.DHCPInterfaceHwaddr = iface.HardwareAddr.String()
i.DHCPInterface = interfaceName
i.DHCPInterfaceHwaddr = iface.HardwareAddr.String()
// Add the client so that we can call it to stop function
i.DHCPClient = client
return nil
}
// FetchLoadBalancerIngressAddresses tries to get the addresses from status.loadBalancerIP
func FetchLoadBalancerIngress(s *v1.Service) ([]string, []string) {
// FetchIngressAddresses tries to get the addresses from status.loadBalancerIP
func FetchLoadBalancerIngressAddresses(s *v1.Service) []string {
// If the service has no status, return empty
lbStatusAddresses := []string{}
lbStatusHostnames := []string{}
if len(s.Status.LoadBalancer.Ingress) == 0 {
return lbStatusAddresses, lbStatusHostnames
return lbStatusAddresses
}
for _, ingress := range s.Status.LoadBalancer.Ingress {
if ingress.IP != "" {
lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
}
if ingress.Hostname != "" {
lbStatusHostnames = append(lbStatusHostnames, ingress.Hostname)
}
// TODO: Handle hostname if needed
}
return lbStatusAddresses, lbStatusHostnames
return lbStatusAddresses
}
// FetchServiceAddresses tries to get the addresses from annotations
// kube-vip.io/loadbalancerIPs, then from spec.loadbalancerIP
func FetchServiceAddresses(s *v1.Service) ([]string, []string) {
func FetchServiceAddresses(s *v1.Service) []string {
annotationAvailable := false
if s.Annotations != nil {
if v, annotationAvailable := s.Annotations[kubevip.LoadbalancerIPAnnotation]; annotationAvailable {
ips := strings.Split(v, ",")
var trimmedIPs []string
var trimmedHostnames []string
for _, a := range ips {
a = strings.TrimSpace(a)
ip := net.ParseIP(a)
if ip == nil {
// this is probably a DNS name
trimmedHostnames = append(trimmedHostnames, a)
} else {
trimmedIPs = append(trimmedIPs, ip.String())
}
for _, ip := range ips {
trimmedIPs = append(trimmedIPs, strings.TrimSpace(ip))
}
return trimmedIPs, trimmedHostnames
return trimmedIPs
}
}
lbStatusAddresses := []string{}
lbStatusHostnames := []string{}
if !annotationAvailable {
lbStatusAddresses, lbStatusHostnames = FetchLoadBalancerIngress(s)
}
// Spec.LoadBalancerIP legacy handling
// if the loadBalancerIP is different from Status.LoadBalancer.Ingress IPs
// return the legacy LB as spec wins over status.
if lbIP := net.ParseIP(s.Spec.LoadBalancerIP); lbIP != nil && len(lbStatusAddresses) > 0 {
isLbIPv4 := utils.IsIPv4(s.Spec.LoadBalancerIP)
for _, a := range lbStatusAddresses {
if lbStatusIP := net.ParseIP(a); lbStatusIP != nil && utils.IsIPv4(a) == isLbIPv4 && !lbIP.Equal(lbStatusIP) {
return []string{s.Spec.LoadBalancerIP}, []string{}
if len(s.Status.LoadBalancer.Ingress) > 0 {
for _, ingress := range s.Status.LoadBalancer.Ingress {
lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
}
}
}
if len(lbStatusAddresses) > 0 || len(lbStatusHostnames) > 0 {
return lbStatusAddresses, lbStatusHostnames
lbIP := net.ParseIP(s.Spec.LoadBalancerIP)
isLbIPv4 := vip.IsIPv4(s.Spec.LoadBalancerIP)
if len(lbStatusAddresses) > 0 {
for _, a := range lbStatusAddresses {
if lbStatusIP := net.ParseIP(a); lbStatusIP != nil && lbIP != nil && vip.IsIPv4(a) == isLbIPv4 && !lbIP.Equal(lbStatusIP) {
return []string{s.Spec.LoadBalancerIP}
}
}
return lbStatusAddresses
}
if s.Spec.LoadBalancerIP != "" {
return []string{s.Spec.LoadBalancerIP}, []string{}
return []string{s.Spec.LoadBalancerIP}
}
return []string{}, []string{}
return []string{}
}
func FindServiceInstance(svc *v1.Service, instances []*Instance) *Instance {
log.Debug("finding service", "namespace", svc.Namespace, "name", svc.Name, "UID", svc.UID)
for index := range instances {
if instances[index].UID() == svc.UID {
return instances[index]
log.Debug("finding service", "UID", svc.UID)
for i := range instances {
log.Debug("saved service", "instance", i, "UID", instances[i].ServiceSnapshot.UID)
if instances[i].ServiceSnapshot.UID == svc.UID {
return instances[i]
}
}
log.Debug("instance not found", "namespace", svc.Namespace, "name", svc.Name, "UID", svc.UID)
return nil
}

View File

@@ -1,62 +0,0 @@
package instance_test
import (
"context"
"sync"
"testing"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/route"
)
func TestNewInstance_PropagatesBGPAttachIPToInterface(t *testing.T) {
tests := []struct {
name string
attach bool
}{
{name: "attach enabled is propagated", attach: true},
{name: "attach disabled is propagated", attach: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
globalConfig := &kubevip.Config{
Interface: "lo",
VIPSubnet: "32",
EnableBGP: true,
BGPAttachIPToInterface: tt.attach,
}
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test-svc",
Namespace: "default",
Annotations: map[string]string{
kubevip.LoadbalancerIPAnnotation: "10.0.1.2",
},
},
}
inst, err := instance.NewInstance(context.Background(), svc, globalConfig,
networkinterface.NewManager(), arp.NewManager(globalConfig), route.NewManager(),
nil, &sync.WaitGroup{})
if err != nil {
t.Fatalf("NewInstance() error = %v", err)
}
if len(inst.VIPConfigs) != 1 {
t.Fatalf("VIPConfigs len = %d, want 1", len(inst.VIPConfigs))
}
if got := inst.VIPConfigs[0].BGPAttachIPToInterface; got != tt.attach {
t.Fatalf("BGPAttachIPToInterface = %t, want %t", got, tt.attach)
}
})
}
}

View File

@@ -1,65 +0,0 @@
package instance
import (
"sync"
"testing"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
func TestUIDUsesImmutableServiceUID(t *testing.T) {
serviceUID := types.UID("original-service")
instance := &Instance{
ServiceUID: serviceUID,
ServiceSnapshot: &v1.Service{ObjectMeta: metav1.ObjectMeta{
UID: types.UID("replacement-service"),
}},
}
if got := instance.UID(); got != serviceUID {
t.Fatalf("UID() = %q, want %q", got, serviceUID)
}
instance.ServiceUID = ""
if got := instance.UID(); got != "" {
t.Fatalf("UID() without ServiceUID = %q, want empty UID", got)
}
}
func TestCleanupStateIsImmutable(t *testing.T) {
original := &v1.Service{ObjectMeta: metav1.ObjectMeta{
Name: "service", Namespace: "default", Annotations: map[string]string{kubevip.ServiceLease: "shared"},
}, Spec: v1.ServiceSpec{LoadBalancerIP: "192.0.2.10", ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeCluster}}
immutableInfo := serviceCleanupInfo(original)
instance := &Instance{
ServiceSnapshot: original,
ServiceAddresses: []string{"192.0.2.10"},
cleanupInfo: &immutableInfo,
}
instance.ServiceSnapshot = &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "changed"}}
cleanupInfo, ok := instance.CleanupInfo()
if !ok || cleanupInfo.Namespace != "default" || cleanupInfo.Name != "service" || cleanupInfo.Lease != "shared" ||
cleanupInfo.ExternalTrafficPolicy != v1.ServiceExternalTrafficPolicyTypeCluster {
t.Fatalf("CleanupInfo() = %+v, %t, want creation-time Service policy", cleanupInfo, ok)
}
}
func TestTransferLinkAttachmentOwnershipConcurrent(t *testing.T) {
target := &Instance{IsVLAN: true, VLANInterface: "eth0.42"}
var wg sync.WaitGroup
for range 100 {
wg.Go(func() {
if !transferLinkAttachmentOwnership("eth0.42", []*Instance{target}, true) {
t.Error("transferLinkAttachmentOwnership() did not find target")
}
})
}
wg.Wait()
if !target.vlanOwned.Load() {
t.Fatal("target did not receive VLAN ownership")
}
}

View File

@@ -1,102 +0,0 @@
//go:build linux
package instance
import (
"errors"
"os"
"runtime"
"testing"
"github.com/vishvananda/netlink"
"github.com/vishvananda/netns"
)
// requireNetworkNamespaces makes the privileged CI job fail instead of silently
// skipping when it cannot enter a network namespace.
var requireNetworkNamespaces = os.Getenv("KUBE_VIP_REQUIRE_NETNS") != ""
func TestCleanupLinkAttachmentsOnlyDeletesOwnedVLAN(t *testing.T) {
for _, test := range []struct {
name string
preexists bool
inUse bool
}{
{name: "owned VLAN", preexists: false},
{name: "adopted VLAN", preexists: true},
{name: "owned VLAN used by another Service", inUse: true},
} {
t.Run(test.name, func(t *testing.T) {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
originalNamespace, err := netns.Get()
if err != nil {
t.Fatalf("getting current network namespace: %v", err)
}
defer originalNamespace.Close()
testNamespace, err := netns.New()
if err != nil {
if requireNetworkNamespaces {
t.Fatalf("creating isolated network namespace: %v", err)
}
t.Skipf("creating isolated network namespace: %v", err)
}
defer testNamespace.Close()
defer func() {
if err := netns.Set(originalNamespace); err != nil {
t.Errorf("restoring network namespace: %v", err)
}
}()
parent := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "kvattach0"}}
if err := netlink.LinkAdd(parent); err != nil {
t.Fatalf("creating parent interface: %v", err)
}
if err := netlink.LinkSetUp(parent); err != nil {
t.Fatalf("bringing up parent interface: %v", err)
}
if test.preexists {
vlan := &netlink.Vlan{LinkAttrs: netlink.LinkAttrs{Name: "kvattach0.42", ParentIndex: parent.Attrs().Index}, VlanId: 42}
if err := netlink.LinkAdd(vlan); err != nil {
t.Fatalf("creating existing VLAN: %v", err)
}
}
instance := &Instance{}
if err := instance.addVLAN(parent.Attrs().Name, 42); err != nil {
t.Fatalf("adding VLAN attachment: %v", err)
}
if instance.vlanOwned.Load() == test.preexists {
t.Fatalf("vlanOwned = %t, want %t", instance.vlanOwned.Load(), !test.preexists)
}
var remaining []*Instance
if test.inUse {
remaining = []*Instance{{IsVLAN: true, VLANInterface: instance.VLANInterface}}
}
if err := instance.CleanupLinkAttachments(remaining...); err != nil {
t.Fatalf("cleaning attachments: %v", err)
}
if test.inUse {
if !remaining[0].vlanOwned.Load() {
t.Fatal("remaining Service did not receive VLAN cleanup ownership")
}
if _, err := netlink.LinkByName("kvattach0.42"); err != nil {
t.Fatalf("VLAN was removed while a Service still used it: %v", err)
}
if err := remaining[0].CleanupLinkAttachments(); err != nil {
t.Fatalf("cleaning transferred attachment: %v", err)
}
}
_, err = netlink.LinkByName("kvattach0.42")
var notFound netlink.LinkNotFoundError
if test.preexists && err != nil {
t.Fatalf("adopted VLAN was removed: %v", err)
}
if !test.preexists && !errors.As(err, &notFound) {
t.Fatalf("owned VLAN remains after cleanup: %v", err)
}
})
}
}

View File

@@ -18,6 +18,7 @@ import (
"bytes"
"fmt"
"io"
"net"
"os/exec"
"regexp"
"strconv"
@@ -86,6 +87,21 @@ type IPTables struct {
nftables bool
}
// Stat represents a structured statistic entry.
type Stat struct {
Packets uint64 `json:"pkts"`
Bytes uint64 `json:"bytes"`
Target string `json:"target"`
Protocol string `json:"prot"`
Opt string `json:"opt"`
Input string `json:"in"`
Output string `json:"out"`
Source *net.IPNet `json:"source"`
Destination *net.IPNet `json:"destination"`
Options string `json:"options"`
}
type Option func(*IPTables)
func IPFamily(proto Protocol) Option {
@@ -235,6 +251,16 @@ func (ipt *IPTables) DeleteIfExists(table, chain string, rulespec ...string) err
return err
}
// List rules in specified table/chain
func (ipt *IPTables) ListByID(table, chain string, id int) (string, error) {
args := []string{"-t", table, "-S", chain, strconv.Itoa(id)}
rule, err := ipt.executeList(args)
if err != nil {
return "", err
}
return rule[0], nil
}
// List rules in specified table/chain
func (ipt *IPTables) List(table, chain string) ([]string, error) {
args := []string{"-t", table, "-S", chain}
@@ -287,6 +313,129 @@ func (ipt *IPTables) ChainExists(table, chain string) (bool, error) {
}
}
// Stats lists rules including the byte and packet counts
func (ipt *IPTables) Stats(table, chain string) ([][]string, error) {
args := []string{"-t", table, "-L", chain, "-n", "-v", "-x"}
lines, err := ipt.executeList(args)
if err != nil {
return nil, err
}
appendSubnet := func(addr string) string {
if strings.IndexByte(addr, byte('/')) < 0 {
if strings.IndexByte(addr, '.') < 0 {
return addr + "/128"
}
return addr + "/32"
}
return addr
}
ipv6 := ipt.proto == ProtocolIPv6
rows := [][]string{}
for i, line := range lines {
// Skip over chain name and field header
if i < 2 {
continue
}
// Fields:
// 0=pkts 1=bytes 2=target 3=prot 4=opt 5=in 6=out 7=source 8=destination 9=options
line = strings.TrimSpace(line)
fields := strings.Fields(line)
// The ip6tables verbose output cannot be naively split due to the default "opt"
// field containing 2 single spaces.
if ipv6 {
// Check if field 6 is "opt" or "source" address
dest := fields[6]
ip, _, _ := net.ParseCIDR(dest)
if ip == nil {
ip = net.ParseIP(dest)
}
// If we detected a CIDR or IP, the "opt" field is empty.. insert it.
if ip != nil {
f := []string{}
f = append(f, fields[:4]...)
f = append(f, " ") // Empty "opt" field for ip6tables
f = append(f, fields[4:]...)
fields = f
}
}
// Adjust "source" and "destination" to include netmask, to match regular
// List output
fields[7] = appendSubnet(fields[7])
fields[8] = appendSubnet(fields[8])
// Combine "options" fields 9... into a single space-delimited field.
options := fields[9:]
fields = fields[:9]
fields = append(fields, strings.Join(options, " "))
rows = append(rows, fields)
}
return rows, nil
}
// ParseStat parses a single statistic row into a Stat struct. The input should
// be a string slice that is returned from calling the Stat method.
func (ipt *IPTables) ParseStat(stat []string) (parsed Stat, err error) {
// For forward-compatibility, expect at least 10 fields in the stat
if len(stat) < 10 {
return parsed, fmt.Errorf("stat contained fewer fields than expected")
}
// Convert the fields that are not plain strings
parsed.Packets, err = strconv.ParseUint(stat[0], 0, 64)
if err != nil {
return parsed, fmt.Errorf(err.Error(), "could not parse packets")
}
parsed.Bytes, err = strconv.ParseUint(stat[1], 0, 64)
if err != nil {
return parsed, fmt.Errorf(err.Error(), "could not parse bytes")
}
_, parsed.Source, err = net.ParseCIDR(stat[7])
if err != nil {
return parsed, fmt.Errorf(err.Error(), "could not parse source")
}
_, parsed.Destination, err = net.ParseCIDR(stat[8])
if err != nil {
return parsed, fmt.Errorf(err.Error(), "could not parse destination")
}
// Put the fields that are strings
parsed.Target = stat[2]
parsed.Protocol = stat[3]
parsed.Opt = stat[4]
parsed.Input = stat[5]
parsed.Output = stat[6]
parsed.Options = stat[9]
return parsed, nil
}
// StructuredStats returns statistics as structured data which may be further
// parsed and marshaled.
func (ipt *IPTables) StructuredStats(table, chain string) ([]Stat, error) {
rawStats, err := ipt.Stats(table, chain)
if err != nil {
return nil, err
}
structStats := []Stat{}
for _, rawStat := range rawStats {
stat, err := ipt.ParseStat(rawStat)
if err != nil {
return nil, err
}
structStats = append(structStats, stat)
}
return structStats, nil
}
func (ipt *IPTables) executeList(args []string) ([]string, error) {
var stdout bytes.Buffer
if err := ipt.runWithOutput(args, &stdout); err != nil {

View File

@@ -31,67 +31,19 @@ const (
// Networks that we wont Egress for
EgressDeniedNetworks = "kube-vip.io/egress-denied-networks"
// EgressNoInternalTraffic, when enabled will prevent any internal traffic from being SNATed to the egress VIP, even if the internal SNAT rule is enabled
EgressNoInternalTraffic = "kube-vip.io/egress-no-internal-traffic"
// EgressDetectAPIServer, this will attempt to detect the API server and add it to the allowed networks for egress
EgressDetectAPIServer = "kube-vip.io/egress-detect-api-server"
// The current active endpoint(pod) for the Egress VIP
ActiveEndpoint = "kube-vip.io/active-endpoint"
// The current active endpoint(pod) for the Egress VIP (v6)
ActiveEndpointIPv6 = "kube-vip.io/active-endpoint-ipv6"
// The nftables egress table base name that owns this Service's SNAT chain
EgressNftablesTable = "kube-vip.io/egress-nftables-table"
// Flush the conntrack rules (remove existing sessions) once Egress is configured
FlushContrack = "kube-vip.io/flush-conntrack"
// Configure LoadBalancer IPs instead of relying on a controller
LoadbalancerIPAnnotation = "kube-vip.io/loadbalancerIPs"
// Ignore the LoadBalancer Service
LoadbalancerIgnore = "kube-vip.io/ignore"
// Used to configure DHCP with a Hostname
LoadbalancerHostname = "kube-vip.io/loadbalancerHostname"
// Define an interface name to bind the address of the LoadBalancer to
ServiceInterface = "kube-vip.io/serviceInterface"
// Specify VLAN subinterface for service (e.g. eth0.200)
ServiceVlan = "kube-vip.io/serviceVLAN"
ServiceSecurityIgnore = "kube-vip.io/ignore-service-security"
// Enable UPNP on a Service
UpnpEnabled = "kube-vip.io/forwardUPNP"
// Set the UPNP lease duration for a specific service using duration format (e.g., "30s", "1h")
UpnpLeaseDuration = "kube-vip.io/upnp-lease-duration"
LoadbalancerHostname = "kube-vip.io/loadbalancerHostname"
ServiceInterface = "kube-vip.io/serviceInterface"
UpnpEnabled = "kube-vip.io/forwardUPNP"
RPFilter = "kube-vip.io/rp_filter" // Set the return path filter for a specific service interface
// Name of the service lease object
ServiceLease = "kube-vip.io/leaseName"
// Versioned kube-vip ownership metadata stored on Kubernetes election Leases
LeaseVIPs = "kube-vip.io/lease-vips"
// Forces kube-vip to use per service election for this particular service
ForcePerServiceElection = "kube-vip.io/forcePerServiceElection"
// Allow service reconciliation even when no endpoints are present (Cluster policy only)
AllowReconcileWithoutEndpoints = "kube-vip.io/allow-reconcile-without-endpoints"
// Enable DDNS for the service
ServiceDDNS = "kube-vip.io/ddns"
// Forces kube-vip to use the specified veth interface when DHCP is being used for a service
MacvlanName = "kube-vip.io/macvlanName"
// Set the BROADCAST flag in DHCP DISCOVER/REQUEST packets
DHCPBroadcast = "kube-vip.io/dhcp-broadcast"
)

View File

@@ -1,334 +0,0 @@
package kubevip
import (
"fmt"
"strconv"
"strings"
"github.com/kube-vip/kube-vip/pkg/utils"
api "github.com/osrg/gobgp/v4/api"
"github.com/vishvananda/netlink"
)
// Peer defines a BGP Peer
type BGPPeer struct {
Address string
Port uint16
Interface string
AS uint32
Password string
MultiHop bool
MpbgpNexthop string
MpbgpIPv4 string
MpbgpIPv6 string
// BFD Configuration
BFDEnabled bool
BFDReceiveInterval uint32
BFDTransmitInterval uint32
BFDDetectMultiplier uint32
}
// Config defines the BGP server configuration
type BGPConfig struct {
AS uint32
RouterID string
SourceIP string
SourceIF string
MpbgpNexthop string
MpbgpIPv4 string
MpbgpIPv6 string
HoldTime uint64
KeepaliveInterval uint64
Peers []BGPPeer
Zebra ZebraConfig
}
// Defines Zebra connection configuration. More on the topic - https://github.com/osrg/gobgp/blob/master/docs/sources/zebra.md#configuration
type ZebraConfig struct {
Enabled bool
URL string
Version uint32
SoftwareName string
}
// BGP Peer layout is as follows:
// <address>:<AS>:<password>:<multihop>:<port>:<optional mpbgp options>:<BFD options>
// <address> - IP address of the peer. For IPv6 addresses, the address should be enclosed in square brackets (e.g. [fd00:100:64::2]). For unnumbered peers, the address should be prefixed with "unnumbered:" followed by the interface name (e.g. unnumbered:eth0).
// <AS> - Autonomous System number of the peer (e.g. 65000)
// <password> - Optional password for BGP authentication (e.g. secret)
// <multihop> - Optional flag to indicate if this is a multihop peer (true/false, default: false)
// <port> - Optional BGP port number (default: 179)
// <optional mpbgp options> - Optional MP-BGP parameters in the format of key=value pairs separated by ';' (e.g. mpbgp_nexthop=auto_sourceif;mpbgp_ipv4=)
// <BFD options> - Optional BFD parameters (if any) in the format of semicolon-separated values (enable, receive_interval, transmit_interval, detect_multiplier) (e.g. true;300;300;3)
// ParseBGPPeerConfig - take a string and parses it into an array of peers
func ParseBGPPeerConfig(config string) (bgpPeers []BGPPeer, err error) {
peers := strings.Split(config, ",")
if len(peers) == 0 || config == "" {
return nil, fmt.Errorf("no BGP Peer configurations found")
}
for x := range peers {
peerStr := peers[x]
if peerStr == "" {
continue
}
// Look at address peer
isV6Peer := peerStr[0] == '['
isUnnumberedPeer := strings.HasPrefix(peerStr, "unnumbered:")
address := ""
if isV6Peer {
addressEndPos := strings.IndexByte(peerStr, ']')
if addressEndPos == -1 {
return nil, fmt.Errorf("no matching ] found for IPv6 BGP Peer")
}
address = peerStr[1:addressEndPos]
peerStr = peerStr[addressEndPos+1:]
} else if isUnnumberedPeer {
unnumberedEndPos := strings.IndexByte(peerStr, ':')
peerStr = peerStr[unnumberedEndPos+1:]
}
peer := strings.Split(peerStr, ":")
if len(peer) < 2 && !isUnnumberedPeer {
return nil, fmt.Errorf("mandatory peering params <host>:<AS> incomplete")
}
iface := ""
if isUnnumberedPeer {
iface = peer[0]
} else if !isV6Peer {
address = peer[0]
}
// Look at peer[1] for AS number
var ASNumber uint64
if len(peer) >= 2 {
ASNumber, err = strconv.ParseUint(peer[1], 10, 32)
if err != nil {
return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
}
}
// Look at peer[2] for password
password := ""
if len(peer) >= 3 {
password = peer[2]
}
// Look at peer[3] for multihop
multiHop := false
if len(peer) >= 4 && peer[3] != "" {
multiHop, err = strconv.ParseBool(peer[3])
if err != nil {
return nil, fmt.Errorf("BGP MultiHop format error (true/false) [%s]", peer[3])
}
}
// Look at peer[4] for BGP port
var port uint64
if len(peer) >= 5 {
if peer[4] == "" {
port = 179
} else {
port, err = strconv.ParseUint(peer[4], 10, 16)
if err != nil {
return nil, fmt.Errorf("BGP Peer Port format error [%s]", peer[4])
}
}
} else if !isUnnumberedPeer {
port = 179
}
// Look at peer[5] for optional MP-BGP parameters
var mpbgpNexthop, mpbgpIPv4, mpbgpIPv6 string
if len(peer) >= 6 && peer[5] != "" {
configData := strings.Split(peer[5], ";")
for _, cfg := range configData {
c := strings.Split(cfg, "=")
if len(c) < 2 {
return nil, fmt.Errorf("peer configuration parameter '%s' is missing a value (expected key=value)", c[0])
}
switch c[0] {
case "mpbgp_nexthop":
mpbgpNexthop = c[1]
case "mpbgp_ipv4":
mpbgpIPv4 = c[1]
case "mpbgp_ipv6":
mpbgpIPv6 = c[1]
default:
return nil, fmt.Errorf("peer configuration parameter '%s' is not supported", c[0])
}
}
}
// Look at peer[6] for optional BFD parameters (if any)
bfdEnabled := false
bfdReceiveInterval := uint64(300)
bfdTransmitInterval := uint64(300)
bfdDetectMultiplier := uint64(3)
if len(peer) >= 7 && peer[6] != "" {
c := strings.Split(peer[6], ";")
if len(c) < 4 {
return nil, fmt.Errorf("BFD configuration error: at least 4 parameters are required (enable, receive_interval, transmit_interval, detect_multiplier) [%s]", peer[6])
}
bfdEnabled, err = strconv.ParseBool(c[0])
if err != nil {
return nil, fmt.Errorf("BFD configuration error: invalid value for bfd_enabled (true/false) [%s]", c[0])
}
if c[1] != "" {
bfdReceiveInterval, err = strconv.ParseUint(c[1], 10, 32)
if err != nil {
return nil, fmt.Errorf("BFD configuration error: invalid value for bfd_receive_interval [%s]", c[1])
}
}
if c[2] != "" {
bfdTransmitInterval, err = strconv.ParseUint(c[2], 10, 32)
if err != nil {
return nil, fmt.Errorf("BFD configuration error: invalid value for bfd_transmit_interval [%s]", c[2])
}
}
if c[3] != "" {
bfdDetectMultiplier, err = strconv.ParseUint(c[3], 10, 32)
if err != nil {
return nil, fmt.Errorf("BFD configuration error: invalid value for bfd_detect_multiplier [%s]", c[3])
}
}
}
peerConfig := BGPPeer{
Address: address,
//nolint:gosec // previously parsed into uint32
AS: uint32(ASNumber),
Port: uint16(port),
Interface: iface,
Password: password,
MultiHop: multiHop,
MpbgpNexthop: mpbgpNexthop,
MpbgpIPv4: mpbgpIPv4,
MpbgpIPv6: mpbgpIPv6,
BFDEnabled: bfdEnabled,
BFDReceiveInterval: uint32(bfdReceiveInterval),
BFDTransmitInterval: uint32(bfdTransmitInterval),
BFDDetectMultiplier: uint32(bfdDetectMultiplier),
}
bgpPeers = append(bgpPeers, peerConfig)
}
return
}
func (p *BGPPeer) FindMpbgpAddresses(ap *api.Peer, server *BGPConfig) (string, string, error) {
var ipv4Address, ipv6Address string
mode := server.MpbgpNexthop
if p.MpbgpNexthop != "" {
mode = p.MpbgpNexthop
}
switch mode {
case "fixed":
ap.Transport.LocalAddress = server.SourceIP
ipv4 := server.MpbgpIPv4
if p.MpbgpIPv4 != "" {
ipv4 = p.MpbgpIPv4
}
ipv6 := server.MpbgpIPv6
if p.MpbgpIPv6 != "" {
ipv6 = p.MpbgpIPv6
}
if ipv4 == "" && ipv6 == "" {
return "", "", fmt.Errorf("to use MP-BGP with fixed address at least one IPv4 or IPv6 address has to be provided [current - IPv4: %s, IPv6: %s]",
ipv4, ipv6)
}
if ipv4 != "" {
if !utils.IsIPv4(ipv4) {
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv4 address", ipv4)
}
}
if ipv6 != "" {
if !utils.IsIPv6(ipv6) {
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv6 address", ipv6)
}
}
ipv4Address = ipv4
ipv6Address = ipv6
case "auto_sourceip":
ap.Transport.LocalAddress = server.SourceIP
// Resolve the local interface by SourceIP
iface, err := utils.GetInterfaceByIP(server.SourceIP)
if err != nil {
return "", "", fmt.Errorf("failed to get interface by IP: %v", err)
}
if utils.IsIPv4(server.SourceIP) {
// Get the non link-local IPv6 address on that interface
ipv6Address, err = utils.GetNonLinkLocalIP(iface, netlink.FAMILY_V6)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
}
} else {
// Get the non link-local IPv4 address on that interface
ipv4Address, err = utils.GetNonLinkLocalIP(iface, netlink.FAMILY_V4)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
}
}
case "auto_sourceif":
ap.Transport.BindInterface = server.SourceIF
iface, err := netlink.LinkByName(server.SourceIF)
if err != nil {
return "", "", fmt.Errorf("failed to get interface by name: %v", err)
}
// Get the non link-local IPv4 address on that interface
ipv4Address, err = utils.GetNonLinkLocalIP(&iface, netlink.FAMILY_V4)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
}
// Get the non link-local IPv6 address on that interface
ipv6Address, err = utils.GetNonLinkLocalIP(&iface, netlink.FAMILY_V6)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
}
default:
return "", "", fmt.Errorf("option %q for MP-BPG nexthop is not supported", mode)
}
return ipv4Address, ipv6Address, nil
}
func (p *BGPPeer) SetMpbgpOptions(server *BGPConfig) {
if p.MpbgpNexthop == "" {
p.MpbgpNexthop = server.MpbgpNexthop
}
if p.MpbgpIPv4 == "" {
p.MpbgpIPv4 = server.MpbgpIPv4
}
if p.MpbgpIPv6 == "" {
p.MpbgpIPv6 = server.MpbgpIPv6
}
}

View File

@@ -1,38 +0,0 @@
package kubevip
import (
"testing"
api "github.com/osrg/gobgp/v4/api"
)
func TestFindMpbgpAddressesRejectsFixedAddressFamilyMismatches(t *testing.T) {
tests := []struct {
name string
peer BGPPeer
}{
{
name: "IPv6 value in IPv4 field",
peer: BGPPeer{
MpbgpNexthop: "fixed",
MpbgpIPv4: "2001:db8::20",
},
},
{
name: "IPv4 value in IPv6 field",
peer: BGPPeer{
MpbgpNexthop: "fixed",
MpbgpIPv6: "192.0.2.20",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, _, err := tt.peer.FindMpbgpAddresses(&api.Peer{Transport: &api.Transport{}}, &BGPConfig{})
if err == nil {
t.Fatal("FindMpbgpAddresses() error = nil, want address-family error")
}
})
}
}

View File

@@ -1,116 +0,0 @@
package kubevip
import (
"reflect"
"testing"
)
func TestParseBGPPeerConfig(t *testing.T) {
type args struct {
config string
}
tests := []struct {
name string
args args
wantBgpPeers []BGPPeer
wantErr bool
}{
{
name: "IPv4, default port",
args: args{config: "192.168.0.10:65000::false,192.168.0.11:65000::false"},
wantBgpPeers: []BGPPeer{
{Address: "192.168.0.10", Port: 179, AS: 65000, MultiHop: false, BFDEnabled: false, BFDReceiveInterval: 300, BFDTransmitInterval: 300, BFDDetectMultiplier: 3},
{Address: "192.168.0.11", Port: 179, AS: 65000, MultiHop: false, BFDEnabled: false, BFDReceiveInterval: 300, BFDTransmitInterval: 300, BFDDetectMultiplier: 3},
},
},
{
name: "IPv4, different port",
args: args{config: "192.168.0.10:65000::false:180,192.168.0.11:65000::false:190"},
wantBgpPeers: []BGPPeer{
{Address: "192.168.0.10", Port: 180, AS: 65000, MultiHop: false, BFDEnabled: false, BFDReceiveInterval: 300, BFDTransmitInterval: 300, BFDDetectMultiplier: 3},
{Address: "192.168.0.11", Port: 190, AS: 65000, MultiHop: false, BFDEnabled: false, BFDReceiveInterval: 300, BFDTransmitInterval: 300, BFDDetectMultiplier: 3},
},
},
{
name: "IPv6, multi-protocol",
args: args{config: "[fd00:1111:2222:3333:c7d9:7235:6bf7:5d52]:65501::false::mpbgp_nexthop=auto_sourceif"},
wantBgpPeers: []BGPPeer{
{Address: "fd00:1111:2222:3333:c7d9:7235:6bf7:5d52", Port: 179, AS: 65501, MultiHop: false, MpbgpNexthop: "auto_sourceif", BFDEnabled: false, BFDReceiveInterval: 300, BFDTransmitInterval: 300, BFDDetectMultiplier: 3},
},
},
{
name: "IPv6, multi-protocol, BFD, no multi-protocol options",
args: args{config: "[fd00:1111:2222:3333:c7d9:7235:6bf7:5d52]:65501::false:::true;300;300;3"},
wantBgpPeers: []BGPPeer{
{Address: "fd00:1111:2222:3333:c7d9:7235:6bf7:5d52", Port: 179, AS: 65501, MultiHop: false, BFDEnabled: true, BFDReceiveInterval: 300, BFDTransmitInterval: 300, BFDDetectMultiplier: 3},
},
},
{
name: "IPv6, multi-protocol, BFD",
args: args{config: "[fd00:1111:2222:3333:c7d9:7235:6bf7:5d52]:65501::false::mpbgp_nexthop=auto_sourceif:true;300;300;3"},
wantBgpPeers: []BGPPeer{
{Address: "fd00:1111:2222:3333:c7d9:7235:6bf7:5d52", Port: 179, AS: 65501, MultiHop: false, MpbgpNexthop: "auto_sourceif", BFDEnabled: true, BFDReceiveInterval: 300, BFDTransmitInterval: 300, BFDDetectMultiplier: 3},
},
},
{
name: "IPv6 bracketed, with password and multihop",
args: args{config: "[fd00:100:64::2]:65000:secret:true"},
wantBgpPeers: []BGPPeer{
{Address: "fd00:100:64::2", Port: 179, AS: 65000, Password: "secret", MultiHop: true, BFDEnabled: false, BFDReceiveInterval: 300, BFDTransmitInterval: 300, BFDDetectMultiplier: 3},
},
},
{
name: "IPv6 bracketed, empty fields",
args: args{config: "[fd00:100:64::2]:65000::false"},
wantBgpPeers: []BGPPeer{
{Address: "fd00:100:64::2", Port: 179, AS: 65000, MultiHop: false, BFDEnabled: false, BFDReceiveInterval: 300, BFDTransmitInterval: 300, BFDDetectMultiplier: 3},
},
},
{
name: "Unnumbered",
args: args{config: "unnumbered:eth0,unnumbered:eth1:65000::true::mpbgp_nexthop=auto_sourceif"},
wantBgpPeers: []BGPPeer{
{Interface: "eth0", MultiHop: false, BFDEnabled: false, BFDReceiveInterval: 300, BFDTransmitInterval: 300, BFDDetectMultiplier: 3},
{Interface: "eth1", Port: 179, AS: 65000, MultiHop: true, MpbgpNexthop: "auto_sourceif", BFDEnabled: false, BFDReceiveInterval: 300, BFDTransmitInterval: 300, BFDDetectMultiplier: 3},
},
},
{
name: "Completely empty config",
args: args{config: ""},
wantErr: true,
},
{
name: "Completely empty config (but with the seperators)",
args: args{config: ":::::::"},
wantErr: true,
},
{
name: "Malformed parameter (no value)",
args: args{config: "1.2.3.4:65000/mpbgp_nexthop"},
wantErr: true,
},
{
name: "Unsupported parameter",
args: args{config: "1.2.3.4:65000;unknown=value"},
wantErr: true,
},
{
name: "Malformed IPv6 (no matching bracket)",
args: args{config: "[fd00:100:64::2:65000"},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotBgpPeers, err := ParseBGPPeerConfig(tt.args.config)
if (err != nil) != tt.wantErr {
t.Errorf("ParseBGPPeerConfig() error = \n%v, wantErr \n%v %v", err, tt.wantErr, gotBgpPeers)
return
}
if !reflect.DeepEqual(gotBgpPeers, tt.wantBgpPeers) {
t.Errorf("ParseBGPPeerConfig() = \n%v, want \n%v", gotBgpPeers, tt.wantBgpPeers)
}
})
}
}

View File

@@ -6,14 +6,11 @@ import (
"math"
"math/bits"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/kube-vip/kube-vip/pkg/debouncer"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/detector"
"github.com/kube-vip/kube-vip/pkg/utils"
"sigs.k8s.io/yaml"
)
// ParseEnvironment - will popultate the configuration from environment variables
@@ -27,18 +24,11 @@ func ParseEnvironment(c *Config) error {
if env != "" {
logLevel, err := strconv.ParseInt(env, 10, 32)
if err != nil {
return fmt.Errorf("unable to parse environment variable [vip_loglevel], should be int: %w", err)
panic("Unable to parse environment variable [vip_loglevel], should be int")
}
c.Logging = int32(logLevel)
}
if env = os.Getenv(instanceName); env == "" {
env = os.Getenv(strings.ToUpper(instanceName))
}
if env != "" {
c.InstanceName = env
}
// Find interface
env = os.Getenv(vipInterface)
if env != "" {
@@ -54,39 +44,12 @@ func ParseEnvironment(c *Config) error {
c.LoInterfaceGlobalScope = b
}
env = os.Getenv(vipLoseLeadership)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.LoseLeadership = b
}
env = os.Getenv(vipLoseLeadershipTimeoutSeconds)
if env != "" {
i, err := strconv.ParseInt(env, 10, 32)
if err != nil {
return fmt.Errorf("parsing env var %s (value: %s): %w", vipLoseLeadershipTimeoutSeconds, env, err)
}
c.LoseLeadershipTimeoutSeconds = int(i)
}
// Find (services) interface
env = os.Getenv(vipServicesInterface)
if env != "" {
c.ServicesInterface = env
}
// Tolerate a down interface
env = os.Getenv(vipAllowInterfaceNotUp)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.AllowInterfaceNotUp = b
}
// Find Kubernetes Leader Election configuration
env = os.Getenv(vipLeaderElection)
if env != "" {
@@ -327,18 +290,6 @@ func ParseEnvironment(c *Config) error {
c.ArpBroadcastRate = 3000
}
// Determine if VIP should be preserved on leadership loss
// true: VIP addresses remain on interface, only ARP/NDP broadcasting stops
// false (default): VIP addresses are deleted on leadership loss (legacy behavior)
env = os.Getenv(vipPreserveOnLeadershipLoss)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.PreserveVIPOnLeadershipLoss = b
}
// Wireguard Mode
env = os.Getenv(vipWireguard)
if env != "" {
@@ -406,46 +357,12 @@ func ParseEnvironment(c *Config) error {
c.CleanRoutingTable = b
}
// Skip Duplicate Address Detection when adding the VIP address
env = os.Getenv(vipSkipDAD)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.SkipDAD = b
}
// DNS mode
env = os.Getenv(dnsMode)
if env != "" {
c.DNSMode = env
}
// DHCP mode
env = os.Getenv(dhcpMode)
if env != "" {
c.DHCPMode = env
} else {
if c.DNSMode != "first" {
c.DHCPMode = c.DNSMode
} else {
c.DHCPMode = strings.ToLower(utils.IPv4Family)
}
}
// DHCP backoff attempts
env = os.Getenv(dhcpBackoffAttempts)
if env != "" {
tmp, err := strconv.ParseInt(env, 10, 32)
if err != nil {
return err
}
if tmp >= 0 {
c.DHCPBackoffAttempts = uint(tmp)
}
}
// Disable updates for services (status.LoadBalancer.Ingress will not be updated)
env = os.Getenv(disableServiceUpdates)
if env != "" {
@@ -466,15 +383,6 @@ func ParseEnvironment(c *Config) error {
c.EnableBGP = b
}
env = os.Getenv(bgpAttachIPToInterface)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.BGPAttachIPToInterface = b
}
// BGP Router interface determines an interface that we can use to find an address for
env = os.Getenv(bgpRouterInterface)
if env != "" {
@@ -514,7 +422,7 @@ func ParseEnvironment(c *Config) error {
// Peer AS
env = os.Getenv(bgpPeers)
if env != "" {
peers, err := ParseBGPPeerConfig(env)
peers, err := bgp.ParseBGPPeerConfig(env)
if err != nil {
return err
}
@@ -593,40 +501,6 @@ func ParseEnvironment(c *Config) error {
c.BGPConfig.KeepaliveInterval = u64
}
// BGP health check options
env = os.Getenv(controlPlaneHealthCheckAddress)
if env != "" {
c.ControlPlaneHealthCheck.Address = env
}
env = os.Getenv(controlPlaneHealthCheckPeriodSeconds)
if env != "" {
i, err := strconv.ParseInt(env, 10, 32)
if err != nil {
return fmt.Errorf("parsing env var %s (value: %s): %w", controlPlaneHealthCheckPeriodSeconds, env, err)
}
c.ControlPlaneHealthCheck.PeriodSeconds = int(i)
}
env = os.Getenv(controlPlaneHealthCheckTimeoutSeconds)
if env != "" {
i, err := strconv.ParseInt(env, 10, 32)
if err != nil {
return fmt.Errorf("parsing env var %s (value: %s): %w", controlPlaneHealthCheckTimeoutSeconds, env, err)
}
c.ControlPlaneHealthCheck.TimeoutSeconds = int(i)
}
env = os.Getenv(controlPlaneHealthCheckFailureThreshold)
if env != "" {
i, err := strconv.ParseInt(env, 10, 32)
if err != nil {
return fmt.Errorf("parsing env var %s (value: %s): %w", controlPlaneHealthCheckFailureThreshold, env, err)
}
c.ControlPlaneHealthCheck.FailureThreshold = int(i)
}
env = os.Getenv(controlPlaneHealthCheckCAPath)
if env != "" {
c.ControlPlaneHealthCheck.CAPath = env
}
env = os.Getenv(zebraEnable)
if env != "" {
result, err := strconv.ParseBool(env)
@@ -727,25 +601,6 @@ func ParseEnvironment(c *Config) error {
c.EgressWithNftables = b
}
env = os.Getenv(perServiceElectionOnDemand)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.PerServiceElectionOnDemand = b
}
// if this is set then we're enabling the internal SNAT rule that kube-vip adds to the egress chain
env = os.Getenv(egressEnableInternalSNAT)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.EnableInternalSNAT = b
}
// check to see if we're using a specific path to the Kubernetes config file
env = os.Getenv(k8sConfigFile)
if env != "" {
@@ -812,332 +667,5 @@ func ParseEnvironment(c *Config) error {
c.EgressClean = b
}
// check for configuration file path
env = os.Getenv(configFile)
if env != "" {
c.ConfigFile = env
}
return nil
}
// LoadConfigFromFile loads configuration from a JSON or YAML file
func LoadConfigFromFile(configFilePath string) (*Config, error) {
if configFilePath == "" {
return nil, fmt.Errorf("config file path is empty")
}
// Check if file exists
if _, err := os.Stat(configFilePath); os.IsNotExist(err) {
return nil, fmt.Errorf("config file does not exist: %s", configFilePath)
}
// Read file content
data, err := os.ReadFile(configFilePath)
if err != nil {
return nil, fmt.Errorf("failed to read config file %s: %v", configFilePath, err)
}
var config Config
ext := strings.ToLower(filepath.Ext(configFilePath))
switch ext {
case ".json":
err = json.Unmarshal(data, &config)
if err != nil {
return nil, fmt.Errorf("failed to parse JSON config file %s: %v", configFilePath, err)
}
case ".yaml", ".yml":
err = yaml.Unmarshal(data, &config)
if err != nil {
return nil, fmt.Errorf("failed to parse YAML config file %s: %v", configFilePath, err)
}
default:
return nil, fmt.Errorf("unsupported config file format %s. Supported formats: .json, .yaml, .yml", ext)
}
return &config, nil
}
// MergeConfigFromFile merges configuration loaded from file with existing config
// Priority: command line flags > environment variables > config file
func MergeConfigFromFile(c *Config, configFilePath string) error {
if configFilePath == "" {
return nil // No config file specified, nothing to merge
}
fileConfig, err := LoadConfigFromFile(configFilePath)
if err != nil {
return err
}
// Merge file config with existing config
// Only set values from file if they haven't been set by flags or env vars
mergeConfigValues(c, fileConfig)
return nil
}
// mergeConfigValues merges values from fileConfig into baseConfig
// Only overwrites zero values in baseConfig
func mergeConfigValues(baseConfig, fileConfig *Config) {
// Basic configuration
if baseConfig.Logging == 0 && fileConfig.Logging != 0 {
baseConfig.Logging = fileConfig.Logging
}
// Network configuration
if baseConfig.Interface == "" && fileConfig.Interface != "" {
baseConfig.Interface = fileConfig.Interface
}
if baseConfig.ServicesInterface == "" && fileConfig.ServicesInterface != "" {
baseConfig.ServicesInterface = fileConfig.ServicesInterface
}
if baseConfig.VIP == "" && fileConfig.VIP != "" {
baseConfig.VIP = fileConfig.VIP
}
if baseConfig.VIPSubnet == "" && fileConfig.VIPSubnet != "" {
baseConfig.VIPSubnet = fileConfig.VIPSubnet
}
if baseConfig.Address == "" && fileConfig.Address != "" {
baseConfig.Address = fileConfig.Address
}
if baseConfig.Port == 0 && fileConfig.Port != 0 {
baseConfig.Port = fileConfig.Port
}
if baseConfig.NodeName == "" && fileConfig.NodeName != "" {
baseConfig.NodeName = fileConfig.NodeName
}
// Boolean flags - only merge if not explicitly set
if !baseConfig.EnableARP && fileConfig.EnableARP {
baseConfig.EnableARP = fileConfig.EnableARP
}
if !baseConfig.EnableBGP && fileConfig.EnableBGP {
baseConfig.EnableBGP = fileConfig.EnableBGP
}
if !baseConfig.BGPAttachIPToInterface && fileConfig.BGPAttachIPToInterface {
baseConfig.BGPAttachIPToInterface = fileConfig.BGPAttachIPToInterface
}
if !baseConfig.EnableWireguard && fileConfig.EnableWireguard {
baseConfig.EnableWireguard = fileConfig.EnableWireguard
}
if !baseConfig.EnableRoutingTable && fileConfig.EnableRoutingTable {
baseConfig.EnableRoutingTable = fileConfig.EnableRoutingTable
}
if !baseConfig.EnableControlPlane && fileConfig.EnableControlPlane {
baseConfig.EnableControlPlane = fileConfig.EnableControlPlane
}
if !baseConfig.DetectControlPlane && fileConfig.DetectControlPlane {
baseConfig.DetectControlPlane = fileConfig.DetectControlPlane
}
if !baseConfig.EnableServices && fileConfig.EnableServices {
baseConfig.EnableServices = fileConfig.EnableServices
}
if !baseConfig.EnableServicesElection && fileConfig.EnableServicesElection {
baseConfig.EnableServicesElection = fileConfig.EnableServicesElection
}
if !baseConfig.EnableNodeLabeling && fileConfig.EnableNodeLabeling {
baseConfig.EnableNodeLabeling = fileConfig.EnableNodeLabeling
}
if !baseConfig.EnableLoadBalancer && fileConfig.EnableLoadBalancer {
baseConfig.EnableLoadBalancer = fileConfig.EnableLoadBalancer
}
if !baseConfig.DDNS && fileConfig.DDNS {
baseConfig.DDNS = fileConfig.DDNS
}
if !baseConfig.SingleNode && fileConfig.SingleNode {
baseConfig.SingleNode = fileConfig.SingleNode
}
if !baseConfig.StartAsLeader && fileConfig.StartAsLeader {
baseConfig.StartAsLeader = fileConfig.StartAsLeader
}
if !baseConfig.PreserveVIPOnLeadershipLoss && fileConfig.PreserveVIPOnLeadershipLoss {
baseConfig.PreserveVIPOnLeadershipLoss = fileConfig.PreserveVIPOnLeadershipLoss
}
// Service configuration
if baseConfig.Namespace == "" && fileConfig.Namespace != "" {
baseConfig.Namespace = fileConfig.Namespace
}
if baseConfig.ServiceNamespace == "" && fileConfig.ServiceNamespace != "" {
baseConfig.ServiceNamespace = fileConfig.ServiceNamespace
}
if baseConfig.ServicesLeaseName == "" && fileConfig.ServicesLeaseName != "" {
baseConfig.ServicesLeaseName = fileConfig.ServicesLeaseName
}
// LoadBalancer configuration
if baseConfig.LoadBalancerPort == 0 && fileConfig.LoadBalancerPort != 0 {
baseConfig.LoadBalancerPort = fileConfig.LoadBalancerPort
}
if baseConfig.LoadBalancerForwardingMethod == "" && fileConfig.LoadBalancerForwardingMethod != "" {
baseConfig.LoadBalancerForwardingMethod = fileConfig.LoadBalancerForwardingMethod
}
if baseConfig.LoadBalancerClassName == "" && fileConfig.LoadBalancerClassName != "" {
baseConfig.LoadBalancerClassName = fileConfig.LoadBalancerClassName
}
// Routing Table configuration
if baseConfig.RoutingTableID == 0 && fileConfig.RoutingTableID != 0 {
baseConfig.RoutingTableID = fileConfig.RoutingTableID
}
if baseConfig.RoutingTableType == 0 && fileConfig.RoutingTableType != 0 {
baseConfig.RoutingTableType = fileConfig.RoutingTableType
}
if baseConfig.RoutingProtocol == 0 && fileConfig.RoutingProtocol != 0 {
baseConfig.RoutingProtocol = fileConfig.RoutingProtocol
}
// BGP configuration
mergeBGPConfig(&baseConfig.BGPConfig, &fileConfig.BGPConfig)
// Kubernetes configuration
if baseConfig.K8sConfigFile == "" && fileConfig.K8sConfigFile != "" {
baseConfig.K8sConfigFile = fileConfig.K8sConfigFile
}
// Leader Election configuration
mergeLeaderElectionConfig(&baseConfig.KubernetesLeaderElection, &fileConfig.KubernetesLeaderElection)
// BGP health check configuration
mergeHealthCheck(&baseConfig.ControlPlaneHealthCheck, &fileConfig.ControlPlaneHealthCheck)
// Prometheus configuration
if baseConfig.PrometheusHTTPServer == "" && fileConfig.PrometheusHTTPServer != "" {
baseConfig.PrometheusHTTPServer = fileConfig.PrometheusHTTPServer
}
// DNS configuration
if baseConfig.DNSMode == "" && fileConfig.DNSMode != "" {
baseConfig.DNSMode = fileConfig.DNSMode
}
// DHCP configuration - mode
if baseConfig.DHCPMode == "" && fileConfig.DHCPMode != "" {
baseConfig.DHCPMode = fileConfig.DHCPMode
}
// DHCP configuration - backoff attempts
if baseConfig.DHCPBackoffAttempts == DefaultDHCPBackoffAttempts && fileConfig.DHCPBackoffAttempts != DefaultDHCPBackoffAttempts {
baseConfig.DHCPBackoffAttempts = fileConfig.DHCPBackoffAttempts
}
// Health check configuration (HTTP listener for kube-vip readiness)
if baseConfig.HealthCheckPort == 0 && fileConfig.HealthCheckPort != 0 {
baseConfig.HealthCheckPort = fileConfig.HealthCheckPort
}
// Instance configuration
if baseConfig.InstanceName == "" && fileConfig.InstanceName != "" {
baseConfig.InstanceName = fileConfig.InstanceName
}
// Egress configuration
if baseConfig.EgressPodCidr == "" && fileConfig.EgressPodCidr != "" {
baseConfig.EgressPodCidr = fileConfig.EgressPodCidr
}
if baseConfig.EgressServiceCidr == "" && fileConfig.EgressServiceCidr != "" {
baseConfig.EgressServiceCidr = fileConfig.EgressServiceCidr
}
// Mirror configuration
if baseConfig.MirrorDestInterface == "" && fileConfig.MirrorDestInterface != "" {
baseConfig.MirrorDestInterface = fileConfig.MirrorDestInterface
}
// Iptables configuration
if baseConfig.IptablesBackend == "" && fileConfig.IptablesBackend != "" {
baseConfig.IptablesBackend = fileConfig.IptablesBackend
}
// Backend health check interval
if baseConfig.BackendHealthCheckInterval == 0 && fileConfig.BackendHealthCheckInterval != 0 {
baseConfig.BackendHealthCheckInterval = fileConfig.BackendHealthCheckInterval
}
// ARP broadcast rate
if baseConfig.ArpBroadcastRate == 0 && fileConfig.ArpBroadcastRate != 0 {
baseConfig.ArpBroadcastRate = fileConfig.ArpBroadcastRate
}
// Annotations
if baseConfig.Annotations == "" && fileConfig.Annotations != "" {
baseConfig.Annotations = fileConfig.Annotations
}
// Load balancers slice
if len(baseConfig.LoadBalancers) == 0 && len(fileConfig.LoadBalancers) > 0 {
baseConfig.LoadBalancers = fileConfig.LoadBalancers
}
// Debounce time for watch events
if baseConfig.DebounceTime == debouncer.DefaultTime && fileConfig.DebounceTime != debouncer.DefaultTime {
baseConfig.DebounceTime = fileConfig.DebounceTime
}
if baseConfig.LoseLeadershipTimeoutSeconds == 0 && fileConfig.LoseLeadershipTimeoutSeconds != 0 {
baseConfig.LoseLeadershipTimeoutSeconds = fileConfig.LoseLeadershipTimeoutSeconds
}
}
// mergeBGPConfig merges BGP configuration
func mergeBGPConfig(base, file *BGPConfig) {
if base.RouterID == "" && file.RouterID != "" {
base.RouterID = file.RouterID
}
if base.AS == 0 && file.AS != 0 {
base.AS = file.AS
}
if base.SourceIF == "" && file.SourceIF != "" {
base.SourceIF = file.SourceIF
}
if base.SourceIP == "" && file.SourceIP != "" {
base.SourceIP = file.SourceIP
}
if base.HoldTime == 0 && file.HoldTime != 0 {
base.HoldTime = file.HoldTime
}
if base.KeepaliveInterval == 0 && file.KeepaliveInterval != 0 {
base.KeepaliveInterval = file.KeepaliveInterval
}
if len(base.Peers) == 0 && len(file.Peers) > 0 {
base.Peers = file.Peers
}
}
// mergeLeaderElectionConfig merges leader election configuration
func mergeLeaderElectionConfig(base, file *KubernetesLeaderElection) {
if base.LeaseName == "" && file.LeaseName != "" {
base.LeaseName = file.LeaseName
}
if base.LeaseDuration == 0 && file.LeaseDuration != 0 {
base.LeaseDuration = file.LeaseDuration
}
if base.RenewDeadline == 0 && file.RenewDeadline != 0 {
base.RenewDeadline = file.RenewDeadline
}
if base.RetryPeriod == 0 && file.RetryPeriod != 0 {
base.RetryPeriod = file.RetryPeriod
}
if len(base.LeaseAnnotations) == 0 && len(file.LeaseAnnotations) > 0 {
base.LeaseAnnotations = file.LeaseAnnotations
}
}
// mergeHealthCheck merges HTTP health check configuration for BGP route advertisement.
func mergeHealthCheck(base, file *HealthCheck) {
if base.Address == "" && file.Address != "" {
base.Address = file.Address
}
if base.PeriodSeconds == 0 && file.PeriodSeconds != 0 {
base.PeriodSeconds = file.PeriodSeconds
}
if base.TimeoutSeconds == 0 && file.TimeoutSeconds != 0 {
base.TimeoutSeconds = file.TimeoutSeconds
}
if base.FailureThreshold == 0 && file.FailureThreshold != 0 {
base.FailureThreshold = file.FailureThreshold
}
if base.CAPath == "" && file.CAPath != "" {
base.CAPath = file.CAPath
}
}

View File

@@ -1,41 +0,0 @@
package kubevip
import (
"testing"
)
func TestParseEnvironmentSkipDAD(t *testing.T) {
cases := []struct {
name string
value string
want bool
wantErr bool
}{
{name: "unset keeps default false", value: "", want: false},
{name: "true enables", value: "true", want: true},
{name: "false disables", value: "false", want: false},
{name: "garbage errors", value: "not-a-bool", wantErr: true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if tc.value != "" {
t.Setenv(vipSkipDAD, tc.value)
}
c := &Config{}
err := ParseEnvironment(c)
if tc.wantErr {
if err == nil {
t.Fatal("expected an error, got nil")
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if c.SkipDAD != tc.want {
t.Fatalf("SkipDAD = %v, want %v", c.SkipDAD, tc.want)
}
})
}
}

View File

@@ -9,51 +9,36 @@ const (
// vip_arpRate - defines the rate of gARP broadcasts
vipArpRate = "vip_arpRate"
// vipPreserveOnLeadershipLoss - if true, VIP addresses will remain on interface when leadership is lost
vipPreserveOnLeadershipLoss = "vip_preserve_on_leadership_loss"
// vipLeaderElection - defines if the kubernetes algorithm should be used
vipLeaderElection = "vip_leaderelection"
// vipLeaseName - defines the name of the lease lock
vipLeaseName = "vip_leasename"
// vipLeaseDuration - defines how long the current leader is considered valid
// vipLeaderElection - defines if the kubernetes algorithm should be used
vipLeaseDuration = "vip_leaseduration"
// vipRenewDeadline - defines how long the leader has to renew the lease before losing leadership
// vipLeaderElection - defines if the kubernetes algorithm should be used
vipRenewDeadline = "vip_renewdeadline"
// vipRetryPeriod - defines the time between attempts to acquire/renew the lease
// vipLeaderElection - defines if the kubernetes algorithm should be used
vipRetryPeriod = "vip_retryperiod"
// vipLeaseAnnotations - defines the annotations given to the lease lock
// vipLeaderElection - defines the annotations given to the lease lock
vipLeaseAnnotations = "vip_leaseannotations"
// vipLogLevel - defines the level of logging to produce (5 being the most verbose)
vipLogLevel = "vip_loglevel"
// instanceName identifies this kube-vip deployment when naming host-global resources
instanceName = "instance_name"
// vipInterface - defines the interface that the vip should bind too
vipInterface = "vip_interface"
// vipLoseLeadership - defines if leader should lose leadership if network interface is down
vipLoseLeadership = "vip_loseleadership"
// vipLoseLeadershipTimeout - defines the timeout for lose leadership
vipLoseLeadershipTimeoutSeconds = "vip_loseleadership_timeout_seconds"
// vipInterfaceLoGlobal - defines if the lo interface (if used) should have a global scope
vipInterfaceLoGlobal = "vip_interfaceloglobal"
// vipServicesInterface - defines the interface that the service vips should bind too
vipServicesInterface = "vip_servicesinterface"
// vipAllowInterfaceNotUp - defines if kube-vip should tolerate a down interface
vipAllowInterfaceNotUp = "vip_allow_interface_not_up"
// vipSubnet - defines the subnet that the vip will use
vipSubnet = "vip_subnet"
@@ -66,11 +51,6 @@ const (
// egressWithNftables - enables using nftables over iptables
egressWithNftables = "egress_withnftables"
// perServiceElectionOnDemand - enables kube-vip to use per-service election for annotated services
perServiceElectionOnDemand = "per_service_election_on_demand"
// egressEnableInternalSNAT - enables the internal SNAT rule that kube-vip adds to the egress chain
egressEnableInternalSNAT = "egress_enableinternalsnat"
/////////////////////////////////////
// TO DO:
// Determine how to tidy this mess up
@@ -105,8 +85,6 @@ const (
// bgpEnable defines if BGP should be enabled
bgpEnable = "bgp_enable"
// bgpAttachIPToInterface defines if BGP service VIPs should be assigned to the configured interface
bgpAttachIPToInterface = "bgp_attach_ip_to_interface"
// bgpRouterID defines the routerID for the BGP server
bgpRouterID = "bgp_routerid"
// bgpRouterInterface defines the interface that we can find the address for
@@ -131,16 +109,6 @@ const (
bgpHoldTime = "bgp_hold_time"
// bgpKeepaliveInterval defines bgp timers keepalive interval
bgpKeepaliveInterval = "bgp_keepalive_interval"
// controlPlaneHealthCheckAddress defines the URL for control-plane health checks (BGP route withdrawal)
controlPlaneHealthCheckAddress = "control_plane_health_check_address"
// controlPlaneHealthCheckPeriodSeconds defines the period between control-plane health checks
controlPlaneHealthCheckPeriodSeconds = "control_plane_health_check_period_seconds"
// controlPlaneHealthCheckTimeoutSeconds defines the timeout for each control-plane health check request
controlPlaneHealthCheckTimeoutSeconds = "control_plane_health_check_timeout_seconds"
// controlPlaneHealthCheckFailureThreshold defines consecutive failures before BGP route withdrawal
controlPlaneHealthCheckFailureThreshold = "control_plane_health_check_failure_threshold"
// controlPlaneHealthCheckCAPath defines the path to a CA certificate for control-plane health check TLS verification
controlPlaneHealthCheckCAPath = "control_plane_health_check_ca_path"
// zebraEnable defines if Zebra integraton should be enabled
zebraEnable = "zebra_enable"
@@ -181,9 +149,6 @@ const (
// vipCleanRoutingTable - defines if routing table will be cleaned of redundant routes on kube-vip's start
vipCleanRoutingTable = "vip_cleanroutingtable" //nolint
// vipSkipDAD - defines if Duplicate Address Detection is skipped when adding the VIP address (IFA_F_NODAD)
vipSkipDAD = "vip_skipdad" //nolint
// cpNamespace defines the namespace the control plane pods will run in
cpNamespace = "cp_namespace"
@@ -244,12 +209,6 @@ const (
// dnsMode defines mode that DNS lookup will be performed with (first, ipv4, ipv6, dual)
dnsMode = "dns_mode"
// dhcpMode defines mode that DHCP lookup will be performed with (ipv4, ipv6, dual)
dhcpMode = "dhcp_mode"
// dhcpBackoffAttempts defines how many times DHCP client will try to obtain an IP address
dhcpBackoffAttempts = "dhcp_backoff_attempts"
// disableServiceUpdates disables service updating
disableServiceUpdates = "disable_service_updates"
@@ -275,10 +234,4 @@ const (
// egressClean enables egress cleaning on kube-vip's start
egressClean = "egress_clean"
// configFile defines the path to a JSON/YAML configuration file
configFile = "config_file"
// debounceTime defines what time should the event debouncer wait for events
debounceTime = "debounce_time"
)

View File

@@ -1,569 +0,0 @@
package kubevip
import (
"os"
"path/filepath"
"testing"
)
func TestLoadConfigFromFile(t *testing.T) {
// Create temporary directory for test files
tmpDir, err := os.MkdirTemp("", "kube-vip-config-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
tests := []struct {
name string
filename string
content string
expectedConfig *Config
wantErr bool
errContains string
}{
{
name: "Valid YAML config",
filename: "config.yaml",
content: `
logging: 2
enableARP: true
enableControlPlane: true
enableServices: true
address: "192.168.1.100"
port: 6443
interface: "eth0"
namespace: "kube-system"
instanceName: "release_a"
vipSubnet: "192.168.1.0/24"
leaseName: "test-lease"
leaseDuration: 15
renewDeadline: 10
retryPeriod: 2
prometheusHTTPServer: ":2112"
`,
expectedConfig: &Config{
Logging: 2,
EnableARP: true,
EnableControlPlane: true,
EnableServices: true,
Address: "192.168.1.100",
Port: 6443,
Interface: "eth0",
Namespace: "kube-system",
InstanceName: "release_a",
VIPSubnet: "192.168.1.0/24",
PrometheusHTTPServer: ":2112",
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "test-lease",
LeaseDuration: 15,
RenewDeadline: 10,
RetryPeriod: 2,
},
},
wantErr: false,
},
{
name: "Valid JSON config",
filename: "config.json",
content: `{
"logging": 3,
"enableBGP": true,
"enableServices": true,
"address": "10.0.0.100",
"port": 8443,
"interface": "ens192",
"namespace": "kube-system",
"loadBalancers": [
{
"name": "control-plane",
"ports": [
{
"type": "TCP",
"port": 6443
}
],
"bindToVip": true,
"forwardingMethod": "local"
}
]
}`,
expectedConfig: &Config{
Logging: 3,
EnableBGP: true,
EnableServices: true,
Address: "10.0.0.100",
Port: 8443,
Interface: "ens192",
Namespace: "kube-system",
LoadBalancers: []LoadBalancer{
{
Name: "control-plane",
Ports: []Port{
{
Type: "TCP",
Port: 6443,
},
},
BindToVip: true,
ForwardingMethod: "local",
},
},
},
wantErr: false,
},
{
name: "Complex BGP config",
filename: "bgp-config.yaml",
content: `
enableBGP: true
bgpConfig:
routerID: "192.168.1.1"
as: 65000
sourceIF: "eth0"
holdTime: 60
keepaliveInterval: 20
peers:
- address: "192.168.1.2"
as: 65001
port: 179
multiHop: false
- address: "192.168.1.3"
as: 65002
port: 179
multiHop: true
`,
expectedConfig: &Config{
EnableBGP: true,
BGPConfig: BGPConfig{
RouterID: "192.168.1.1",
AS: 65000,
SourceIF: "eth0",
HoldTime: 60,
KeepaliveInterval: 20,
Peers: []BGPPeer{
{
Address: "192.168.1.2",
AS: 65001,
Port: 179,
MultiHop: false,
},
{
Address: "192.168.1.3",
AS: 65002,
Port: 179,
MultiHop: true,
},
},
},
},
wantErr: false,
},
{
name: "Invalid JSON",
filename: "invalid.json",
content: `{"logging": 2, "invalid": }`,
wantErr: true,
errContains: "failed to parse JSON config file",
},
{
name: "Invalid YAML",
filename: "invalid.yaml",
content: "logging: 2\ninvalid: [unclosed",
wantErr: true,
errContains: "failed to parse YAML config file",
},
{
name: "Unsupported format",
filename: "config.txt",
content: "logging=2",
wantErr: true,
errContains: "unsupported config file format",
},
{
name: "Empty path",
filename: "",
content: "",
wantErr: true,
errContains: "config file path is empty",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var filePath string
if tt.filename != "" {
filePath = filepath.Join(tmpDir, tt.filename)
if err := os.WriteFile(filePath, []byte(tt.content), 0600); err != nil {
t.Fatalf("Failed to write test file: %v", err)
}
}
config, err := LoadConfigFromFile(filePath)
if tt.wantErr {
if err == nil {
t.Errorf("LoadConfigFromFile() expected error, got nil")
return
}
if tt.errContains != "" && !containsString(err.Error(), tt.errContains) {
t.Errorf("LoadConfigFromFile() error = %v, expected to contain %v", err, tt.errContains)
}
return
}
if err != nil {
t.Errorf("LoadConfigFromFile() unexpected error = %v", err)
return
}
if config == nil {
t.Errorf("LoadConfigFromFile() returned nil config")
return
}
// Compare key fields
if config.Logging != tt.expectedConfig.Logging {
t.Errorf("Logging = %v, expected %v", config.Logging, tt.expectedConfig.Logging)
}
if config.EnableARP != tt.expectedConfig.EnableARP {
t.Errorf("EnableARP = %v, expected %v", config.EnableARP, tt.expectedConfig.EnableARP)
}
if config.EnableBGP != tt.expectedConfig.EnableBGP {
t.Errorf("EnableBGP = %v, expected %v", config.EnableBGP, tt.expectedConfig.EnableBGP)
}
if config.Address != tt.expectedConfig.Address {
t.Errorf("Address = %v, expected %v", config.Address, tt.expectedConfig.Address)
}
if config.Port != tt.expectedConfig.Port {
t.Errorf("Port = %v, expected %v", config.Port, tt.expectedConfig.Port)
}
if config.Interface != tt.expectedConfig.Interface {
t.Errorf("Interface = %v, expected %v", config.Interface, tt.expectedConfig.Interface)
}
if config.InstanceName != tt.expectedConfig.InstanceName {
t.Errorf("InstanceName = %v, expected %v", config.InstanceName, tt.expectedConfig.InstanceName)
}
// Test BGP config if present
if tt.expectedConfig.EnableBGP {
if config.BGPConfig.RouterID != tt.expectedConfig.BGPConfig.RouterID {
t.Errorf("BGPConfig.RouterID = %v, expected %v", config.BGPConfig.RouterID, tt.expectedConfig.BGPConfig.RouterID)
}
if config.BGPConfig.AS != tt.expectedConfig.BGPConfig.AS {
t.Errorf("BGPConfig.AS = %v, expected %v", config.BGPConfig.AS, tt.expectedConfig.BGPConfig.AS)
}
if len(config.BGPConfig.Peers) != len(tt.expectedConfig.BGPConfig.Peers) {
t.Errorf("BGPConfig.Peers length = %v, expected %v", len(config.BGPConfig.Peers), len(tt.expectedConfig.BGPConfig.Peers))
}
}
// Test LoadBalancers if present
if len(tt.expectedConfig.LoadBalancers) > 0 {
if len(config.LoadBalancers) != len(tt.expectedConfig.LoadBalancers) {
t.Errorf("LoadBalancers length = %v, expected %v", len(config.LoadBalancers), len(tt.expectedConfig.LoadBalancers))
}
}
})
}
}
func TestMergeConfigFromFile(t *testing.T) {
// Create temporary directory for test files
tmpDir, err := os.MkdirTemp("", "kube-vip-merge-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
// Create test config file
configFile := filepath.Join(tmpDir, "test-config.yaml")
configContent := `
logging: 3
enableARP: true
enableServices: true
address: "192.168.1.200"
port: 6443
interface: "eth1"
namespace: "test-namespace"
leaseName: "file-lease"
leaseDuration: 20
prometheusHTTPServer: ":3000"
`
if err := os.WriteFile(configFile, []byte(configContent), 0600); err != nil {
t.Fatalf("Failed to write test config file: %v", err)
}
tests := []struct {
name string
baseConfig *Config
configFilePath string
expected *Config
wantErr bool
}{
{
name: "Merge with empty base config",
baseConfig: &Config{},
configFilePath: configFile,
expected: &Config{
Logging: 3,
EnableARP: true,
EnableServices: true,
Address: "192.168.1.200",
Port: 6443,
Interface: "eth1",
Namespace: "test-namespace",
PrometheusHTTPServer: ":3000",
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "file-lease",
LeaseDuration: 20,
},
},
wantErr: false,
},
{
name: "Merge respects existing values (priority test)",
baseConfig: &Config{
Logging: 5, // Should not be overridden
Port: 8443, // Should not be overridden
Interface: "eth0", // Should not be overridden
},
configFilePath: configFile,
expected: &Config{
Logging: 5, // From base (higher priority)
EnableARP: true, // From file
EnableServices: true, // From file
Address: "192.168.1.200", // From file
Port: 8443, // From base (higher priority)
Interface: "eth0", // From base (higher priority)
Namespace: "test-namespace", // From file
PrometheusHTTPServer: ":3000", // From file
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "file-lease", // From file
LeaseDuration: 20, // From file
},
},
wantErr: false,
},
{
name: "Empty config file path",
baseConfig: &Config{
Logging: 1,
},
configFilePath: "",
expected: &Config{
Logging: 1, // Unchanged
},
wantErr: false,
},
{
name: "Non-existent config file",
baseConfig: &Config{
Logging: 1,
},
configFilePath: "/non/existent/file.yaml",
expected: &Config{
Logging: 1,
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := MergeConfigFromFile(tt.baseConfig, tt.configFilePath)
if tt.wantErr {
if err == nil {
t.Errorf("MergeConfigFromFile() expected error, got nil")
}
return
}
if err != nil {
t.Errorf("MergeConfigFromFile() unexpected error = %v", err)
return
}
// Compare key fields
if tt.baseConfig.Logging != tt.expected.Logging {
t.Errorf("Logging = %v, expected %v", tt.baseConfig.Logging, tt.expected.Logging)
}
if tt.baseConfig.EnableARP != tt.expected.EnableARP {
t.Errorf("EnableARP = %v, expected %v", tt.baseConfig.EnableARP, tt.expected.EnableARP)
}
if tt.baseConfig.Address != tt.expected.Address {
t.Errorf("Address = %v, expected %v", tt.baseConfig.Address, tt.expected.Address)
}
if tt.baseConfig.Port != tt.expected.Port {
t.Errorf("Port = %v, expected %v", tt.baseConfig.Port, tt.expected.Port)
}
if tt.baseConfig.Interface != tt.expected.Interface {
t.Errorf("Interface = %v, expected %v", tt.baseConfig.Interface, tt.expected.Interface)
}
if tt.baseConfig.Namespace != tt.expected.Namespace {
t.Errorf("Namespace = %v, expected %v", tt.baseConfig.Namespace, tt.expected.Namespace)
}
if tt.baseConfig.PrometheusHTTPServer != tt.expected.PrometheusHTTPServer {
t.Errorf("PrometheusHTTPServer = %v, expected %v", tt.baseConfig.PrometheusHTTPServer, tt.expected.PrometheusHTTPServer)
}
})
}
}
func TestMergeConfigValues(t *testing.T) {
tests := []struct {
name string
baseConfig *Config
fileConfig *Config
expectedBase *Config
}{
{
name: "Merge basic configuration",
baseConfig: &Config{
Logging: 5, // Should not be overridden
Port: 0, // Should be overridden
},
fileConfig: &Config{
Logging: 2,
Port: 6443,
Interface: "eth0",
Address: "192.168.1.100",
InstanceName: "release_a",
},
expectedBase: &Config{
Logging: 5, // From base (non-zero)
Port: 6443, // From file (base was zero)
Interface: "eth0", // From file (base was empty)
Address: "192.168.1.100", // From file (base was empty)
InstanceName: "release_a", // From file (base was empty)
},
},
{
name: "Merge boolean flags",
baseConfig: &Config{
EnableARP: true, // Should not be overridden
},
fileConfig: &Config{
EnableARP: false, // Should not override true
EnableBGP: true, // Should be set
EnableServices: true, // Should be set
EnableWireguard: false, // Should not be set (false doesn't override false)
},
expectedBase: &Config{
EnableARP: true, // From base (true has priority)
EnableBGP: true, // From file
EnableServices: true, // From file
EnableWireguard: false, // Remains false
},
},
{
name: "Merge BGP configuration",
baseConfig: &Config{
BGPConfig: BGPConfig{
RouterID: "1.1.1.1", // Should not be overridden
},
},
fileConfig: &Config{
BGPConfig: BGPConfig{
RouterID: "2.2.2.2", // Should not override
AS: 65000, // Should be set
SourceIF: "eth0", // Should be set
HoldTime: 30, // Should be set
KeepaliveInterval: 10, // Should be set
},
},
expectedBase: &Config{
BGPConfig: BGPConfig{
RouterID: "1.1.1.1", // From base (non-empty)
AS: 65000, // From file (base was zero)
SourceIF: "eth0", // From file (base was empty)
HoldTime: 30, // From file (base was zero)
KeepaliveInterval: 10, // From file (base was zero)
},
},
},
{
name: "Merge leader election configuration",
baseConfig: &Config{
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "base-lease", // Should not be overridden
},
},
fileConfig: &Config{
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "file-lease", // Should not override
LeaseDuration: 15, // Should be set
RenewDeadline: 10, // Should be set
RetryPeriod: 2, // Should be set
},
},
expectedBase: &Config{
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "base-lease", // From base (non-empty)
LeaseDuration: 15, // From file (base was zero)
RenewDeadline: 10, // From file (base was zero)
RetryPeriod: 2, // From file (base was zero)
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mergeConfigValues(tt.baseConfig, tt.fileConfig)
// Compare results
if tt.baseConfig.Logging != tt.expectedBase.Logging {
t.Errorf("Logging = %v, expected %v", tt.baseConfig.Logging, tt.expectedBase.Logging)
}
if tt.baseConfig.Port != tt.expectedBase.Port {
t.Errorf("Port = %v, expected %v", tt.baseConfig.Port, tt.expectedBase.Port)
}
if tt.baseConfig.Interface != tt.expectedBase.Interface {
t.Errorf("Interface = %v, expected %v", tt.baseConfig.Interface, tt.expectedBase.Interface)
}
if tt.baseConfig.InstanceName != tt.expectedBase.InstanceName {
t.Errorf("InstanceName = %v, expected %v", tt.baseConfig.InstanceName, tt.expectedBase.InstanceName)
}
if tt.baseConfig.EnableARP != tt.expectedBase.EnableARP {
t.Errorf("EnableARP = %v, expected %v", tt.baseConfig.EnableARP, tt.expectedBase.EnableARP)
}
if tt.baseConfig.EnableBGP != tt.expectedBase.EnableBGP {
t.Errorf("EnableBGP = %v, expected %v", tt.baseConfig.EnableBGP, tt.expectedBase.EnableBGP)
}
if tt.baseConfig.BGPConfig.RouterID != tt.expectedBase.BGPConfig.RouterID {
t.Errorf("BGPConfig.RouterID = %v, expected %v", tt.baseConfig.BGPConfig.RouterID, tt.expectedBase.BGPConfig.RouterID)
}
if tt.baseConfig.BGPConfig.AS != tt.expectedBase.BGPConfig.AS {
t.Errorf("BGPConfig.AS = %v, expected %v", tt.baseConfig.BGPConfig.AS, tt.expectedBase.BGPConfig.AS)
}
if tt.baseConfig.KubernetesLeaderElection.LeaseName != tt.expectedBase.KubernetesLeaderElection.LeaseName {
t.Errorf("KubernetesLeaderElection.LeaseName = %v, expected %v", tt.baseConfig.KubernetesLeaderElection.LeaseName, tt.expectedBase.KubernetesLeaderElection.LeaseName)
}
})
}
}
func TestLoadConfigFromFile_FileNotExists(t *testing.T) {
_, err := LoadConfigFromFile("/non/existent/path/config.yaml")
if err == nil {
t.Error("LoadConfigFromFile() expected error for non-existent file, got nil")
}
if !containsString(err.Error(), "config file does not exist") {
t.Errorf("LoadConfigFromFile() error = %v, expected to contain 'config file does not exist'", err)
}
}
// Helper function to check if a string contains a substring
func containsString(str, substr string) bool {
return len(str) >= len(substr) && (str == substr || len(substr) == 0 ||
(len(substr) > 0 && func() bool {
for i := 0; i <= len(str)-len(substr); i++ {
if str[i:i+len(substr)] == substr {
return true
}
}
return false
}()))
}

View File

@@ -5,8 +5,6 @@ import (
"log"
"strconv"
"github.com/google/go-containerregistry/pkg/name"
"github.com/kube-vip/kube-vip/pkg/debouncer"
appv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -15,6 +13,7 @@ import (
applyCoreV1 "k8s.io/client-go/applyconfigurations/core/v1"
applyMetaV1 "k8s.io/client-go/applyconfigurations/meta/v1"
applyRbacV1 "k8s.io/client-go/applyconfigurations/rbac/v1"
"sigs.k8s.io/yaml"
)
@@ -125,13 +124,6 @@ func GenerateRole(c *Config, role bool) *applyRbacV1.RoleApplyConfiguration {
},
},
}
if !role {
newManifest.Rules = append(newManifest.Rules, applyRbacV1.PolicyRuleApplyConfiguration{
APIGroups: []string{"networking.k8s.io"},
Resources: []string{"servicecidrs"},
Verbs: []string{"list", "get", "watch"},
})
}
return newManifest
}
@@ -175,11 +167,7 @@ func GenerateRoleBinding(rolebinding bool, saCfg *applyCoreV1.ServiceAccountAppl
}
// generatePodSpec will take a kube-vip config and generate a Pod spec
func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*corev1.Pod, error) {
imageRef, err := name.NewTag(image, name.WeakValidation, name.WithDefaultTag(imageVersion))
if err != nil {
return nil, fmt.Errorf("cannot parse %q: %w", image, err)
}
func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod {
command := "manager"
// Determine where the pods should be living (for multi-tenancy)
@@ -209,12 +197,6 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
},
},
}
if c.InstanceName != "" {
newEnvironment = append(newEnvironment, corev1.EnvVar{
Name: instanceName,
Value: c.InstanceName,
})
}
// If we're specifically saying which interface to use then add it to the manifest
if c.Interface != "" {
@@ -246,17 +228,6 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
newEnvironment = append(newEnvironment, svcInterface...)
}
// Tolerate a down interface
if c.AllowInterfaceNotUp {
allowIface := []corev1.EnvVar{
{
Name: vipAllowInterfaceNotUp,
Value: strconv.FormatBool(c.AllowInterfaceNotUp),
},
}
newEnvironment = append(newEnvironment, allowIface...)
}
// If a subnet is required for the VIP
if c.VIPSubnet != "" {
// build environment variables
@@ -280,28 +251,6 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
newEnvironment = append(newEnvironment, dnsModeSelector...)
}
if c.DHCPMode != "" {
// build environment variables
dhcpModeSelector := []corev1.EnvVar{
{
Name: dhcpMode,
Value: c.DHCPMode,
},
}
newEnvironment = append(newEnvironment, dhcpModeSelector...)
}
if c.DHCPBackoffAttempts != DefaultDHCPBackoffAttempts {
// build environment variables
dhcpBackoff := []corev1.EnvVar{
{
Name: dhcpBackoffAttempts,
Value: strconv.FormatUint(uint64(c.DHCPBackoffAttempts), 10),
},
}
newEnvironment = append(newEnvironment, dhcpBackoff...)
}
// If we're doing the hybrid mode
if c.EnableControlPlane {
cp := []corev1.EnvVar{
@@ -414,23 +363,6 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
newEnvironment = append(newEnvironment, leaderElection...)
}
if c.LoseLeadership {
loseLeadership := []corev1.EnvVar{
{
Name: vipLoseLeadership,
Value: strconv.FormatBool(c.LoseLeadership),
},
}
if c.LoseLeadershipTimeoutSeconds > 0 {
loseLeadership = append(loseLeadership, corev1.EnvVar{
Name: vipLoseLeadership,
Value: fmt.Sprintf("%d", c.LoseLeadershipTimeoutSeconds),
})
}
newEnvironment = append(newEnvironment, loseLeadership...)
}
// If we're enabling node labeling on leader election
if c.EnableNodeLabeling {
EnableNodeLabeling := []corev1.EnvVar{
@@ -483,12 +415,6 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
Value: strconv.FormatBool(c.EnableBGP),
},
}
if c.BGPAttachIPToInterface {
bgp = append(bgp, corev1.EnvVar{
Name: bgpAttachIPToInterface,
Value: strconv.FormatBool(c.BGPAttachIPToInterface),
})
}
newEnvironment = append(newEnvironment, bgp...)
}
@@ -555,40 +481,6 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
}
if c.ControlPlaneHealthCheck.Address != "" {
healthCheckVars := []corev1.EnvVar{
{
Name: controlPlaneHealthCheckAddress,
Value: c.ControlPlaneHealthCheck.Address,
},
}
if c.ControlPlaneHealthCheck.PeriodSeconds > 0 {
healthCheckVars = append(healthCheckVars, corev1.EnvVar{
Name: controlPlaneHealthCheckPeriodSeconds,
Value: fmt.Sprintf("%d", c.ControlPlaneHealthCheck.PeriodSeconds),
})
}
if c.ControlPlaneHealthCheck.TimeoutSeconds > 0 {
healthCheckVars = append(healthCheckVars, corev1.EnvVar{
Name: controlPlaneHealthCheckTimeoutSeconds,
Value: fmt.Sprintf("%d", c.ControlPlaneHealthCheck.TimeoutSeconds),
})
}
if c.ControlPlaneHealthCheck.FailureThreshold > 0 {
healthCheckVars = append(healthCheckVars, corev1.EnvVar{
Name: controlPlaneHealthCheckFailureThreshold,
Value: fmt.Sprintf("%d", c.ControlPlaneHealthCheck.FailureThreshold),
})
}
if c.ControlPlaneHealthCheck.CAPath != "" {
healthCheckVars = append(healthCheckVars, corev1.EnvVar{
Name: controlPlaneHealthCheckCAPath,
Value: c.ControlPlaneHealthCheck.CAPath,
})
}
newEnvironment = append(newEnvironment, healthCheckVars...)
}
// If the load-balancer is enabled then add the configuration to the manifest
if c.EnableLoadBalancer {
lb := []corev1.EnvVar{
@@ -669,7 +561,7 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
var securityContext *corev1.SecurityContext
if c.LoadBalancerForwardingMethod == "masquerade" {
privileged := true
var privileged = true
securityContext = &corev1.SecurityContext{
Privileged: &privileged,
}
@@ -687,25 +579,6 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
}
}
if c.PreserveVIPOnLeadershipLoss {
preserveVIPOnLeadershipLoss := []corev1.EnvVar{
{
Name: vipPreserveOnLeadershipLoss,
Value: strconv.FormatBool(c.PreserveVIPOnLeadershipLoss),
},
}
newEnvironment = append(newEnvironment, preserveVIPOnLeadershipLoss...)
}
if c.DebounceTime != debouncer.DefaultTime {
debTime := corev1.EnvVar{
Name: debounceTime,
Value: c.DebounceTime,
}
newEnvironment = append(newEnvironment, debTime)
}
newManifest := &corev1.Pod{
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
@@ -719,7 +592,7 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
Containers: []corev1.Container{
{
Name: "kube-vip",
Image: imageRef.Name(),
Image: fmt.Sprintf("ghcr.io/kube-vip/kube-vip:%s", imageVersion),
ImagePullPolicy: corev1.PullIfNotPresent,
SecurityContext: securityContext,
Args: []string{
@@ -760,24 +633,18 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) (*co
newManifest.Spec.HostAliases = append(newManifest.Spec.HostAliases, hostAlias)
}
return newManifest, nil
return newManifest
}
// GeneratePodManifestFromConfig will take a kube-vip config and generate a manifest
func GeneratePodManifestFromConfig(c *Config, image, imageVersion string, inCluster bool) (string, error) {
newManifest, err := generatePodSpec(c, image, imageVersion, inCluster)
if err != nil {
return "", err
}
b, err := yaml.Marshal(newManifest)
if err != nil {
return "", fmt.Errorf("failed to marshal manifest: %w", err)
}
return string(b), nil
func GeneratePodManifestFromConfig(c *Config, imageVersion string, inCluster bool) string {
newManifest := generatePodSpec(c, imageVersion, inCluster)
b, _ := yaml.Marshal(newManifest)
return string(b)
}
// GenerateDaemonsetManifestFromConfig will take a kube-vip config and generate a manifest
func GenerateDaemonsetManifestFromConfig(c *Config, image, imageVersion string, inCluster, taint bool) (string, error) {
func GenerateDaemonsetManifestFromConfig(c *Config, imageVersion string, inCluster, taint bool) string {
// Determine where the pod should be deployed
var namespace string
if c.ServiceNamespace != "" {
@@ -786,11 +653,7 @@ func GenerateDaemonsetManifestFromConfig(c *Config, image, imageVersion string,
namespace = metav1.NamespaceSystem
}
pod, err := generatePodSpec(c, image, imageVersion, inCluster)
if err != nil {
return "", err
}
podSpec := generatePodSpec(c, imageVersion, inCluster).Spec
newManifest := &appv1.DaemonSet{
TypeMeta: metav1.TypeMeta{
Kind: "DaemonSet",
@@ -817,7 +680,7 @@ func GenerateDaemonsetManifestFromConfig(c *Config, image, imageVersion string,
"app.kubernetes.io/version": imageVersion,
},
},
Spec: pod.Spec,
Spec: podSpec,
},
},
}
@@ -867,5 +730,5 @@ func GenerateDaemonsetManifestFromConfig(c *Config, image, imageVersion string,
delete(m, "status")
b, _ = yaml.Marshal(m)
return string(b), nil
return string(b)
}

View File

@@ -1,38 +1,6 @@
package kubevip
import (
"os"
"slices"
"strings"
"testing"
applyRbacV1 "k8s.io/client-go/applyconfigurations/rbac/v1"
)
func TestGenerateRoleServiceCIDRAccess(t *testing.T) {
clusterRole := GenerateRole(&Config{}, false)
if !hasServiceCIDRRule(clusterRole) {
t.Fatal("generated ClusterRole is missing ServiceCIDR access")
}
role := GenerateRole(&Config{ServiceNamespace: "kube-vip"}, true)
if hasServiceCIDRRule(role) {
t.Fatal("generated namespaced Role contains ineffective ServiceCIDR access")
}
}
func hasServiceCIDRRule(role *applyRbacV1.RoleApplyConfiguration) bool {
for _, rule := range role.Rules {
if slices.Contains(rule.APIGroups, "networking.k8s.io") &&
slices.Contains(rule.Resources, "servicecidrs") &&
slices.Contains(rule.Verbs, "get") &&
slices.Contains(rule.Verbs, "list") &&
slices.Contains(rule.Verbs, "watch") {
return true
}
}
return false
}
import "testing"
func TestParseEnvironment(t *testing.T) {
@@ -41,8 +9,8 @@ func TestParseEnvironment(t *testing.T) {
c *Config
wantErr bool
}{
{"nil config", nil, false},
{"basic config", &Config{Interface: "eth0", ServicesInterface: "eth1"}, false},
{"", nil, false},
{"", &Config{Interface: "eth0", ServicesInterface: "eth1"}, false},
}
for _, tt := range tests {
t.Logf("%v", tt.c)
@@ -53,146 +21,3 @@ func TestParseEnvironment(t *testing.T) {
})
}
}
func TestParseEnvironmentInstanceName(t *testing.T) {
tests := []struct {
name string
lowercase string
uppercase string
want string
}{
{name: "lowercase", lowercase: "release_a", want: "release_a"},
{name: "uppercase fallback", uppercase: "release_b", want: "release_b"},
{name: "lowercase takes precedence", lowercase: "release_a", uppercase: "release_b", want: "release_a"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Setenv(instanceName, tt.lowercase)
t.Setenv(strings.ToUpper(instanceName), tt.uppercase)
config := &Config{}
if err := ParseEnvironment(config); err != nil {
t.Fatalf("ParseEnvironment() error = %v", err)
}
if config.InstanceName != tt.want {
t.Fatalf("InstanceName = %q, want %q", config.InstanceName, tt.want)
}
})
}
}
func TestParseEnvironmentBGPAttachIPToInterface(t *testing.T) {
t.Setenv(bgpAttachIPToInterface, "true")
config := &Config{}
if err := ParseEnvironment(config); err != nil {
t.Fatalf("ParseEnvironment() error = %v", err)
}
if !config.BGPAttachIPToInterface {
t.Fatal("BGPAttachIPToInterface = false, want true")
}
}
func TestGeneratePodSpecBGPAttachIPToInterface(t *testing.T) {
pod, err := generatePodSpec(&Config{
EnableBGP: true,
BGPAttachIPToInterface: true,
}, "ghcr.io/kube-vip/kube-vip", "v0.0.0", true)
if err != nil {
t.Fatalf("generatePodSpec() error = %v", err)
}
for _, env := range pod.Spec.Containers[0].Env {
if env.Name == bgpAttachIPToInterface && env.Value == "true" {
return
}
}
t.Fatalf("%s=true is missing from generated pod environment", bgpAttachIPToInterface)
}
func TestGeneratePodSpecInstanceName(t *testing.T) {
tests := []struct {
name string
instanceName string
wantPresent bool
}{
{name: "configured", instanceName: "release_a", wantPresent: true},
{name: "empty", wantPresent: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pod, err := generatePodSpec(&Config{InstanceName: tt.instanceName}, "ghcr.io/kube-vip/kube-vip", "v0.0.0", true)
if err != nil {
t.Fatalf("generatePodSpec() error = %v", err)
}
var value string
found := false
for _, env := range pod.Spec.Containers[0].Env {
if env.Name == instanceName {
found = true
value = env.Value
break
}
}
if found != tt.wantPresent {
t.Fatalf("instance_name present = %t, want %t", found, tt.wantPresent)
}
if found && value != tt.instanceName {
t.Fatalf("instance_name = %q, want %q", value, tt.instanceName)
}
})
}
}
func TestParseEnvironmentConfigFile(t *testing.T) {
// Save original environment
originalConfigFile := os.Getenv("config_file")
defer func() {
if originalConfigFile != "" {
os.Setenv("config_file", originalConfigFile)
} else {
os.Unsetenv("config_file")
}
}()
tests := []struct {
name string
envValue string
expectedConfig string
}{
{
name: "config_file environment variable set",
envValue: "/etc/kube-vip/config.yaml",
expectedConfig: "/etc/kube-vip/config.yaml",
},
{
name: "config_file environment variable empty",
envValue: "",
expectedConfig: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Set environment variable
if tt.envValue != "" {
os.Setenv("config_file", tt.envValue)
} else {
os.Unsetenv("config_file")
}
config := &Config{}
err := ParseEnvironment(config)
if err != nil {
t.Errorf("ParseEnvironment() unexpected error = %v", err)
}
if config.ConfigFile != tt.expectedConfig {
t.Errorf("ConfigFile = %v, expected %v", config.ConfigFile, tt.expectedConfig)
}
})
}
}

View File

@@ -1,7 +1,6 @@
package kubevip
import (
"errors"
"fmt"
log "log/slog"
@@ -13,34 +12,16 @@ const (
Auto = "auto"
)
var ErrInterfaceNotUp = errors.New("interface is not up")
func (c *Config) CheckSubnetExists() error {
if c.VIPSubnet == "" && c.VIP != "" && c.Address == "" {
return fmt.Errorf("vip_subnet must be set if using vip_address instead of address environment variable")
}
return nil
}
func (c *Config) CheckInterface() error {
if c.Interface != "" {
if err := isValidInterface(c.Interface); err != nil {
if errors.Is(err, ErrInterfaceNotUp) && c.AllowInterfaceNotUp {
log.Warn("interface is not up, continuing as allowInterfaceNotUp is set", "interface", c.Interface)
} else {
return fmt.Errorf("%s is not valid interface, reason: %w", c.Interface, err)
}
return fmt.Errorf("%s is not valid interface, reason: %w", c.Interface, err)
}
}
if c.ServicesInterface != "" {
if err := isValidInterface(c.ServicesInterface); err != nil {
if errors.Is(err, ErrInterfaceNotUp) && c.AllowInterfaceNotUp {
log.Warn("interface is not up, continuing as allowInterfaceNotUp is set", "interface", c.ServicesInterface)
} else {
return fmt.Errorf("%s is not valid interface, reason: %w", c.ServicesInterface, err)
}
return fmt.Errorf("%s is not valid interface, reason: %w", c.ServicesInterface, err)
}
}
@@ -69,7 +50,7 @@ func isValidInterface(iface string) error {
iface,
)
} else if attrs.OperState != netlink.OperUp {
return fmt.Errorf("%s %w", iface, ErrInterfaceNotUp)
return fmt.Errorf("%s is not up", iface)
}
return nil

View File

@@ -1,56 +0,0 @@
package kubevip
import "testing"
func TestCheckSubnetExists(t *testing.T) {
tests := []struct {
name string
config Config
wantErr bool
}{
{
name: "vip only without subnet returns error",
config: Config{
VIP: "172.18.0.20",
},
wantErr: true,
},
{
name: "vip with subnet does not return error",
config: Config{
VIP: "172.18.0.20",
VIPSubnet: "32",
},
wantErr: false,
},
{
name: "address only without subnet does not return error",
config: Config{
Address: "172.18.0.20",
},
wantErr: false,
},
{
name: "address overrides vip without subnet",
config: Config{
VIP: "172.18.0.20",
Address: "172.18.0.30",
},
wantErr: false,
},
{
name: "empty config does not return error",
config: Config{},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.config.CheckSubnetExists()
if (err != nil) != tt.wantErr {
t.Errorf("CheckSubnetExists() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

View File

@@ -1,5 +1,9 @@
package kubevip
import (
"github.com/kube-vip/kube-vip/pkg/bgp"
)
// Config defines all of the settings for the Kube-Vip Pod
type Config struct {
// Logging, settings
@@ -11,9 +15,6 @@ type Config struct {
// EnableBGP, will use BGP to advertise the VIP address
EnableBGP bool `yaml:"enableBGP"`
// BGPAttachIPToInterface assigns BGP-advertised service VIPs to the configured interface
BGPAttachIPToInterface bool `yaml:"bgpAttachIPToInterface"`
// EnableWireguard, will use wireguard to advertise the VIP address
EnableWireguard bool `yaml:"enableWireguard"`
@@ -53,16 +54,6 @@ type Config struct {
// ArpBroadcastRate, defines how often kube-vip will update the network about updates to the network
ArpBroadcastRate int64 `yaml:"arpBroadcastRate"`
// PreserveVIPOnLeadershipLoss, if true, VIP addresses will remain on interface when leadership is lost (only ARP/NDP broadcasting stops)
// If false, VIP addresses are deleted on leadership loss (legacy behavior)
PreserveVIPOnLeadershipLoss bool `yaml:"preserveVipOnLeadershipLoss"`
// LoseLeadership enables leadership loss if VIP interface(physical) is down
LoseLeadership bool `yaml:"loseLeadership"`
// LoseLeadershipTimeoutSeconds defines the timeout after which interface will be considered down. Default is 30s
LoseLeadershipTimeoutSeconds int `yaml:"loseLeadershipTimeoutSeconds"`
// Annotations will define if we're going to wait and lookup configuration from Kubernetes node annotations
Annotations string
@@ -94,9 +85,6 @@ type Config struct {
// Namespace will define which namespace the control plane pods will run in
Namespace string `yaml:"namespace"`
// InstanceName identifies this kube-vip deployment when naming host-global resources.
InstanceName string `yaml:"instanceName"`
// Namespace will define which namespace the control plane pods will run in
ServiceNamespace string `yaml:"serviceNamespace"`
@@ -118,9 +106,6 @@ type Config struct {
// ServicesInterface is the network interface to bind to for services (optional)
ServicesInterface string `yaml:"servicesInterface,omitempty"`
// AllowInterfaceNotUp allows kube-vip to start even when the interface is not up
AllowInterfaceNotUp bool `yaml:"allowInterfaceNotUp,omitempty"`
// EnableLoadBalancer, provides the flexibility to make the load-balancer optional
EnableLoadBalancer bool `yaml:"enableLoadBalancer"`
@@ -142,18 +127,11 @@ type Config struct {
// Clean routing table of redundant routes on start
CleanRoutingTable bool `yaml:"cleanRoutingTable"`
// Skip Duplicate Address Detection when adding the VIP address (IFA_F_NODAD)
SkipDAD bool `yaml:"skipDAD"`
// BGP Configuration
BGPConfig BGPConfig
BGPPeerConfig BGPPeer
BGPConfig bgp.Config
BGPPeerConfig bgp.Peer
BGPPeers []string
// ControlPlaneHealthCheck configures HTTP polling of the control plane when using BGP without
// leader election. If the health check fails, the BGP route will be withdrawn.
ControlPlaneHealthCheck HealthCheck `yaml:"controlPlaneHealthCheck,omitempty"`
// LoadBalancers are the various services we can load balance over
LoadBalancers []LoadBalancer `yaml:"loadBalancers,omitempty"`
@@ -168,9 +146,6 @@ type Config struct {
// EgressServiceCidr, this contains the service cidr range to ignore
EgressServiceCidr string
// EnableInternalSNAT, this will enable the internal SNAT rule that kube-vip adds to the egress chain
EnableInternalSNAT bool
// EgressWithNftables, this will use the iptables-nftables OVER iptables
EgressWithNftables bool
@@ -183,16 +158,6 @@ type Config struct {
// DNSMode, this will set the mode DSN lookup will be performed (first, ipv4, ipv6, dual)
DNSMode string `yaml:"dnsDualStackMode"`
// IsDualStack reports if service is DualStack.
IsDualStack bool
// RequireDualStack defines if DualStack is required for the service. Based on service's Spec.ipFamilyPolicy field.
RequireDualStack bool
// DNSMode, this will set the mode DHCP lookup will be performed for DDNS (ipv4, ipv6, dual). By default will be the same as DNSMode.
// If DNSMode is 'first', IPv4 will be used.
DHCPMode string `yaml:"dhcpDualStackMode"`
// DisableServiceUpdates, if true, kube-vip will only advertise service, but it will not update service's Status.LoadBalancer.Ingress slice
DisableServiceUpdates bool `yaml:"disableServiceUpdates"`
@@ -221,18 +186,6 @@ type Config struct {
// EgressClean, enables egress cleaning on Kube-vip's start
EgressClean bool `yaml:"egressClean"`
// ConfigFile defines the path to a JSON/YAML configuration file
ConfigFile string `yaml:"configFile"`
// DHCPBackoffAttempts defines how many times will DHCP client try to obtain address (unlimited when 0)
DHCPBackoffAttempts uint `yaml:"dhcpBackoffAttempts"`
// DebounceTime defines how long will event debouncer wait for the events to arrive
DebounceTime string `yaml:"debounceTime"`
// PerServiceElectionOnDemand will enable kube-vip to handle services with per-service election when annotation is used
PerServiceElectionOnDemand bool `yaml:"perServiceElectionOnDemand"`
}
// KubernetesLeaderElection defines all of the settings for Kubernetes KubernetesLeaderElection
@@ -264,29 +217,12 @@ type Etcd struct {
Endpoints []string
}
// HealthCheck defines HTTP health-check settings for control-plane polling when using BGP
// without leader election.
type HealthCheck struct {
// Address is the URL to poll to check the health of the control-plane. If the health
// check fails, the BGP route will be withdrawn.
Address string `yaml:"address"`
// PeriodSeconds is the interval in seconds between health checks.
PeriodSeconds int `yaml:"periodSeconds"`
// TimeoutSeconds is the timeout per health check request. If a request takes longer
// than this timeout, the health check is considered failed.
TimeoutSeconds int `yaml:"timeoutSeconds"`
// FailureThreshold is the number of consecutive failures before route withdrawal.
FailureThreshold int `yaml:"failureThreshold"`
// CAPath is the CA certificate path used for TLS verification when Address is an HTTPS URL.
CAPath string `yaml:"caPath"`
}
// LoadBalancer contains the configuration of a load balancing instance
type LoadBalancer struct {
// Name of a LoadBalancer
Name string `yaml:"name"`
// Ports exposed by a LoadBalancer
//Ports exposed by a LoadBalancer
Ports []Port
// BindToVip will bind the load balancer port to the VIP itself

View File

@@ -1,87 +0,0 @@
package kubevip
import (
"fmt"
"math"
"net/url"
"strings"
)
const (
// nftables object names are limited to 255 bytes. Reserve space for the
// prefix and address-family suffix added to the instance name.
nftablesNameMaxLength = 255
egressNftablesTablePrefix = "kube_vip_"
egressNftablesTableSuffix = "_v4"
instanceNameMaxLength = nftablesNameMaxLength - len(egressNftablesTablePrefix) - len(egressNftablesTableSuffix)
)
// Validate runs configuration checks that are independent of host state.
// This should be called after all config sources (flags, file, env vars) are merged.
func (c *Config) Validate() error {
if err := validateHealthCheckAddress(c.ControlPlaneHealthCheck.Address); err != nil {
return err
}
if err := validateInstanceName(c.InstanceName); err != nil {
return err
}
if err := validateRoutingProtocol(c.RoutingProtocol); err != nil {
return err
}
return nil
}
// validateRoutingProtocol rejects values the kernel cannot represent: netlink
// carries the address and route protocol in a single byte, so a larger value is
// silently truncated on the wire and never matches on readback.
func validateRoutingProtocol(protocol int) error {
if protocol < 0 || protocol > math.MaxUint8 {
return fmt.Errorf("routingProtocol %d is out of range, must be between 0 and %d", protocol, math.MaxUint8)
}
return nil
}
func validateInstanceName(name string) error {
if name == "" {
return nil
}
if len(name) > instanceNameMaxLength {
return fmt.Errorf("instance_name is %d bytes, must not exceed %d bytes so the %q prefix and %q or %q suffix fit within the nftables %d-byte name limit",
len(name), instanceNameMaxLength, egressNftablesTablePrefix, "_v4", "_v6", nftablesNameMaxLength)
}
for position, char := range name {
if isValidNftablesNameCharacter(char) {
continue
}
return fmt.Errorf("instance_name %q contains invalid character %q at byte %d; only ASCII letters, digits, '.', '-' and '_' are allowed",
name, char, position)
}
return nil
}
func isValidNftablesNameCharacter(char rune) bool {
return char >= 'a' && char <= 'z' ||
char >= 'A' && char <= 'Z' ||
char >= '0' && char <= '9' ||
char == '_' || char == '-' || char == '.'
}
func validateHealthCheckAddress(address string) error {
if address == "" {
return nil
}
parsedURL, err := url.ParseRequestURI(address)
if err != nil {
return fmt.Errorf("control_plane_health_check_address %q is not a valid URL: %w", address, err)
}
scheme := strings.ToLower(parsedURL.Scheme)
if scheme != "http" && scheme != "https" {
return fmt.Errorf("control_plane_health_check_address %q has unsupported scheme %q, expected http or https", address, parsedURL.Scheme)
}
return nil
}

View File

@@ -1,94 +0,0 @@
package kubevip
import (
"strings"
"testing"
)
func TestValidate_HealthCheckAddress(t *testing.T) {
tests := []struct {
name string
address string
wantErr bool
}{
{"empty address (disabled)", "", false},
{"valid http URL", "http://localhost:6443/livez", false},
{"valid https URL", "https://localhost:6443/livez", false},
{"https with path", "https://127.0.0.1:6443/readyz?verbose", false},
{"invalid URL", "not-a-url", true},
{"ftp scheme", "ftp://localhost/file", true},
{"tcp scheme", "tcp://localhost:6443", true},
{"missing scheme", "localhost:6443/livez", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := &Config{ControlPlaneHealthCheck: HealthCheck{Address: tt.address}}
err := c.Validate()
if (err != nil) != tt.wantErr {
t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestValidate_InstanceName(t *testing.T) {
tests := []struct {
name string
instanceName string
wantErr bool
}{
{name: "empty uses legacy default", instanceName: "", wantErr: false},
{name: "letters digits and separators", instanceName: "release_01.prod-a", wantErr: false},
{name: "exact maximum length", instanceName: strings.Repeat("a", instanceNameMaxLength), wantErr: false},
{name: "exceeds maximum length", instanceName: strings.Repeat("a", instanceNameMaxLength+1), wantErr: true},
{name: "space", instanceName: "release a", wantErr: true},
{name: "slash", instanceName: "namespace/release", wantErr: true},
{name: "dollar sign", instanceName: "release$a", wantErr: true},
{name: "at sign", instanceName: "release@a", wantErr: true},
{name: "newline", instanceName: "release\na", wantErr: true},
{name: "null byte", instanceName: "release\x00a", wantErr: true},
{name: "unicode", instanceName: "rilascio-à", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config := &Config{InstanceName: tt.instanceName}
err := config.Validate()
if (err != nil) != tt.wantErr {
t.Fatalf("Validate() error = %v, wantErr %t", err, tt.wantErr)
}
})
}
}
func TestValidate_RoutingProtocol(t *testing.T) {
tests := []struct {
name string
protocol int
wantErr bool
}{
{name: "unset", protocol: 0, wantErr: false},
{name: "kube-vip default", protocol: 248, wantErr: false},
{name: "maximum byte value", protocol: 255, wantErr: false},
{name: "truncated on the wire", protocol: 256, wantErr: true},
{name: "negative", protocol: -1, wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config := &Config{RoutingProtocol: tt.protocol}
err := config.Validate()
if (err != nil) != tt.wantErr {
t.Fatalf("Validate() error = %v, wantErr %t", err, tt.wantErr)
}
})
}
}
func TestInstanceNameLimitReservesNftablesPrefixAndFamilySuffix(t *testing.T) {
name := strings.Repeat("a", instanceNameMaxLength)
if got := len(egressNftablesTablePrefix + name + egressNftablesTableSuffix); got != nftablesNameMaxLength {
t.Fatalf("family-specific table name length = %d, want %d", got, nftablesNameMaxLength)
}
}

View File

@@ -1,7 +0,0 @@
package kubevip
const (
LBClassName = "kube-vip.io/kube-vip-class"
DefaultDHCPBackoffAttempts = 3
)

View File

@@ -1,22 +0,0 @@
package kubevip
import (
"slices"
)
const (
// ServiceProvided is the name of the label that will be added to the node
ServiceProvided = "service-provided.kube-vip.io"
// label used on nodes, which announce the LoadBalancer IP
HasIP = "kube-vip.io/has-ip"
)
var kubevipLabelKeys = []string{
ServiceProvided,
HasIP,
}
func GetKeysForCleanup() []string {
return slices.Clone(kubevipLabelKeys)
}

View File

@@ -1,134 +0,0 @@
package kubevip
import (
"encoding/json"
"fmt"
"net/netip"
"slices"
"strings"
)
const LeaseVIPsVersion = "v1"
// LeaseVIPKind distinguishes literal addresses from names that resolve to one.
type LeaseVIPKind string
const (
LeaseVIPKindAddress LeaseVIPKind = "address"
LeaseVIPKindName LeaseVIPKind = "name"
)
type LeaseVIPsValue struct {
Version string `json:"version"`
InstanceName string `json:"instance_name"`
IFAProto int `json:"ifa_proto"`
VIPs []LeaseVIP `json:"vips"`
}
type LeaseVIP struct {
Index int `json:"index"`
Value string `json:"value"`
Kind LeaseVIPKind `json:"kind"`
}
func WithLeaseVIPs(annotations map[string]string, instanceName string, ifaProto int, vips []string) (map[string]string, error) {
result := make(map[string]string, len(annotations)+1)
for key, value := range annotations {
result[key] = value
}
encoded, err := json.Marshal(LeaseVIPsValue{
Version: LeaseVIPsVersion,
InstanceName: instanceName,
IFAProto: ifaProto,
VIPs: normalizeLeaseVIPs(vips),
})
if err != nil {
return nil, fmt.Errorf("encode %s annotation: %w", LeaseVIPs, err)
}
result[LeaseVIPs] = string(encoded)
return result, nil
}
func ParseLeaseVIPs(value string) (LeaseVIPsValue, error) {
var parsed LeaseVIPsValue
if err := json.Unmarshal([]byte(value), &parsed); err != nil {
return LeaseVIPsValue{}, fmt.Errorf("decode %s annotation: %w", LeaseVIPs, err)
}
if parsed.Version != LeaseVIPsVersion {
return LeaseVIPsValue{}, fmt.Errorf("unsupported %s annotation version %q", LeaseVIPs, parsed.Version)
}
for index, vip := range parsed.VIPs {
if vip.Index != index {
return LeaseVIPsValue{}, fmt.Errorf("invalid %s VIP index %d at position %d", LeaseVIPs, vip.Index, index)
}
switch vip.Kind {
case LeaseVIPKindAddress, LeaseVIPKindName:
default:
return LeaseVIPsValue{}, fmt.Errorf("invalid %s VIP kind %q at index %d", LeaseVIPs, vip.Kind, vip.Index)
}
}
return parsed, nil
}
func normalizeLeaseVIPs(values []string) []LeaseVIP {
unique := make(map[string]struct{}, len(values))
addresses := make([]string, 0, len(values))
for _, value := range values {
for candidate := range strings.SplitSeq(value, ",") {
candidate = strings.TrimSpace(candidate)
if candidate == "" {
continue
}
if _, exists := unique[candidate]; exists {
continue
}
unique[candidate] = struct{}{}
addresses = append(addresses, candidate)
}
}
// Sorting keeps the annotation byte-identical however callers happen to order VIPs.
slices.SortFunc(addresses, compareLeaseVIPs)
result := make([]LeaseVIP, 0, len(addresses))
for _, address := range addresses {
kind := LeaseVIPKindName
if _, isAddress := leaseVIPAddress(address); isAddress {
kind = LeaseVIPKindAddress
}
result = append(result, LeaseVIP{Index: len(result), Value: address, Kind: kind})
}
return result
}
// compareLeaseVIPs orders addresses numerically and ahead of names, which keeps VIPs like
// 10.0.0.2 and 10.0.0.10 in the order an operator expects. Values that are not addresses,
// such as DNS records, are kept and ordered lexically.
func compareLeaseVIPs(a, b string) int {
addressA, isAddressA := leaseVIPAddress(a)
addressB, isAddressB := leaseVIPAddress(b)
switch {
case isAddressA && isAddressB:
if order := addressA.Compare(addressB); order != 0 {
return order
}
// Distinct spellings of one address still need a stable order.
return strings.Compare(a, b)
case isAddressA:
return -1
case isAddressB:
return 1
default:
return strings.Compare(a, b)
}
}
func leaseVIPAddress(value string) (netip.Addr, bool) {
if address, err := netip.ParseAddr(value); err == nil {
return address.Unmap(), true
}
if prefix, err := netip.ParsePrefix(value); err == nil {
return prefix.Addr().Unmap(), true
}
return netip.Addr{}, false
}

View File

@@ -1,94 +0,0 @@
package kubevip
import (
"slices"
"testing"
)
func TestWithLeaseVIPsEncodesVersionedInstanceOwnership(t *testing.T) {
base := map[string]string{"example.test/preserved": "true", LeaseVIPs: "stale"}
annotations, err := WithLeaseVIPs(base, "release_a", 248, []string{
"2001:db8::10/128", "192.0.2.10", "192.0.2.10/32", "api.example.test",
})
if err != nil {
t.Fatalf("WithLeaseVIPs() error = %v", err)
}
if annotations["example.test/preserved"] != "true" {
t.Fatal("WithLeaseVIPs() dropped an existing annotation")
}
if base[LeaseVIPs] != "stale" {
t.Fatal("WithLeaseVIPs() mutated the input annotations")
}
value, err := ParseLeaseVIPs(annotations[LeaseVIPs])
if err != nil {
t.Fatalf("ParseLeaseVIPs() error = %v", err)
}
if value.Version != LeaseVIPsVersion || value.InstanceName != "release_a" || value.IFAProto != 248 {
t.Fatalf("Lease VIP metadata = %+v", value)
}
// Values are stored verbatim so DNS records survive alongside addresses.
want := []LeaseVIP{
{Index: 0, Value: "192.0.2.10", Kind: LeaseVIPKindAddress},
{Index: 1, Value: "192.0.2.10/32", Kind: LeaseVIPKindAddress},
{Index: 2, Value: "2001:db8::10/128", Kind: LeaseVIPKindAddress},
{Index: 3, Value: "api.example.test", Kind: LeaseVIPKindName},
}
if !slices.Equal(value.VIPs, want) {
t.Fatalf("Lease VIPs = %v, want %v", value.VIPs, want)
}
}
// The annotation is rewritten whenever a node starts campaigning, so the encoding
// has to be stable even when callers collect the same VIPs in a different order.
func TestWithLeaseVIPsIsIndependentOfInputOrder(t *testing.T) {
first, err := WithLeaseVIPs(nil, "release_a", 248, []string{
"2001:db8::10", "192.0.2.10", "10.0.0.2", "10.0.0.10",
})
if err != nil {
t.Fatalf("WithLeaseVIPs() error = %v", err)
}
second, err := WithLeaseVIPs(nil, "release_a", 248, []string{
"10.0.0.10", "192.0.2.10", "2001:db8::10", "10.0.0.2",
})
if err != nil {
t.Fatalf("WithLeaseVIPs() error = %v", err)
}
if first[LeaseVIPs] != second[LeaseVIPs] {
t.Fatalf("annotation changed with input order:\n%s\n%s", first[LeaseVIPs], second[LeaseVIPs])
}
value, err := ParseLeaseVIPs(first[LeaseVIPs])
if err != nil {
t.Fatalf("ParseLeaseVIPs() error = %v", err)
}
want := []string{"10.0.0.2", "10.0.0.10", "192.0.2.10", "2001:db8::10"}
if len(value.VIPs) != len(want) {
t.Fatalf("Lease VIPs = %v, want %v", value.VIPs, want)
}
for index, address := range want {
if value.VIPs[index] != (LeaseVIP{Index: index, Value: address, Kind: LeaseVIPKindAddress}) {
t.Fatalf("Lease VIPs = %v, want %v", value.VIPs, want)
}
}
}
func TestParseLeaseVIPsRejectsUnknownVersion(t *testing.T) {
if _, err := ParseLeaseVIPs(`{"version":"v2","instance_name":"release_a","ifa_proto":248,"vips":[]}`); err == nil {
t.Fatal("ParseLeaseVIPs() accepted an unknown version")
}
}
func TestParseLeaseVIPsRejectsUnknownKind(t *testing.T) {
if _, err := ParseLeaseVIPs(
`{"version":"v1","instance_name":"release_a","ifa_proto":248,"vips":[{"index":0,"value":"192.0.2.10","kind":"cidr"}]}`,
); err == nil {
t.Fatal("ParseLeaseVIPs() accepted an unknown VIP kind")
}
}
func TestParseLeaseVIPsRejectsOutOfOrderIndexes(t *testing.T) {
if _, err := ParseLeaseVIPs(`{"version":"v1","instance_name":"release_a","ifa_proto":248,"vips":[{"index":1,"value":"192.0.2.10"}]}`); err == nil {
t.Fatal("ParseLeaseVIPs() accepted an out-of-order VIP index")
}
}

View File

@@ -1,270 +0,0 @@
package lease
import (
"context"
"fmt"
log "log/slog"
"os"
"strings"
"sync"
"sync/atomic"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
)
// Manager is used to manage leases.
type Manager struct {
leases map[string]*Lease
lock sync.Mutex
}
// NewManager creates new lease manager.
func NewManager() *Manager {
return &Manager{
leases: make(map[string]*Lease),
}
}
// Add adds lease to the manager.
// It returns three values:
// - lease for the object
// - isNewObject, which reports if it is a new object that is being handled
// - isSharedLease, which is true if object shares the lease with another object
// If object is new but not shared, we should start leaderelection and sync it
// If object is new and shared, we should only sync it as the leaderelection should be already handled
// If object is not new we should do nothing
func (m *Manager) Add(ctx context.Context, id ID) *Lease {
m.lock.Lock()
defer m.lock.Unlock()
// A lease whose context is already cancelled cannot be handed out again:
// anything derived from it would be cancelled straight away. Replace it.
if l, exists := m.leases[id.NamespacedName()]; !exists || l.Ctx.Err() != nil {
leaseCtx, leaseCancel := context.WithCancel(ctx)
m.leases[id.NamespacedName()] = newLease(leaseCtx, leaseCancel)
}
return m.leases[id.NamespacedName()]
}
// Delete removes the object from the lease it was added to and cancels that lease
// once its last object is gone. With a common lease, the siblings that still use
// it keep it alive.
//
// The lease the caller was given has to be passed in, because cleanup is usually
// deferred to a goroutine that runs long after the object went away. By then the
// lease of that name may already have been replaced, for instance because the
// service was torn down and rebuilt, and cancelling the replacement would leave
// the service unhandled. A stale caller is therefore ignored.
//
// Teardown paths have to call this synchronously rather than leaving it to the
// deferred cleanup: until the lease is out of the map, Add hands the same
// instance back, so a service that is rebuilt straight away gets parented to a
// lease that the pending cleanup is about to cancel.
func (m *Manager) Delete(id ID, objectName string, l *Lease) {
m.lock.Lock()
defer m.lock.Unlock()
current := m.currentFor(id, l)
if current == nil {
return
}
current.delete(objectName)
if current.cnt.Load() < 1 {
m.retire(id, current)
}
}
// currentFor returns the registered lease for id, or nil when the caller is
// stale, meaning the lease it holds is no longer the registered one. Callers have
// to hold m.lock.
func (m *Manager) currentFor(id ID, l *Lease) *Lease {
current, exist := m.leases[id.NamespacedName()]
if !exist || (l != nil && current != l) {
return nil
}
return current
}
// retire cancels the lease and drops it from the manager. Callers have to hold
// m.lock.
func (m *Manager) retire(id ID, l *Lease) {
l.Cancel()
delete(m.leases, id.NamespacedName())
}
// Get returns lease for the service.
func (m *Manager) Get(id ID) *Lease {
m.lock.Lock()
defer m.lock.Unlock()
if lease, exist := m.leases[id.NamespacedName()]; exist {
return lease
}
return nil
}
// Lease holds lease data.
type Lease struct {
Ctx context.Context
Cancel context.CancelFunc
Started chan any
services sync.Map
cnt atomic.Int64
Elected atomic.Bool
Mtx sync.Mutex
locked bool
}
func newLease(ctx context.Context, cancel context.CancelFunc) *Lease {
return &Lease{
Ctx: ctx,
Cancel: cancel,
Started: make(chan any),
}
}
// Add adds the object to the lease and increments counter
// it will return true if object was added
func (l *Lease) Add(name string) bool {
if _, exists := l.services.LoadOrStore(name, true); !exists {
l.cnt.Add(1)
return true
}
return false
}
// delete removes the service from the lease and decrements the counter
func (l *Lease) delete(service string) {
if _, exists := l.services.Load(service); exists {
l.services.Delete(service)
l.cnt.Add(-1)
}
}
func (l *Lease) Lock() {
l.Mtx.Lock()
l.locked = true
}
func (l *Lease) Unlock() {
if l.locked {
l.locked = false
l.Mtx.Unlock()
}
}
// ServiceName gets lease name and id for the service.
func ServiceName(service *v1.Service) (string, string) {
name, exists := service.Annotations[kubevip.ServiceLease]
if !exists || name == "" {
name = fmt.Sprintf("kubevip-%s", service.Name)
}
serviceLeaseParts := strings.Split(name, "/")
namespace := service.Namespace
if len(serviceLeaseParts) > 1 {
namespace = serviceLeaseParts[0]
name = serviceLeaseParts[1]
}
return namespace, name
}
func ServiceNamespacedName(service *v1.Service) string {
return fmt.Sprintf("%s/%s", service.Namespace, service.Name)
}
func ObjectName(id ID, suffix string) string {
return fmt.Sprintf("%s-%s", id.NamespacedName(), suffix)
}
func NamespaceName(lease string, c *kubevip.Config) (string, string) {
leaseName := lease
leasnameParts := strings.Split(lease, "/")
var ns string
var err error
if len(leasnameParts) > 1 {
ns = leasnameParts[0]
leaseName = leasnameParts[1]
} else {
ns, err = returnNamespace()
if err != nil {
log.Warn("unable to auto-detect namespace, dropping to config", "namespace", c.Namespace)
ns = c.Namespace
}
}
return ns, leaseName
}
func returnNamespace() (string, error) {
if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
return ns, nil
}
return "", err
}
return "", fmt.Errorf("unable to find Namespace")
}
type ID interface {
Name() string
Namespace() string
NamespacedName() string
}
type CommonID struct {
namespace string
name string
}
func NewID(leaseType, namespace, name string) ID {
if leaseType == "etcd" {
return newEtcdID(namespace, name)
}
return newKubernetesID(namespace, name)
}
func newKubernetesID(namespace, name string) ID {
return &KubernetesID{
CommonID: CommonID{
namespace: namespace,
name: name,
},
}
}
func newEtcdID(namespace, name string) ID {
return &EtcdID{
CommonID: CommonID{
namespace: namespace,
name: name,
},
}
}
func (c *CommonID) Name() string {
return c.name
}
func (c *CommonID) Namespace() string {
return c.namespace
}
type KubernetesID struct {
CommonID
}
func (k *KubernetesID) NamespacedName() string {
return fmt.Sprintf("%s/%s", k.namespace, k.name)
}
type EtcdID struct {
CommonID
}
func (e *EtcdID) NamespacedName() string {
return fmt.Sprintf("%s-%s", e.namespace, e.name)
}

File diff suppressed because it is too large Load Diff

View File

@@ -16,7 +16,6 @@ import (
"github.com/cloudflare/ipvs/netmask"
"github.com/kube-vip/kube-vip/pkg/backend"
"github.com/kube-vip/kube-vip/pkg/sysctl"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
)
@@ -51,61 +50,49 @@ type IPVSLoadBalancer struct {
backendMap backend.Map
interval int
lock sync.Mutex
killFunc func()
stop chan struct{}
networkInterface string
leaderCancel context.CancelFunc
signal chan os.Signal
address string
family ipvs.AddressFamily
}
func NewIPVSLB(ctx context.Context, network vip.Network, port uint16, forwardingMethod string, backendHealthCheckInterval int,
nftables bool, killFunc func(), wg *sync.WaitGroup) (*IPVSLoadBalancer, error) {
log.Info("Starting IPVS LoadBalancer", "network", network)
address := network.IP()
func NewIPVSLB(address string, port uint16, forwardingMethod string, backendHealthCheckInterval int, networkInterface string, leaderCancel context.CancelFunc, signal chan os.Signal) (*IPVSLoadBalancer, error) {
log.Info("Starting IPVS LoadBalancer", "address", address)
// Create IPVS client
c, err := ipvs.New()
if err != nil {
log.Error("ensure IPVS kernel modules are loaded")
log.Error("error starting IPVS", "err", err)
return nil, fmt.Errorf("starting IPVS: %w", err)
log.Error("Error starting IPVS", "err", err)
panic("")
}
i, err := c.Info()
if err != nil {
log.Error("ensure IPVS kernel modules are loaded")
log.Error("error retrieving IPVS info", "err", err)
log.Error("Error retrieving IPVS info", "err", err)
if errors.Is(err, os.ErrPermission) {
log.Error("no permission to get IPVS info - please ensure that kube-vip is running with proper capabilities/privileged mode")
}
return nil, fmt.Errorf("retrieving IPVS: %w", err)
panic("")
}
log.Info("IPVS Loadbalancer enabled", "version", fmt.Sprintf("%d.%d.%d", i.Version[0], i.Version[1], i.Version[2]))
ip, family := ipAndFamily(address)
if strings.ToLower(forwardingMethod) == "masquerade" {
if err := enableProcSys("/proc/sys/net/ipv4/vs/conntrack", "net.ipv4.vs.conntrack"); err != nil {
return nil, err
}
enableProcSys("/proc/sys/net/ipv4/vs/conntrack", "net.ipv4.vs.conntrack")
if family == ipvs.INET6 {
if err := enableProcSys("/proc/sys/net/ipv6/conf/all/forwarding", "net.ipv6.conf.all.forwarding"); err != nil {
return nil, err
}
enableProcSys("/proc/sys/net/ipv6/conf/all/forwarding", "net.ipv6.conf.all.forwarding")
} else {
if err := enableProcSys("/proc/sys/net/ipv4/ip_forward", "net.ipv4.ip_forward"); err != nil {
return nil, err
}
enableProcSys("/proc/sys/net/ipv4/ip_forward", "net.ipv4.ip_forward")
}
}
netMask := netmask.MaskFrom(31, vip.DefaultMaskIPv4) // For ipv4
netMask := netmask.MaskFrom(31, 32) // For ipv4
if family == ipvs.INET6 {
netMask = netmask.MaskFrom(128, vip.DefaultMaskIPv6) // For ipv6
}
var fwmark uint32
if nftables && forwardingMethod == "masquerade" && family == ipvs.INET {
fwmark = network.IPVSMark()
netMask = netmask.MaskFrom(128, 128) // For ipv6
}
// Generate out API Server LoadBalancer instance
@@ -116,7 +103,6 @@ func NewIPVSLB(ctx context.Context, network vip.Network, port uint16, forwarding
Port: port,
Address: ip,
Scheduler: ROUNDROBIN,
FWMark: fwmark,
}
var m ipvs.ForwardType
@@ -147,32 +133,34 @@ func NewIPVSLB(ctx context.Context, network vip.Network, port uint16, forwarding
forwardingMethod: m,
interval: backendHealthCheckInterval,
backendMap: make(backend.Map),
killFunc: killFunc,
stop: make(chan struct{}),
networkInterface: networkInterface,
leaderCancel: leaderCancel,
signal: signal,
address: address,
family: family,
}
wg.Go(func() {
lb.healthCheck(ctx)
})
go lb.healthCheck()
// Return our created load-balancer
return lb, nil
}
func enableProcSys(path, name string) error {
func enableProcSys(path, name string) {
isSet, err := sysctl.EnableProcSys(path)
if err != nil {
return fmt.Errorf("ensuring %s enabled: %w", name, err)
log.Error(fmt.Sprintf("ensuring %s enabled", name), "err", err)
panic("")
}
if isSet {
log.Info(fmt.Sprintf("sysctl set %s to 1", name))
}
return nil
}
func (lb *IPVSLoadBalancer) RemoveIPVSLB() error {
log.Info("Stopping IPVS LoadBalancer", "address", lb.address)
close(lb.stop)
err := lb.client.RemoveService(lb.loadBalancerService)
if err != nil {
return fmt.Errorf("error removing existing IPVS service: %v", err)
@@ -195,7 +183,6 @@ func (lb *IPVSLoadBalancer) AddBackend(address string, port uint16) error {
if err != nil {
log.Error("checking if backend is local", "err", err)
}
log.Info("checked if backend is local", "addr", address, "local", isLocal)
}
backend := backend.Entry{Addr: address, Port: port, IsLocal: isLocal}
@@ -240,7 +227,7 @@ func (lb *IPVSLoadBalancer) addBackend(address string, port uint16) error {
// Fatal error at this point as IPVS is probably not working
log.Error("Unable to create an IPVS service, ensure IPVS kernel modules are loaded")
log.Error("IPVS service", "err", err)
return utils.WrapPanicError(err, "unable to create an IPVS service")
panic("")
}
log.Info("load-Balancer services created", "address", lb.addrString(), "port", lb.Port)
@@ -325,8 +312,8 @@ func ipAndFamily(address string) (netip.Addr, ipvs.AddressFamily) {
return netip.AddrFrom4([4]byte(ipAddr.To4())), ipvs.INET
}
func (lb *IPVSLoadBalancer) healthCheck(ctx context.Context) {
backend.Watch(ctx, lb.interval, func() {
func (lb *IPVSLoadBalancer) healthCheck() {
backend.Watch(func() {
lb.lock.Lock()
defer lb.lock.Unlock()
for backend, oldStatus := range lb.backendMap {
@@ -351,50 +338,43 @@ func (lb *IPVSLoadBalancer) healthCheck(ctx context.Context) {
lb.backendMap[backend] = newStatus
}
if lb.forwardingMethod == ipvs.Local && !lb.localBackendExists() {
if lb.killFunc != nil {
log.Error("no local backends available, restarting kube-vip")
lb.killFunc()
if lb.signal != nil {
close(lb.signal)
}
if lb.leaderCancel != nil {
lb.leaderCancel()
}
}
}
}
})
}, lb.interval, lb.stop)
}
func (lb *IPVSLoadBalancer) isLocal(address string) (bool, error) {
link, err := netlink.LinkByName(lb.networkInterface)
if err != nil {
return false, fmt.Errorf("getting link '%s': %w", lb.networkInterface, err)
}
family := netlink.FAMILY_V6
if vip.IsIPv4(address) {
family = netlink.FAMILY_V4
}
target := net.ParseIP(address)
if target == nil {
return false, fmt.Errorf("unable to parse IP address %s", address)
return false, fmt.Errorf("address '%s' is not a valid IP address", address)
}
links, err := netlink.LinkList()
addrs, err := netlink.AddrList(link, family)
if err != nil {
return false, fmt.Errorf("listing links: %w", err)
return false, fmt.Errorf("listing addresses for link '%s': %w", lb.networkInterface, err)
}
family := netlink.FAMILY_V4
if utils.IsIPv6(address) {
family = netlink.FAMILY_V6
}
for _, link := range links {
if link.Type() == "veth" {
continue
}
addrs, err := netlink.AddrList(link, family)
if err != nil {
log.Error("listing addresses", "link", link.Attrs().Name, "error", err.Error())
continue
}
for _, addr := range addrs {
if addr.Scope != int(netlink.SCOPE_UNIVERSE) {
continue
}
if addr.IP.Equal(target) {
return true, nil
}
for _, addr := range addrs {
if addr.IP.Equal(target) {
return true, nil
}
}

View File

@@ -6,7 +6,6 @@ import (
"testing"
"github.com/cloudflare/ipvs"
"github.com/kube-vip/kube-vip/pkg/utils"
)
func Test_ipAndFamily(t *testing.T) {
@@ -20,7 +19,7 @@ func Test_ipAndFamily(t *testing.T) {
want1 ipvs.AddressFamily
}{
{
name: utils.IPv4Family,
name: "IPv4",
args: args{
address: "192.168.0.20",
},
@@ -28,7 +27,7 @@ func Test_ipAndFamily(t *testing.T) {
want1: ipvs.INET,
},
{
name: utils.IPv6Family,
name: "IPv6",
args: args{
address: "ff02::3",
},

30
pkg/manager/cluster.go Normal file
View File

@@ -0,0 +1,30 @@
package manager
import (
"github.com/pkg/errors"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/etcd"
)
func initClusterManager(sm *Manager) (*cluster.Manager, error) {
m := &cluster.Manager{
SignalChan: sm.signalChan,
}
switch sm.config.LeaderElectionType {
case "kubernetes", "":
m.KubernetesClient = sm.clientSet
m.RetryWatcherClient = sm.rwClientSet
case "etcd":
client, err := etcd.NewClient(sm.config)
if err != nil {
return nil, err
}
m.EtcdClient = client
default:
return nil, errors.Errorf("invalid LeaderElectionMode %s not supported", sm.config.LeaderElectionType)
}
return m, nil
}

View File

@@ -9,7 +9,6 @@ import (
"path/filepath"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
@@ -17,25 +16,19 @@ import (
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/election"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/manager/worker"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/node"
"github.com/kube-vip/kube-vip/pkg/route"
"github.com/kube-vip/kube-vip/pkg/services"
"github.com/kube-vip/kube-vip/pkg/upnp"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
const plunderLock = "plndr-svcs-lock"
// Manager degines the manager of the load-balancing services
type Manager struct {
clientSet *kubernetes.Clientset
@@ -52,10 +45,19 @@ type Manager struct {
// This channel is used to catch an OS signal and trigger a shutdown
signalChan chan os.Signal
sigint sync.Once
// This channel is used to signal a shutdown
shutdownChan chan struct{}
svcProcessor *services.Processor
// This is a prometheus counter used to count the number of events received
// from the service watcher
countServiceWatchEvent *prometheus.CounterVec
// This is a prometheus gauge indicating the state of the sessions.
// 1 means "ESTABLISHED", 0 means "NOT ESTABLISHED"
bgpSessionInfoGauge *prometheus.GaugeVec
// This mutex is to protect calls from various goroutines
mutex sync.Mutex
@@ -64,27 +66,10 @@ type Manager struct {
// This tracks VIPs and performs ARP/NDP advertisement.
arpMgr *arp.Manager
// This tracks node labels and performs label management
// implementation will be decided in constructor
// based on config.EnableNodeLabeling
nodeLabelManager node.LabelManager
// This variable reports if manager is being closed
closing atomic.Bool
// Will be used for leaderelection when required
electionMgr *election.Manager
// Will handle leases
leaseMgr *lease.Manager
// Will handle routes
routeMgr *route.Manager
}
// New will create a new managing object
func New(ctx context.Context, configMap string, config *kubevip.Config) (*Manager, error) {
func New(configMap string, config *kubevip.Config) (*Manager, error) {
// Instance identity should be the same as k8s node name to ensure better compatibility.
// By default k8s sets node name to `hostname -s`,
@@ -99,7 +84,6 @@ func New(ctx context.Context, configMap string, config *kubevip.Config) (*Manage
}
config.NodeName = hostname
}
config.NodeName = normalizeNodeName(config.NodeName)
log.Info("using node name", "name", config.NodeName)
adminConfigPath := "/etc/kubernetes/admin.conf"
@@ -112,21 +96,6 @@ func New(ctx context.Context, configMap string, config *kubevip.Config) (*Manage
switch {
case config.LeaderElectionType == "etcd":
// Do nothing, we don't construct a k8s client for etcd leader election
case config.K8sConfigFile != "" && config.K8sConfigFile != adminConfigPath &&
config.K8sConfigFile != homeConfigPath && utils.FileExists(config.K8sConfigFile):
// An explicitly configured kubeconfig (k8s_config_file env or
// --k8sConfigPath) takes precedence over the well-known host paths.
// KubernetesAddr, when set, overrides the API endpoint - static pods
// on control plane hosts use it to reach their local API server
// instead of a VIP that may not be up yet.
clientConfig, err = k8s.NewRestConfig(config.K8sConfigFile, false, config.KubernetesAddr)
if err != nil {
return nil, fmt.Errorf("could not create k8s REST config from file %q: %w", config.K8sConfigFile, err)
}
if clientset, err = k8s.NewClientset(clientConfig); err != nil {
return nil, fmt.Errorf("could not create k8s clientset: %w", err)
}
log.Info("Using Kubernetes configuration from explicit file", "path", config.K8sConfigFile, "address", config.KubernetesAddr)
case utils.FileExists(adminConfigPath):
if config.KubernetesAddr != "" {
log.Info("k8s address", "address", config.KubernetesAddr)
@@ -212,68 +181,62 @@ func New(ctx context.Context, configMap string, config *kubevip.Config) (*Manage
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(signalChan, syscall.SIGTERM)
// Add Notification for SIGUSR1 (for configuration dump)
signal.Notify(signalChan, syscall.SIGUSR1)
// All watchers and other goroutines should have an additional goroutine that blocks on this, to shut things down
shutdownChan := make(chan struct{})
intfMgr := networkinterface.NewManager()
arpMgr := arp.NewManager(config)
// create the node label manager
// constructor will decide if it should be a noop or not
nodeLabelManager := node.NewManager(config, clientset)
var bgpServer *bgp.Server
// If BGP is enabled then we start a server instance that will broadcast VIPs
if config.EnableBGP {
var err error
// If Annotations have been set then we will look them up
if config.Annotations != "" {
err = annotationsWatcher(ctx, clientset, rwClientSet, config)
} else {
log.Debug("No Node annotations to parse")
}
if err != nil {
return nil, err
}
bgpServer, err = bgp.NewBGPServer(config.BGPConfig, log.Level(config.Logging))
bgpServer, err = bgp.NewBGPServer(&config.BGPConfig)
if err != nil {
return nil, fmt.Errorf("creating BGP server: %w", err)
}
}
electionMgr, err := election.NewManager(config, clientset, rwClientSet)
if err != nil {
return nil, fmt.Errorf("creating election manager: %w", err)
}
leaseMgr := lease.NewManager()
routeMgr := route.NewManager()
svcProcessor := services.NewServicesProcessor(config, bgpServer, clientset, rwClientSet,
intfMgr, arpMgr, nodeLabelManager, electionMgr, leaseMgr, routeMgr)
svcProcessor := services.NewServicesProcessor(config, bgpServer, clientset, rwClientSet, shutdownChan, intfMgr, arpMgr)
return &Manager{
clientSet: clientset,
rwClientSet: rwClientSet,
configMap: configMap,
config: config,
signalChan: signalChan,
svcProcessor: svcProcessor,
intfMgr: intfMgr,
arpMgr: arpMgr,
bgpServer: bgpServer,
nodeLabelManager: nodeLabelManager,
electionMgr: electionMgr,
leaseMgr: leaseMgr,
routeMgr: routeMgr,
clientSet: clientset,
rwClientSet: rwClientSet,
configMap: configMap,
config: config,
countServiceWatchEvent: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "kube_vip",
Subsystem: "manager",
Name: "all_services_events",
Help: "Count all events fired by the service watcher categorised by event type",
}, []string{"type"}),
bgpSessionInfoGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "kube_vip",
Subsystem: "manager",
Name: "bgp_session_info",
Help: "Display state of session by setting metric for label value with current state to 1",
}, []string{"state", "peer"}),
signalChan: signalChan,
shutdownChan: shutdownChan,
svcProcessor: svcProcessor,
intfMgr: intfMgr,
arpMgr: arpMgr,
bgpServer: bgpServer,
}, nil
}
// Start will begin the Manager, which will start services and watch the configmap
func (sm *Manager) Start(ctx context.Context) error {
wg := sync.WaitGroup{}
defer wg.Wait()
func (sm *Manager) Start() error {
// listen for interrupts or the Linux SIGTERM signal and cancel
// our context, which the leader election code will observe and
// step down
sm.signalChan = make(chan os.Signal, 1)
// Add Notification for Userland interrupt
signal.Notify(sm.signalChan, syscall.SIGINT)
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(sm.signalChan, syscall.SIGTERM)
// All watchers and other goroutines should have an additional goroutine that blocks on this, to shut things down
sm.shutdownChan = make(chan struct{})
// HealthCheck
if sm.config.HealthCheckPort != 0 {
@@ -283,7 +246,7 @@ func (sm *Manager) Start(ctx context.Context) error {
http.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
fmt.Fprintf(w, "OK")
})
wg.Go(func() {
go func() {
server := &http.Server{
Addr: fmt.Sprintf(":%d", sm.config.HealthCheckPort),
ReadHeaderTimeout: 3 * time.Second,
@@ -292,19 +255,25 @@ func (sm *Manager) Start(ctx context.Context) error {
if err != nil {
log.Error("healthcheck", "unable to start", err)
}
})
}()
}
// on exit, clean up the node labels
defer func() {
if err := sm.nodeLabelManager.CleanUpLabels(10 * time.Second); err != nil {
log.Error("CleanUpNodeLabels", "unable to cleanup node labels", err)
// If BGP is enabled then we start a server instance that will broadcast VIPs
if sm.config.EnableBGP {
// If Annotations have been set then we will look them up
err := sm.parseAnnotations()
if err != nil {
return err
}
}()
log.Info("Starting Kube-vip Manager with the BGP engine")
return sm.startBGP()
}
if sm.config.EnableARP || sm.config.EnableWireguard {
if sm.config.EnableUPNP {
clients := upnp.GetConnectionClients(ctx)
clients := upnp.GetConnectionClients(context.TODO())
if len(clients) == 0 {
log.Error("Error Enabling UPNP. No Clients found")
// Set the struct to false so nothing should use it in future
@@ -318,132 +287,50 @@ func (sm *Manager) Start(ctx context.Context) error {
log.Info("Found UPNP IGD2 Gateway address", "ip", ip)
}
}
// TODO: It would be nice to run the UPNP refresh only on the leader.
wg.Go(func() {
sm.svcProcessor.RefreshUPNPForwards(ctx)
})
}
// TODO: It would be nice to run the UPNP refresh only on the leader.
go sm.svcProcessor.RefreshUPNPForwards()
}
return sm.startMode(ctx)
}
// Start will begin the Manager, which will start services and watch the configmap
func (sm *Manager) startMode(ctx context.Context) error {
var cpCluster *cluster.Cluster
var err error
w := worker.New(sm.arpMgr, sm.intfMgr, sm.config, &sm.closing, sm.Kill,
sm.svcProcessor, &sm.mutex, sm.clientSet, sm.bgpServer, sm.electionMgr,
sm.leaseMgr, sm.routeMgr, sm.nodeLabelManager)
// use a Go context so we can tell the leaderelection code when we
// want to step down
wg := sync.WaitGroup{}
modeCtx, cancel := context.WithCancel(ctx)
defer func() {
wg.Wait()
w.Cleanup()
cancel()
log.Info("Shutting down Kube-Vip")
}()
log.Info("starting Kube-vip Manager", "mode", w.Name())
if err := w.Configure(modeCtx, &wg); err != nil {
defer cancel()
return fmt.Errorf("failed to configure %s mode: %w", w.Name(), err)
// If ARP is enabled then we start a LeaderElection that will use ARP to advertise VIPs
if sm.config.EnableARP {
log.Info("Starting Kube-vip Manager with the ARP engine")
return sm.startARP(sm.config.NodeName)
}
if sm.config.EnableControlPlane {
err = w.InitControlPlane()
if err != nil {
defer cancel()
return err
}
if sm.config.EnableWireguard {
log.Info("Starting Kube-vip Manager with the Wireguard engine")
return sm.startWireguard(sm.config.NodeName)
}
// Shutdown function that will wait on this signal, unless we call it ourselves
wg.Go(func() {
sm.waitForShutdown(modeCtx, cancel, cpCluster)
})
if sm.config.EnableControlPlane {
wg.Go(func() {
w.StartControlPlane(modeCtx, sm.electionMgr)
})
}
if sm.config.EnableServices {
// This will tidy any dangling kube-vip iptables rules
if sm.config.EgressClean {
tableName := nftables.EgressTableBaseNameForInstance(sm.config.InstanceName)
err := nftables.ClearTablesWithName(tableName)
if err != nil {
log.Warn("[egress]", "mode", "nftables-internal", "clearing error", err)
} else {
log.Info("[egress]", "mode", "nftables-internal", "tables", "cleared")
}
// TODO: Deprecate the iptables code v1.2.x
err = vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
if err != nil {
log.Info("[egress]", "legacy-iptables", sm.config.EgressWithNftables, "mode", "IPv4", "error", err)
}
err = vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv6)
if err != nil {
log.Info("[egress]", "legacy-iptables", sm.config.EgressWithNftables, "mode", "IPv6", "error", err)
}
}
w.ConfigureServices()
for {
select {
case <-modeCtx.Done():
return nil
default:
if err = w.StartServices(modeCtx); err != nil {
if utils.IsPanicError(err) {
sm.Kill()
return fmt.Errorf("failed to reconcile services, non-recoverable error: %w", err)
} else {
log.Error("failed to reconcile services, restarting", "error", err)
}
}
}
}
if sm.config.EnableRoutingTable {
log.Info("Starting Kube-vip Manager with the Routing Table engine")
return sm.startTableMode(sm.config.NodeName)
}
log.Error("prematurely exiting Load-balancer as no modes [ARP/BGP/Wireguard] are enabled")
return nil
}
func (sm *Manager) waitForShutdown(ctx context.Context, cancel context.CancelFunc, cpCluster *cluster.Cluster) {
for {
sig := <-sm.signalChan
switch sig {
case syscall.SIGUSR1:
log.Info("Received SIGUSR1, dumping configuration")
sm.dumpConfiguration(ctx)
case syscall.SIGINT, syscall.SIGTERM:
sm.closing.Store(true)
log.Info("Received kube-vip termination, signaling shutdown")
if cpCluster != nil {
cpCluster.Stop()
}
// Cancel the context, which will in turn cancel the leadership and all goroutines
cancel()
return
func returnNameSpace() (string, error) {
if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
return ns, nil
}
return "", err
}
return "", fmt.Errorf("unable to find Namespace")
}
func (sm *Manager) Kill() {
sm.sigint.Do(func() {
sm.signalChan <- syscall.SIGINT
})
}
func (sm *Manager) parseAnnotations() error {
if sm.config.Annotations == "" {
log.Debug("No Node annotations to parse")
return nil
}
// normalizeNodeName ensures the local machine hostname conforms to
// Kubernetes RFC1123 node naming conventions (lowercase).
func normalizeNodeName(name string) string {
return strings.ToLower(name)
err := sm.annotationsWatcher()
if err != nil {
return err
}
return nil
}

159
pkg/manager/manager_arp.go Normal file
View File

@@ -0,0 +1,159 @@
package manager
import (
"context"
"syscall"
"time"
log "log/slog"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/vip"
)
// Start will begin the Manager, which will start services and watch the configmap
func (sm *Manager) startARP(id string) error {
var cpCluster *cluster.Cluster
var ns string
var err error
// use a Go context so we can tell the leaderelection code when we
// want to step down
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
log.Info("Start ARP/NDP advertisement")
go sm.arpMgr.StartAdvertisement(ctx)
// Shutdown function that will wait on this signal, unless we call it ourselves
go func() {
<-sm.signalChan
log.Info("Received kube-vip termination, signaling shutdown")
if sm.config.EnableControlPlane {
cpCluster.Stop()
}
// Close all go routines
close(sm.shutdownChan)
// Cancel the context, which will in turn cancel the leadership
cancel()
}()
if sm.config.EnableControlPlane {
cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
if err != nil {
return err
}
clusterManager, err := initClusterManager(sm)
if err != nil {
return err
}
go func() {
err := cpCluster.StartCluster(sm.config, clusterManager, nil)
if err != nil {
log.Error("starting control plane", "err", err)
// Trigger the shutdown of this manager instance
sm.signalChan <- syscall.SIGINT
}
}()
// Check if we're also starting the services, if not we can sit and wait on the closing channel and return here
if !sm.config.EnableServices {
<-sm.shutdownChan
log.Info("Shutting down Kube-Vip")
return nil
}
ns = sm.config.Namespace
} else {
ns, err = returnNameSpace()
if err != nil {
log.Warn("unable to auto-detect namespace, dropping to config", "namespace", sm.config.Namespace)
ns = sm.config.Namespace
}
}
// This will tidy any dangling kube-vip iptables rules
if sm.config.EgressClean {
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
}
// Start a services watcher (all kube-vip pods will watch services), upon a new service
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
} else {
log.Info("beginning services leadership", "namespace", ns, "lock name", sm.config.ServicesLeaseName, "id", id)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: sm.config.ServicesLeaseName,
Namespace: ns,
},
Client: sm.clientSet.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: id,
},
}
// start the leader election code loop
leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(sm.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(sm.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
log.Error("service watcher", "err", err)
panic("") // TODO: - emulating log.fatal here
}
},
OnStoppedLeading: func() {
// we can do cleanup here
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "new leader", id)
sm.svcProcessor.Stop()
log.Error("lost leadership, restarting kube-vip")
panic("") // TODO: - emulating log.fatal here
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if sm.config.EnableNodeLabeling {
applyNodeLabel(sm.clientSet, sm.config.Address, id, identity)
}
if identity == id {
// I just got the lock
return
}
log.Info("new leader elected", "new leader", identity)
},
},
})
}
return nil
}

Some files were not shown because too many files have changed in this diff Show More