Compare commits

..

4 Commits

Author SHA1 Message Date
Marcel Fest
65b61dee57 Update golang-ci-lint to use kubernetes 1.24.x 2025-04-04 16:43:58 +02:00
dependabot[bot]
451eb43c8f Bump k8s.io/client-go from 0.32.1 to 0.32.3
Bumps [k8s.io/client-go](https://github.com/kubernetes/client-go) from 0.32.1 to 0.32.3.
- [Changelog](https://github.com/kubernetes/client-go/blob/master/CHANGELOG.md)
- [Commits](https://github.com/kubernetes/client-go/compare/v0.32.1...v0.32.3)

---
updated-dependencies:
- dependency-name: k8s.io/client-go
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-04-04 16:43:56 +02:00
dependabot[bot]
f52849f254 Bump k8s.io/api from 0.32.1 to 0.32.3
Bumps [k8s.io/api](https://github.com/kubernetes/api) from 0.32.1 to 0.32.3.
- [Commits](https://github.com/kubernetes/api/compare/v0.32.1...v0.32.3)

---
updated-dependencies:
- dependency-name: k8s.io/api
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-04-04 16:43:09 +02:00
Marcel Fest
7032ba0c31 bump golang 2025-04-04 16:23:20 +02:00
112 changed files with 4237 additions and 11590 deletions

View File

@@ -22,10 +22,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@v4
with:
ref: ${{ github.ref_name }}
- name: Anchore SBOM Action
uses: anchore/sbom-action@v0.20.10
uses: anchore/sbom-action@v0.18.0
with:
format: cyclonedx-json

View File

@@ -1,71 +0,0 @@
name: For each PR
on:
pull_request:
jobs:
e2e-tests:
runs-on: ubuntu-latest
name: E2E tests
strategy:
matrix:
mode: ["arp", "rt", "bgp"]
fail-fast: true
steps:
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H-%M')"
- name: Ensure fs wont cause issues
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Checkout code
uses: actions/checkout@v6
- name: Install Go
uses: actions/setup-go@v6
with:
go-version-file: go.mod
- name: Build image locally
run: make dockerx86Local
- name: Run Manifest generation tests
run: make manifest-test
- name: Run ARP mode tests v1.29.0 onwards
run: E2E_KEEP_LOGS=true make e2e-tests129-arp
if: matrix.mode== 'arp'
- name: Run RT mode tests v1.29.0 onwards
run: E2E_KEEP_LOGS=true make e2e-tests129-rt
if: matrix.mode== 'rt'
- name: Get GoBGP binaries
run: make get-gobgp
if: matrix.mode== 'bgp'
- name: Run BGP mode tests v1.29.0 onwards
run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 E2E_KEEP_LOGS=true make e2e-tests129-bgp
if: matrix.mode== 'bgp'
- name: Change log directory permissions
run: sudo chmod -R 755 /tmp/kube-vip-test-${{ matrix.mode }}*
if: matrix.mode== 'bgp' && always()
- name: Save logs
uses: actions/upload-artifact@v5
with:
name: e2e-test-logs-${{ matrix.mode }}-${{ steps.date.outputs.date }}
path: /tmp/kube-vip-test-${{ matrix.mode }}*
if: always()
service-e2e-tests:
runs-on: ubuntu-latest
name: E2E service tests
steps:
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y-%m-%d-%H-%M')"
- name: Checkout code
uses: actions/checkout@v6
- name: Install Go
uses: actions/setup-go@v6
with:
go-version-file: go.mod
- name: Build image with iptables
run: make dockerx86ActionIPTables
- name: Run tests
run: DOCKERTAG=action E2E_KEEP_LOGS=true make service-tests
- name: Save logs
uses: actions/upload-artifact@v5
with:
name: services-test-logs-${{ steps.date.outputs.date }}
path: /tmp/kube-vip-service-tests*
if: always()

View File

@@ -1,4 +1,4 @@
name: For each commit
name: For each commit and PR
on:
push:
pull_request:
@@ -12,9 +12,9 @@ jobs:
- name: Install golangci-lint
run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.64.8
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: All checks
@@ -24,9 +24,9 @@ jobs:
name: Unit tests
steps:
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Run tests
@@ -36,19 +36,53 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Run tests
run: make integration-tests
e2e-tests:
runs-on: ubuntu-latest
name: E2E ARP tests
steps:
- name: Ensure fs wont cause issues
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Checkout code
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image locally
run: make dockerx86Local
- name: Run Manifest generation tests
run: make manifest-test
# - name: Run Control plane tests
# run: make e2e-tests
- name: Run Control plane tests v1.29.0 onwards
run: make e2e-tests129
service-e2e-tests:
runs-on: ubuntu-latest
name: E2E service tests
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image with iptables
run: make dockerx86ActionIPTables
- name: Run tests
run: DOCKERTAG=action make service-tests
image-vul-check:
runs-on: ubuntu-latest
name: Image vulnerability scan
steps:
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Build image with iptables
run: make dockerx86ActionIPTables
- name: Run Trivy vulnerability scanner

View File

@@ -38,16 +38,16 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version-file: go.mod
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v4
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@@ -58,7 +58,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v4
uses: github/codeql-action/autobuild@v3
# Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
@@ -72,4 +72,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v4
uses: github/codeql-action/analyze@v3

View File

@@ -11,7 +11,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx

View File

@@ -11,7 +11,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
@@ -27,12 +27,6 @@ jobs:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Generate Metadata
uses: docker/metadata-action@v5.9.0
id: metadata
with:
labels: |
org.opencontainers.image.documentation=https://kube-vip.io/docs/
- name: Build and push main branch
id: docker_build
uses: docker/build-push-action@v6
@@ -40,7 +34,6 @@ jobs:
context: .
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
push: ${{ github.event_name != 'pull_request' }}
labels: ${{ steps.metadata.outputs.labels }}
tags: >-
plndr/kube-vip:${{ github.ref_name }},
plndr/kube-vip:latest,
@@ -54,7 +47,6 @@ jobs:
file: Dockerfile_iptables
platforms: linux/amd64,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x
push: ${{ github.event_name != 'pull_request' }}
labels: ${{ steps.metadata.outputs.labels }}
tags: >-
plndr/kube-vip-iptables:${{ github.ref_name }},
plndr/kube-vip-iptables:latest,

View File

@@ -1,33 +0,0 @@
# Changelog
All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
### Added
- SIGUSR1 signal handler for runtime configuration dumps (#1301)
- Send SIGUSR1 to kube-vip process to dump current configuration to stdout
- Configuration dump includes:
- Basic configuration (VIP, interface, port, namespace settings)
- BGP configuration (enabled status, AS number, router ID, peers)
- ARP/NDP configuration (enabled status, broadcast rate)
- Services configuration (enabled status, load balancer settings)
- Network interfaces status
- Leader election configuration (type, lease details)
- Runtime statistics (load balancer, Prometheus, health check settings)
- Output format: Human-readable plaintext via fmt.Printf()
- Thread-safe implementation using mutex protection
- Non-disruptive: Process continues running after configuration dump
- Added comprehensive unit tests for all dump methods
- Added E2E tests for signal handling
### Changed
- Updated signal handlers in manager_arp.go, manager_bgp.go, manager_wireguard.go, and manager_table.go to use switch statement pattern for handling multiple signals (SIGUSR1, SIGINT, SIGTERM)
## [v1.0.1] - Previous Release
### Previous changes
- See git history for changes prior to CHANGELOG.md introduction

View File

@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:experimental
FROM golang:1.25.4-alpine3.22 as dev
FROM golang:1.24.2-alpine3.20 as dev
RUN apk add --no-cache git ca-certificates make
RUN adduser -D appuser
COPY . /src/

View File

@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:experimental
FROM golang:1.25.4-alpine3.22 as dev
FROM golang:1.24.2-alpine3.20 as dev
RUN apk add --no-cache git make
RUN adduser -D appuser
COPY . /src/
@@ -11,7 +11,7 @@ RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
--mount=type=cache,sharing=locked,id=goroot,target=/root/.cache/go-build \
CGO_ENABLED=0 GOOS=linux make build
FROM alpine:3.22.2
FROM alpine:3.21.3
# Update pkgs and add iptables
RUN apk upgrade && \
apk add --no-cache iptables iptables-legacy

View File

@@ -5,7 +5,7 @@ TARGET := kube-vip
.DEFAULT_GOAL := $(TARGET)
# These will be provided to the target
VERSION := v1.0.2
VERSION := v0.8.10
BUILD := `git rev-parse HEAD`
@@ -15,7 +15,7 @@ TARGETOS=linux
# Use linker flags to provide version/build settings to the target
LDFLAGS=-ldflags "-s -w -X=main.Version=$(VERSION) -X=main.Build=$(BUILD) -extldflags -static"
DOCKERTAG ?= $(VERSION)
REPOSITORY ?= docker.io/plndr
REPOSITORY ?= plndr
.PHONY: all build clean install uninstall simplify check run e2e-tests
@@ -109,22 +109,22 @@ run: install
manifests:
@make build
@mkdir -p ./docs/manifests/$(VERSION)/
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-arp.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-lb.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-bgp.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-arp.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-lb.yaml
@./kube-vip manifest pod --interface eth0 --vip 192.168.0.1 --bgp --controlplane --services > ./docs/manifests/$(VERSION)/kube-vip-bgp.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-arp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer > ./docs/manifests/$(VERSION)/kube-vip-arp-ds-lb.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster > ./docs/manifests/$(VERSION)/kube-vip-bgp-ds.yaml
@./kube-vip manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster --provider-config /etc/cloud-sa/cloud-sa.json > ./docs/manifests/$(VERSION)/kube-vip-bgp-em-ds.yaml
@-rm ./kube-vip
manifest-test:
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --enableLoadBalancer
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --controlplane --services
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --bgp --leaderElection --controlplane --services --inCluster
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --enableLoadBalancer
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest pod --interface eth0 --vip 192.168.0.1 --bgp --controlplane --services
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --arp --leaderElection --controlplane --services --inCluster --enableLoadBalancer
docker run $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --interface eth0 --vip 192.168.0.1 --bgp --leaderElection --controlplane --services --inCluster
unit-tests:
go test ./...
@@ -133,18 +133,10 @@ integration-tests:
go test -tags=integration,e2e -v ./pkg/etcd
e2e-tests:
GOMAXPROCS=4 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd
E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e ./testing/e2e/etcd
e2e-tests129-arp:
GOMAXPROCS=4 TEST_MODE=arp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129-rt:
GOMAXPROCS=4 TEST_MODE=rt V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129-bgp:
GOMAXPROCS=4 TEST_MODE=bgp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129: e2e-tests129-arp e2e-tests129-rt e2e-tests129-bgp
e2e-tests129:
V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
service-tests:
E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/services -Services -simple -deployments -leaderActive -leaderFailover -localDeploy -egress -egressIPv6 -dualStack
@@ -166,14 +158,8 @@ kind-quick:
kubectl create configmap --namespace kube-system kubevip --from-literal range-global=172.18.100.10-172.18.100.30
kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
kind load docker-image --name kube-vip $(REPOSITORY)/$(TARGET):$(DOCKERTAG)
docker run --network host --rm $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --services --inCluster --image "$(REPOSITORY)/$(TARGET):$(DOCKERTAG)" --arp --servicesElection --interface eth0 | kubectl apply -f -
docker run --network host --rm $(REPOSITORY)/$(TARGET):$(DOCKERTAG) manifest daemonset --services --inCluster --arp --servicesElection --interface eth0 | kubectl apply -f -
kind-reload:
kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name services
kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name kube-vip
kubectl rollout restart -n kube-system daemonset/kube-vip-ds
get-gobgp:
mkdir -p bin
wget -nc --directory-prefix=bin https://github.com/osrg/gobgp/releases/download/v3.37.0/gobgp_3.37.0_linux_amd64.tar.gz
tar -xvzf bin/gobgp_3.37.0_linux_amd64.tar.gz -C bin

View File

@@ -4,7 +4,7 @@ High Availability and Load-Balancing
![](https://github.com/kube-vip/kube-vip/raw/main/kube-vip.png)
[![Build and publish main image regularly](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml/badge.svg)](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml) [![LFX Health Score](https://img.shields.io/static/v1?label=Health%20Score&message=Healthy&color=A7F3D0&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip) [![LFX Active Contributors](https://img.shields.io/static/v1?label=Active%20contributors%20(1Y)&message=212&color=0094FF&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip)
[![Build and publish main image regularly](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml/badge.svg)](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml)
## Overview
Kubernetes Virtual IP and Load-Balancer for both control plane and Kubernetes services

View File

@@ -57,15 +57,15 @@ var kubeKubeadmInit = &cobra.Command{
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating VIPSubnet", "err", err)
log.Error("generating CIDR", "err", err)
return
}
}
cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
fmt.Println(cfg) // output manifest to stdout
},
}
@@ -102,15 +102,15 @@ var kubeKubeadmJoin = &cobra.Command{
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating VIPSubnet", "err", err)
log.Error("generating CIDR", "err", err)
return
}
}
cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
fmt.Println(cfg) // output manifest to stdout
},
}

View File

@@ -7,22 +7,20 @@ import (
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
)
// manifests will eventually deprecate the kubeadm set of subcommands
// manifests will be used to generate:
// - Pod spec manifest, mainly used for a static pod (kubeadm)
// - Daemonset manifest, mainly used to run kube-vip as a deamonset within Kubernetes (k3s/rke)
// - RBAC manifest, used to generate the RBAC permissions for kube-vip
var taint, role, rolebinding bool
// var inCluster bool
var taint bool
func init() {
kubeManifest.PersistentFlags().BoolVar(&inCluster, "inCluster", false, "Use the incluster token to authenticate to Kubernetes")
kubeManifest.PersistentFlags().StringVar(&image, "image", "ghcr.io/kube-vip/kube-vip", "Define a hardcoded image with or without tag for the manifest")
kubeManifestDaemon.PersistentFlags().BoolVar(&taint, "taint", false, "Taint the manifest for only running on control planes")
kubeManifestRbac.PersistentFlags().BoolVar(&role, "role", false, "Generate only a Role inside the serviceNamespace access")
kubeManifestRbac.PersistentFlags().BoolVar(&rolebinding, "rolebinding", false, "Generate only a RoleBinding for namespaced access")
kubeManifest.AddCommand(kubeManifestPod)
kubeManifest.AddCommand(kubeManifestDaemon)
@@ -59,15 +57,15 @@ var kubeManifestPod = &cobra.Command{
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("config parse", "err", err)
return
}
}
cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, image, Release.Version, inCluster)
cfg := kubevip.GeneratePodManifestFromConfig(&initConfig, Release.Version, inCluster)
fmt.Println(cfg) // output manifest to stdout
},
}
@@ -92,15 +90,15 @@ var kubeManifestDaemon = &cobra.Command{
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("config parse", "err", err)
return
}
}
cfg := kubevip.GenerateDaemonsetManifestFromConfig(&initConfig, image, Release.Version, inCluster, taint)
cfg := kubevip.GenerateDaemonsetManifestFromConfig(&initConfig, Release.Version, inCluster, taint)
fmt.Println(cfg) // output manifest to stdout
},
}
@@ -126,26 +124,16 @@ var kubeManifestRbac = &cobra.Command{
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating VIPSubnet", "err", err)
log.Error("generating CIDR", "err", err)
return
}
}
saCfg := kubevip.GenerateSA(&initConfig)
roleCfg := kubevip.GenerateRole(&initConfig, role)
if role {
rolebinding = true
}
roleBindingCfg := kubevip.GenerateRoleBinding(rolebinding, saCfg, roleCfg)
// Output the YAML manifests to stdout
fmt.Println("---") // Separator for YAML documents
fmt.Println(kubevip.TransformApplyObjectToManifest(saCfg))
fmt.Println("---") // Separator for YAML documents
fmt.Println(kubevip.TransformApplyObjectToManifest(roleCfg))
fmt.Println("---") // Separator for YAML documents
fmt.Println(kubevip.TransformApplyObjectToManifest(roleBindingCfg))
cfg := kubevip.GenerateSA()
b, _ := yaml.Marshal(cfg)
fmt.Println(string(b)) // output manifest to stdout
},
}

View File

@@ -6,7 +6,6 @@ import (
"net"
"net/http"
"os"
"slices"
"strings"
"time"
@@ -16,23 +15,22 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/spf13/cobra"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
"github.com/kube-vip/kube-vip/pkg/equinixmetal"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/manager"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
// Is an option to set the image
var image string
// Is kube-vip running within cluster
var inCluster bool
// ConfigMap name within a Kubernetes cluster
var configMap string
// Provider Config
var providerConfig string
// Points to a kubernetes configuration file
var kubeConfigPath string
@@ -58,16 +56,16 @@ func init() {
kubeVipCmd.PersistentFlags().StringVar(&initConfig.Interface, "interface", "", "Name of the interface to bind to")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesInterface, "serviceInterface", "", "Name of the interface to bind to (for services)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIP, "vip", "", "The Virtual IP address")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPSubnet, "vipSubnet", "", "The Virtual IP address subnet e.g. /32 /24 /8 etc.. (Default to 32 for IPv4 and 128 for IPv6)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPSubnet, "vipSubnet", "", "The Virtual IP address subnet e.g. /32 /24 /8 etc..")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.NodeName, "nodeName", "", "Name to be used for lease holder. Must be unique for each node/instance")
// VIP flags
kubeVipCmd.PersistentFlags().StringVar(&initConfig.VIPCIDR, "cidr", "", "The CIDR range for the virtual IP address. Default to 32 for IPv4 and 128 for IPv6") // todo: deprecate
kubeVipCmd.PersistentFlags().StringVar(&initConfig.Address, "address", "", "an address (IP or DNS name) to use as a VIP")
kubeVipCmd.PersistentFlags().Uint16Var(&initConfig.Port, "port", 6443, "Port for the VIP")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableARP, "arp", false, "Enable Arp for VIP changes")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableWireguard, "wireguard", false, "Enable Wireguard for services VIPs")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableRoutingTable, "table", false, "Enable Routing Table for services VIPs")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.PreserveVIPOnLeadershipLoss, "preserveVipOnLeadershipLoss", false, "Preserve ARP VIP addresses on interface when leadership is lost (default: false for backward compatibility)")
// LoadBalancer flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableLoadBalancer, "enableLoadBalancer", false, "enable loadbalancing on the VIP with IPVS")
@@ -84,6 +82,13 @@ func init() {
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RenewDeadline, "leaseRenewDuration", 3, "Length of time (in seconds) a Kubernetes leader can attempt to renew its lease")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RetryPeriod, "leaseRetry", 1, "Length of time (in seconds) the LeaderElector clients should wait between tries of actions")
// Equinix Metal flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableMetal, "metal", false, "This will use the Equinix Metal API (requires the token ENV) to update the EIP <-> VIP")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalAPIKey, "metalKey", "", "The API token for authenticating with the Equinix Metal API")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalProject, "metalProject", "", "The name of project already created within Equinix Metal")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.MetalProjectID, "metalProjectID", "", "The ID of project already created within Equinix Metal")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ProviderConfig, "provider-config", "", "The path to a provider configuration")
// BGP flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableBGP, "bgp", false, "This will enable BGP support within kube-vip")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.RouterID, "bgpRouterID", "", "The routerID for the bgp server")
@@ -99,11 +104,6 @@ func init() {
kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.BGPPeers, "bgppeers", []string{}, "Comma separated BGP Peer, format: address:as:password:multihop")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.Annotations, "annotations", "", "Set Node annotations prefix for parsing")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPConfig.Zebra.Enabled, "zebra", false, "This will enable Zebra support within kube-vip")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.URL, "zebraUrl", "unix:/var/run/frr/zserv.api", "Path to the unix domain socket for connecting to Zebra daemon")
kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.Zebra.Version, "zebraVersion", 6, "Zebra API Version")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.SoftwareName, "zebraSoftwareName", "frr8.3", "Software Name for Zebra")
// Namespace for kube-vip
kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The namespace for the configmap defined within the cluster")
@@ -115,7 +115,7 @@ func init() {
// Routing Table flags
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableID, "tableID", 198, "The routing table used for all table entries")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableType, "tableType", unix.RTN_UNICAST, "The type of route that will be added to the routing table")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingTableType, "tableType", 0, "The type of route that will be added to the routing table")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.RoutingProtocol, "routingProtocol", 248, "The routing protocol value used to create routes")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.CleanRoutingTable, "cleanRoutingTable", false, "Clean routing table of redundant routes on start")
@@ -126,15 +126,15 @@ func init() {
// Extended behaviour flags
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServicesElection, "servicesElection", false, "Enable leader election per kubernetes service")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassOnly, "lbClassOnly", false, fmt.Sprintf("Enable load balancing only for services with LoadBalancerClass %q", kubevip.LBClassName))
kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerClassName, "lbClassName", kubevip.LBClassName, fmt.Sprintf("Name of load balancer class for kube-VIP, defaults to %q", kubevip.LBClassName))
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassOnly, "lbClassOnly", false, "Enable load balancing only for services with LoadBalancerClass \"kube-vip.io/kube-vip-class\"")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.LoadBalancerClassName, "lbClassName", "kube-vip.io/kube-vip-class", "Name of load balancer class for kube-VIP, defaults to \"kube-vip.io/kube-vip-class\"")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoadBalancerClassLegacyHandling, "lbClassNameLegacyHandling", true, "Use legacy LoadBalancer class name handling (e.g. accepting services both with empty and non-empty class)")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableServiceSecurity, "onlyAllowTrafficServicePorts", false, "Only allow traffic to service ports, others will be dropped, defaults to false")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableNodeLabeling, "enableNodeLabeling", false, fmt.Sprintf("Enable leader node labeling with %q, defaults to false", kubevip.HasIP))
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableNodeLabeling, "enableNodeLabeling", false, "Enable leader node labeling with \"kube-vip.io/has-ip=<VIP address>\", defaults to false")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesLeaseName, "servicesLeaseName", "plndr-svcs-lock", "Name of the lease that is used for leader election for services (in arp mode)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.DNSMode, "dnsMode", "first", "Name of the mode that DNS lookup will be performed (first, ipv4, ipv6, dual)")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DisableServiceUpdates, "disableServiceUpdates", false, "If true, kube-vip will process services as usual, but will not update service's Status.LoadBalancer.Ingress slice")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpoints, "enableEndpoints", false, "If enabled, kube-vip will only advertise services, but will use the (deprecated since v1.33) endpoints for IP addresses")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpointSlices, "enableEndpointSlices", false, "If enabled, kube-vip will only advertise services, but will use EndpointSlices instead of endpoints to get IPs of Pods")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoInterfaceGlobalScope, "loInterfaceGlobalScope", false, "If true, kube-vip will set global scope when using the lo interface, otherwise a host scope will be used by default")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.HealthCheckPort, "healthCheckPort", 0, "If set to non-zero (> 1024), then this is the port that the healthcheck will listen on")
@@ -150,9 +150,6 @@ func init() {
// Kubernetes client specific flags
kubeVipCmd.PersistentFlags().StringVar(&initConfig.K8sConfigFile, "k8sConfigPath", "/etc/kubernetes/admin.conf", "Path to the configuration file used with the Kubernetes client")
// Configuration file flag
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ConfigFile, "config-file", "", "Path to a JSON/YAML configuration file to load settings from")
kubeVipCmd.AddCommand(kubeKubeadm)
kubeVipCmd.AddCommand(kubeManifest)
kubeVipCmd.AddCommand(kubeVipManager)
@@ -195,27 +192,13 @@ var kubeVipService = &cobra.Command{
Short: "Start the Virtual IP / Load balancer as a service within a Kubernetes cluster",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
// Load configuration from file if specified (lowest priority)
if initConfig.ConfigFile != "" {
err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
if err != nil {
log.Error("loading config file", "err", err)
return
}
}
// parse environment variables, these will overwrite anything loaded from config file
// parse environment variables, these will overwrite anything loaded or flags
err := kubevip.ParseEnvironment(&initConfig)
if err != nil {
log.Error("parsing env", "err", err)
return
}
// Change RTN_UNSPEC to default type
if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
initConfig.RoutingProtocol = unix.RTN_UNICAST
}
// Set the logging level for all subsequent functions
log.SetLogLoggerLevel(log.Level(initConfig.Logging))
@@ -231,8 +214,8 @@ var kubeVipService = &cobra.Command{
}
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("generating CIDR", "err", err)
return
@@ -259,33 +242,19 @@ var kubeVipManager = &cobra.Command{
Use: "manager",
Short: "Start the kube-vip manager",
Run: func(cmd *cobra.Command, args []string) { //nolint TODO
// Load configuration from file if specified (lowest priority)
if initConfig.ConfigFile != "" {
err := kubevip.MergeConfigFromFile(&initConfig, initConfig.ConfigFile)
if err != nil {
log.Error("loading config file", "err", err)
return
}
}
// parse environment variables, these will overwrite anything loaded from config file
// parse environment variables, these will overwrite anything loaded or flags
err := kubevip.ParseEnvironment(&initConfig)
if err != nil {
log.Error("parsing environment", "err", err)
return
}
// Change RTN_UNSPEC to default type
if initConfig.RoutingProtocol == unix.RTN_UNSPEC {
initConfig.RoutingProtocol = unix.RTN_UNICAST
}
// Set the logging level for all subsequent functions
log.SetLogLoggerLevel(log.Level(initConfig.Logging))
// Ensure there is an address to generate the CIDR from
if initConfig.VIPSubnet == "" && initConfig.Address != "" {
initConfig.VIPSubnet, err = GenerateCidrRange(initConfig.Address, initConfig.DNSMode)
if initConfig.VIPCIDR == "" && initConfig.Address != "" {
initConfig.VIPCIDR, err = GenerateCidrRange(initConfig.Address)
if err != nil {
log.Error("No interface is specified for kube-vip to bind to")
return
@@ -329,12 +298,12 @@ var kubeVipManager = &cobra.Command{
return
}
if !initConfig.EnableARP && strings.Contains(initConfig.VIPSubnet, kubevip.Auto) {
if !initConfig.EnableARP && strings.Contains(initConfig.VIPCIDR, kubevip.Auto) {
log.Error("auto subnet discovery cannot be used outside ARP mode")
return
}
if strings.Contains(initConfig.VIPSubnet, kubevip.Auto) && initConfig.Address != "" {
if strings.Contains(initConfig.VIPCIDR, kubevip.Auto) && initConfig.Address != "" {
log.Error("auto subnet discovery cannot be used if VIP address was provided")
return
}
@@ -403,6 +372,19 @@ var kubeVipManager = &cobra.Command{
configMap = envConfigMap
}
// If Equinix Metal is enabled and there is a provider configuration passed
if initConfig.EnableMetal {
if providerConfig != "" {
providerAPI, providerProject, err := equinixmetal.GetPacketConfig(providerConfig)
if err != nil {
log.Error("retrieving equinix metal config", "err", err)
return
}
initConfig.MetalAPIKey = providerAPI
initConfig.MetalProject = providerProject
}
}
// Define the new service manager
mgr, err := manager.New(configMap, &initConfig)
if err != nil {
@@ -475,37 +457,26 @@ func servePrometheusHTTPServer(ctx context.Context, config PrometheusHTTPServerC
}
}
func GenerateCidrRange(address string, dnsMode string) (string, error) {
func GenerateCidrRange(address string) (string, error) {
var cidrs []string
addresses := strings.Split(address, ",")
for _, a := range addresses {
ip := net.ParseIP(a)
if ip == nil {
// we probably are a DNS name
ips, err := utils.LookupHost(a, dnsMode)
ips, err := net.LookupIP(a)
if len(ips) == 0 || err != nil {
return "", fmt.Errorf("invalid IP address: %s from [%s], %v", a, address, err)
}
for _, addr := range ips {
ip = net.ParseIP(addr)
if ip.To4() != nil {
cidrs = append(cidrs, "32")
} else {
cidrs = append(cidrs, "128")
}
}
ip = ips[0]
}
if ip.To4() != nil {
cidrs = append(cidrs, "32")
} else {
if ip.To4() != nil {
cidrs = append(cidrs, "32")
} else {
cidrs = append(cidrs, "128")
}
cidrs = append(cidrs, "128")
}
}
// compact as DNS could have a lot of addresses
slices.Sort(cidrs)
cidrs = slices.Compact(cidrs)
slices.Reverse(cidrs)
return strings.Join(cidrs, ","), nil
}

134
go.mod
View File

@@ -1,121 +1,108 @@
module github.com/kube-vip/kube-vip
go 1.24.4
go 1.24.2
require (
github.com/cloudflare/ipvs v0.11.0
github.com/cloudflare/ipvs v0.10.3
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/docker/docker v28.5.2+incompatible
github.com/florianl/go-conntrack v0.4.0
github.com/golang/protobuf v1.5.4
github.com/google/go-cmp v0.7.0
github.com/google/go-containerregistry v0.20.6
github.com/google/nftables v0.3.0
github.com/gookit/slog v0.6.0
github.com/gookit/slog v0.5.7
github.com/huin/goupnp v1.3.0
github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d
github.com/jpillora/backoff v1.0.0
github.com/mdlayher/ndp v1.1.0
github.com/onsi/ginkgo/v2 v2.27.2
github.com/onsi/gomega v1.38.2
github.com/osrg/gobgp/v3 v3.37.0
github.com/onsi/ginkgo/v2 v2.22.2
github.com/onsi/gomega v1.36.2
github.com/osrg/gobgp/v3 v3.35.0
github.com/packethost/packngo v0.31.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.23.2
github.com/prometheus/client_golang v1.21.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.10.1
github.com/stretchr/testify v1.11.1
github.com/vishvananda/netlink v1.3.1
go.etcd.io/etcd/api/v3 v3.6.6
go.etcd.io/etcd/client/pkg/v3 v3.6.6
go.etcd.io/etcd/client/v3 v3.6.6
go.uber.org/zap v1.27.1
github.com/spf13/cobra v1.9.1
github.com/stretchr/testify v1.10.0
github.com/vishvananda/netlink v1.3.0
go.etcd.io/etcd/api/v3 v3.5.18
go.etcd.io/etcd/client/pkg/v3 v3.5.18
go.etcd.io/etcd/client/v3 v3.5.18
go.uber.org/zap v1.27.0
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
golang.org/x/sync v0.18.0
golang.org/x/sys v0.38.0
golang.org/x/sync v0.11.0
golang.org/x/sys v0.30.0
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
google.golang.org/grpc v1.77.0
google.golang.org/protobuf v1.36.10
k8s.io/api v0.34.2
k8s.io/apimachinery v0.34.2
k8s.io/client-go v0.34.1
google.golang.org/protobuf v1.36.5
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.32.3
k8s.io/apimachinery v0.32.3
k8s.io/client-go v0.32.3
k8s.io/klog/v2 v2.130.1
sigs.k8s.io/kind v0.30.0
sigs.k8s.io/yaml v1.6.0
sigs.k8s.io/kind v0.27.0
sigs.k8s.io/yaml v1.4.0
)
require (
al.essio.dev/pkg/shellescape v1.5.1 // indirect
github.com/BurntSushi/toml v1.4.0 // indirect
github.com/Masterminds/semver/v3 v3.4.0 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/containerd/errdefs v1.0.0 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect
github.com/coreos/go-semver v0.3.1 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/dlclark/regexp2 v1.11.4 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/eapache/channels v1.1.0 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/emicklei/go-restful/v3 v3.12.2 // indirect
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.8.0 // indirect
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.21.0 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.7.0 // indirect
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
github.com/google/gnostic-models v0.6.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect
github.com/google/safetext v0.0.0-20240722112252-5a72de7e7962 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gookit/color v1.6.0 // indirect
github.com/gookit/goutil v0.7.1 // indirect
github.com/gookit/gsr v0.1.1 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
github.com/gookit/color v1.5.4 // indirect
github.com/gookit/goutil v0.6.18 // indirect
github.com/gookit/gsr v0.1.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/josharian/native v1.1.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/k-sone/critbitgo v1.4.0 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/magiconair/properties v1.8.9 // indirect
github.com/mailru/easyjson v0.9.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mdlayher/genetlink v1.3.2 // indirect
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect
github.com/mdlayher/netlink v1.7.2 // indirect
github.com/mdlayher/packet v1.1.2 // indirect
github.com/mdlayher/socket v0.5.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/sys/sequential v0.6.0 // indirect
github.com/moby/term v0.5.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/pborman/uuid v1.2.1 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/pierrec/lz4/v4 v4.1.22 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.62.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/sagikazarmark/locafero v0.6.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.7.1 // indirect
github.com/spf13/pflag v1.0.9 // indirect
github.com/spf13/pflag v1.0.6 // indirect
github.com/spf13/viper v1.19.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tj/go-spin v1.1.0 // indirect
@@ -126,33 +113,25 @@ require (
github.com/xlab/c-for-go v1.3.0 // indirect
github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
go.opentelemetry.io/otel v1.38.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 // indirect
go.opentelemetry.io/otel/metric v1.38.0 // indirect
go.opentelemetry.io/otel/trace v1.38.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.45.0 // indirect
golang.org/x/mod v0.29.0 // indirect
golang.org/x/net v0.47.0 // indirect
golang.org/x/oauth2 v0.32.0 // indirect
golang.org/x/term v0.37.0 // indirect
golang.org/x/text v0.31.0 // indirect
golang.org/x/crypto v0.35.0 // indirect
golang.org/x/mod v0.22.0 // indirect
golang.org/x/net v0.36.0 // indirect
golang.org/x/oauth2 v0.25.0 // indirect
golang.org/x/term v0.29.0 // indirect
golang.org/x/text v0.22.0 // indirect
golang.org/x/time v0.9.0 // indirect
golang.org/x/tools v0.38.0 // indirect
golang.org/x/tools v0.28.0 // indirect
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d // indirect
google.golang.org/grpc v1.69.2 // indirect
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect
k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 // indirect
k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect
modernc.org/cc/v4 v4.24.4 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/opt v0.1.4 // indirect
@@ -160,6 +139,5 @@ require (
modernc.org/strutil v1.2.1 // indirect
modernc.org/token v1.1.0 // indirect
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
sigs.k8s.io/randfill v1.0.0 // indirect
sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.5.0 // indirect
)

329
go.sum
View File

@@ -1,31 +1,17 @@
al.essio.dev/pkg/shellescape v1.5.1 h1:86HrALUujYS/h+GtqoB26SBEdkWfmMI6FubjXlsXyho=
al.essio.dev/pkg/shellescape v1.5.1/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8=
github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
github.com/cloudflare/ipvs v0.11.0 h1:niLcbqfv8+RSYk+yI+jhiCIHobEdEhO++mcwCX0Znsw=
github.com/cloudflare/ipvs v0.11.0/go.mod h1:XsbuKcQpqb3rpjYPtsFsM8BNZlgLw+Z0iC6RhocqDd8=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
github.com/cloudflare/ipvs v0.10.3 h1:YFhvFFRzRxV1A1+RBa28AoC0XS4cg1QSmxBr4xb2pu4=
github.com/cloudflare/ipvs v0.10.3/go.mod h1:XsbuKcQpqb3rpjYPtsFsM8BNZlgLw+Z0iC6RhocqDd8=
github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
@@ -37,28 +23,20 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k=
github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0=
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU=
github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU=
github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/florianl/go-conntrack v0.4.0 h1:TlYkxytdwgVayfU0cKwkHurQA0Rd1ZSEBRckRYDUu18=
github.com/florianl/go-conntrack v0.4.0/go.mod h1:iPDx4oIats2T7X7Jm3PFyRCJM1GfZhJaSHOWROYOrE8=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
@@ -66,17 +44,10 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs=
github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo=
github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M=
github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk=
github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE=
github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
@@ -89,15 +60,13 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw=
github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -105,31 +74,29 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-containerregistry v0.20.6 h1:cvWX87UxxLgaH76b4hIvya6Dzz9qHB31qAwjAohdSTU=
github.com/google/go-containerregistry v0.20.6/go.mod h1:T0x8MuoAoKX/873bkeSfLD2FAkwCDf9/HZgsFJ02E2Y=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg=
github.com/google/nftables v0.3.0/go.mod h1:BCp9FsrbF1Fn/Yu6CLUc9GGZFw/+hsxfluNXXmxBfRM=
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg=
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
github.com/google/safetext v0.0.0-20240722112252-5a72de7e7962 h1:+9C/TgFfcCmZBV7Fjb3kQCGlkpFrhtvFDgbdQHB9RaA=
github.com/google/safetext v0.0.0-20240722112252-5a72de7e7962/go.mod h1:H3K1Iu/utuCfa10JO+GsmKUYSWi7ug57Rk6GaDRHaaQ=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gookit/assert v0.1.1 h1:lh3GcawXe/p+cU7ESTZ5Ui3Sm/x8JWpIis4/1aF0mY0=
github.com/gookit/assert v0.1.1/go.mod h1:jS5bmIVQZTIwk42uXl4lyj4iaaxx32tqH16CFj0VX2E=
github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA=
github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs=
github.com/gookit/goutil v0.7.1 h1:AaFJPN9mrdeYBv8HOybri26EHGCC34WJVT7jUStGJsI=
github.com/gookit/goutil v0.7.1/go.mod h1:vJS9HXctYTCLtCsZot5L5xF+O1oR17cDYO9R0HxBmnU=
github.com/gookit/gsr v0.1.1 h1:TaHD3M7qa6lcAf9D2J4mGNg+QjgDtD1bw7uctF8RXOM=
github.com/gookit/gsr v0.1.1/go.mod h1:7wv4Y4WCnil8+DlDYHBjidzrEzfHhXEoFjEA0pPPWpI=
github.com/gookit/slog v0.6.0 h1:KEQxOJxbTtk7oyqah6nJOEKjOdI0z5qoqkX7I6G65g4=
github.com/gookit/slog v0.6.0/go.mod h1:hPlpNi/WIcGmkEjHzQTS7s5JZkHmmnGy9sYo6csa08s=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
github.com/gookit/goutil v0.6.18 h1:MUVj0G16flubWT8zYVicIuisUiHdgirPAkmnfD2kKgw=
github.com/gookit/goutil v0.6.18/go.mod h1:AY/5sAwKe7Xck+mEbuxj0n/bc3qwrGNe3Oeulln7zBA=
github.com/gookit/gsr v0.1.0 h1:0gadWaYGU4phMs0bma38t+Do5OZowRMEVlHv31p0Zig=
github.com/gookit/gsr v0.1.0/go.mod h1:7wv4Y4WCnil8+DlDYHBjidzrEzfHhXEoFjEA0pPPWpI=
github.com/gookit/slog v0.5.7 h1:n3Dhgmr3NP+KppkNg95+vpFcI4YD1csu9VTQwgcEYTs=
github.com/gookit/slog v0.5.7/go.mod h1:uWCRB4YO+FmgwXEq3s8U7ob1wWP7RStOuY/2a4yC/8o=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8CrSJVmaolDVOxTfS9kc36uB6H40kdbQq8=
@@ -145,8 +112,6 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF
github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE=
github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung=
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
@@ -164,8 +129,8 @@ github.com/k-sone/critbitgo v1.4.0 h1:l71cTyBGeh6X5ATh6Fibgw3+rtNT80BA0uNNWgkPrb
github.com/k-sone/critbitgo v1.4.0/go.mod h1:7E6pyoyADnFxlUBEKcnfS49b7SUAQGMK+OAp/UQvo0s=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -179,8 +144,6 @@ github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a
github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo=
@@ -201,8 +164,8 @@ github.com/mdlayher/netlink v1.3.0/go.mod h1:xK/BssKuwcRXHrtN04UBkwQ6dY9VviGGuri
github.com/mdlayher/netlink v1.4.0/go.mod h1:dRJi5IABcZpBD2A3D0Mv/AiX8I9uDEu5oGkAVrekmf8=
github.com/mdlayher/netlink v1.4.1/go.mod h1:e4/KuJ+s8UhfUpO9z00/fDZZmhSrs+oxyqAS9cNgn6Q=
github.com/mdlayher/netlink v1.5.0/go.mod h1:1Kr8BBFxGyUyNmztC9WLOayqYVAd2wsgOZm18nqGuzQ=
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 h1:A1Cq6Ysb0GM0tpKMbdCXCIfBclan4oHk1Jb+Hrejirg=
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42/go.mod h1:BB4YCPDOzfy7FniQ/lxuYQ3dgmM2cZumHbK8RpTjN2o=
github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g=
github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY=
github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4=
github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00/go.mod h1:GAFlyu4/XV68LkQKYzKhIo/WW7j3Zi0YRAz/BOoanUc=
@@ -210,42 +173,30 @@ github.com/mdlayher/socket v0.0.0-20211007213009-516dcbdf0267/go.mod h1:nFZ1EtZY
github.com/mdlayher/socket v0.1.0/go.mod h1:mYV5YIZAfHh4dzDVzI8x8tWLWCliuX8Mon5Awbj+qDs=
github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos=
github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ=
github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE=
github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A=
github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws=
github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw=
github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs=
github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns=
github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A=
github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
github.com/osrg/gobgp/v3 v3.37.0 h1:+ObuOdvj7G7nxrT0fKFta+EAupdWf/q1WzbXydr8IOY=
github.com/osrg/gobgp/v3 v3.37.0/go.mod h1:kVHVFy1/fyZHJ8P32+ctvPeJogn9qKwa1YCeMRXXrP0=
github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU=
github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk=
github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8=
github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY=
github.com/osrg/gobgp/v3 v3.35.0 h1:8HQyCCoByAJyXiSzTFBl77wuOyVdDXozPdXX+tjYyHQ=
github.com/osrg/gobgp/v3 v3.35.0/go.mod h1:l2nPaHaLmIoKbFxMUzKon/h6c9BTzCp5zJI9Dhnrx5c=
github.com/packethost/packngo v0.31.0 h1:LLH90ardhULWbagBIc3I3nl2uU75io0a7AwY6hyi0S4=
github.com/packethost/packngo v0.31.0/go.mod h1:Io6VJqzkiqmIEQbpOjeIw9v8q9PfcTEq8TEY/tMQsfw=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
@@ -257,18 +208,18 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/prometheus/client_golang v1.21.0 h1:DIsaGmiaBkSangBgMtWdNfxbMNdku5IK6iNhrEqWvdA=
github.com/prometheus/client_golang v1.21.0/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk=
github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0=
@@ -282,37 +233,33 @@ github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds=
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 h1:pyC9PaHYZFgEKFdlp3G8RaCKgVpHZnecvArXvPXcFkM=
github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk=
github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs=
github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
@@ -326,55 +273,42 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJu
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.etcd.io/etcd/api/v3 v3.6.6 h1:mcaMp3+7JawWv69p6QShYWS8cIWUOl32bFLb6qf8pOQ=
go.etcd.io/etcd/api/v3 v3.6.6/go.mod h1:f/om26iXl2wSkcTA1zGQv8reJRSLVdoEBsi4JdfMrx4=
go.etcd.io/etcd/client/pkg/v3 v3.6.6 h1:uoqgzSOv2H9KlIF5O1Lsd8sW+eMLuV6wzE3q5GJGQNs=
go.etcd.io/etcd/client/pkg/v3 v3.6.6/go.mod h1:YngfUVmvsvOJ2rRgStIyHsKtOt9SZI2aBJrZiWJhCbI=
go.etcd.io/etcd/client/v3 v3.6.6 h1:G5z1wMf5B9SNexoxOHUGBaULurOZPIgGPsW6CN492ec=
go.etcd.io/etcd/client/v3 v3.6.6/go.mod h1:36Qv6baQ07znPR3+n7t+Rk5VHEzVYPvFfGmfF4wBHV8=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 h1:bDMKF3RUSxshZ5OjOTi8rsHGaPKsAt76FaqgvIUySLc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0/go.mod h1:dDT67G/IkA46Mr2l9Uj7HsQVwsjASyV9SjGofsiUZDA=
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
go.etcd.io/etcd/api/v3 v3.5.18 h1:Q4oDAKnmwqTo5lafvB+afbgCDF7E35E4EYV2g+FNGhs=
go.etcd.io/etcd/api/v3 v3.5.18/go.mod h1:uY03Ob2H50077J7Qq0DeehjM/A9S8PhVfbQ1mSaMopU=
go.etcd.io/etcd/client/pkg/v3 v3.5.18 h1:mZPOYw4h8rTk7TeJ5+3udUkfVGBqc+GCjOJYd68QgNM=
go.etcd.io/etcd/client/pkg/v3 v3.5.18/go.mod h1:BxVf2o5wXG9ZJV+/Cu7QNUiJYk4A29sAhoI5tIRsCu4=
go.etcd.io/etcd/client/v3 v3.5.18 h1:nvvYmNHGumkDjZhTHgVU36A9pykGa2K4lAJ0yY7hcXA=
go.etcd.io/etcd/client/v3 v3.5.18/go.mod h1:kmemwOsPU9broExyhYsBxX4spCTDX3yLgPMWtpBXG6E=
go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=
go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=
go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=
go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0=
go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc=
go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=
go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329 h1:9kj3STMvgqy3YA4VQXBrN7925ICMxD5wzMRcgA30588=
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -395,16 +329,16 @@ golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY=
golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA=
golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I=
golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70=
golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -435,16 +369,16 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -453,8 +387,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8=
golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -463,16 +397,14 @@ golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uI
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA=
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10 h1:3GDAcqdIg1ozBNLgPy4SLT84nfcBjr6rhGtXYtrkWLU=
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10/go.mod h1:T97yPqesLiNrOYxkwmhMI0ZIlJDm+p0PMR8eRVeR5tQ=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 h1:mepRgnBZa07I4TRuomDE4sTIYieg/osKmzIf4USdWS4=
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 h1:M1rk8KBnUsBDg1oPGHNCxG4vc1f49epmTO7xscSajMk=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM=
google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d h1:H8tOf8XM88HvKqLTxe755haY6r1fqqzLbEnfrmLXlSA=
google.golang.org/genproto/googleapis/api v0.0.0-20250102185135-69823020774d/go.mod h1:2v7Z7gP2ZUOGsaFyxATQSRoBnKygqVq2Cwnvom7QiqY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d h1:xJJRGY7TJcvIlpSrN3K6LAWgNFUILlO+OMAqtg9aqnw=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d/go.mod h1:3ENsm/5D1mzDyhpzeRi1NR784I0BcofWBoSc5QqqMK4=
google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU=
google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4=
google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
@@ -482,6 +414,7 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -491,18 +424,18 @@ gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
k8s.io/api v0.34.2 h1:fsSUNZhV+bnL6Aqrp6O7lMTy6o5x2C4XLjnh//8SLYY=
k8s.io/api v0.34.2/go.mod h1:MMBPaWlED2a8w4RSeanD76f7opUoypY8TFYkSM+3XHw=
k8s.io/apimachinery v0.34.2 h1:zQ12Uk3eMHPxrsbUJgNF8bTauTVR2WgqJsTmwTE/NW4=
k8s.io/apimachinery v0.34.2/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw=
k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY=
k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8=
k8s.io/api v0.32.3 h1:Hw7KqxRusq+6QSplE3NYG4MBxZw1BZnq4aP4cJVINls=
k8s.io/api v0.32.3/go.mod h1:2wEDTXADtm/HA7CCMD8D8bK4yuBUptzaRhYcYEEYA3k=
k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U=
k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE=
k8s.io/client-go v0.32.3 h1:RKPVltzopkSgHS7aS98QdscAgtgah/+zmpAogooIqVU=
k8s.io/client-go v0.32.3/go.mod h1:3v0+3k4IcT9bXTc4V2rt+d2ZPPG700Xy6Oi0Gdl2PaY=
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA=
k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts=
k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y=
k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 h1:hcha5B1kVACrLujCKLbr8XWMxCxzQx42DY8QKYJrDLg=
k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7/go.mod h1:GewRfANuJ70iYzvn+i4lezLDAFzvjxZYK1gn1lWcfas=
k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0=
k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
modernc.org/cc/v4 v4.24.4 h1:TFkx1s6dCkQpd6dKurBNmpo+G8Zl4Sq/ztJ+2+DEsh0=
modernc.org/cc/v4 v4.24.4/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
modernc.org/ccorpus2 v1.5.2 h1:Ui+4tc58mf/W+2arcYCJR903y3zl3ecsI7Fpaaqozyw=
@@ -521,11 +454,9 @@ pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
sigs.k8s.io/kind v0.30.0 h1:2Xi1KFEfSMm0XDcvKnUt15ZfgRPCT0OnCBbpgh8DztY=
sigs.k8s.io/kind v0.30.0/go.mod h1:FSqriGaoTPruiXWfRnUXNykF8r2t+fHtK0P0m1AbGF8=
sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco=
sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
sigs.k8s.io/kind v0.27.0 h1:PQ3f0iAWNIj66LYkZ1ivhEg/+Zb6UPMbO+qVei/INZA=
sigs.k8s.io/kind v0.27.0/go.mod h1:RZVFmy6qcwlSWwp6xeIUv7kXCPF3i8MXsEXxW/J+gJY=
sigs.k8s.io/structured-merge-diff/v4 v4.5.0 h1:nbCitCK2hfnhyiKo6uf2HxUPTCodY6Qaf85SbDIaMBk=
sigs.k8s.io/structured-merge-diff/v4 v4.5.0/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4=
sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

View File

@@ -1,207 +0,0 @@
package arp
import (
"context"
"fmt"
log "log/slog"
"sync"
"time"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
type Manager struct {
instances sync.Map
config *kubevip.Config
}
type Instance struct {
network vip.Network
ndp *vip.NdpResponder
mu sync.Mutex
counter int
}
func NewManager(config *kubevip.Config) *Manager {
return &Manager{
config: config,
}
}
func NewInstance(network vip.Network, ndp *vip.NdpResponder) *Instance {
return &Instance{
ndp: ndp,
network: network,
counter: 1,
}
}
func (i *Instance) Name() string {
return i.network.ARPName()
}
func (m *Manager) Insert(instance *Instance) {
i, err := m.get(instance.Name())
if err != nil {
log.Error("[ARP manager] unable to insert instance", "err", err)
return
}
if i == nil {
log.Info("[ARP manager] inserting ARP/NDP instance", "name", instance.Name())
m.instances.Store(instance.Name(), instance)
} else {
i.mu.Lock()
defer i.mu.Unlock()
i.counter++
}
}
func (m *Manager) Remove(instance *Instance) {
m.RemoveWithIPDelete(instance, true)
}
// RemoveOnLeadershipLoss removes an ARP instance when leadership is lost
func (m *Manager) RemoveOnLeadershipLoss(instance *Instance) {
// Use the inverse of PreserveVIPOnLeadershipLoss to decide whether to delete the IP
// If preserve is true, don't delete IP (deleteIP = false)
// If preserve is false, delete IP (deleteIP = true), This is the legacy behavior
deleteIP := !m.config.PreserveVIPOnLeadershipLoss
m.RemoveWithIPDelete(instance, deleteIP)
}
func (m *Manager) RemoveWithIPDelete(instance *Instance, deleteIP bool) {
i, err := m.get(instance.Name())
if err != nil {
log.Error("[ARP manager] unable to remove the instance", "err", err)
return
}
if i != nil {
i.mu.Lock()
defer i.mu.Unlock()
if i.counter > 1 {
i.counter--
} else {
log.Info("[ARP manager] removing ARP/NDP instance", "name", instance.Name())
if deleteIP {
if _, err := instance.network.DeleteIP(); err != nil {
log.Error("failed to delete IP", "address", instance.network.IP(), "err", err)
}
}
m.instances.Delete(instance.Name())
}
} else {
log.Warn("[ARP manager] unable to remove the instance - instance not found", "name", instance.Name())
}
}
func (m *Manager) Count(name string) int {
i, err := m.get(name)
if err != nil {
log.Error("[ARP manager] unable to count instance", "err", err)
return -1
}
if i != nil {
i.mu.Lock()
defer i.mu.Unlock()
return i.counter
}
return 0
}
func (m *Manager) StartAdvertisement(ctx context.Context) {
log.Info("[ARP manager] starting ARP/NDP advertisement")
for {
select {
case <-ctx.Done(): // if cancel() execute
return
default:
m.instances.Range(func(_ any, instance any) bool {
if i, ok := instance.(*Instance); ok {
if i.counter > 0 {
ensureIPAndSendGratuitous(i)
} else {
// this instance should not be advertised - delete the IP just in case...
if _, err := i.network.DeleteIP(); err != nil {
log.Error("[ARP manager] failed to delete IP", "address", i.network.IP(), "err", err)
}
}
}
return true
})
}
if m.config.ArpBroadcastRate < 500 {
log.Warn("[ARP manager] arp broadcast rate is too low", "rate (ms)", m.config.ArpBroadcastRate, "setting to (ms)", "3000")
m.config.ArpBroadcastRate = 3000
}
time.Sleep(time.Duration(m.config.ArpBroadcastRate) * time.Millisecond)
}
}
func (m *Manager) get(name string) (*Instance, error) {
i, exists := m.instances.Load(name)
if !exists {
return nil, nil
}
inst, ok := i.(*Instance)
if !ok {
return nil, fmt.Errorf("value for name %q is not of Instance pointer type", name)
}
return inst, nil
}
// ensureIPAndSendGratuitous - adds IP to the interface if missing, and send
// either a gratuitous ARP or gratuitous NDP. Re-adds the interface if it is IPv6
// and in a dadfailed state.
func ensureIPAndSendGratuitous(instance *Instance) {
iface := instance.network.Interface()
ipString := instance.network.IP()
// Check if IP is dadfailed
if instance.network.IsDADFAIL() {
log.Warn("IP address is in dadfailed state, removing config", "ip", ipString, "interface", iface)
deleted, err := instance.network.DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted and recreating address with NODAD flag to skip DAD", "IP", ipString, "interface", iface)
// Re-add immediately without DAD check since we're recovering from DADFAILED
// The AddIP function will set IFA_F_NODAD flag for IPv6 addresses when skipDAD=true
if _, err := instance.network.AddIP(false, true); err != nil {
log.Error("failed to recreate address after DADFAILED", "IP", ipString, "interface", iface, "err", err)
} else {
log.Info("successfully recreated address after DADFAILED recovery", "IP", ipString, "interface", iface)
}
}
// Return early after DADFAILED recovery to avoid double IP addition
return
}
// Normal case: add IP with precheck and normal DAD process
if added, err := instance.network.AddIP(true, false); err != nil {
log.Warn(err.Error())
} else if added {
log.Warn("Re-applied the VIP configuration", "ip", ipString, "interface", iface)
}
if utils.IsIPv6(ipString) {
// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
if instance.ndp == nil {
log.Error("NDP responder was not created")
} else {
err := instance.ndp.SendGratuitous(ipString)
if err != nil {
log.Warn(err.Error())
}
}
} else {
// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
err := vip.ARPSendGratuitous(ipString, iface)
if err != nil {
log.Warn(err.Error())
}
}
}

View File

@@ -8,6 +8,7 @@ import (
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
@@ -30,7 +31,7 @@ func (e *Entry) Check() bool {
// homeConfigPath := filepath.Join(os.Getenv("HOME"), ".kube", "config")
var k8sAddr string
if utils.IsIPv4(e.Addr) {
if vip.IsIPv4(e.Addr) {
k8sAddr = fmt.Sprintf("%s:%v", e.Addr, e.Port)
} else {
k8sAddr = fmt.Sprintf("[%s]:%v", e.Addr, e.Port)
@@ -39,6 +40,7 @@ func (e *Entry) Check() bool {
switch {
case utils.FileExists(adminConfigPath):
config, err = k8s.NewRestConfig(adminConfigPath, false, k8sAddr)
// client, err = k8s.NewClientset(adminConfigPath, false, k8sAddr)
if err != nil {
log.Error("create k8s REST config", "path", adminConfigPath, "err", err)
return false

View File

@@ -4,19 +4,19 @@ import (
"context"
"fmt"
"net"
"strconv"
"strings"
//nolint
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/golang/protobuf/ptypes/any"
"github.com/kube-vip/kube-vip/pkg/vip"
api "github.com/osrg/gobgp/v3/api"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/osrg/gobgp/v3/pkg/server"
"google.golang.org/protobuf/types/known/anypb"
)
// AddPeer will add peers to the BGP configuration
func (b *Server) AddPeer(peer kubevip.BGPPeer) (err error) {
func (b *Server) AddPeer(peer Peer) (err error) {
p := &api.Peer{
Conf: &api.PeerConf{
NeighborAddress: peer.Address,
@@ -67,9 +67,9 @@ func (b *Server) AddPeer(peer kubevip.BGPPeer) (err error) {
},
}
peer.SetMpbgpOptions(b.c)
peer.setMpbgpOptions(b.c)
ipv4Address, ipv6Address, err := peer.FindMpbgpAddresses(p, b.c)
ipv4Address, ipv6Address, err := peer.findMpbgpAddresses(p, b.c)
if err != nil {
return fmt.Errorf("failed to get MP-BGP addresses: %w", err)
}
@@ -77,7 +77,7 @@ func (b *Server) AddPeer(peer kubevip.BGPPeer) (err error) {
mask := "128"
address := ipv4Address
family := api.Family_AFI_IP
if utils.IsIPv4(p.Conf.NeighborAddress) {
if vip.IsIPv4(p.Conf.NeighborAddress) {
mask = "32"
address = ipv6Address
family = api.Family_AFI_IP6
@@ -142,7 +142,7 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
Safi: api.Family_SAFI_UNICAST,
},
Nlri: nlri,
Pattrs: []*anypb.Any{originAttr, nhAttr},
Pattrs: []*any.Any{originAttr, nhAttr},
}
} else {
//nolint
@@ -160,18 +160,116 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
mpAttr, _ := anypb.New(&api.MpReachNLRIAttribute{
Family: v6Family,
NextHops: []string{"::"}, // gobgp will fill this
Nlris: []*anypb.Any{nlri},
Nlris: []*any.Any{nlri},
})
path = &api.Path{
Family: v6Family,
Nlri: nlri,
Pattrs: []*anypb.Any{originAttr, mpAttr},
Pattrs: []*any.Any{originAttr, mpAttr},
}
}
return
}
// ParseBGPPeerConfig - take a string and parses it into an array of peers
func ParseBGPPeerConfig(config string) (bgpPeers []Peer, err error) {
peers := strings.Split(config, ",")
if len(peers) == 0 {
return nil, fmt.Errorf("no BGP Peer configurations found")
}
for x := range peers {
peerStr := peers[x]
config := strings.Split(peerStr, "/")
peerStr = config[0]
if peerStr == "" {
continue
}
isV6Peer := peerStr[0] == '['
address := ""
if isV6Peer {
addressEndPos := strings.IndexByte(peerStr, ']')
if addressEndPos == -1 {
return nil, fmt.Errorf("no matching ] found for IPv6 BGP Peer")
}
address = peerStr[1:addressEndPos]
peerStr = peerStr[addressEndPos+1:]
}
peer := strings.Split(peerStr, ":")
if len(peer) < 2 {
return nil, fmt.Errorf("mandatory peering params <host>:<AS> incomplete")
}
if !isV6Peer {
address = peer[0]
}
ASNumber, err := strconv.ParseUint(peer[1], 10, 32)
if err != nil {
return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
}
password := ""
if len(peer) >= 3 {
password = peer[2]
}
multiHop := false
if len(peer) >= 4 {
multiHop, err = strconv.ParseBool(peer[3])
if err != nil {
return nil, fmt.Errorf("BGP MultiHop format error (true/false) [%s]", peer[1])
}
}
var port uint64
if len(peer) >= 5 {
port, err = strconv.ParseUint(peer[4], 10, 16)
if err != nil {
return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
}
} else {
port = 179
}
var mpbgpNexthop, mpbgpIPv4, mpbgpIPv6 string
if len(config) > 1 {
configData := strings.Split(config[1], ";")
for _, cfg := range configData {
c := strings.Split(cfg, "=")
switch c[0] {
case "mpbgp_nexthop":
mpbgpNexthop = c[1]
case "mpbgp_ipv4":
mpbgpIPv4 = c[1]
case "mpbgp_ipv6":
mpbgpIPv6 = c[1]
default:
return nil, fmt.Errorf("peer configuration parameter '%s' is not supported", c[0])
}
}
}
peerConfig := Peer{
Address: address,
AS: uint32(ASNumber),
Port: uint16(port),
Password: password,
MultiHop: multiHop,
MpbgpNexthop: mpbgpNexthop,
MpbgpIPv4: mpbgpIPv4,
MpbgpIPv6: mpbgpIPv6,
}
bgpPeers = append(bgpPeers, peerConfig)
}
return
}
func insertPolicy(s *server.BgpServer, address string, p *api.Peer, family api.Family_Afi) error {
familyType := "v4"
if family == api.Family_AFI_IP6 {

View File

@@ -3,8 +3,6 @@ package bgp
import (
"reflect"
"testing"
"github.com/kube-vip/kube-vip/pkg/kubevip"
)
func TestParseBGPPeerConfig(t *testing.T) {
@@ -14,13 +12,13 @@ func TestParseBGPPeerConfig(t *testing.T) {
tests := []struct {
name string
args args
wantBgpPeers []kubevip.BGPPeer
wantBgpPeers []Peer
wantErr bool
}{
{
name: "IPv4, default port",
args: args{config: "192.168.0.10:65000::false,192.168.0.11:65000::false"},
wantBgpPeers: []kubevip.BGPPeer{
wantBgpPeers: []Peer{
{Address: "192.168.0.10", Port: 179, AS: 65000, MultiHop: false},
{Address: "192.168.0.11", Port: 179, AS: 65000, MultiHop: false},
},
@@ -28,7 +26,7 @@ func TestParseBGPPeerConfig(t *testing.T) {
{
name: "IPv4, different port",
args: args{config: "192.168.0.10:65000::false:180,192.168.0.11:65000::false:190"},
wantBgpPeers: []kubevip.BGPPeer{
wantBgpPeers: []Peer{
{Address: "192.168.0.10", Port: 180, AS: 65000, MultiHop: false},
{Address: "192.168.0.11", Port: 190, AS: 65000, MultiHop: false},
},
@@ -36,14 +34,14 @@ func TestParseBGPPeerConfig(t *testing.T) {
{
name: "IPv6, multi-protocol",
args: args{config: "[fd00:1111:2222:3333:c7d9:7235:6bf7:5d52]:65501::false/mpbgp_nexthop=auto_sourceif"},
wantBgpPeers: []kubevip.BGPPeer{
wantBgpPeers: []Peer{
{Address: "fd00:1111:2222:3333:c7d9:7235:6bf7:5d52", Port: 179, AS: 65501, MultiHop: false, MpbgpNexthop: "auto_sourceif"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotBgpPeers, err := kubevip.ParseBGPPeerConfig(tt.args.config)
gotBgpPeers, err := ParseBGPPeerConfig(tt.args.config)
if (err != nil) != tt.wantErr {
t.Errorf("ParseBGPPeerConfig() error = %v, wantErr %v", err, tt.wantErr)
return

View File

@@ -7,24 +7,12 @@ import (
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
api "github.com/osrg/gobgp/v3/api"
gobgp "github.com/osrg/gobgp/v3/pkg/server"
"github.com/prometheus/client_golang/prometheus"
)
// Server manages a server object
type Server struct {
s *gobgp.BgpServer
c *kubevip.BGPConfig
// This is a prometheus gauge indicating the state of the sessions.
// 1 means "ESTABLISHED", 0 means "NOT ESTABLISHED"
BGPSessionInfoGauge *prometheus.GaugeVec
}
// NewBGPServer takes a configuration and returns a running BGP server instance
func NewBGPServer(c kubevip.BGPConfig) (b *Server, err error) {
func NewBGPServer(c *Config, peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (b *Server, err error) {
if c.AS == 0 {
return nil, fmt.Errorf("you need to provide AS")
}
@@ -39,26 +27,14 @@ func NewBGPServer(c kubevip.BGPConfig) (b *Server, err error) {
b = &Server{
s: gobgp.NewBgpServer(),
c: &c,
BGPSessionInfoGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "kube_vip",
Subsystem: "manager",
Name: "bgp_session_info",
Help: "Display state of session by setting metric for label value with current state to 1",
}, []string{"state", "peer"}),
c: c,
}
return
}
// Start starts the BGP server
func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (err error) {
go b.s.Serve()
if err = b.s.StartBgp(context.Background(), &api.StartBgpRequest{
Global: &api.Global{
Asn: b.c.AS,
RouterId: b.c.RouterID,
Asn: c.AS,
RouterId: c.RouterID,
ListenPort: -1,
},
}); err != nil {
@@ -76,23 +52,12 @@ func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_Peer
return
}
for _, p := range b.c.Peers {
for _, p := range c.Peers {
if err = b.AddPeer(p); err != nil {
return
}
}
if b.c.Zebra.Enabled {
if err = b.s.EnableZebra(context.Background(), &api.EnableZebraRequest{
Url: b.c.Zebra.URL,
Version: b.c.Zebra.Version,
SoftwareName: b.c.Zebra.SoftwareName,
}); err != nil {
log.Error(err.Error())
return
}
}
return
}

130
pkg/bgp/types.go Normal file
View File

@@ -0,0 +1,130 @@
package bgp
import (
"fmt"
"net"
"github.com/kube-vip/kube-vip/pkg/vip"
api "github.com/osrg/gobgp/v3/api"
gobgp "github.com/osrg/gobgp/v3/pkg/server"
"github.com/vishvananda/netlink"
)
// Peer defines a BGP Peer
type Peer struct {
Address string
Port uint16
AS uint32
Password string
MultiHop bool
MpbgpNexthop string
MpbgpIPv4 string
MpbgpIPv6 string
}
func (p *Peer) setMpbgpOptions(server *Config) {
if p.MpbgpNexthop == "" {
p.MpbgpNexthop = server.MpbgpNexthop
}
if p.MpbgpIPv4 == "" {
p.MpbgpIPv4 = server.MpbgpIPv4
}
if p.MpbgpIPv6 == "" {
p.MpbgpIPv6 = server.MpbgpIPv6
}
}
func (p *Peer) findMpbgpAddresses(ap *api.Peer, server *Config) (string, string, error) {
var ipv4Address, ipv6Address string
switch p.MpbgpNexthop {
case "fixed":
ap.Transport.LocalAddress = server.SourceIP
if p.MpbgpIPv4 == "" && p.MpbgpIPv6 == "" {
return "", "", fmt.Errorf("to use MP-BGP with fixed address at least one IPv4 or IPv6 address has to be provided [current - IPv4: %s, IPv6: %s]",
p.MpbgpIPv4, p.MpbgpIPv6)
}
if p.MpbgpIPv4 != "" {
if net.ParseIP(p.MpbgpIPv4) == nil {
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv4 address", p.MpbgpIPv4)
}
}
if p.MpbgpIPv6 != "" {
if net.ParseIP(p.MpbgpIPv6) == nil {
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv6 address", p.MpbgpIPv6)
}
}
ipv4Address = p.MpbgpIPv4
ipv6Address = p.MpbgpIPv6
case "auto_sourceip":
ap.Transport.LocalAddress = server.SourceIP
// Resolve the local interface by SourceIP
iface, err := vip.GetInterfaceByIP(server.SourceIP)
if err != nil {
return "", "", fmt.Errorf("failed to get interface by IP: %v", err)
}
if vip.IsIPv4(server.SourceIP) {
// Get the non link-local IPv6 address on that interface
ipv6Address, err = vip.GetNonLinkLocalIP(iface, netlink.FAMILY_V6)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
}
} else {
// Get the non link-local IPv4 address on that interface
ipv4Address, err = vip.GetNonLinkLocalIP(iface, netlink.FAMILY_V4)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
}
}
case "auto_sourceif":
ap.Transport.BindInterface = server.SourceIF
iface, err := netlink.LinkByName(server.SourceIF)
if err != nil {
return "", "", fmt.Errorf("failed to get interface by name: %v", err)
}
// Get the non link-local IPv4 address on that interface
ipv4Address, err = vip.GetNonLinkLocalIP(&iface, netlink.FAMILY_V4)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
}
// Get the non link-local IPv6 address on that interface
ipv6Address, err = vip.GetNonLinkLocalIP(&iface, netlink.FAMILY_V6)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
}
default:
return "", "", fmt.Errorf("option %s for MP-BPG nexthop is not supported", server.MpbgpNexthop)
}
return ipv4Address, ipv6Address, nil
}
// Config defines the BGP server configuration
type Config struct {
AS uint32
RouterID string
SourceIP string
SourceIF string
MpbgpNexthop string
MpbgpIPv4 string
MpbgpIPv6 string
HoldTime uint64
KeepaliveInterval uint64
Peers []Peer
}
// Server manages a server object
type Server struct {
s *gobgp.BgpServer
c *Config
}

View File

@@ -5,9 +5,7 @@ import (
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/vip"
)
@@ -17,17 +15,16 @@ type Cluster struct {
completed chan bool
once sync.Once
Network []vip.Network
arpMgr *arp.Manager
}
// InitCluster - Will attempt to initialise all of the required settings for the cluster
func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Cluster, error) {
func InitCluster(c *kubevip.Config, disableVIP bool) (*Cluster, error) {
var networks []vip.Network
var err error
if !disableVIP {
// Start the Virtual IP Networking configuration
networks, err = startNetworking(c, intfMgr)
networks, err = startNetworking(c)
if err != nil {
return nil, err
}
@@ -35,7 +32,6 @@ func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.M
// Initialise the Cluster structure
newCluster := &Cluster{
Network: networks,
arpMgr: arpMgr,
}
log.Debug("service security", "enabled", c.EnableServiceSecurity)
@@ -43,7 +39,7 @@ func InitCluster(c *kubevip.Config, disableVIP bool, intfMgr *networkinterface.M
return newCluster, nil
}
func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vip.Network, error) {
func startNetworking(c *kubevip.Config) ([]vip.Network, error) {
address := c.VIP
if c.Address != "" {
@@ -55,8 +51,7 @@ func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vi
networks := []vip.Network{}
for _, addr := range addresses {
network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.RoutingTableID,
c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend,
c.EnableLoadBalancer, c.EnableServiceSecurity, intfMgr)
c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend, c.EnableLoadBalancer)
if err != nil {
return nil, err
}

View File

@@ -12,14 +12,16 @@ import (
// during runtime if IP changes, startDDNS don't have to do reconfigure because
// dnsUpdater already have the functionality to keep trying resolve the IP
// and update the VIP configuration if it changes
func (cluster *Cluster) StartDDNS(ctx context.Context, network vip.Network) error {
ddnsMgr := vip.NewDDNSManager(ctx, network)
ip, err := ddnsMgr.Start()
if err != nil {
return err
}
if err = network.SetIP(ip); err != nil {
return err
func (cluster *Cluster) StartDDNS(ctx context.Context) error {
for i := range cluster.Network {
ddnsMgr := vip.NewDDNSManager(ctx, cluster.Network[i])
ip, err := ddnsMgr.Start()
if err != nil {
return err
}
if err = cluster.Network[i].SetIP(ip); err != nil {
return err
}
}
return nil

View File

@@ -11,11 +11,13 @@ import (
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/equinixmetal"
"github.com/kube-vip/kube-vip/pkg/etcd"
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/loadbalancer"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/packethost/packngo"
log "log/slog"
@@ -133,13 +135,10 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
// (attempt to) Remove the virtual IP, in case it already exists
for i := range cluster.Network {
deleted, err := cluster.Network[i].DeleteIP()
err = cluster.Network[i].DeleteIP()
if err != nil {
log.Error("could not delete virtualIP", "err", err)
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
}
// Defer a function to check if the bgpServer has been created and if so attempt to close it
@@ -149,16 +148,42 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
}
}()
// If Equinix Metal is enabled then we can begin our preparation work
var packetClient *packngo.Client
if c.EnableMetal {
if c.ProviderConfig != "" {
key, project, err := equinixmetal.GetPacketConfig(c.ProviderConfig)
if err != nil {
log.Error("retrieve Equinix metal config", "err", err)
} else {
// Set the environment variable with the key for the project
os.Setenv("PACKET_AUTH_TOKEN", key)
// Update the configuration with the project key
c.MetalProjectID = project
}
}
packetClient, err = packngo.NewClient()
if err != nil {
log.Error("create Equinix metal client", "err", err)
}
// We're using Equinix Metal with BGP, populate the Peer information from the API
if c.EnableBGP {
log.Info("Looking up the BGP configuration from Equinix Metal")
err = equinixmetal.BGPLookup(packetClient, c)
if err != nil {
log.Error("retrieve Equinix metal BPG config", "err", err)
}
}
}
if c.EnableBGP && bgpServer == nil {
// Lets start BGP
log.Info("Starting the BGP server to advertise VIP routes to VGP peers")
bgpServer, err = bgp.NewBGPServer(c.BGPConfig)
bgpServer, err = bgp.NewBGPServer(&c.BGPConfig, nil)
if err != nil {
log.Error("new BGP server", "err", err)
}
if err := bgpServer.Start(nil); err != nil {
log.Error("starting BGP server", "err", err)
}
}
run := &runConfig{
@@ -166,28 +191,8 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
leaseID: c.NodeName,
sm: sm,
onStartedLeading: func(ctx context.Context) { //nolint TODO: potential clean code
// When we become leader, ensure we can take over VIPs even if they're preserved on other nodes
if c.PreserveVIPOnLeadershipLoss {
log.Info("Becoming leader with VIP preservation enabled - ensuring VIP takeover")
// Force add the VIPs (this will work even if they exist due to the precheck logic)
for i := range cluster.Network {
added, err := cluster.Network[i].AddIP(true, false)
if err != nil {
log.Error("failed to ensure VIP on leader takeover", "vip", cluster.Network[i].IP(), "err", err)
} else if added {
log.Info("took over VIP as new leader", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
} else {
log.Info("VIP already configured on interface", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
}
}
// Start ARP advertisements now that we have leadership
log.Info("Start ARP/NDP advertisement")
go cluster.arpMgr.StartAdvertisement(ctxArp)
// As we're leading lets start the vip service
err := cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, cancel)
err := cluster.vipService(ctxArp, ctxDNS, c, sm, bgpServer, packetClient, cancel)
if err != nil {
log.Error("starting VIP service on leader", "err", err)
}
@@ -209,36 +214,10 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
}
}
// Handle VIP cleanup based on configuration
if c.PreserveVIPOnLeadershipLoss {
// For IPv6, we must remove VIPs immediately to avoid DAD failures on the new leader
// IPv6 Duplicate Address Detection will fail if the new leader tries to add an IP that is still present on this node's interface
// We need to check each VIP individually and only remove IPv6 VIPs
for i := range cluster.Network {
if utils.IsIPv6(cluster.Network[i].IP()) {
log.Info("Removing IPv6 VIP immediately (required to prevent DAD failures on new leader)", "ip", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("delete VIP", "err", err)
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
} else {
log.Info("Preserving IPv4 VIP address on interface, only stopped ARP broadcasting", "ip", cluster.Network[i].IP())
}
}
} else {
// Legacy behavior: delete VIP addresses on leadership loss
log.Info("Deleting VIP addresses on leadership loss (legacy behavior)")
for i := range cluster.Network {
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("delete VIP", "err", err)
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
for i := range cluster.Network {
err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("delete VIP", "err", err)
}
}
@@ -248,23 +227,6 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
onNewLeader: func(identity string) {
// we're notified when new leader elected
log.Info("New leader", "leader", identity)
// If we're not the new leader and we have VIPs preserved from previous leadership,
// we need to clean them up to avoid conflicts.
if identity != c.NodeName && c.PreserveVIPOnLeadershipLoss {
log.Info("Cleaning up preserved VIPs as another node became leader", "new_leader", identity)
for i := range cluster.Network {
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("failed to cleanup preserved VIP", "vip", cluster.Network[i].IP(), "err", err)
}
if deleted {
log.Info("cleaned up preserved VIP to avoid conflict", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface(), "new_leader", identity)
} else {
log.Debug("VIP was not present on this node", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
}
}
},
}
@@ -345,7 +307,7 @@ func (cluster *Cluster) runEtcdLeaderElectionOrDie(ctx context.Context, run *run
})
}
func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
func (sm *Manager) NodeWatcher(lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
log.Info("Kube-Vip is watching nodes for control-plane labels")
@@ -353,7 +315,7 @@ func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoad
LabelSelector: "node-role.kubernetes.io/control-plane",
}
rw, err := watchtools.NewRetryWatcherWithContext(ctxArp, "1", &cache.ListWatch{
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.RetryWatcherClient.CoreV1().Nodes().Watch(context.Background(), listOptions)
},

View File

@@ -8,26 +8,25 @@ import (
"net"
"os"
"os/signal"
"sync"
"syscall"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/backend"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/equinixmetal"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/loadbalancer"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
"github.com/packethost/packngo"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Config, sm *Manager, bgpServer *bgp.Server, cancelLeaderElection context.CancelFunc) error {
func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Config, sm *Manager, bgpServer *bgp.Server, packetClient *packngo.Client, cancelLeaderElection context.CancelFunc) error {
var err error
// listen for interrupts or the Linux SIGTERM signal and cancel
@@ -42,53 +41,75 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
loadbalancers := []*loadbalancer.IPVSLoadBalancer{}
var arpWG sync.WaitGroup
for i := range cluster.Network {
network := cluster.Network[i]
if network.IsDDNS() {
if err := cluster.StartDDNS(ctxDNS, cluster.Network[i]); err != nil {
log.Error("failed to start DDNS", "err", err)
if cluster.Network[i].IsDDNS() {
if err := cluster.StartDDNS(ctxDNS); err != nil {
log.Error(err.Error())
}
}
if err := network.SetMask(c.VIPSubnet); err != nil {
log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
panic("")
}
// start the dns updater if address is dns
if network.IsDNS() {
log.Info("starting the DNS updater", "address", network.DNSName())
ipUpdater := vip.NewIPUpdater(network)
if cluster.Network[i].IsDNS() {
log.Info("starting the DNS updater", "address", cluster.Network[i].DNSName())
ipUpdater := vip.NewIPUpdater(cluster.Network[i])
ipUpdater.Run(ctxDNS)
}
if !c.EnableRoutingTable {
// Normal VIP addition, use skipDAD=false for normal DAD process
if _, err = network.AddIP(false, false); err != nil {
if c.EnableARP {
subnets := vip.Split(c.VIPCIDR)
subnet := ""
if len(subnets) > 0 {
subnet = subnets[0]
}
if vip.IsIPv6(cluster.Network[i].IP()) && len(subnets) > 1 {
subnet = subnets[1]
}
if subnet == "" {
log.Error("no subnet provided", "IP", cluster.Network[i].IP())
panic("")
}
if err = cluster.Network[i].SetMask(subnet); err != nil {
log.Error("failed to set mask", "subnet", subnet, "err", err)
panic("")
}
}
if err = cluster.Network[i].AddIP(false); err != nil {
log.Error(err.Error())
}
}
if c.EnableMetal {
// We're not using Equinix Metal with BGP
if !c.EnableBGP {
// Attempt to attach the EIP in the standard manner
log.Debug("Attaching the Equinix Metal EIP through the API to this host")
err = equinixmetal.AttachEIP(packetClient, c, c.NodeName)
if err != nil {
log.Error(err.Error())
}
}
}
if c.EnableBGP {
// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
log.Debug("Attempting to advertise over BGP", "address", network.CIDR())
err = bgpServer.AddHost(network.CIDR())
cidrVip := fmt.Sprintf("%s/%s", cluster.Network[i].IP(), c.VIPCIDR)
log.Debug("Attempting to advertise over BGP", "address", cidrVip)
err = bgpServer.AddHost(cidrVip)
if err != nil {
log.Error(err.Error())
}
}
if c.EnableLoadBalancer {
lb, err := loadbalancer.NewIPVSLB(network.IP(), c.LoadBalancerPort, c.LoadBalancerForwardingMethod, c.BackendHealthCheckInterval, c.Interface, cancelLeaderElection, signalChan)
lb, err := loadbalancer.NewIPVSLB(cluster.Network[i].IP(), c.LoadBalancerPort, c.LoadBalancerForwardingMethod, c.BackendHealthCheckInterval, c.Interface, cancelLeaderElection, signalChan)
if err != nil {
log.Error("Error creating IPVS LoadBalancer", "err", err)
}
go func() {
err = sm.NodeWatcher(ctxArp, lb, c.Port) //TODO: We're using the ctxARP as the context this will change when rkatz finishes his change
err = sm.NodeWatcher(lb, c.Port)
if err != nil {
log.Error("Error watching node labels", "err", err)
}
@@ -98,8 +119,7 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
}
if c.EnableARP {
arpWG.Add(1)
go cluster.layer2Update(ctxArp, network, c, &arpWG)
go cluster.layer2Update(ctxArp, cluster.Network[i], c)
}
}
@@ -170,14 +190,12 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
backend.Watch(func() {
for i := range cluster.Network {
network := cluster.Network[i]
networkIP := network.IP()
networkIP := cluster.Network[i].IP()
isNetworkV6, err := isV6(networkIP)
if err != nil {
log.Error("failed to check IP type", "IP", networkIP, "error", err)
continue
}
log.Debug("current ip to process", "ip", networkIP)
backendMap := &backendMapV4
if isNetworkV6 {
@@ -185,23 +203,20 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
}
for entry := range *backendMap {
log.Debug("entry.Check() for entry", "entry", entry)
if entry.Check() {
log.Debug("entry.Check() true")
// Normal VIP addition with precheck, use skipDAD=false for normal DAD process
_, err = network.AddIP(true, false)
err = cluster.Network[i].AddIP(true)
if err != nil {
log.Error("error adding address", "err", err)
}
if !(*backendMap)[entry] {
log.Info("added backend", "ip", network.IP())
log.Info("added backend", "ip", cluster.Network[i].IP())
}
err = network.AddRoute(true)
err = cluster.Network[i].AddRoute(true)
if err != nil && !errors.Is(err, fs.ErrExist) && !errors.Is(err, syscall.ESRCH) {
log.Warn(err.Error())
} else if err == nil && !(*backendMap)[entry] {
log.Info("added route", "route", network.PrepareRoute().String())
log.Info("added route", "route", cluster.Network[i].PrepareRoute().String())
}
(*backendMap)[entry] = true
@@ -219,20 +234,24 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
}
if deleteAddress {
err = network.DeleteRoute()
err = cluster.Network[i].DeleteRoute()
if err != nil && !errors.Is(err, fs.ErrNotExist) && !errors.Is(err, syscall.ESRCH) {
log.Warn("deleting route", "err", err)
} else if err == nil {
log.Info("deleted route", "route", network.PrepareRoute().String())
log.Info("deleted route", "route", cluster.Network[i].PrepareRoute().String())
}
deleted, err := network.DeleteIP()
isSet, err := cluster.Network[i].IsSet()
if err != nil {
log.Error("error deleting IP", "err", err)
panic("")
log.Error("failed to check IP address", "error", err)
}
if deleted {
log.Info("deleted address", "IP", network.IP(), "interface", network.Interface())
if isSet {
err = cluster.Network[i].DeleteIP()
if err != nil {
log.Error("error deleting IP", "err", err)
panic("")
}
log.Info("deleted address", "ip", cluster.Network[i].IP())
}
}
}
@@ -265,56 +284,60 @@ func getNodeIPs(ctx context.Context, nodename string, client *kubernetes.Clients
}
// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp *bgp.Server, name string, CountRouteReferences func(*netlink.Route) int) {
func (cluster *Cluster) StartLoadBalancerService(c *kubevip.Config, bgp *bgp.Server) {
// use a Go context so we can tell the arp loop code when we
// want to step down
//nolint
ctxArp, cancelArp := context.WithCancel(ctx)
ctxArp, cancelArp := context.WithCancel(context.Background())
cluster.stop = make(chan bool, 1)
cluster.completed = make(chan bool, 1)
var arpWG sync.WaitGroup
log.Debug("StartLoadBalancerService")
for i := range cluster.Network {
network := cluster.Network[i]
log.Debug("current ip to process", "ip", network.IP(), "mask", c.VIPSubnet)
if err := network.SetMask(c.VIPSubnet); err != nil {
log.Error("failed to set mask", "subnet", c.VIPSubnet, "err", err)
panic("")
}
_, err := network.DeleteIP()
if err != nil {
log.Warn("attempted to clean existing VIP", "err", err)
}
log.Debug("config flags", "enable_routing_table", c.EnableRoutingTable, "enable_leader_election", c.EnableLeaderElection, "enable_services_election", c.EnableServicesElection)
err := network.DeleteIP()
if err != nil {
log.Warn("Attempted to clean existing VIP", "err", err)
}
if c.EnableRoutingTable && (c.EnableLeaderElection || c.EnableServicesElection) {
err = network.AddRoute(false)
if err != nil {
log.Warn(err.Error())
} else {
log.Info("successful add Route")
}
} else if !c.EnableRoutingTable {
if c.EnableARP {
subnets := vip.Split(c.VIPCIDR)
subnet := ""
if len(subnets) > 0 {
subnet = subnets[0]
}
if vip.IsIPv6(cluster.Network[i].IP()) && len(subnets) > 1 {
subnet = subnets[1]
}
if subnet == "" {
log.Error("no subnet provided for address", "ip", cluster.Network[i].IP())
panic("")
}
if err = network.SetMask(subnet); err != nil {
log.Error("failed to set mask", "subnet", subnet, "err", err)
panic("")
}
}
if err = network.AddIP(false); err != nil {
log.Warn(err.Error())
}
}
// Normal VIP addition, use skipDAD=false for normal DAD process
if _, err = network.AddIP(false, false); err != nil {
log.Warn(err.Error())
} else {
log.Info("successful add IP")
}
if c.EnableARP {
arpWG.Add(1)
go cluster.layer2Update(ctxArp, network, c, &arpWG)
go cluster.layer2Update(ctxArp, cluster.Network[i], c)
}
if c.EnableBGP && (c.EnableLeaderElection || c.EnableServicesElection) {
// Lets advertise the VIP over BGP, the host needs to be passed using CIDR notation
log.Debug("(svcs) attempting to advertise over BGP", "address", network.CIDR())
err = bgp.AddHost(network.CIDR())
cidrVip := fmt.Sprintf("%s/%s", network.IP(), c.VIPCIDR)
log.Debug("(svcs) attempting to advertise over BGP", "address", cidrVip)
err = bgp.AddHost(cidrVip)
if err != nil {
log.Error(err.Error())
}
@@ -326,19 +349,13 @@ func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip
// Stop the Arp context if it is running
cancelArp()
arpWG.Wait() // wait for all cluster ARP/NDP to be finished
log.Info("[LOADBALANCER] Stopping load balancers", "name", name)
log.Info("[LOADBALANCER] Stopping load balancers")
if c.EnableRoutingTable {
for i := range cluster.Network {
// chek if route is not referenced by another service
r := cluster.Network[i].PrepareRoute()
if CountRouteReferences(r) < 1 {
log.Info("[VIP] Deleting Route for VIP", "IP", cluster.Network[i].IP())
if err := cluster.Network[i].DeleteRoute(); err != nil {
log.Warn(err.Error())
}
log.Info("[VIP] Deleting Route for VIP [%s]", "ip", cluster.Network[i].IP())
if err := cluster.Network[i].DeleteRoute(); err != nil {
log.Warn(err.Error())
}
}
@@ -346,37 +363,9 @@ func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip
return
}
for i := range cluster.Network {
if c.EnableARP && cluster.arpMgr.Count(cluster.Network[i].ARPName()) > 1 {
continue
}
// Handle VIP cleanup based on configuration
if c.PreserveVIPOnLeadershipLoss {
// For IPv6, we must remove VIPs immediately to avoid DAD failures on the new leader
// IPv6 Duplicate Address Detection will fail if the new leader tries to add an IP
// that is still present on this node's interface
if utils.IsIPv6(cluster.Network[i].IP()) {
log.Info("[VIP] Removing IPv6 VIP immediately (required to prevent DAD failures on new leader)", "ip", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
} else {
log.Info("[VIP] Preserving IPv4 VIP address on interface, only stopped ARP broadcasting", "ip", cluster.Network[i].IP())
}
} else {
// Legacy behavior: delete VIP addresses on leadership loss
log.Info("[VIP] Deleting VIP", "ip", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
log.Info("[VIP] Deleting VIP", "ip", cluster.Network[i].IP())
if err := cluster.Network[i].DeleteIP(); err != nil {
log.Warn(err.Error())
}
}
@@ -385,13 +374,12 @@ func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip
}
// Layer2Update, handles the creation of the
func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config, arpWG *sync.WaitGroup) {
defer arpWG.Done()
func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c *kubevip.Config) {
log.Info("layer 2 broadcaster starting")
var ndp *vip.NdpResponder
var err error
ipString := network.IP()
if utils.IsIPv6(ipString) {
if vip.IsIPv6(ipString) {
if network.IPisLinkLocal() {
log.Error("layer2 is link-local can't use NDP", "address", ipString)
@@ -409,10 +397,68 @@ func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c
log.Debug("layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
arpInstance := arp.NewInstance(network, ndp)
cluster.arpMgr.Insert(arpInstance)
<-ctx.Done() // if cancel() execute
log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
cluster.arpMgr.RemoveOnLeadershipLoss(arpInstance)
for {
select {
case <-ctx.Done(): // if cancel() execute
log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
return
default:
cluster.ensureIPAndSendGratuitous(network, ndp)
}
if c.ArpBroadcastRate < 500 {
log.Error("arp broadcast rate is too low", "rate (ms)", c.ArpBroadcastRate, "setting to (ms)", "3000")
c.ArpBroadcastRate = 3000
}
time.Sleep(time.Duration(c.ArpBroadcastRate) * time.Millisecond)
}
}
// ensureIPAndSendGratuitous - adds IP to the interface if missing, and send
// either a gratuitous ARP or gratuitous NDP. Re-adds the interface if it is IPv6
// and in a dadfailed state.
func (cluster *Cluster) ensureIPAndSendGratuitous(network vip.Network, ndp *vip.NdpResponder) {
iface := network.Interface()
ipString := network.IP()
// Check if IP is dadfailed
if network.IsDADFAIL() {
log.Warn("IP address is in dadfailed state, removing config", "ip", ipString, "interface", iface)
err := network.DeleteIP()
if err != nil {
log.Warn(err.Error())
}
}
// Ensure the address exists on the interface before attempting to ARP
set, err := network.IsSet()
if err != nil {
log.Warn(err.Error())
}
if !set {
log.Warn("Re-applying the VIP configuration", "ip", ipString, "interface", iface)
err = network.AddIP(false)
if err != nil {
log.Warn(err.Error())
}
}
if vip.IsIPv6(ipString) {
// Gratuitous NDP, will broadcast new MAC <-> IPv6 address
if ndp == nil {
log.Error("NDP responder was not created")
} else {
err := ndp.SendGratuitous(ipString)
if err != nil {
log.Warn(err.Error())
}
}
} else {
// Gratuitous ARP, will broadcast to new MAC <-> IPv4 address
err := vip.ARPSendGratuitous(ipString, iface)
if err != nil {
log.Warn(err.Error())
}
}
}

View File

@@ -5,6 +5,8 @@ import (
log "log/slog"
"github.com/packethost/packngo"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/vip"
@@ -24,16 +26,12 @@ func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) erro
for i := range cluster.Network {
if !disableVIP {
deleted, err := cluster.Network[i].DeleteIP()
err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn("Attempted to clean existing VIP", "err", err)
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
// Normal VIP addition for single node, use skipDAD=false for normal DAD process
_, err = cluster.Network[i].AddIP(false, false)
err = cluster.Network[i].AddIP(false)
if err != nil {
log.Warn(err.Error())
}
@@ -55,13 +53,10 @@ func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) erro
if !disableVIP {
for i := range cluster.Network {
log.Info("[VIP] Releasing the VIP", "address", cluster.Network[i].IP())
deleted, err := cluster.Network[i].DeleteIP()
err := cluster.Network[i].DeleteIP()
if err != nil {
log.Warn(err.Error())
}
if deleted {
log.Info("deleted address", "IP", cluster.Network[i].IP(), "interface", cluster.Network[i].Interface())
}
}
}
close(cluster.completed)
@@ -70,7 +65,7 @@ func (cluster *Cluster) StartSingleNode(c *kubevip.Config, disableVIP bool) erro
return nil
}
func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp.Server) error {
func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp.Server, packetClient *packngo.Client) error {
// use a Go context so we can tell the arp loop code when we
// want to step down
ctxArp, cancelArp := context.WithCancel(context.Background())
@@ -81,5 +76,5 @@ func (cluster *Cluster) StartVipService(c *kubevip.Config, sm *Manager, bgp *bgp
ctxDNS, cancelDNS := context.WithCancel(context.Background())
defer cancelDNS()
return cluster.vipService(ctxArp, ctxDNS, c, sm, bgp, nil)
return cluster.vipService(ctxArp, ctxDNS, c, sm, bgp, packetClient, nil)
}

View File

@@ -1,100 +0,0 @@
package egress
import (
"fmt"
"strings"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
func Teardown(podIP, vipIP, namespace, serviceUUID string, annotations map[string]string, useNftables bool) error {
// Look up the destination ports from the annotations on the service
destinationPorts := annotations[kubevip.EgressDestinationPorts]
deniedNetworks := annotations[kubevip.EgressDeniedNetworks]
allowedNetworks := annotations[kubevip.EgressAllowedNetworks]
internalEgress := annotations[kubevip.EgressInternal]
protocol := iptables.ProtocolIPv4
IPv6 := false
if utils.IsIPv6(podIP) {
protocol = iptables.ProtocolIPv6
IPv6 = true
}
// Use the internal egress implementation
if internalEgress != "" {
return nftables.DeleteSNAT(IPv6, serviceUUID)
}
i, err := vip.CreateIptablesClient(useNftables, namespace, protocol)
if err != nil {
return fmt.Errorf("error Creating iptables client [%s]", err)
}
if deniedNetworks != "" {
networks := strings.Split(deniedNetworks, ",")
for x := range networks {
err = i.DeleteMangleReturnForNetwork(vip.MangleChainName, networks[x])
if err != nil {
return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
}
if allowedNetworks != "" {
networks := strings.Split(allowedNetworks, ",")
for x := range networks {
err = i.DeleteMangleMarkingForNetwork(podIP, vip.MangleChainName, networks[x])
if err != nil {
return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
} else {
// Remove the marking of egress packets
err = i.DeleteMangleMarking(podIP, vip.MangleChainName)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
// Clear up SNAT rules
if destinationPorts != "" {
fixedPorts := strings.Split(destinationPorts, ",")
for _, fixedPort := range fixedPorts {
var proto, port string
data := strings.Split(fixedPort, ":")
if len(data) == 0 {
continue
} else if len(data) == 1 {
proto = "tcp"
port = data[0]
} else {
proto = data[0]
port = data[1]
}
err = i.DeleteSourceNatForDestinationPort(podIP, vipIP, port, proto)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
} else {
err = i.DeleteSourceNat(podIP, vipIP)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
err = vip.DeleteExistingSessions(podIP, false, destinationPorts, "")
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
return nil
}

View File

@@ -1,165 +0,0 @@
package endpoints
import (
"context"
"fmt"
"net"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
"k8s.io/apimachinery/pkg/watch"
)
type Processor struct {
config *kubevip.Config
provider providers.Provider
bgpServer *bgp.Server
worker endpointWorker
instances *[]*instance.Instance
}
func NewEndpointProcessor(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
instances *[]*instance.Instance) *Processor {
return &Processor{
config: config,
provider: provider,
bgpServer: bgpServer,
instances: instances,
worker: newEndpointWorker(config, provider, bgpServer, instances),
}
}
func (p *Processor) AddOrModify(ctx *servicecontext.Context, event watch.Event,
lastKnownGoodEndpoint *string, service *v1.Service, id string, leaderElectionActive *bool,
serviceFunc func(context.Context, *v1.Service) error,
leaderCtx *context.Context, cancel *context.CancelFunc) (bool, error) {
var err error
if err = p.provider.LoadObject(event.Object, *cancel); err != nil {
return false, fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
}
endpoints, err := p.worker.getEndpoints(service, id)
if err != nil {
return false, err
}
if err := p.worker.setInstanceEndpointsStatus(service, endpoints); err != nil {
log.Error("updating instance", "err", err)
}
// Find out if we have any local endpoints
// if out endpoint is empty then populate it
// if not, go through the endpoints and see if ours still exists
// If we have a local endpoint then begin the leader Election, unless it's already running
//
// Check that we have local endpoints
if len(endpoints) != 0 {
// Ignore IPv4
if service.Annotations[kubevip.EgressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
return true, nil
}
p.updateLastKnownGoodEndpoint(lastKnownGoodEndpoint, endpoints, service, leaderElectionActive, *cancel)
// start leader election if it's enabled and not already started
if !*leaderElectionActive && p.config.EnableServicesElection {
go func() {
*leaderCtx, *cancel = context.WithCancel(ctx.Ctx)
startLeaderElection(*leaderCtx, leaderElectionActive, service, serviceFunc)
}()
}
// There are local endpoints available on the node
if !p.config.EnableServicesElection && !p.config.EnableLeaderElection {
if err := p.worker.processInstance(ctx, service, leaderElectionActive); err != nil {
return false, fmt.Errorf("failed to process non-empty instance: %w", err)
}
}
} else {
// There are no local endpoints
p.worker.clear(ctx, lastKnownGoodEndpoint, service, *cancel, leaderElectionActive)
}
// Set the service accordingly
p.updateAnnotations(service, lastKnownGoodEndpoint)
log.Debug("watcher", "provider",
p.provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", *lastKnownGoodEndpoint, "active leader election", *leaderElectionActive)
return false, nil
}
func (p *Processor) Delete(service *v1.Service, id string) error {
if err := p.worker.delete(service, id); err != nil {
return fmt.Errorf("[%s] error deleting service: %w", p.provider.GetLabel(), err)
}
return nil
}
func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service, leaderElectionActive *bool, cancel context.CancelFunc) {
// if we haven't populated one, then do so
if *lastKnownGoodEndpoint == "" {
*lastKnownGoodEndpoint = endpoints[0]
return
}
// check out previous endpoint exists
stillExists := false
for x := range endpoints {
if endpoints[x] == *lastKnownGoodEndpoint {
stillExists = true
}
}
// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
if !stillExists {
p.worker.removeEgress(service, lastKnownGoodEndpoint)
if *leaderElectionActive && (p.config.EnableServicesElection || p.config.EnableLeaderElection) {
log.Warn("existing endpoint has been removed, restarting leaderElection", "provider", p.provider.GetLabel(), "endpoint", *lastKnownGoodEndpoint)
// Stop the existing leaderElection
cancel()
// disable last leaderElection flag
*leaderElectionActive = false
}
// Set our active endpoint to an existing one
*lastKnownGoodEndpoint = endpoints[0]
}
}
func (p *Processor) updateAnnotations(service *v1.Service, lastKnownGoodEndpoint *string) {
// Set the service accordingly
if service.Annotations[kubevip.Egress] == "true" {
activeEndpointAnnotation := kubevip.ActiveEndpoint
if !p.config.EnableEndpoints && p.provider.GetProtocol() == string(discoveryv1.AddressTypeIPv6) {
activeEndpointAnnotation = kubevip.ActiveEndpointIPv6
}
service.Annotations[activeEndpointAnnotation] = *lastKnownGoodEndpoint
}
}
func startLeaderElection(ctx context.Context, leaderElectionActive *bool, service *v1.Service, serviceFunc func(context.Context, *v1.Service) error) {
// This is a blocking function, that will restart (in the event of failure)
for {
// if the context isn't cancelled restart
if ctx.Err() != context.Canceled {
*leaderElectionActive = true
err := serviceFunc(ctx, service)
if err != nil {
log.Error(err.Error())
}
*leaderElectionActive = false
} else {
*leaderElectionActive = false
break
}
}
}

View File

@@ -1,129 +0,0 @@
package endpoints
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
)
type BGP struct {
generic
bgpServer *bgp.Server
}
func newBGP(generic generic, bgpServer *bgp.Server) endpointWorker {
return &BGP{
generic: generic,
bgpServer: bgpServer,
}
}
func (b *BGP) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) {
log.Debug("attempting to advertise BGP service", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP())
err := b.bgpServer.AddHost(cluster.Network[i].CIDR())
if err != nil {
log.Error("error adding BGP host", "provider", b.provider.GetLabel(), "err", err)
} else {
log.Info("added BGP host", "provider",
b.provider.GetLabel(), "ip", cluster.Network[i].CIDR(), "service name", service.Name, "namespace", service.Namespace)
ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
*leaderElectionActive = true
}
}
}
}
}
return nil
}
func (b *BGP) clear(ctx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
// If BGP mode is enabled - routes should be deleted
if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
err := b.bgpServer.DelHost(cluster.Network[i].CIDR())
if err != nil {
log.Error("deleting BGP host", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "err", err)
} else {
log.Info("deleted BGP host", "provider",
b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace)
ctx.ConfiguredNetworks.Delete(cluster.Network[i].IP())
*leaderElectionActive = false
}
}
}
}
}
b.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (b *BGP) getEndpoints(service *v1.Service, id string) ([]string, error) {
return b.getAllEndpoints(service, id)
}
func (b *BGP) delete(service *v1.Service, id string) error {
// When no-leader-elecition mode
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
// find all existing local endpoints
endpoints, err := b.getEndpoints(service, id)
if err != nil {
return fmt.Errorf("[%s] error getting endpoints: %w", b.provider.GetLabel(), err)
}
// If there were local endpoints deleted
if len(endpoints) > 0 {
b.deleteAction(service)
}
}
return nil
}
func (b *BGP) deleteAction(service *v1.Service) {
b.clearBGPHosts(service)
}
func (b *BGP) clearBGPHosts(service *v1.Service) {
ClearBGPHosts(service, b.instances, b.bgpServer)
}
func (b *BGP) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
return nil
}
func ClearBGPHosts(service *v1.Service, instances *[]*instance.Instance, bgpServer *bgp.Server) {
if instance := instance.FindServiceInstance(service, *instances); instance != nil {
ClearBGPHostsByInstance(instance, bgpServer)
}
}
func ClearBGPHostsByInstance(instance *instance.Instance, bgpServer *bgp.Server) {
if instance == nil {
log.Error("failed to clear BGP host for nil instance")
return
}
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
network := cluster.Network[i]
err := bgpServer.DelHost(network.CIDR())
if err != nil {
log.Error("[endpoint] error deleting BGP host", "err", err)
} else {
log.Debug("[endpoint] deleted BGP host", "ip",
network.CIDR(), "service name", instance.ServiceSnapshot.Name, "namespace", instance.ServiceSnapshot.Namespace)
}
}
}
}

View File

@@ -1,118 +0,0 @@
package endpoints
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/egress"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
)
type endpointWorker interface {
processInstance(svcCtx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error
clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool)
getEndpoints(service *v1.Service, id string) ([]string, error)
removeEgress(service *v1.Service, lastKnownGoodEndpoint *string)
delete(service *v1.Service, id string) error
setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error
}
func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server, instances *[]*instance.Instance) endpointWorker {
generic := newGeneric(config, provider, instances)
if config.EnableRoutingTable {
return newRoutingTable(generic)
}
if config.EnableBGP {
return newBGP(generic, bgpServer)
}
return &generic
}
type generic struct {
config *kubevip.Config
provider providers.Provider
instances *[]*instance.Instance
}
func newGeneric(config *kubevip.Config, provider providers.Provider, instances *[]*instance.Instance) generic {
return generic{
config: config,
provider: provider,
instances: instances,
}
}
func (g *generic) processInstance(_ *servicecontext.Context, _ *v1.Service, _ *bool) error {
return nil
}
func (g *generic) clear(_ *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
g.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if *lastKnownGoodEndpoint != "" {
log.Warn("existing endpoint has been removed, no remaining endpoints for leaderElection", "provider", g.provider.GetLabel(), "endpoint", lastKnownGoodEndpoint)
if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP, service.Namespace, string(service.UID), service.Annotations, g.config.EgressWithNftables); err != nil {
log.Error("error removing redundant egress rules", "err", err)
}
*lastKnownGoodEndpoint = "" // reset endpoint
if g.config.EnableServicesElection || g.config.EnableLeaderElection {
cancel() // stop services watcher
}
*leaderElectionActive = false
}
}
func (g *generic) getEndpoints(_ *v1.Service, id string) ([]string, error) {
return g.getLocalEndpoints(id)
}
func (g *generic) getLocalEndpoints(id string) ([]string, error) {
// Build endpoints
var endpoints []string
var err error
if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
}
return endpoints, nil
}
func (g *generic) getAllEndpoints(service *v1.Service, id string) ([]string, error) {
// Build endpoints
var err error
var endpoints []string
if !g.config.EnableLeaderElection && !g.config.EnableServicesElection &&
service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
if endpoints, err = g.provider.GetAllEndpoints(); err != nil {
return nil, fmt.Errorf("[%s] error getting all endpoints: %w", g.provider.GetLabel(), err)
}
} else {
if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
}
}
return endpoints, nil
}
func (g *generic) removeEgress(_ *v1.Service, _ *string) {
}
func (g *generic) delete(_ *v1.Service, _ string) error {
return nil
}
func (g *generic) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
return nil
}

View File

@@ -1,189 +0,0 @@
package endpoints
import (
"context"
"errors"
"fmt"
"net"
"syscall"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/egress"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
)
type RoutingTable struct {
generic
}
func newRoutingTable(generic generic) endpointWorker {
return &RoutingTable{
generic: generic,
}
}
func (rt *RoutingTable) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
instance := instance.FindServiceInstance(service, *rt.instances)
if instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) && cluster.Network[i].HasEndpoints() {
err := cluster.Network[i].AddRoute(false)
if err != nil {
if errors.Is(err, syscall.EEXIST) {
// If route exists, but protocol is not set (e.g. the route was created by the older version
// of kube-vip) try to update it if necessary
isUpdated, err := cluster.Network[i].UpdateRoutes()
if err != nil {
return fmt.Errorf("[%s] error updating existing routes: %w", rt.provider.GetLabel(), err)
}
if isUpdated {
log.Info("updated route", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
} else {
log.Info("route already present", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
}
} else {
// If other error occurs, return error
return fmt.Errorf("[%s] error adding route: %s", rt.provider.GetLabel(), err.Error())
}
} else {
log.Info("added route", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
*leaderElectionActive = true
}
}
}
}
}
return nil
}
func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
if errs := ClearRoutes(service, rt.instances); len(errs) == 0 {
svcCtx.ConfiguredNetworks.Clear()
} else {
for _, err := range errs {
log.Error("error while clearing routes", "err", err)
}
}
}
rt.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (rt *RoutingTable) getEndpoints(service *v1.Service, id string) ([]string, error) {
return rt.getAllEndpoints(service, id)
}
func (rt *RoutingTable) removeEgress(service *v1.Service, lastKnownGoodEndpoint *string) {
if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
service.Namespace, string(service.UID), service.Annotations, rt.config.EgressWithNftables); err != nil {
log.Warn("removing redundant egress rules", "err", err)
}
}
func (rt *RoutingTable) delete(service *v1.Service, id string) error {
// When no-leader-elecition mode
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
// find all existing local endpoints
endpoints, err := rt.getEndpoints(service, id)
if err != nil {
return fmt.Errorf("[%s] error getting endpoints: %w", rt.provider.GetLabel(), err)
}
// If there were local endpoints deleted
if len(endpoints) > 0 {
rt.deleteAction(service)
}
}
return nil
}
func (rt *RoutingTable) deleteAction(service *v1.Service) {
ClearRoutes(service, rt.instances)
}
func (rt *RoutingTable) setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error {
instance := instance.FindServiceInstanceWithTimeout(service, *rt.instances)
if instance == nil {
log.Error("failed to find the instance", "namespace", service.Namespace, "name", service.Name, "uid", service.UID, "provider", rt.provider.GetLabel())
} else {
for _, c := range instance.Clusters {
for n := range c.Network {
// if there are no endpoints set HasEndpoints false just in case
if len(endpoints) < 1 {
c.Network[n].SetHasEndpoints(false)
}
// check if endpoint are available and are of same IP family as service
if len(endpoints) > 0 && ((net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(endpoints[0]).To4() == nil)) {
c.Network[n].SetHasEndpoints(true)
}
}
}
}
return nil
}
func ClearRoutes(service *v1.Service, instances *[]*instance.Instance) []error {
errs := []error{}
if svcInst := instance.FindServiceInstance(service, *instances); svcInst != nil {
clearErrs := ClearRoutesByInstance(service, svcInst, instances)
errs = append(errs, clearErrs...)
}
return errs
}
func ClearRoutesByInstance(service *v1.Service, svcInst *instance.Instance, instances *[]*instance.Instance) []error {
if svcInst == nil {
return []error{fmt.Errorf("failed to remove routes for nil instance of service %s/%s, uid: %s", service.Namespace, service.Name, service.UID)}
}
errs := []error{}
for _, cluster := range svcInst.Clusters {
for i := range cluster.Network {
route := cluster.Network[i].PrepareRoute()
// check if route we are about to delete is not referenced by more than one service
if CountRouteReferences(route, instances) <= 1 {
err := cluster.Network[i].DeleteRoute()
if err != nil && !errors.Is(err, syscall.ESRCH) {
log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
errs = append(errs, err)
}
log.Debug("deleted route", "ip",
cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface())
}
}
}
return errs
}
func CountRouteReferences(route *netlink.Route, instances *[]*instance.Instance) int {
cnt := 0
for _, instance := range *instances {
for _, cluster := range instance.Clusters {
for n := range cluster.Network {
if cluster.Network[n].HasEndpoints() {
r := cluster.Network[n].PrepareRoute()
if r.Dst.String() == route.Dst.String() {
cnt++
}
}
}
}
}
return cnt
}

View File

@@ -1,137 +0,0 @@
package providers
import (
"context"
"fmt"
"strings"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/client-go/util/retry"
log "log/slog"
)
type Endpoints struct {
label string
//nolint:staticcheck // SA1019 endpoints are moving to an opt-in only
endpoints *v1.Endpoints
}
func NewEndpoints() Provider {
return &Endpoints{
label: "endpoints",
}
}
func (ep *Endpoints) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
service *v1.Service) (*watchtools.RetryWatcher, error) {
opts := metav1.ListOptions{
FieldSelector: fields.OneTermEqualSelector("metadata.name", service.Name).String(),
}
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return clientSet.CoreV1().Endpoints(service.Namespace).Watch(ctx, opts)
},
})
if err != nil {
return nil, fmt.Errorf("error creating endpoint watcher: %s", err.Error())
}
return rw, nil
}
func (ep *Endpoints) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
eps, ok := endpoints.(*v1.Endpoints)
if !ok {
cancel()
return fmt.Errorf("[%s] unable to parse Kubernetes services from API watcher", ep.GetLabel())
}
ep.endpoints = eps
return nil
}
func (ep *Endpoints) GetAllEndpoints() ([]string, error) {
result := []string{}
for subset := range ep.endpoints.Subsets {
for address := range ep.endpoints.Subsets[subset].Addresses {
addr := strings.Split(ep.endpoints.Subsets[subset].Addresses[address].IP, "/")
result = append(result, addr[0])
}
}
return result, nil
}
func (ep *Endpoints) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
var localEndpoints []string
for _, subset := range ep.endpoints.Subsets {
for _, address := range subset.Addresses {
log.Debug("processing endpoint", "label", ep.label, "ip", address.IP)
// 1. Compare the Nodename
if address.NodeName != nil && id == *address.NodeName {
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname, "nodename", *address.NodeName)
localEndpoints = append(localEndpoints, address.IP)
continue
}
// 2. Compare the Hostname (only useful if address.NodeName is not available)
if id == address.Hostname {
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname)
localEndpoints = append(localEndpoints, address.IP)
continue
}
}
}
return localEndpoints, nil
}
func (ep *Endpoints) UpdateServiceAnnotation(endpoint string, _ string, service *v1.Service, clientSet *kubernetes.Clientset) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
currentServiceCopy := currentService.DeepCopy()
if currentServiceCopy.Annotations == nil {
currentServiceCopy.Annotations = make(map[string]string)
}
currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("error updating Service Spec", "label", ep.GetLabel(), "name", currentServiceCopy.Name, "err", err)
return err
}
return nil
})
if retryErr != nil {
log.Error("failed to set Services", "label", ep.GetLabel(), "err", retryErr)
return retryErr
}
return nil
}
func (ep *Endpoints) GetLabel() string {
return ep.label
}
func (ep *Endpoints) GetProtocol() string {
return ""
}

View File

@@ -1,22 +0,0 @@
package providers
import (
"context"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
watchtools "k8s.io/client-go/tools/watch"
)
type Provider interface {
CreateRetryWatcher(context.Context, *kubernetes.Clientset,
*v1.Service) (*watchtools.RetryWatcher, error)
GetAllEndpoints() ([]string, error)
GetLocalEndpoints(string, *kubevip.Config) ([]string, error)
GetLabel() string
UpdateServiceAnnotation(string, string, *v1.Service, *kubernetes.Clientset) error
LoadObject(runtime.Object, context.CancelFunc) error
GetProtocol() string
}

64
pkg/equinixmetal/bgp.go Normal file
View File

@@ -0,0 +1,64 @@
package equinixmetal
import (
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/packethost/packngo"
)
// BGPLookup will use the Equinix Metal API functions to populate the BGP information
func BGPLookup(c *packngo.Client, k *kubevip.Config) error {
var thisDevice *packngo.Device
if k.MetalProjectID == "" {
proj := findProject(k.MetalProject, c)
if proj == nil {
return fmt.Errorf("Unable to find Project [%s]", k.MetalProject)
}
thisDevice = findSelf(c, proj.ID)
} else {
thisDevice = findSelf(c, k.MetalProjectID)
}
if thisDevice == nil {
return fmt.Errorf("Unable to find local/this device in Equinix Metal API")
}
log.Info("Querying BGP settings", "hostname", thisDevice.Hostname)
neighbours, _, err := c.Devices.ListBGPNeighbors(thisDevice.ID, &packngo.ListOptions{})
if err != nil {
return err
}
// Ensure neighbours exist (and it's enabled)
if len(neighbours) == 0 {
return fmt.Errorf("The server [%s]/[%s] has no BGP neighbours, ensure BGP is enabled", thisDevice.Hostname, thisDevice.ID)
}
// Add a warning (TODO)
if len(neighbours) > 1 {
log.Warn(fmt.Sprintf("There are [%d] neighbours, only designed to manage one", len(neighbours)))
}
// Ensure a peer exists
if len(neighbours[0].PeerIps) == 0 {
return fmt.Errorf("The server [%s]/[%s] has no BGP peers, ensure BGP is enabled", thisDevice.Hostname, thisDevice.ID)
}
k.BGPConfig.RouterID = neighbours[0].CustomerIP
k.BGPConfig.AS = uint32(neighbours[0].CustomerAs) //nolint (being removed)
// Add the peer(s)
for x := range neighbours[0].PeerIps {
peer := bgp.Peer{
Address: neighbours[0].PeerIps[x],
AS: uint32(neighbours[0].PeerAs), //nolint (being removed)
MultiHop: neighbours[0].Multihop,
Password: neighbours[0].Md5Password,
}
k.BGPConfig.Peers = append(k.BGPConfig.Peers, peer)
}
return nil
}

66
pkg/equinixmetal/eip.go Normal file
View File

@@ -0,0 +1,66 @@
package equinixmetal
import (
"fmt"
"path"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/packethost/packngo"
)
// AttachEIP will use the Equinix Metal APIs to move an EIP and attach to a host
func AttachEIP(c *packngo.Client, k *kubevip.Config, _ string) error {
// Use MetalProjectID if it is defined
projID := k.MetalProjectID
if projID == "" {
// Fallback to attempting to find the project by name
proj := findProject(k.MetalProject, c)
if proj == nil {
return fmt.Errorf("unable to find Project [%s]", k.MetalProject)
}
projID = proj.ID
}
// Prefer Address over VIP
vip := k.Address
if vip == "" {
vip = k.VIP
}
ips, _, _ := c.ProjectIPs.List(projID, &packngo.ListOptions{})
for _, ip := range ips {
// Find the device id for our EIP
if ip.Address == vip {
log.Info("Found EIP", "address", ip.Address, "id", ip.ID)
// If attachments already exist then remove them
if len(ip.Assignments) != 0 {
hrefID := path.Base(ip.Assignments[0].Href)
_, err := c.DeviceIPs.Unassign(hrefID)
if err != nil {
return fmt.Errorf("unable to unassign deviceIP %q: %v", hrefID, err)
}
}
}
}
// Lookup this server through the Equinix Metal API
thisDevice := findSelf(c, projID)
if thisDevice == nil {
return fmt.Errorf("unable to find local/this device in Equinix Metal API")
}
// Assign the EIP to this device
log.Info("Assigning EIP", "hostname", thisDevice.Hostname)
_, _, err := c.DeviceIPs.Assign(thisDevice.ID, &packngo.AddressStruct{
Address: vip,
})
if err != nil {
return err
}
return nil
}

60
pkg/equinixmetal/utils.go Normal file
View File

@@ -0,0 +1,60 @@
package equinixmetal
import (
"encoding/json"
"fmt"
"os"
log "log/slog"
"github.com/packethost/packngo"
)
func findProject(project string, c *packngo.Client) *packngo.Project {
l := &packngo.ListOptions{Includes: []string{project}}
ps, _, err := c.Projects.List(l)
if err != nil {
log.Error(err.Error())
}
for _, p := range ps {
// Find our project
if p.Name == project {
return &p
}
}
return nil
}
func findSelf(c *packngo.Client, projectID string) *packngo.Device {
// Go through devices
dev, _, _ := c.Devices.List(projectID, &packngo.ListOptions{})
for _, d := range dev {
// TODO do we need to replace os.Hostname with config.NodeName here?
me, _ := os.Hostname()
if me == d.Hostname {
return &d
}
}
return nil
}
// GetPacketConfig will lookup the configuration from a file path
func GetPacketConfig(providerConfig string) (string, string, error) {
var config struct {
AuthToken string `json:"apiKey"`
ProjectID string `json:"projectId"`
}
// get our token and project
if providerConfig != "" {
configBytes, err := os.ReadFile(providerConfig)
if err != nil {
return "", "", fmt.Errorf("failed to get read configuration file at path %s: %v", providerConfig, err)
}
err = json.Unmarshal(configBytes, &config)
if err != nil {
return "", "", fmt.Errorf("failed to process json of configuration file at path %s: %v", providerConfig, err)
}
}
return config.AuthToken, config.ProjectID, nil
}

View File

@@ -221,7 +221,7 @@ func (m *member) tryToBeLeader(ctx context.Context) {
func (m *member) resignOnCancel(ctx context.Context) {
<-ctx.Done()
if err := m.election.Resign(m.client.Ctx()); err != nil && !errors.Is(err, context.Canceled) {
if err := m.election.Resign(m.client.Ctx()); err != nil {
log.Error("Failed to resign after the context was canceled", "err", err)
}
}

View File

@@ -32,7 +32,7 @@ func TestRunElectionWithMemberIDCollision(t *testing.T) {
Client: cli,
},
Name: electionName,
MemberID: randomElectionNameForTest("my-host"),
MemberID: "my-host",
LeaseDurationSeconds: 1,
Callbacks: etcd.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
@@ -90,21 +90,19 @@ func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
config1 := configBase
config1.EtcdConfig.Client = cliMember1
config1.MemberID = randomElectionNameForTest("my-host")
config1.MemberID = "my-host"
uniqueID := rand.Uint64()
config1.MemberUniqueID = &uniqueID
config1.Callbacks = baseCallbacksForName(config1.MemberID)
syncMembers := make(chan (any))
config1.Callbacks.OnStartedLeading = func(_ context.Context) {
log.Println("I'm my-host, the new leader!!!!")
close(syncMembers)
log.Println("Losing the leadership on purpose by stopping renewing the lease")
g.Expect(cliMember1.Lease.Close()).To(Succeed())
log.Println("Member1 leases closed")
}
config2 := configBase
config2.MemberID = randomElectionNameForTest("my-other-host")
config2.MemberID = "my-other-host"
config2.Callbacks = baseCallbacksForName(config2.MemberID)
config2.Callbacks.OnStartedLeading = func(_ context.Context) {
log.Println("I'm my-other-host, the new leader!!!!")
@@ -118,14 +116,14 @@ func TestRunElectionWithTwoMembersAndReelection(t *testing.T) {
go func() {
defer wg.Done()
g.Expect(etcd.RunElection(member1Ctx, &config1)).To(Succeed())
log.Printf("%s routine done\n", config1.MemberID)
log.Println("Member1 routine done")
}()
go func() {
defer wg.Done()
<-syncMembers
time.Sleep(time.Millisecond * 50) // Make sure member1 becomes leader
g.Expect(etcd.RunElection(member2Ctx, &config2)).To(Succeed())
log.Printf("%s routine done\n", config2.MemberID)
log.Println("Member2 routine done")
}()
wg.Wait()

View File

@@ -1,63 +0,0 @@
package kubevip
const (
// Hardware address of the host that has the VIP
HwAddrKey = "kube-vip.io/hwaddr"
// The IP address that is requested
RequestedIP = "kube-vip.io/requestedIP"
// The host that has the VIP
VipHost = "kube-vip.io/vipHost"
// Enable Egress on a service
Egress = "kube-vip.io/egress"
// Enable internal Egress
EgressInternal = "kube-vip.io/egress-internal"
// Egress should be IPv6
EgressIPv6 = "kube-vip.io/egress-ipv6"
// Ports that traffic is allowed to access from the egress VIP
EgressDestinationPorts = "kube-vip.io/egress-destination-ports"
// Allowed incoming ports to the VIP
EgressSourcePorts = "kube-vip.io/egress-source-ports"
// Allowed networks for the Egress to be enabled for
EgressAllowedNetworks = "kube-vip.io/egress-allowed-networks"
// Networks that we wont Egress for
EgressDeniedNetworks = "kube-vip.io/egress-denied-networks"
// The current active endpoint(pod) for the Egress VIP
ActiveEndpoint = "kube-vip.io/active-endpoint"
// The current active endpoint(pod) for the Egress VIP (v6)
ActiveEndpointIPv6 = "kube-vip.io/active-endpoint-ipv6"
// Flush the conntrack rules (remove existing sessions) once Egress is configured
FlushContrack = "kube-vip.io/flush-conntrack"
// Configure LoadBalancer IPs instead of relying on a controller
LoadbalancerIPAnnotation = "kube-vip.io/loadbalancerIPs"
// Ignore the LoadBalancer Service
LoadbalancerIgnore = "kube-vip.io/ignore"
// Used to configure DHCP with a Hostname
LoadbalancerHostname = "kube-vip.io/loadbalancerHostname"
// Define an interface name to bind the address of the LoadBalancer to
ServiceInterface = "kube-vip.io/serviceInterface"
ServiceSecurityIgnore = "kube-vip.io/ignore-service-security"
// Enable UPNP on a Service
UpnpEnabled = "kube-vip.io/forwardUPNP"
RPFilter = "kube-vip.io/rp_filter" // Set the return path filter for a specific service interface
ServiceLease = "kube-vip.io/leaseName"
)

View File

@@ -1,234 +0,0 @@
package kubevip
import (
"fmt"
"net"
"strconv"
"strings"
"github.com/kube-vip/kube-vip/pkg/utils"
api "github.com/osrg/gobgp/v3/api"
"github.com/vishvananda/netlink"
)
// Peer defines a BGP Peer
type BGPPeer struct {
Address string
Port uint16
AS uint32
Password string
MultiHop bool
MpbgpNexthop string
MpbgpIPv4 string
MpbgpIPv6 string
}
// Config defines the BGP server configuration
type BGPConfig struct {
AS uint32
RouterID string
SourceIP string
SourceIF string
MpbgpNexthop string
MpbgpIPv4 string
MpbgpIPv6 string
HoldTime uint64
KeepaliveInterval uint64
Peers []BGPPeer
Zebra ZebraConfig
}
// Defines Zebra connection configuration. More on the topic - https://github.com/osrg/gobgp/blob/master/docs/sources/zebra.md#configuration
type ZebraConfig struct {
Enabled bool
URL string
Version uint32
SoftwareName string
}
// ParseBGPPeerConfig - take a string and parses it into an array of peers
func ParseBGPPeerConfig(config string) (bgpPeers []BGPPeer, err error) {
peers := strings.Split(config, ",")
if len(peers) == 0 {
return nil, fmt.Errorf("no BGP Peer configurations found")
}
for x := range peers {
peerStr := peers[x]
config := strings.Split(peerStr, "/")
peerStr = config[0]
if peerStr == "" {
continue
}
isV6Peer := peerStr[0] == '['
address := ""
if isV6Peer {
addressEndPos := strings.IndexByte(peerStr, ']')
if addressEndPos == -1 {
return nil, fmt.Errorf("no matching ] found for IPv6 BGP Peer")
}
address = peerStr[1:addressEndPos]
peerStr = peerStr[addressEndPos+1:]
}
peer := strings.Split(peerStr, ":")
if len(peer) < 2 {
return nil, fmt.Errorf("mandatory peering params <host>:<AS> incomplete")
}
if !isV6Peer {
address = peer[0]
}
ASNumber, err := strconv.ParseUint(peer[1], 10, 32)
if err != nil {
return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
}
password := ""
if len(peer) >= 3 {
password = peer[2]
}
multiHop := false
if len(peer) >= 4 {
multiHop, err = strconv.ParseBool(peer[3])
if err != nil {
return nil, fmt.Errorf("BGP MultiHop format error (true/false) [%s]", peer[1])
}
}
var port uint64
if len(peer) >= 5 {
port, err = strconv.ParseUint(peer[4], 10, 16)
if err != nil {
return nil, fmt.Errorf("BGP Peer AS format error [%s]", peer[1])
}
} else {
port = 179
}
var mpbgpNexthop, mpbgpIPv4, mpbgpIPv6 string
if len(config) > 1 {
configData := strings.Split(config[1], ";")
for _, cfg := range configData {
c := strings.Split(cfg, "=")
switch c[0] {
case "mpbgp_nexthop":
mpbgpNexthop = c[1]
case "mpbgp_ipv4":
mpbgpIPv4 = c[1]
case "mpbgp_ipv6":
mpbgpIPv6 = c[1]
default:
return nil, fmt.Errorf("peer configuration parameter '%s' is not supported", c[0])
}
}
}
peerConfig := BGPPeer{
Address: address,
AS: uint32(ASNumber),
Port: uint16(port),
Password: password,
MultiHop: multiHop,
MpbgpNexthop: mpbgpNexthop,
MpbgpIPv4: mpbgpIPv4,
MpbgpIPv6: mpbgpIPv6,
}
bgpPeers = append(bgpPeers, peerConfig)
}
return
}
func (p *BGPPeer) FindMpbgpAddresses(ap *api.Peer, server *BGPConfig) (string, string, error) {
var ipv4Address, ipv6Address string
switch p.MpbgpNexthop {
case "fixed":
ap.Transport.LocalAddress = server.SourceIP
if p.MpbgpIPv4 == "" && p.MpbgpIPv6 == "" {
return "", "", fmt.Errorf("to use MP-BGP with fixed address at least one IPv4 or IPv6 address has to be provided [current - IPv4: %s, IPv6: %s]",
p.MpbgpIPv4, p.MpbgpIPv6)
}
if p.MpbgpIPv4 != "" {
if net.ParseIP(p.MpbgpIPv4) == nil {
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv4 address", p.MpbgpIPv4)
}
}
if p.MpbgpIPv6 != "" {
if net.ParseIP(p.MpbgpIPv6) == nil {
return "", "", fmt.Errorf("provided address '%s' is not a valid IPv6 address", p.MpbgpIPv6)
}
}
ipv4Address = p.MpbgpIPv4
ipv6Address = p.MpbgpIPv6
case "auto_sourceip":
ap.Transport.LocalAddress = server.SourceIP
// Resolve the local interface by SourceIP
iface, err := utils.GetInterfaceByIP(server.SourceIP)
if err != nil {
return "", "", fmt.Errorf("failed to get interface by IP: %v", err)
}
if utils.IsIPv4(server.SourceIP) {
// Get the non link-local IPv6 address on that interface
ipv6Address, err = utils.GetNonLinkLocalIP(iface, netlink.FAMILY_V6)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
}
} else {
// Get the non link-local IPv4 address on that interface
ipv4Address, err = utils.GetNonLinkLocalIP(iface, netlink.FAMILY_V4)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
}
}
case "auto_sourceif":
ap.Transport.BindInterface = server.SourceIF
iface, err := netlink.LinkByName(server.SourceIF)
if err != nil {
return "", "", fmt.Errorf("failed to get interface by name: %v", err)
}
// Get the non link-local IPv4 address on that interface
ipv4Address, err = utils.GetNonLinkLocalIP(&iface, netlink.FAMILY_V4)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv4 address: %v", err)
}
// Get the non link-local IPv6 address on that interface
ipv6Address, err = utils.GetNonLinkLocalIP(&iface, netlink.FAMILY_V6)
if err != nil {
return "", "", fmt.Errorf("failed to get non link-local IPv6 address: %v", err)
}
default:
return "", "", fmt.Errorf("option %s for MP-BPG nexthop is not supported", server.MpbgpNexthop)
}
return ipv4Address, ipv6Address, nil
}
func (p *BGPPeer) SetMpbgpOptions(server *BGPConfig) {
if p.MpbgpNexthop == "" {
p.MpbgpNexthop = server.MpbgpNexthop
}
if p.MpbgpIPv4 == "" {
p.MpbgpIPv4 = server.MpbgpIPv4
}
if p.MpbgpIPv6 == "" {
p.MpbgpIPv6 = server.MpbgpIPv6
}
}

View File

@@ -6,12 +6,10 @@ import (
"math"
"math/bits"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/detector"
"sigs.k8s.io/yaml"
)
// ParseEnvironment - will popultate the configuration from environment variables
@@ -51,6 +49,12 @@ func ParseEnvironment(c *Config) error {
c.ServicesInterface = env
}
// Find provider configuration
env = os.Getenv(providerConfig)
if env != "" {
c.ProviderConfig = env
}
// Find Kubernetes Leader Election configuration
env = os.Getenv(vipLeaderElection)
if env != "" {
@@ -234,6 +238,12 @@ func ParseEnvironment(c *Config) error {
}
}
// Find vip address cidr range
env = os.Getenv(vipCidr)
if env != "" {
c.VIPCIDR = env
}
// Find vip address subnet
env = os.Getenv(vipSubnet)
if env != "" {
@@ -291,21 +301,6 @@ func ParseEnvironment(c *Config) error {
c.ArpBroadcastRate = 3000
}
// Determine if VIP should be preserved on leadership loss
// true: VIP addresses remain on interface, only ARP/NDP broadcasting stops
// false (default): VIP addresses are deleted on leadership loss (legacy behavior)
env = os.Getenv(vipPreserveOnLeadershipLoss)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.PreserveVIPOnLeadershipLoss = b
} else {
// Default to false for backward compatibility
c.PreserveVIPOnLeadershipLoss = false
}
// Wireguard Mode
env = os.Getenv(vipWireguard)
if env != "" {
@@ -438,7 +433,7 @@ func ParseEnvironment(c *Config) error {
// Peer AS
env = os.Getenv(bgpPeers)
if env != "" {
peers, err := ParseBGPPeerConfig(env)
peers, err := bgp.ParseBGPPeerConfig(env)
if err != nil {
return err
}
@@ -517,32 +512,28 @@ func ParseEnvironment(c *Config) error {
c.BGPConfig.KeepaliveInterval = u64
}
env = os.Getenv(zebraEnable)
// Enable the Equinix Metal API calls
env = os.Getenv(vipPacket)
if env != "" {
result, err := strconv.ParseBool(env)
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.BGPConfig.Zebra.Enabled = result
c.EnableMetal = b
}
env = os.Getenv(zebraURL)
// Find the Equinix Metal project name
env = os.Getenv(vipPacketProject)
if env != "" {
c.BGPConfig.Zebra.URL = env
// TODO - parse address net.Host()
c.MetalProject = env
}
env = os.Getenv(zebraVersion)
// Find the Equinix Metal project ID
env = os.Getenv(vipPacketProjectID)
if env != "" {
u64, err := strconv.ParseUint(env, 10, 32)
if err != nil {
return err
}
c.BGPConfig.Zebra.Version = uint32(u64)
}
env = os.Getenv(zebraSoftwareName)
if env != "" {
c.BGPConfig.Zebra.SoftwareName = env
// TODO - parse address net.Host()
c.MetalProjectID = env
}
// Enable the load-balancer
@@ -623,13 +614,13 @@ func ParseEnvironment(c *Config) error {
c.K8sConfigFile = env
}
env = os.Getenv(enableEndpoints)
env = os.Getenv(enableEndpointSlices)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.EnableEndpoints = b
c.EnableEndpointSlices = b
}
env = os.Getenv(mirrorDestInterface)
@@ -663,305 +654,5 @@ func ParseEnvironment(c *Config) error {
c.HealthCheckPort = int(i)
}
env = os.Getenv(enableUPNP)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.EnableUPNP = b
}
if env = os.Getenv(egressClean); env == "" {
env = os.Getenv(strings.ToUpper(egressClean))
}
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.EgressClean = b
}
// check for configuration file path
env = os.Getenv(configFile)
if env != "" {
c.ConfigFile = env
}
return nil
}
// LoadConfigFromFile loads configuration from a JSON or YAML file
func LoadConfigFromFile(configFilePath string) (*Config, error) {
if configFilePath == "" {
return nil, fmt.Errorf("config file path is empty")
}
// Check if file exists
if _, err := os.Stat(configFilePath); os.IsNotExist(err) {
return nil, fmt.Errorf("config file does not exist: %s", configFilePath)
}
// Read file content
data, err := os.ReadFile(configFilePath)
if err != nil {
return nil, fmt.Errorf("failed to read config file %s: %v", configFilePath, err)
}
var config Config
ext := strings.ToLower(filepath.Ext(configFilePath))
switch ext {
case ".json":
err = json.Unmarshal(data, &config)
if err != nil {
return nil, fmt.Errorf("failed to parse JSON config file %s: %v", configFilePath, err)
}
case ".yaml", ".yml":
err = yaml.Unmarshal(data, &config)
if err != nil {
return nil, fmt.Errorf("failed to parse YAML config file %s: %v", configFilePath, err)
}
default:
return nil, fmt.Errorf("unsupported config file format %s. Supported formats: .json, .yaml, .yml", ext)
}
return &config, nil
}
// MergeConfigFromFile merges configuration loaded from file with existing config
// Priority: command line flags > environment variables > config file
func MergeConfigFromFile(c *Config, configFilePath string) error {
if configFilePath == "" {
return nil // No config file specified, nothing to merge
}
fileConfig, err := LoadConfigFromFile(configFilePath)
if err != nil {
return err
}
// Merge file config with existing config
// Only set values from file if they haven't been set by flags or env vars
mergeConfigValues(c, fileConfig)
return nil
}
// mergeConfigValues merges values from fileConfig into baseConfig
// Only overwrites zero values in baseConfig
func mergeConfigValues(baseConfig, fileConfig *Config) {
// Basic configuration
if baseConfig.Logging == 0 && fileConfig.Logging != 0 {
baseConfig.Logging = fileConfig.Logging
}
// Network configuration
if baseConfig.Interface == "" && fileConfig.Interface != "" {
baseConfig.Interface = fileConfig.Interface
}
if baseConfig.ServicesInterface == "" && fileConfig.ServicesInterface != "" {
baseConfig.ServicesInterface = fileConfig.ServicesInterface
}
if baseConfig.VIP == "" && fileConfig.VIP != "" {
baseConfig.VIP = fileConfig.VIP
}
if baseConfig.VIPSubnet == "" && fileConfig.VIPSubnet != "" {
baseConfig.VIPSubnet = fileConfig.VIPSubnet
}
if baseConfig.Address == "" && fileConfig.Address != "" {
baseConfig.Address = fileConfig.Address
}
if baseConfig.Port == 0 && fileConfig.Port != 0 {
baseConfig.Port = fileConfig.Port
}
if baseConfig.NodeName == "" && fileConfig.NodeName != "" {
baseConfig.NodeName = fileConfig.NodeName
}
// Boolean flags - only merge if not explicitly set
if !baseConfig.EnableARP && fileConfig.EnableARP {
baseConfig.EnableARP = fileConfig.EnableARP
}
if !baseConfig.EnableBGP && fileConfig.EnableBGP {
baseConfig.EnableBGP = fileConfig.EnableBGP
}
if !baseConfig.EnableWireguard && fileConfig.EnableWireguard {
baseConfig.EnableWireguard = fileConfig.EnableWireguard
}
if !baseConfig.EnableRoutingTable && fileConfig.EnableRoutingTable {
baseConfig.EnableRoutingTable = fileConfig.EnableRoutingTable
}
if !baseConfig.EnableControlPlane && fileConfig.EnableControlPlane {
baseConfig.EnableControlPlane = fileConfig.EnableControlPlane
}
if !baseConfig.DetectControlPlane && fileConfig.DetectControlPlane {
baseConfig.DetectControlPlane = fileConfig.DetectControlPlane
}
if !baseConfig.EnableServices && fileConfig.EnableServices {
baseConfig.EnableServices = fileConfig.EnableServices
}
if !baseConfig.EnableServicesElection && fileConfig.EnableServicesElection {
baseConfig.EnableServicesElection = fileConfig.EnableServicesElection
}
if !baseConfig.EnableNodeLabeling && fileConfig.EnableNodeLabeling {
baseConfig.EnableNodeLabeling = fileConfig.EnableNodeLabeling
}
if !baseConfig.EnableLoadBalancer && fileConfig.EnableLoadBalancer {
baseConfig.EnableLoadBalancer = fileConfig.EnableLoadBalancer
}
if !baseConfig.DDNS && fileConfig.DDNS {
baseConfig.DDNS = fileConfig.DDNS
}
if !baseConfig.SingleNode && fileConfig.SingleNode {
baseConfig.SingleNode = fileConfig.SingleNode
}
if !baseConfig.StartAsLeader && fileConfig.StartAsLeader {
baseConfig.StartAsLeader = fileConfig.StartAsLeader
}
if !baseConfig.PreserveVIPOnLeadershipLoss && fileConfig.PreserveVIPOnLeadershipLoss {
baseConfig.PreserveVIPOnLeadershipLoss = fileConfig.PreserveVIPOnLeadershipLoss
}
// Service configuration
if baseConfig.Namespace == "" && fileConfig.Namespace != "" {
baseConfig.Namespace = fileConfig.Namespace
}
if baseConfig.ServiceNamespace == "" && fileConfig.ServiceNamespace != "" {
baseConfig.ServiceNamespace = fileConfig.ServiceNamespace
}
if baseConfig.ServicesLeaseName == "" && fileConfig.ServicesLeaseName != "" {
baseConfig.ServicesLeaseName = fileConfig.ServicesLeaseName
}
// LoadBalancer configuration
if baseConfig.LoadBalancerPort == 0 && fileConfig.LoadBalancerPort != 0 {
baseConfig.LoadBalancerPort = fileConfig.LoadBalancerPort
}
if baseConfig.LoadBalancerForwardingMethod == "" && fileConfig.LoadBalancerForwardingMethod != "" {
baseConfig.LoadBalancerForwardingMethod = fileConfig.LoadBalancerForwardingMethod
}
if baseConfig.LoadBalancerClassName == "" && fileConfig.LoadBalancerClassName != "" {
baseConfig.LoadBalancerClassName = fileConfig.LoadBalancerClassName
}
// Routing Table configuration
if baseConfig.RoutingTableID == 0 && fileConfig.RoutingTableID != 0 {
baseConfig.RoutingTableID = fileConfig.RoutingTableID
}
if baseConfig.RoutingTableType == 0 && fileConfig.RoutingTableType != 0 {
baseConfig.RoutingTableType = fileConfig.RoutingTableType
}
if baseConfig.RoutingProtocol == 0 && fileConfig.RoutingProtocol != 0 {
baseConfig.RoutingProtocol = fileConfig.RoutingProtocol
}
// BGP configuration
mergeBGPConfig(&baseConfig.BGPConfig, &fileConfig.BGPConfig)
// Kubernetes configuration
if baseConfig.K8sConfigFile == "" && fileConfig.K8sConfigFile != "" {
baseConfig.K8sConfigFile = fileConfig.K8sConfigFile
}
// Leader Election configuration
mergeLeaderElectionConfig(&baseConfig.KubernetesLeaderElection, &fileConfig.KubernetesLeaderElection)
// Prometheus configuration
if baseConfig.PrometheusHTTPServer == "" && fileConfig.PrometheusHTTPServer != "" {
baseConfig.PrometheusHTTPServer = fileConfig.PrometheusHTTPServer
}
// DNS configuration
if baseConfig.DNSMode == "" && fileConfig.DNSMode != "" {
baseConfig.DNSMode = fileConfig.DNSMode
}
// Health check configuration
if baseConfig.HealthCheckPort == 0 && fileConfig.HealthCheckPort != 0 {
baseConfig.HealthCheckPort = fileConfig.HealthCheckPort
}
// Egress configuration
if baseConfig.EgressPodCidr == "" && fileConfig.EgressPodCidr != "" {
baseConfig.EgressPodCidr = fileConfig.EgressPodCidr
}
if baseConfig.EgressServiceCidr == "" && fileConfig.EgressServiceCidr != "" {
baseConfig.EgressServiceCidr = fileConfig.EgressServiceCidr
}
// Mirror configuration
if baseConfig.MirrorDestInterface == "" && fileConfig.MirrorDestInterface != "" {
baseConfig.MirrorDestInterface = fileConfig.MirrorDestInterface
}
// Iptables configuration
if baseConfig.IptablesBackend == "" && fileConfig.IptablesBackend != "" {
baseConfig.IptablesBackend = fileConfig.IptablesBackend
}
// Backend health check interval
if baseConfig.BackendHealthCheckInterval == 0 && fileConfig.BackendHealthCheckInterval != 0 {
baseConfig.BackendHealthCheckInterval = fileConfig.BackendHealthCheckInterval
}
// ARP broadcast rate
if baseConfig.ArpBroadcastRate == 0 && fileConfig.ArpBroadcastRate != 0 {
baseConfig.ArpBroadcastRate = fileConfig.ArpBroadcastRate
}
// Annotations
if baseConfig.Annotations == "" && fileConfig.Annotations != "" {
baseConfig.Annotations = fileConfig.Annotations
}
// Load balancers slice
if len(baseConfig.LoadBalancers) == 0 && len(fileConfig.LoadBalancers) > 0 {
baseConfig.LoadBalancers = fileConfig.LoadBalancers
}
}
// mergeBGPConfig merges BGP configuration
func mergeBGPConfig(base, file *BGPConfig) {
if base.RouterID == "" && file.RouterID != "" {
base.RouterID = file.RouterID
}
if base.AS == 0 && file.AS != 0 {
base.AS = file.AS
}
if base.SourceIF == "" && file.SourceIF != "" {
base.SourceIF = file.SourceIF
}
if base.SourceIP == "" && file.SourceIP != "" {
base.SourceIP = file.SourceIP
}
if base.HoldTime == 0 && file.HoldTime != 0 {
base.HoldTime = file.HoldTime
}
if base.KeepaliveInterval == 0 && file.KeepaliveInterval != 0 {
base.KeepaliveInterval = file.KeepaliveInterval
}
if len(base.Peers) == 0 && len(file.Peers) > 0 {
base.Peers = file.Peers
}
}
// mergeLeaderElectionConfig merges leader election configuration
func mergeLeaderElectionConfig(base, file *KubernetesLeaderElection) {
if base.LeaseName == "" && file.LeaseName != "" {
base.LeaseName = file.LeaseName
}
if base.LeaseDuration == 0 && file.LeaseDuration != 0 {
base.LeaseDuration = file.LeaseDuration
}
if base.RenewDeadline == 0 && file.RenewDeadline != 0 {
base.RenewDeadline = file.RenewDeadline
}
if base.RetryPeriod == 0 && file.RetryPeriod != 0 {
base.RetryPeriod = file.RetryPeriod
}
if len(base.LeaseAnnotations) == 0 && len(file.LeaseAnnotations) > 0 {
base.LeaseAnnotations = file.LeaseAnnotations
}
}

View File

@@ -9,9 +9,6 @@ const (
// vip_arpRate - defines the rate of gARP broadcasts
vipArpRate = "vip_arpRate"
// vipPreserveOnLeadershipLoss - if true, VIP addresses will remain on interface when leadership is lost
vipPreserveOnLeadershipLoss = "vip_preserve_on_leadership_loss"
// vipLeaderElection - defines if the kubernetes algorithm should be used
vipLeaderElection = "vip_leaderelection"
@@ -42,6 +39,9 @@ const (
// vipServicesInterface - defines the interface that the service vips should bind too
vipServicesInterface = "vip_servicesinterface"
// vipCidr - defines the cidr that the vip will use (for BGP)
vipCidr = "vip_cidr"
// vipSubnet - defines the subnet that the vip will use
vipSubnet = "vip_subnet"
@@ -86,6 +86,18 @@ const (
// vipStartLeader - will start this instance as the leader of the cluster
vipStartLeader = "vip_startleader"
// vipPacket defines that the packet API will be used for EIP
vipPacket = "vip_packet"
// vipPacketProject defines which project within Packet to use
vipPacketProject = "vip_packetproject"
// vipPacketProjectID defines which projectID within Packet to use
vipPacketProjectID = "vip_packetprojectid"
// providerConfig defines a path to a configuration that should be parsed
providerConfig = "provider_config"
// bgpEnable defines if BGP should be enabled
bgpEnable = "bgp_enable"
// bgpRouterID defines the routerID for the BGP server
@@ -113,15 +125,6 @@ const (
// bgpKeepaliveInterval defines bgp timers keepalive interval
bgpKeepaliveInterval = "bgp_keepalive_interval"
// zebraEnable defines if Zebra integraton should be enabled
zebraEnable = "zebra_enable"
// zebraUrl specifies path to the unix domain socket for connecting to Zebra daemon
zebraURL = "zebra_url"
// zebraVersion specifies Zebra API Version
zebraVersion = "zebra_version"
// zebraSoftwareName specifies Software Name for Zebra
zebraSoftwareName = "zebra_software_name"
// mpbgpNexthop defines MPBGP mode
mpbgpNexthop = "mpbgp_nexthop"
// mpbgpIPv4 defines fixed IPv4 to be used with MPBGP
@@ -215,8 +218,8 @@ const (
// disableServiceUpdates disables service updating
disableServiceUpdates = "disable_service_updates"
// enableEndpoints enables use of Endpoints instead of EndpointSlices
enableEndpoints = "enable_endpoints"
// enableEndpointSlices enables use of EndpointSlices instead of Endpoints
enableEndpointSlices = "enable_endpointslices"
// mirrorDestInterface is the network interface where all traffics that go through service interface
// will be mirrored to. The source interface is ServicesInterface by default, fall back to Interface if not set.
@@ -231,13 +234,4 @@ const (
// healthCheckPort, if set to non-zero will be the port the health check will listen on
healthCheckPort = "health_check_port"
// enableUPNP enables UPNP functions
enableUPNP = "enable_upnp"
// egressClean enables egress cleaning on kube-vip's start
egressClean = "egress_clean"
// configFile defines the path to a JSON/YAML configuration file
configFile = "config_file"
)

View File

@@ -1,559 +0,0 @@
package kubevip
import (
"os"
"path/filepath"
"testing"
)
func TestLoadConfigFromFile(t *testing.T) {
// Create temporary directory for test files
tmpDir, err := os.MkdirTemp("", "kube-vip-config-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
tests := []struct {
name string
filename string
content string
expectedConfig *Config
wantErr bool
errContains string
}{
{
name: "Valid YAML config",
filename: "config.yaml",
content: `
logging: 2
enableARP: true
enableControlPlane: true
enableServices: true
address: "192.168.1.100"
port: 6443
interface: "eth0"
namespace: "kube-system"
vipSubnet: "192.168.1.0/24"
leaseName: "test-lease"
leaseDuration: 15
renewDeadline: 10
retryPeriod: 2
prometheusHTTPServer: ":2112"
`,
expectedConfig: &Config{
Logging: 2,
EnableARP: true,
EnableControlPlane: true,
EnableServices: true,
Address: "192.168.1.100",
Port: 6443,
Interface: "eth0",
Namespace: "kube-system",
VIPSubnet: "192.168.1.0/24",
PrometheusHTTPServer: ":2112",
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "test-lease",
LeaseDuration: 15,
RenewDeadline: 10,
RetryPeriod: 2,
},
},
wantErr: false,
},
{
name: "Valid JSON config",
filename: "config.json",
content: `{
"logging": 3,
"enableBGP": true,
"enableServices": true,
"address": "10.0.0.100",
"port": 8443,
"interface": "ens192",
"namespace": "kube-system",
"loadBalancers": [
{
"name": "control-plane",
"ports": [
{
"type": "TCP",
"port": 6443
}
],
"bindToVip": true,
"forwardingMethod": "local"
}
]
}`,
expectedConfig: &Config{
Logging: 3,
EnableBGP: true,
EnableServices: true,
Address: "10.0.0.100",
Port: 8443,
Interface: "ens192",
Namespace: "kube-system",
LoadBalancers: []LoadBalancer{
{
Name: "control-plane",
Ports: []Port{
{
Type: "TCP",
Port: 6443,
},
},
BindToVip: true,
ForwardingMethod: "local",
},
},
},
wantErr: false,
},
{
name: "Complex BGP config",
filename: "bgp-config.yaml",
content: `
enableBGP: true
bgpConfig:
routerID: "192.168.1.1"
as: 65000
sourceIF: "eth0"
holdTime: 60
keepaliveInterval: 20
peers:
- address: "192.168.1.2"
as: 65001
port: 179
multiHop: false
- address: "192.168.1.3"
as: 65002
port: 179
multiHop: true
`,
expectedConfig: &Config{
EnableBGP: true,
BGPConfig: BGPConfig{
RouterID: "192.168.1.1",
AS: 65000,
SourceIF: "eth0",
HoldTime: 60,
KeepaliveInterval: 20,
Peers: []BGPPeer{
{
Address: "192.168.1.2",
AS: 65001,
Port: 179,
MultiHop: false,
},
{
Address: "192.168.1.3",
AS: 65002,
Port: 179,
MultiHop: true,
},
},
},
},
wantErr: false,
},
{
name: "Invalid JSON",
filename: "invalid.json",
content: `{"logging": 2, "invalid": }`,
wantErr: true,
errContains: "failed to parse JSON config file",
},
{
name: "Invalid YAML",
filename: "invalid.yaml",
content: "logging: 2\ninvalid: [unclosed",
wantErr: true,
errContains: "failed to parse YAML config file",
},
{
name: "Unsupported format",
filename: "config.txt",
content: "logging=2",
wantErr: true,
errContains: "unsupported config file format",
},
{
name: "Empty path",
filename: "",
content: "",
wantErr: true,
errContains: "config file path is empty",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var filePath string
if tt.filename != "" {
filePath = filepath.Join(tmpDir, tt.filename)
if err := os.WriteFile(filePath, []byte(tt.content), 0600); err != nil {
t.Fatalf("Failed to write test file: %v", err)
}
}
config, err := LoadConfigFromFile(filePath)
if tt.wantErr {
if err == nil {
t.Errorf("LoadConfigFromFile() expected error, got nil")
return
}
if tt.errContains != "" && !containsString(err.Error(), tt.errContains) {
t.Errorf("LoadConfigFromFile() error = %v, expected to contain %v", err, tt.errContains)
}
return
}
if err != nil {
t.Errorf("LoadConfigFromFile() unexpected error = %v", err)
return
}
if config == nil {
t.Errorf("LoadConfigFromFile() returned nil config")
return
}
// Compare key fields
if config.Logging != tt.expectedConfig.Logging {
t.Errorf("Logging = %v, expected %v", config.Logging, tt.expectedConfig.Logging)
}
if config.EnableARP != tt.expectedConfig.EnableARP {
t.Errorf("EnableARP = %v, expected %v", config.EnableARP, tt.expectedConfig.EnableARP)
}
if config.EnableBGP != tt.expectedConfig.EnableBGP {
t.Errorf("EnableBGP = %v, expected %v", config.EnableBGP, tt.expectedConfig.EnableBGP)
}
if config.Address != tt.expectedConfig.Address {
t.Errorf("Address = %v, expected %v", config.Address, tt.expectedConfig.Address)
}
if config.Port != tt.expectedConfig.Port {
t.Errorf("Port = %v, expected %v", config.Port, tt.expectedConfig.Port)
}
if config.Interface != tt.expectedConfig.Interface {
t.Errorf("Interface = %v, expected %v", config.Interface, tt.expectedConfig.Interface)
}
// Test BGP config if present
if tt.expectedConfig.EnableBGP {
if config.BGPConfig.RouterID != tt.expectedConfig.BGPConfig.RouterID {
t.Errorf("BGPConfig.RouterID = %v, expected %v", config.BGPConfig.RouterID, tt.expectedConfig.BGPConfig.RouterID)
}
if config.BGPConfig.AS != tt.expectedConfig.BGPConfig.AS {
t.Errorf("BGPConfig.AS = %v, expected %v", config.BGPConfig.AS, tt.expectedConfig.BGPConfig.AS)
}
if len(config.BGPConfig.Peers) != len(tt.expectedConfig.BGPConfig.Peers) {
t.Errorf("BGPConfig.Peers length = %v, expected %v", len(config.BGPConfig.Peers), len(tt.expectedConfig.BGPConfig.Peers))
}
}
// Test LoadBalancers if present
if len(tt.expectedConfig.LoadBalancers) > 0 {
if len(config.LoadBalancers) != len(tt.expectedConfig.LoadBalancers) {
t.Errorf("LoadBalancers length = %v, expected %v", len(config.LoadBalancers), len(tt.expectedConfig.LoadBalancers))
}
}
})
}
}
func TestMergeConfigFromFile(t *testing.T) {
// Create temporary directory for test files
tmpDir, err := os.MkdirTemp("", "kube-vip-merge-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
// Create test config file
configFile := filepath.Join(tmpDir, "test-config.yaml")
configContent := `
logging: 3
enableARP: true
enableServices: true
address: "192.168.1.200"
port: 6443
interface: "eth1"
namespace: "test-namespace"
leaseName: "file-lease"
leaseDuration: 20
prometheusHTTPServer: ":3000"
`
if err := os.WriteFile(configFile, []byte(configContent), 0600); err != nil {
t.Fatalf("Failed to write test config file: %v", err)
}
tests := []struct {
name string
baseConfig *Config
configFilePath string
expected *Config
wantErr bool
}{
{
name: "Merge with empty base config",
baseConfig: &Config{},
configFilePath: configFile,
expected: &Config{
Logging: 3,
EnableARP: true,
EnableServices: true,
Address: "192.168.1.200",
Port: 6443,
Interface: "eth1",
Namespace: "test-namespace",
PrometheusHTTPServer: ":3000",
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "file-lease",
LeaseDuration: 20,
},
},
wantErr: false,
},
{
name: "Merge respects existing values (priority test)",
baseConfig: &Config{
Logging: 5, // Should not be overridden
Port: 8443, // Should not be overridden
Interface: "eth0", // Should not be overridden
},
configFilePath: configFile,
expected: &Config{
Logging: 5, // From base (higher priority)
EnableARP: true, // From file
EnableServices: true, // From file
Address: "192.168.1.200", // From file
Port: 8443, // From base (higher priority)
Interface: "eth0", // From base (higher priority)
Namespace: "test-namespace", // From file
PrometheusHTTPServer: ":3000", // From file
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "file-lease", // From file
LeaseDuration: 20, // From file
},
},
wantErr: false,
},
{
name: "Empty config file path",
baseConfig: &Config{
Logging: 1,
},
configFilePath: "",
expected: &Config{
Logging: 1, // Unchanged
},
wantErr: false,
},
{
name: "Non-existent config file",
baseConfig: &Config{
Logging: 1,
},
configFilePath: "/non/existent/file.yaml",
expected: &Config{
Logging: 1,
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := MergeConfigFromFile(tt.baseConfig, tt.configFilePath)
if tt.wantErr {
if err == nil {
t.Errorf("MergeConfigFromFile() expected error, got nil")
}
return
}
if err != nil {
t.Errorf("MergeConfigFromFile() unexpected error = %v", err)
return
}
// Compare key fields
if tt.baseConfig.Logging != tt.expected.Logging {
t.Errorf("Logging = %v, expected %v", tt.baseConfig.Logging, tt.expected.Logging)
}
if tt.baseConfig.EnableARP != tt.expected.EnableARP {
t.Errorf("EnableARP = %v, expected %v", tt.baseConfig.EnableARP, tt.expected.EnableARP)
}
if tt.baseConfig.Address != tt.expected.Address {
t.Errorf("Address = %v, expected %v", tt.baseConfig.Address, tt.expected.Address)
}
if tt.baseConfig.Port != tt.expected.Port {
t.Errorf("Port = %v, expected %v", tt.baseConfig.Port, tt.expected.Port)
}
if tt.baseConfig.Interface != tt.expected.Interface {
t.Errorf("Interface = %v, expected %v", tt.baseConfig.Interface, tt.expected.Interface)
}
if tt.baseConfig.Namespace != tt.expected.Namespace {
t.Errorf("Namespace = %v, expected %v", tt.baseConfig.Namespace, tt.expected.Namespace)
}
if tt.baseConfig.PrometheusHTTPServer != tt.expected.PrometheusHTTPServer {
t.Errorf("PrometheusHTTPServer = %v, expected %v", tt.baseConfig.PrometheusHTTPServer, tt.expected.PrometheusHTTPServer)
}
})
}
}
func TestMergeConfigValues(t *testing.T) {
tests := []struct {
name string
baseConfig *Config
fileConfig *Config
expectedBase *Config
}{
{
name: "Merge basic configuration",
baseConfig: &Config{
Logging: 5, // Should not be overridden
Port: 0, // Should be overridden
},
fileConfig: &Config{
Logging: 2,
Port: 6443,
Interface: "eth0",
Address: "192.168.1.100",
},
expectedBase: &Config{
Logging: 5, // From base (non-zero)
Port: 6443, // From file (base was zero)
Interface: "eth0", // From file (base was empty)
Address: "192.168.1.100", // From file (base was empty)
},
},
{
name: "Merge boolean flags",
baseConfig: &Config{
EnableARP: true, // Should not be overridden
},
fileConfig: &Config{
EnableARP: false, // Should not override true
EnableBGP: true, // Should be set
EnableServices: true, // Should be set
EnableWireguard: false, // Should not be set (false doesn't override false)
},
expectedBase: &Config{
EnableARP: true, // From base (true has priority)
EnableBGP: true, // From file
EnableServices: true, // From file
EnableWireguard: false, // Remains false
},
},
{
name: "Merge BGP configuration",
baseConfig: &Config{
BGPConfig: BGPConfig{
RouterID: "1.1.1.1", // Should not be overridden
},
},
fileConfig: &Config{
BGPConfig: BGPConfig{
RouterID: "2.2.2.2", // Should not override
AS: 65000, // Should be set
SourceIF: "eth0", // Should be set
HoldTime: 30, // Should be set
KeepaliveInterval: 10, // Should be set
},
},
expectedBase: &Config{
BGPConfig: BGPConfig{
RouterID: "1.1.1.1", // From base (non-empty)
AS: 65000, // From file (base was zero)
SourceIF: "eth0", // From file (base was empty)
HoldTime: 30, // From file (base was zero)
KeepaliveInterval: 10, // From file (base was zero)
},
},
},
{
name: "Merge leader election configuration",
baseConfig: &Config{
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "base-lease", // Should not be overridden
},
},
fileConfig: &Config{
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "file-lease", // Should not override
LeaseDuration: 15, // Should be set
RenewDeadline: 10, // Should be set
RetryPeriod: 2, // Should be set
},
},
expectedBase: &Config{
KubernetesLeaderElection: KubernetesLeaderElection{
LeaseName: "base-lease", // From base (non-empty)
LeaseDuration: 15, // From file (base was zero)
RenewDeadline: 10, // From file (base was zero)
RetryPeriod: 2, // From file (base was zero)
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mergeConfigValues(tt.baseConfig, tt.fileConfig)
// Compare results
if tt.baseConfig.Logging != tt.expectedBase.Logging {
t.Errorf("Logging = %v, expected %v", tt.baseConfig.Logging, tt.expectedBase.Logging)
}
if tt.baseConfig.Port != tt.expectedBase.Port {
t.Errorf("Port = %v, expected %v", tt.baseConfig.Port, tt.expectedBase.Port)
}
if tt.baseConfig.Interface != tt.expectedBase.Interface {
t.Errorf("Interface = %v, expected %v", tt.baseConfig.Interface, tt.expectedBase.Interface)
}
if tt.baseConfig.EnableARP != tt.expectedBase.EnableARP {
t.Errorf("EnableARP = %v, expected %v", tt.baseConfig.EnableARP, tt.expectedBase.EnableARP)
}
if tt.baseConfig.EnableBGP != tt.expectedBase.EnableBGP {
t.Errorf("EnableBGP = %v, expected %v", tt.baseConfig.EnableBGP, tt.expectedBase.EnableBGP)
}
if tt.baseConfig.BGPConfig.RouterID != tt.expectedBase.BGPConfig.RouterID {
t.Errorf("BGPConfig.RouterID = %v, expected %v", tt.baseConfig.BGPConfig.RouterID, tt.expectedBase.BGPConfig.RouterID)
}
if tt.baseConfig.BGPConfig.AS != tt.expectedBase.BGPConfig.AS {
t.Errorf("BGPConfig.AS = %v, expected %v", tt.baseConfig.BGPConfig.AS, tt.expectedBase.BGPConfig.AS)
}
if tt.baseConfig.KubernetesLeaderElection.LeaseName != tt.expectedBase.KubernetesLeaderElection.LeaseName {
t.Errorf("KubernetesLeaderElection.LeaseName = %v, expected %v", tt.baseConfig.KubernetesLeaderElection.LeaseName, tt.expectedBase.KubernetesLeaderElection.LeaseName)
}
})
}
}
func TestLoadConfigFromFile_FileNotExists(t *testing.T) {
_, err := LoadConfigFromFile("/non/existent/path/config.yaml")
if err == nil {
t.Error("LoadConfigFromFile() expected error for non-existent file, got nil")
}
if !containsString(err.Error(), "config file does not exist") {
t.Errorf("LoadConfigFromFile() error = %v, expected to contain 'config file does not exist'", err)
}
}
// Helper function to check if a string contains a substring
func containsString(str, substr string) bool {
return len(str) >= len(substr) && (str == substr || len(substr) == 0 ||
(len(substr) > 0 && func() bool {
for i := 0; i <= len(str)-len(substr); i++ {
if str[i:i+len(substr)] == substr {
return true
}
}
return false
}()))
}

View File

@@ -2,58 +2,23 @@ package kubevip
import (
"fmt"
"log"
"strconv"
"github.com/google/go-containerregistry/pkg/name"
appv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
applyCoreV1 "k8s.io/client-go/applyconfigurations/core/v1"
applyMetaV1 "k8s.io/client-go/applyconfigurations/meta/v1"
applyRbacV1 "k8s.io/client-go/applyconfigurations/rbac/v1"
"sigs.k8s.io/yaml"
)
// TransformApplyObjectToManifest transforms an apply object into a normal Kubernetes manifest
func TransformApplyObjectToManifest(applyObject interface{}) string {
// Convert the apply object to an unstructured object
unstructuredObj := &unstructured.Unstructured{}
err := runtime.DefaultUnstructuredConverter.FromUnstructured(applyConfigToMap(applyObject), unstructuredObj)
if err != nil {
log.Fatalf("Error converting apply object to unstructured: %v", err)
}
// Marshal the unstructured object into YAML
yamlData, err := yaml.Marshal(unstructuredObj.Object)
if err != nil {
log.Fatalf("Error marshaling unstructured object to YAML: %v", err)
}
return string(yamlData)
}
// Helper function to convert apply configuration to a map
func applyConfigToMap(applyConfig interface{}) map[string]interface{} {
data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(applyConfig)
if err != nil {
log.Fatalf("Error converting apply configuration to map: %v", err)
}
return data
}
// GenerateSA will create the service account for kube-vip
func GenerateSA(c *Config) *applyCoreV1.ServiceAccountApplyConfiguration {
func GenerateSA() *applyCoreV1.ServiceAccountApplyConfiguration {
kind := "ServiceAccount"
name := "kube-vip"
var namespace string
if c.ServiceNamespace != "" {
namespace = c.ServiceNamespace
} else {
namespace = metav1.NamespaceSystem
}
namespace := "kube-system"
newManifest := &applyCoreV1.ServiceAccountApplyConfiguration{
TypeMetaApplyConfiguration: applyMetaV1.TypeMetaApplyConfiguration{APIVersion: &corev1.SchemeGroupVersion.Version, Kind: &kind},
ObjectMetaApplyConfiguration: &applyMetaV1.ObjectMetaApplyConfiguration{
@@ -65,31 +30,15 @@ func GenerateSA(c *Config) *applyCoreV1.ServiceAccountApplyConfiguration {
}
// GenerateCR will generate the Cluster role for kube-vip
func GenerateRole(c *Config, role bool) *applyRbacV1.RoleApplyConfiguration {
var kind, name string
var namespace *string
if role {
kind = "Role"
name = "kube-vip"
if c.ServiceNamespace != "" {
namespace = &c.ServiceNamespace
} else {
// If the namespace is empty then we need to set it to the system namespace
copiedNamespace := metav1.NamespaceSystem
namespace = &copiedNamespace
}
} else {
kind = "ClusterRole"
name = "system:kube-vip-role"
}
func GenerateCR() *applyRbacV1.ClusterRoleApplyConfiguration {
name := "system:kube-vip-role"
roleRefKind := "ClusterRole"
apiVersion := "rbac.authorization.k8s.io/v1"
newManifest := &applyRbacV1.RoleApplyConfiguration{
TypeMetaApplyConfiguration: applyMetaV1.TypeMetaApplyConfiguration{APIVersion: &apiVersion, Kind: &kind},
newManifest := &applyRbacV1.ClusterRoleApplyConfiguration{
TypeMetaApplyConfiguration: applyMetaV1.TypeMetaApplyConfiguration{APIVersion: &apiVersion, Kind: &roleRefKind},
ObjectMetaApplyConfiguration: &applyMetaV1.ObjectMetaApplyConfiguration{
Name: &name,
Namespace: namespace,
Name: &name,
},
Rules: []applyRbacV1.PolicyRuleApplyConfiguration{
{
@@ -100,7 +49,7 @@ func GenerateRole(c *Config, role bool) *applyRbacV1.RoleApplyConfiguration {
{
APIGroups: []string{""},
Resources: []string{"services", "endpoints"},
Verbs: []string{"list", "get", "watch", "update"},
Verbs: []string{"list", "get", "watch", "endoints"},
},
{
APIGroups: []string{""},
@@ -112,54 +61,38 @@ func GenerateRole(c *Config, role bool) *applyRbacV1.RoleApplyConfiguration {
Resources: []string{"leases"},
Verbs: []string{"list", "get", "watch", "update", "create"},
},
{
APIGroups: []string{"discovery.k8s.io"},
Resources: []string{"endpointslices"},
Verbs: []string{"list", "get", "watch", "update"},
},
{
APIGroups: []string{""},
Resources: []string{"pods"},
Verbs: []string{"list"},
},
},
}
return newManifest
}
// GenerateCRB will generate the clusterRoleBinding or rolebinding
func GenerateRoleBinding(rolebinding bool, saCfg *applyCoreV1.ServiceAccountApplyConfiguration, crCfg *applyRbacV1.RoleApplyConfiguration) *applyRbacV1.RoleBindingApplyConfiguration {
// GenerateCRB will generate the clusterRoleBinding
func GenerateCRB() *applyRbacV1.ClusterRoleBindingApplyConfiguration {
kind := "ClusterRoleBinding"
apiVersion := "rbac.authorization.k8s.io/v1"
subjectKind := "ServiceAccount"
apiGroup := "rbac.authorization.k8s.io"
var kind, bindName string
var namespace, objectNamespace *string
if rolebinding {
kind = "RoleBinding"
bindName = "kube-vip"
namespace = nil
objectNamespace = saCfg.Namespace
} else {
kind = "ClusterRoleBinding"
bindName = "system:kube-vip-binding"
namespace = saCfg.Namespace
objectNamespace = nil
}
newManifest := &applyRbacV1.RoleBindingApplyConfiguration{
roleRefKind := "ClusterRole"
roleRefName := "system:kube-vip-role"
name := "kube-vip"
bindName := "system:kube-vip-role-binding"
namespace := "kube-system"
newManifest := &applyRbacV1.ClusterRoleBindingApplyConfiguration{
TypeMetaApplyConfiguration: applyMetaV1.TypeMetaApplyConfiguration{APIVersion: &apiVersion, Kind: &kind},
ObjectMetaApplyConfiguration: &applyMetaV1.ObjectMetaApplyConfiguration{
Name: &bindName,
Namespace: objectNamespace,
Name: &bindName,
},
RoleRef: &applyRbacV1.RoleRefApplyConfiguration{
APIGroup: &apiGroup,
Kind: crCfg.Kind,
Name: crCfg.Name,
Kind: &roleRefKind,
Name: &roleRefName,
},
Subjects: []applyRbacV1.SubjectApplyConfiguration{
{
Kind: saCfg.Kind,
Name: saCfg.Name,
Namespace: namespace,
Kind: &subjectKind,
Name: &name,
Namespace: &namespace,
},
},
}
@@ -167,11 +100,7 @@ func GenerateRoleBinding(rolebinding bool, saCfg *applyCoreV1.ServiceAccountAppl
}
// generatePodSpec will take a kube-vip config and generate a Pod spec
func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) *corev1.Pod {
imageRef, err := name.NewTag(image, name.WeakValidation, name.WithDefaultTag(imageVersion))
if err != nil {
panic(fmt.Errorf("Cannot parse %q: %w", image, err))
}
func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod {
command := "manager"
// Determine where the pods should be living (for multi-tenancy)
@@ -232,6 +161,18 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) *cor
newEnvironment = append(newEnvironment, svcInterface...)
}
// If a CIDR is used add it to the manifest
if c.VIPCIDR != "" {
// build environment variables
cidr := []corev1.EnvVar{
{
Name: vipCidr,
Value: c.VIPCIDR,
},
}
newEnvironment = append(newEnvironment, cidr...)
}
// If a subnet is required for the VIP
if c.VIPSubnet != "" {
// build environment variables
@@ -390,6 +331,40 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) *cor
}
// If we're specifying a configuration
if c.ProviderConfig != "" {
provider := []corev1.EnvVar{
{
Name: providerConfig,
Value: c.ProviderConfig,
},
}
newEnvironment = append(newEnvironment, provider...)
}
// If Equinix Metal is enabled then add it to the manifest
if c.EnableMetal {
packet := []corev1.EnvVar{
{
Name: vipPacket,
Value: strconv.FormatBool(c.EnableMetal),
},
{
Name: vipPacketProject,
Value: c.MetalProject,
},
{
Name: vipPacketProjectID,
Value: c.MetalProjectID,
},
{
Name: "PACKET_AUTH_TOKEN",
Value: c.MetalAPIKey,
},
}
newEnvironment = append(newEnvironment, packet...)
}
// Detect and enable wireguard mode
if c.EnableWireguard {
wireguard := []corev1.EnvVar{
@@ -411,7 +386,8 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) *cor
}
newEnvironment = append(newEnvironment, routingtable...)
}
// If BGP
// If BGP, but we're not using Equinix Metal
if c.EnableBGP {
bgp := []corev1.EnvVar{
{
@@ -421,9 +397,8 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) *cor
}
newEnvironment = append(newEnvironment, bgp...)
}
// If BGP
if c.EnableBGP {
// If BGP, but we're not using Equinix Metal
if c.EnableBGP && !c.EnableMetal {
bgpConfig := []corev1.EnvVar{
{
Name: bgpRouterID,
@@ -525,10 +500,10 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) *cor
}
newEnvironment = append(newEnvironment, prometheus...)
if c.EnableEndpoints {
if c.EnableEndpointSlices {
newEnvironment = append(newEnvironment, corev1.EnvVar{
Name: enableEndpoints,
Value: strconv.FormatBool(c.EnableEndpoints),
Name: enableEndpointSlices,
Value: strconv.FormatBool(c.EnableEndpointSlices),
})
}
@@ -596,7 +571,7 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) *cor
Containers: []corev1.Container{
{
Name: "kube-vip",
Image: imageRef.Name(),
Image: fmt.Sprintf("ghcr.io/kube-vip/kube-vip:%s", imageVersion),
ImagePullPolicy: corev1.PullIfNotPresent,
SecurityContext: securityContext,
Args: []string{
@@ -637,18 +612,38 @@ func generatePodSpec(c *Config, image, imageVersion string, inCluster bool) *cor
newManifest.Spec.HostAliases = append(newManifest.Spec.HostAliases, hostAlias)
}
if c.ProviderConfig != "" {
providerConfigMount := corev1.VolumeMount{
Name: "cloud-sa-volume",
MountPath: "/etc/cloud-sa",
ReadOnly: true,
}
newManifest.Spec.Containers[0].VolumeMounts = append(newManifest.Spec.Containers[0].VolumeMounts, providerConfigMount)
providerConfigVolume := corev1.Volume{
Name: "cloud-sa-volume",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: "metal-cloud-config",
},
},
}
newManifest.Spec.Volumes = append(newManifest.Spec.Volumes, providerConfigVolume)
}
return newManifest
}
// GeneratePodManifestFromConfig will take a kube-vip config and generate a manifest
func GeneratePodManifestFromConfig(c *Config, image, imageVersion string, inCluster bool) string {
newManifest := generatePodSpec(c, image, imageVersion, inCluster)
func GeneratePodManifestFromConfig(c *Config, imageVersion string, inCluster bool) string {
newManifest := generatePodSpec(c, imageVersion, inCluster)
b, _ := yaml.Marshal(newManifest)
return string(b)
}
// GenerateDaemonsetManifestFromConfig will take a kube-vip config and generate a manifest
func GenerateDaemonsetManifestFromConfig(c *Config, image, imageVersion string, inCluster, taint bool) string {
func GenerateDaemonsetManifestFromConfig(c *Config, imageVersion string, inCluster, taint bool) string {
// Determine where the pod should be deployed
var namespace string
if c.ServiceNamespace != "" {
@@ -657,7 +652,7 @@ func GenerateDaemonsetManifestFromConfig(c *Config, image, imageVersion string,
namespace = metav1.NamespaceSystem
}
podSpec := generatePodSpec(c, image, imageVersion, inCluster).Spec
podSpec := generatePodSpec(c, imageVersion, inCluster).Spec
newManifest := &appv1.DaemonSet{
TypeMeta: metav1.TypeMeta{
Kind: "DaemonSet",

View File

@@ -1,9 +1,6 @@
package kubevip
import (
"os"
"testing"
)
import "testing"
func TestParseEnvironment(t *testing.T) {
@@ -12,8 +9,8 @@ func TestParseEnvironment(t *testing.T) {
c *Config
wantErr bool
}{
{"nil config", nil, false},
{"basic config", &Config{Interface: "eth0", ServicesInterface: "eth1"}, false},
{"", nil, false},
{"", &Config{Interface: "eth0", ServicesInterface: "eth1"}, false},
}
for _, tt := range tests {
t.Logf("%v", tt.c)
@@ -24,53 +21,3 @@ func TestParseEnvironment(t *testing.T) {
})
}
}
func TestParseEnvironmentConfigFile(t *testing.T) {
// Save original environment
originalConfigFile := os.Getenv("config_file")
defer func() {
if originalConfigFile != "" {
os.Setenv("config_file", originalConfigFile)
} else {
os.Unsetenv("config_file")
}
}()
tests := []struct {
name string
envValue string
expectedConfig string
}{
{
name: "config_file environment variable set",
envValue: "/etc/kube-vip/config.yaml",
expectedConfig: "/etc/kube-vip/config.yaml",
},
{
name: "config_file environment variable empty",
envValue: "",
expectedConfig: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Set environment variable
if tt.envValue != "" {
os.Setenv("config_file", tt.envValue)
} else {
os.Unsetenv("config_file")
}
config := &Config{}
err := ParseEnvironment(config)
if err != nil {
t.Errorf("ParseEnvironment() unexpected error = %v", err)
}
if config.ConfigFile != tt.expectedConfig {
t.Errorf("ConfigFile = %v, expected %v", config.ConfigFile, tt.expectedConfig)
}
})
}
}

View File

@@ -1,5 +1,9 @@
package kubevip
import (
"github.com/kube-vip/kube-vip/pkg/bgp"
)
// Config defines all of the settings for the Kube-Vip Pod
type Config struct {
// Logging, settings
@@ -50,10 +54,6 @@ type Config struct {
// ArpBroadcastRate, defines how often kube-vip will update the network about updates to the network
ArpBroadcastRate int64 `yaml:"arpBroadcastRate"`
// PreserveVIPOnLeadershipLoss, if true, VIP addresses will remain on interface when leadership is lost (only ARP/NDP broadcasting stops)
// If false, VIP addresses are deleted on leadership loss (legacy behavior)
PreserveVIPOnLeadershipLoss bool `yaml:"preserveVipOnLeadershipLoss"`
// Annotations will define if we're going to wait and lookup configuration from Kubernetes node annotations
Annotations string
@@ -76,6 +76,9 @@ type Config struct {
// VipSubnet is the Subnet that is applied to the VIP
VIPSubnet string `yaml:"vipSubnet"`
// VIPCIDR is cidr range for the VIP (primarily needed for BGP)
VIPCIDR string `yaml:"vipCidr"`
// Address is the IP or DNS Name to use as a VirtualIP
Address string `yaml:"address"`
@@ -128,10 +131,25 @@ type Config struct {
CleanRoutingTable bool `yaml:"cleanRoutingTable"`
// BGP Configuration
BGPConfig BGPConfig
BGPPeerConfig BGPPeer
BGPConfig bgp.Config
BGPPeerConfig bgp.Peer
BGPPeers []string
// EnableMetal, will use the metal API to update the EIP <-> VIP (if BGP is enabled then BGP will be used)
EnableMetal bool `yaml:"enableMetal"`
// MetalAPIKey, is the API token used to authenticate to the API
MetalAPIKey string
// MetalProject, is the name of a particular defined project
MetalProject string
// MetalProjectID, is the name of a particular defined project
MetalProjectID string
// ProviderConfig, is the path to a provider configuration file
ProviderConfig string
// LoadBalancers are the various services we can load balance over
LoadBalancers []LoadBalancer `yaml:"loadBalancers,omitempty"`
@@ -161,8 +179,8 @@ type Config struct {
// DisableServiceUpdates, if true, kube-vip will only advertise service, but it will not update service's Status.LoadBalancer.Ingress slice
DisableServiceUpdates bool `yaml:"disableServiceUpdates"`
// EnableEndpoints, if enabled, Endpoints will be used instead of EndpointSlices
EnableEndpoints bool `yaml:"enableEndpoints"`
// EnableEndpointSlices, if enabled, EndpointSlices will be used instead of Endpoints
EnableEndpointSlices bool `yaml:"enableEndpointSlices"`
// MirrorDestInterface is the network interface where all traffics that go through service interface
// will be mirrored to. If ServicesInterface is not set, fall back to Interface.
@@ -180,15 +198,6 @@ type Config struct {
// HealthCheckPort, if non-zero then will enable the healthcheck to return ok on this port
HealthCheckPort int `yaml:"healthCheckPort"`
// EnableUPNP, enables UPNP functions
EnableUPNP bool `yaml:"enableUPNP"`
// EgressClean, enables egress cleaning on Kube-vip's start
EgressClean bool `yaml:"egressClean"`
// ConfigFile defines the path to a JSON/YAML configuration file
ConfigFile string `yaml:"configFile"`
}
// KubernetesLeaderElection defines all of the settings for Kubernetes KubernetesLeaderElection

View File

@@ -1,5 +0,0 @@
package kubevip
const (
LBClassName = "kube-vip.io/kube-vip-class"
)

View File

@@ -1,7 +0,0 @@
package kubevip
const (
// label used on nodes, which announce the LoadBalancer IP
HasIP = "kube-vip.io/has-ip"
HasIPJSONPath = `kube-vip.io~1has-ip`
)

View File

@@ -1,127 +0,0 @@
package lease
import (
"context"
"fmt"
"sync"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
)
// Manager is used to manage leases.
type Manager struct {
leases map[string]*Lease
lock sync.Mutex
}
// NewManager creates new lease manager.
func NewManager() *Manager {
return &Manager{
leases: make(map[string]*Lease),
}
}
// Add adds lease or incerements counter if lease is alreay used.
func (m *Manager) Add(service *v1.Service) (*Lease, bool) {
m.lock.Lock()
defer m.lock.Unlock()
_, id := GetName(service)
if _, exist := m.leases[id]; !exist {
ctx, cancel := context.WithCancel(context.Background())
m.leases[id] = newLease(ctx, cancel)
return m.leases[id], true
}
m.leases[id].increment()
return m.leases[id], false
}
// Delete decrements lease counter and removes the lease if counter equals 0.
func (m *Manager) Delete(service *v1.Service) {
m.lock.Lock()
defer m.lock.Unlock()
_, id := GetName(service)
if _, exist := m.leases[id]; exist {
m.leases[id].decrement()
if m.leases[id].cnt < 1 {
delete(m.leases, id)
}
}
}
// Get returns lease for the service.
func (m *Manager) Get(service *v1.Service) *Lease {
m.lock.Lock()
defer m.lock.Unlock()
_, id := GetName(service)
if lease, exist := m.leases[id]; exist {
return lease
}
return nil
}
// GetLeaderContext returns leder context for the service.
func (m *Manager) GetLeaderContext(service *v1.Service) context.Context {
m.lock.Lock()
defer m.lock.Unlock()
_, id := GetName(service)
if _, ok := m.leases[id]; !ok {
return nil
}
return m.leases[id].Ctx
}
// Lease holds lease data.
type Lease struct {
cnt uint
Lock *sync.Mutex
Ctx context.Context
Cancel context.CancelFunc
Started chan any
}
func newLease(ctx context.Context, cancel context.CancelFunc) *Lease {
return &Lease{
Ctx: ctx,
Cancel: cancel,
cnt: 1,
Lock: new(sync.Mutex),
Started: make(chan any),
}
}
func (l *Lease) increment() {
l.Lock.Lock()
defer l.Lock.Unlock()
l.cnt++
}
func (l *Lease) decrement() {
l.Lock.Lock()
defer l.Lock.Unlock()
if l.cnt == 0 {
return
}
l.cnt--
if l.cnt < 1 {
l.Cancel()
}
}
// GetName gets lease name and id for the service.
func GetName(service *v1.Service) (string, string) {
serviceLease, exists := service.Annotations[kubevip.ServiceLease]
if !exists || serviceLease == "" {
serviceLease = fmt.Sprintf("kubevip-%s", service.Name)
}
serviceLeaseID := fmt.Sprintf("%s/%s", serviceLease, service.Namespace)
return serviceLease, serviceLeaseID
}
// UsesCommon checks if service uses common lease feature.
func UsesCommon(service *v1.Service) bool {
_, common := service.Annotations[kubevip.ServiceLease]
return common
}

View File

@@ -16,7 +16,7 @@ import (
"github.com/cloudflare/ipvs/netmask"
"github.com/kube-vip/kube-vip/pkg/backend"
"github.com/kube-vip/kube-vip/pkg/sysctl"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
)
@@ -358,7 +358,7 @@ func (lb *IPVSLoadBalancer) isLocal(address string) (bool, error) {
}
family := netlink.FAMILY_V6
if utils.IsIPv4(address) {
if vip.IsIPv4(address) {
family = netlink.FAMILY_V4
}

View File

@@ -6,7 +6,6 @@ import (
"testing"
"github.com/cloudflare/ipvs"
"github.com/kube-vip/kube-vip/pkg/utils"
)
func Test_ipAndFamily(t *testing.T) {
@@ -20,7 +19,7 @@ func Test_ipAndFamily(t *testing.T) {
want1 ipvs.AddressFamily
}{
{
name: utils.IPv4Family,
name: "IPv4",
args: args{
address: "192.168.0.20",
},
@@ -28,7 +27,7 @@ func Test_ipAndFamily(t *testing.T) {
want1: ipvs.INET,
},
{
name: utils.IPv6Family,
name: "IPv6",
args: args{
address: "ff02::3",
},

View File

@@ -0,0 +1,44 @@
package manager
const (
// Hardware address of the host that has the VIP
hwAddrKey = "kube-vip.io/hwaddr"
// The IP address that is requested
requestedIP = "kube-vip.io/requestedIP"
// The host that has the VIP
vipHost = "kube-vip.io/vipHost"
// Enable egress on a service
egress = "kube-vip.io/egress"
// Egress should be IPv6
egressIPv6 = "kube-vip.io/egress-ipv6"
// Ports that traffic is allowed to access from the egress VIP
egressDestinationPorts = "kube-vip.io/egress-destination-ports"
// Allowed incoming ports to the VIP
egressSourcePorts = "kube-vip.io/egress-source-ports"
// Allowed networks for the Egress to be enabled for
egressAllowedNetworks = "kube-vip.io/egress-allowed-networks"
// Networks that we wont Egress for
egressDeniedNetworks = "kube-vip.io/egress-denied-networks"
// The current active endpoint(pod) for the Egress VIP
activeEndpoint = "kube-vip.io/active-endpoint"
// The current active endpoint(pod) for the Egress VIP (v6)
activeEndpointIPv6 = "kube-vip.io/active-endpoint-ipv6"
// Flush the conntrack rules (remove existing sessions) once Egress is configured
flushContrack = "kube-vip.io/flush-conntrack"
loadbalancerIPAnnotation = "kube-vip.io/loadbalancerIPs"
loadbalancerHostname = "kube-vip.io/loadbalancerHostname"
serviceInterface = "kube-vip.io/serviceInterface"
upnpEnabled = "kube-vip.io/forwardUPNP"
)

View File

@@ -1,48 +1,42 @@
package instance
package manager
import (
"fmt"
"net"
"strconv"
"strings"
"time"
"log/slog"
log "log/slog"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/sysctl"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
// Instance defines an instance of everything needed to manage vips
type Instance struct {
// Virtual IP / Load Balancer configuration
VIPConfigs []*kubevip.Config
vipConfigs []*kubevip.Config
// cluster instances
Clusters []*cluster.Cluster
clusters []*cluster.Cluster
// Service uses DHCP
IsDHCP bool
DHCPInterface string
DHCPInterfaceHwaddr string
DHCPInterfaceIP string
DHCPHostname string
DHCPClient *vip.DHCPClient
isDHCP bool
dhcpInterface string
dhcpInterfaceHwaddr string
dhcpInterfaceIP string
dhcpHostname string
dhcpClient *vip.DHCPClient
HasEndpoints bool
// External Gateway IP the service is forwarded from
UPNPGatewayIPs []string
upnpGatewayIPs []string
// Kubernetes service mapping
ServiceSnapshot *v1.Service
serviceSnapshot *v1.Service
}
type Port struct {
@@ -50,8 +44,9 @@ type Port struct {
Type string
}
func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterface.Manager, arpMgr *arp.Manager) (*Instance, error) {
instanceAddresses, _ := FetchServiceAddresses(svc)
func NewInstance(svc *v1.Service, config *kubevip.Config) (*Instance, error) {
instanceAddresses := fetchServiceAddresses(svc)
//instanceUID := string(svc.UID)
var newVips []*kubevip.Config
var link netlink.Link
@@ -60,7 +55,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
for _, address := range instanceAddresses {
// Detect if we're using a specific interface for services
var svcInterface string
svcInterface = svc.Annotations[kubevip.ServiceInterface] // If the service has a specific interface defined, then use it
svcInterface = svc.Annotations[serviceInterface] // If the service has a specific interface defined, then use it
if svcInterface == kubevip.Auto {
link, err = autoFindInterface(address)
if err != nil {
@@ -103,7 +98,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
}
}
cidrs := vip.Split(config.VIPSubnet)
cidrs := vip.Split(config.VIPCIDR)
ipv4AutoSubnet := false
ipv6AutoSubnet := false
@@ -121,14 +116,14 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
subnet := ""
var err error
if utils.IsIPv4(address) {
if vip.IsIPv4(address) {
if ipv4AutoSubnet {
subnet, err = autoFindSubnet(link, address)
if err != nil {
return nil, fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
}
} else {
if cidrs[0] != "" && cidrs[0] != kubevip.Auto {
if cidrs[0] != "" {
subnet = cidrs[0]
} else {
subnet = "32"
@@ -141,7 +136,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
return nil, fmt.Errorf("failed to automatically find subnet for service %s/%s with IP address %s on interface %s: %w", svc.Namespace, svc.Name, address, svcInterface, err)
}
} else {
if len(cidrs) > 1 && cidrs[1] != "" && cidrs[1] != kubevip.Auto {
if len(cidrs) > 1 {
subnet = cidrs[1]
} else {
subnet = "128"
@@ -149,24 +144,26 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
}
}
//log.Info("new instance", "svc", *svc, "interface", svcInterface)
// Generate new Virtual IP configuration
newVips = append(newVips, &kubevip.Config{
VIP: address,
Interface: svcInterface,
SingleNode: true,
EnableARP: config.EnableARP,
EnableBGP: config.EnableBGP,
VIPSubnet: subnet,
EnableRoutingTable: config.EnableRoutingTable,
RoutingTableID: config.RoutingTableID,
RoutingTableType: config.RoutingTableType,
RoutingProtocol: config.RoutingProtocol,
ArpBroadcastRate: config.ArpBroadcastRate,
EnableServiceSecurity: config.EnableServiceSecurity,
DNSMode: config.DNSMode,
DisableServiceUpdates: config.DisableServiceUpdates,
EnableServicesElection: config.EnableServicesElection,
PreserveVIPOnLeadershipLoss: config.PreserveVIPOnLeadershipLoss,
VIP: address,
Interface: svcInterface,
SingleNode: true,
EnableARP: config.EnableARP,
EnableBGP: config.EnableBGP,
VIPCIDR: subnet,
VIPSubnet: config.VIPSubnet,
EnableRoutingTable: config.EnableRoutingTable,
RoutingTableID: config.RoutingTableID,
RoutingTableType: config.RoutingTableType,
RoutingProtocol: config.RoutingProtocol,
ArpBroadcastRate: config.ArpBroadcastRate,
EnableServiceSecurity: config.EnableServiceSecurity,
DNSMode: config.DNSMode,
DisableServiceUpdates: config.DisableServiceUpdates,
EnableServicesElection: config.EnableServicesElection,
KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
EnableLeaderElection: config.EnableLeaderElection,
},
@@ -177,7 +174,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
instance := &Instance{
//UID: instanceUID,
//VIPs: instanceAddresses,
ServiceSnapshot: svc,
serviceSnapshot: svc,
}
// for _, port := range svc.Spec.Ports {
// instance.ExternalPorts = append(instance.ExternalPorts, Port{
@@ -187,9 +184,9 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
// }
if svc.Annotations != nil {
instance.DHCPInterfaceHwaddr = svc.Annotations[kubevip.HwAddrKey]
instance.DHCPInterfaceIP = svc.Annotations[kubevip.RequestedIP]
instance.DHCPHostname = svc.Annotations[kubevip.LoadbalancerHostname]
instance.dhcpInterfaceHwaddr = svc.Annotations[hwAddrKey]
instance.dhcpInterfaceIP = svc.Annotations[requestedIP]
instance.dhcpHostname = svc.Annotations[loadbalancerHostname]
}
configPorts := make([]kubevip.Port, 0)
@@ -210,7 +207,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
vip.LoadBalancers = append(vip.LoadBalancers, newLB)
}
// Create Add configuration to the new service
instance.VIPConfigs = newVips
instance.vipConfigs = newVips
// If this was purposely created with the address 0.0.0.0,
// we will create a macvlan on the main interface and a DHCP client
@@ -221,18 +218,18 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
return nil, err
}
select {
case err := <-instance.DHCPClient.ErrorChannel():
case err := <-instance.dhcpClient.ErrorChannel():
return nil, fmt.Errorf("error starting DHCP for %s/%s: error: %s",
instance.ServiceSnapshot.Namespace, instance.ServiceSnapshot.Name, err)
case ip := <-instance.DHCPClient.IPChannel():
instance.VIPConfigs[0].Interface = instance.DHCPInterface
instance.VIPConfigs[0].VIP = ip
instance.DHCPInterfaceIP = ip
instance.serviceSnapshot.Namespace, instance.serviceSnapshot.Name, err)
case ip := <-instance.dhcpClient.IPChannel():
instance.vipConfigs[0].Interface = instance.dhcpInterface
instance.vipConfigs[0].VIP = ip
instance.dhcpInterfaceIP = ip
}
}
for _, vipConfig := range instance.VIPConfigs {
c, err := cluster.InitCluster(vipConfig, false, intfMgr, arpMgr)
for _, vipConfig := range instance.vipConfigs {
c, err := cluster.InitCluster(vipConfig, false)
if err != nil {
log.Error("Failed to add Service %s/%s", svc.Namespace, svc.Name)
return nil, err
@@ -242,8 +239,9 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
c.Network[i].SetServicePorts(svc)
}
instance.Clusters = append(instance.Clusters, c)
instance.clusters = append(instance.clusters, c)
log.Info("(svcs) adding VIP", "ip", vipConfig.VIP, "interface", vipConfig.Interface, "namespace", svc.Namespace, "name", svc.Name)
}
return instance, nil
@@ -307,24 +305,24 @@ func getAutoInterfaceName(link netlink.Link, defaultInterface string) string {
}
func (i *Instance) startDHCP() error {
if len(i.VIPConfigs) != 1 {
return fmt.Errorf("DHCP requires exactly 1 VIP config, got: %v", len(i.VIPConfigs))
if len(i.vipConfigs) != 1 {
return fmt.Errorf("DHCP requires exactly 1 VIP config, got: %v", len(i.vipConfigs))
}
parent, err := netlink.LinkByName(i.VIPConfigs[0].Interface)
parent, err := netlink.LinkByName(i.vipConfigs[0].Interface)
if err != nil {
return fmt.Errorf("error finding VIP Interface, for building DHCP Link : %v", err)
}
// Generate name from UID
interfaceName := fmt.Sprintf("vip-%s", i.ServiceSnapshot.UID[0:8])
interfaceName := fmt.Sprintf("vip-%s", i.serviceSnapshot.UID[0:8])
// Check if the interface doesn't exist first
iface, err := net.InterfaceByName(interfaceName)
if err != nil {
log.Info("creating new macvlan interface for DHCP", "interface", interfaceName)
hwaddr, err := net.ParseMAC(i.DHCPInterfaceHwaddr)
if i.DHCPInterfaceHwaddr != "" && err != nil {
hwaddr, err := net.ParseMAC(i.dhcpInterfaceHwaddr)
if i.dhcpInterfaceHwaddr != "" && err != nil {
return err
} else if hwaddr == nil {
hwaddr, err = net.ParseMAC(vip.GenerateMac())
@@ -361,156 +359,28 @@ func (i *Instance) startDHCP() error {
log.Info("Using existing macvlan interface for DHCP", "interface", interfaceName)
}
// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
rpfilterSetting := "0"
// Check if we need to set an override rp_filter value for the interface
if i.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
// Check the rp_filter value
rpFilter, err := strconv.Atoi(i.ServiceSnapshot.Annotations[kubevip.RPFilter])
if err != nil {
slog.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
} else {
if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
rpfilterSetting = i.ServiceSnapshot.Annotations[kubevip.RPFilter]
} else {
slog.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
}
}
}
err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
if err != nil {
slog.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
}
var initRebootFlag bool
if i.DHCPInterfaceIP != "" {
if i.dhcpInterfaceIP != "" {
initRebootFlag = true
}
client := vip.NewDHCPClient(iface, initRebootFlag, i.DHCPInterfaceIP)
client := vip.NewDHCPClient(iface, initRebootFlag, i.dhcpInterfaceIP)
// Add hostname to dhcp client if annotated
if i.DHCPHostname != "" {
log.Info("Hostname specified for dhcp lease", "interface", interfaceName, "hostname", i.DHCPHostname)
client.WithHostName(i.DHCPHostname)
if i.dhcpHostname != "" {
log.Info("Hostname specified for dhcp lease", "interface", interfaceName, "hostname", i.dhcpHostname)
client.WithHostName(i.dhcpHostname)
}
go client.Start()
// Set that DHCP is enabled
i.IsDHCP = true
i.isDHCP = true
// Set the name of the interface so that it can be removed on Service deletion
i.DHCPInterface = interfaceName
i.DHCPInterfaceHwaddr = iface.HardwareAddr.String()
i.dhcpInterface = interfaceName
i.dhcpInterfaceHwaddr = iface.HardwareAddr.String()
// Add the client so that we can call it to stop function
i.DHCPClient = client
i.dhcpClient = client
return nil
}
// FetchLoadBalancerIngressAddresses tries to get the addresses from status.loadBalancerIP
func FetchLoadBalancerIngress(s *v1.Service) ([]string, []string) {
// If the service has no status, return empty
lbStatusAddresses := []string{}
lbStatusHostnames := []string{}
if len(s.Status.LoadBalancer.Ingress) == 0 {
return lbStatusAddresses, lbStatusHostnames
}
for _, ingress := range s.Status.LoadBalancer.Ingress {
if ingress.IP != "" {
lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
}
if ingress.Hostname != "" {
lbStatusHostnames = append(lbStatusHostnames, ingress.Hostname)
}
}
return lbStatusAddresses, lbStatusHostnames
}
// FetchServiceAddresses tries to get the addresses from annotations
// kube-vip.io/loadbalancerIPs, then from spec.loadbalancerIP
func FetchServiceAddresses(s *v1.Service) ([]string, []string) {
annotationAvailable := false
if s.Annotations != nil {
if v, annotationAvailable := s.Annotations[kubevip.LoadbalancerIPAnnotation]; annotationAvailable {
ips := strings.Split(v, ",")
var trimmedIPs []string
var trimmedHostnames []string
for _, a := range ips {
a = strings.TrimSpace(a)
ip := net.ParseIP(a)
if ip == nil {
// this is probably a DNS name
trimmedHostnames = append(trimmedHostnames, a)
} else {
trimmedIPs = append(trimmedIPs, ip.String())
}
}
return trimmedIPs, trimmedHostnames
}
}
lbStatusAddresses := []string{}
lbStatusHostnames := []string{}
if !annotationAvailable {
lbStatusAddresses, lbStatusHostnames = FetchLoadBalancerIngress(s)
}
// Spec.LoadBalancerIP legacy handling
// if the loadBalancerIP is different from Status.LoadBalancer.Ingress IPs
// return the legacy LB as spec wins over status.
if lbIP := net.ParseIP(s.Spec.LoadBalancerIP); lbIP != nil && len(lbStatusAddresses) > 0 {
isLbIPv4 := utils.IsIPv4(s.Spec.LoadBalancerIP)
for _, a := range lbStatusAddresses {
if lbStatusIP := net.ParseIP(a); lbStatusIP != nil && utils.IsIPv4(a) == isLbIPv4 && !lbIP.Equal(lbStatusIP) {
return []string{s.Spec.LoadBalancerIP}, []string{}
}
}
}
if len(lbStatusAddresses) > 0 || len(lbStatusHostnames) > 0 {
return lbStatusAddresses, lbStatusHostnames
}
if s.Spec.LoadBalancerIP != "" {
return []string{s.Spec.LoadBalancerIP}, []string{}
}
return []string{}, []string{}
}
func FindServiceInstance(svc *v1.Service, instances []*Instance) *Instance {
log.Debug("finding service", "namespace", svc.Namespace, "name", svc.Name, "UID", svc.UID)
for i := range instances {
log.Debug("saved service", "instance", i, "UID", instances[i].ServiceSnapshot.UID)
if instances[i].ServiceSnapshot.UID == svc.UID {
return instances[i]
}
}
log.Debug("instance not found", "namespace", svc.Namespace, "name", svc.Name, "UID", svc.UID)
return nil
}
func FindServiceInstanceWithTimeout(svc *v1.Service, instances []*Instance) *Instance {
log.Debug("finding service with timeout", "namespace", svc.Namespace, "name", svc.Name, "UID", svc.UID)
ticker := time.NewTicker(time.Millisecond * 200)
defer ticker.Stop()
to := time.NewTimer(time.Second * 60)
defer to.Stop()
for {
select {
case <-to.C:
return nil
case <-ticker.C:
for i := range instances {
log.Debug("saved service", "instance", i, "UID", instances[i].ServiceSnapshot.UID)
if instances[i].ServiceSnapshot.UID == svc.UID {
return instances[i]
}
}
}
}
}

View File

@@ -7,6 +7,7 @@ import (
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
@@ -14,16 +15,14 @@ import (
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/node"
"github.com/kube-vip/kube-vip/pkg/services"
"github.com/kube-vip/kube-vip/pkg/trafficmirror"
"github.com/kube-vip/kube-vip/pkg/upnp"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/prometheus/client_golang/prometheus"
v1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
@@ -40,6 +39,11 @@ type Manager struct {
// Manager services
// service bool
// Keeps track of all running instances
serviceInstances []*Instance
// UPNP functionality
upnp bool
// BGP Manager, this is a singleton that manages all BGP advertisements
bgpServer *bgp.Server
@@ -49,8 +53,6 @@ type Manager struct {
// This channel is used to signal a shutdown
shutdownChan chan struct{}
svcProcessor *services.Processor
// This is a prometheus counter used to count the number of events received
// from the service watcher
countServiceWatchEvent *prometheus.CounterVec
@@ -61,17 +63,6 @@ type Manager struct {
// This mutex is to protect calls from various goroutines
mutex sync.Mutex
// This tracks used network interfaces and guards them with mutex for concurrent changes.
intfMgr *networkinterface.Manager
// This tracks VIPs and performs ARP/NDP advertisement.
arpMgr *arp.Manager
// This tracks node labels and performs label management
// implementation will be decided in constructor
// based on config.EnableNodeLabeling
nodeLabelManager node.LabelManager
}
// New will create a new managing object
@@ -177,36 +168,6 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
// }
// }
// listen for interrupts or the Linux SIGTERM signal and cancel
// our context, which the leader election code will observe and
// step down
signalChan := make(chan os.Signal, 1)
// Add Notification for Userland interrupt
signal.Notify(signalChan, syscall.SIGINT)
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(signalChan, syscall.SIGTERM)
// All watchers and other goroutines should have an additional goroutine that blocks on this, to shut things down
shutdownChan := make(chan struct{})
intfMgr := networkinterface.NewManager()
arpMgr := arp.NewManager(config)
// create the node label manager
// constructor will decide if it should be a noop or not
nodeLabelManager := node.NewManager(config, clientset)
var bgpServer *bgp.Server
if config.EnableBGP {
bgpServer, err = bgp.NewBGPServer(config.BGPConfig)
if err != nil {
return nil, fmt.Errorf("creating BGP server: %w", err)
}
}
svcProcessor := services.NewServicesProcessor(config, bgpServer, clientset, rwClientSet, shutdownChan, intfMgr, arpMgr, nodeLabelManager)
return &Manager{
clientSet: clientset,
rwClientSet: rwClientSet,
@@ -224,13 +185,6 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
Name: "bgp_session_info",
Help: "Display state of session by setting metric for label value with current state to 1",
}, []string{"state", "peer"}),
signalChan: signalChan,
shutdownChan: shutdownChan,
svcProcessor: svcProcessor,
intfMgr: intfMgr,
arpMgr: arpMgr,
bgpServer: bgpServer,
nodeLabelManager: nodeLabelManager,
}, nil
}
@@ -246,9 +200,6 @@ func (sm *Manager) Start() error {
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(sm.signalChan, syscall.SIGTERM)
// Add Notification for SIGUSR1 (for configuration dump)
signal.Notify(sm.signalChan, syscall.SIGUSR1)
// All watchers and other goroutines should have an additional goroutine that blocks on this, to shut things down
sm.shutdownChan = make(chan struct{})
@@ -272,13 +223,6 @@ func (sm *Manager) Start() error {
}()
}
// on exit, clean up the node labels
defer func() {
if err := sm.nodeLabelManager.CleanUpLabels(10 * time.Second); err != nil {
log.Error("CleanUpNodeLabels", "unable to cleanup node labels", err)
}
}()
// If BGP is enabled then we start a server instance that will broadcast VIPs
if sm.config.EnableBGP {
@@ -293,12 +237,16 @@ func (sm *Manager) Start() error {
}
if sm.config.EnableARP || sm.config.EnableWireguard {
if sm.config.EnableUPNP {
// Before starting the leader Election enable any additional functionality
upnpEnabled, _ := strconv.ParseBool(os.Getenv("enableUPNP"))
if upnpEnabled {
sm.upnp = true
clients := upnp.GetConnectionClients(context.TODO())
if len(clients) == 0 {
log.Error("Error Enabling UPNP. No Clients found")
// Set the struct to false so nothing should use it in future
sm.config.EnableUPNP = false
sm.upnp = false
} else {
for _, c := range clients {
ip, err := c.GetExternalIPAddress()
@@ -306,11 +254,12 @@ func (sm *Manager) Start() error {
log.Error("unable to find IGD2 Gateway address", "err", err)
}
log.Info("Found UPNP IGD2 Gateway address", "ip", ip)
}
}
// TODO: It would be nice to run the UPNP refresh only on the leader.
go sm.svcProcessor.RefreshUPNPForwards()
}
// TODO: It would be nice to run the UPNP refresh only on the leader.
go sm.refreshUPNPForwards()
}
// If ARP is enabled then we start a LeaderElection that will use ARP to advertise VIPs
@@ -355,3 +304,89 @@ func (sm *Manager) parseAnnotations() error {
}
return nil
}
func (sm *Manager) serviceInterface() string {
svcIf := sm.config.Interface
if sm.config.ServicesInterface != "" {
svcIf = sm.config.ServicesInterface
}
return svcIf
}
func (sm *Manager) startTrafficMirroringIfEnabled() error {
if sm.config.MirrorDestInterface != "" {
svcIf := sm.serviceInterface()
log.Info("mirroring traffic", "src", svcIf, "dest", sm.config.MirrorDestInterface)
if err := trafficmirror.MirrorTrafficFromNIC(svcIf, sm.config.MirrorDestInterface); err != nil {
return err
}
} else {
log.Debug("skip starting traffic mirroring since it's not enabled.")
}
return nil
}
func (sm *Manager) stopTrafficMirroringIfEnabled() error {
if sm.config.MirrorDestInterface != "" {
svcIf := sm.serviceInterface()
log.Info("clean up qdisc config", "interface", svcIf)
if err := trafficmirror.CleanupQDSICFromNIC(svcIf); err != nil {
return err
}
} else {
log.Debug("skip stopping traffic mirroring since it's not enabled.")
}
return nil
}
func (sm *Manager) findServiceInstance(svc *v1.Service) *Instance {
log.Debug("finding service", "UID", svc.UID)
for i := range sm.serviceInstances {
log.Debug("saved service", "instance", i, "UID", sm.serviceInstances[i].serviceSnapshot.UID)
if sm.serviceInstances[i].serviceSnapshot.UID == svc.UID {
return sm.serviceInstances[i]
}
}
return nil
}
func (sm *Manager) findServiceInstanceWithTimeout(ctx context.Context, svc *v1.Service) (*Instance, error) {
log.Debug("finding service with timeout", "UID", svc.UID)
ctxTimeout, ctxTimeoutCancel := context.WithTimeout(ctx, time.Minute)
defer ctxTimeoutCancel()
for {
for i := range sm.serviceInstances {
log.Debug("saved service", "instance", i, "UID", sm.serviceInstances[i].serviceSnapshot.UID)
if sm.serviceInstances[i].serviceSnapshot.UID == svc.UID {
return sm.serviceInstances[i], nil
}
}
t := time.NewTimer(time.Millisecond * 100)
select {
case <-ctxTimeout.Done():
t.Stop()
return nil, fmt.Errorf("failed to wait for the service instance: %w", ctx.Err())
case <-t.C:
log.Debug("waiting for the service to be discovered", "UID", svc.UID)
}
}
}
// Refresh UPNP Port Forwards for all Service Instances registered in the SM
func (sm *Manager) refreshUPNPForwards() {
log.Info("Starting UPNP Port Refresher")
for {
time.Sleep(300 * time.Second)
log.Info("[UPNP] Refreshing Instances", "number of instances", len(sm.serviceInstances))
for i := range sm.serviceInstances {
sm.upnpMap(context.TODO(), sm.serviceInstances[i])
if err := sm.updateStatus(sm.serviceInstances[i]); err != nil {
log.Warn("[UPNP] Error updating service", "ip", sm.serviceInstances[i].serviceSnapshot.Name, "err", err)
}
}
}
}

View File

@@ -2,6 +2,7 @@ package manager
import (
"context"
"os"
"syscall"
"time"
@@ -27,33 +28,21 @@ func (sm *Manager) startARP(id string) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
log.Info("Start ARP/NDP advertisement")
go sm.arpMgr.StartAdvertisement(ctx)
// Shutdown function that will wait on this signal, unless we call it ourselves
go func() {
for {
sig := <-sm.signalChan
switch sig {
case syscall.SIGUSR1:
log.Info("Received SIGUSR1, dumping configuration")
sm.dumpConfiguration()
case syscall.SIGINT, syscall.SIGTERM:
log.Info("Received kube-vip termination, signaling shutdown")
if sm.config.EnableControlPlane {
cpCluster.Stop()
}
// Close all go routines
close(sm.shutdownChan)
// Cancel the context, which will in turn cancel the leadership
cancel()
return
}
<-sm.signalChan
log.Info("Received kube-vip termination, signaling shutdown")
if sm.config.EnableControlPlane {
cpCluster.Stop()
}
// Close all go routines
close(sm.shutdownChan)
// Cancel the context, which will in turn cancel the leadership
cancel()
}()
if sm.config.EnableControlPlane {
cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
cpCluster, err = cluster.InitCluster(sm.config, false)
if err != nil {
return err
}
@@ -91,7 +80,7 @@ func (sm *Manager) startARP(id string) error {
}
// This will tidy any dangling kube-vip iptables rules
if sm.config.EgressClean {
if os.Getenv("EGRESS_CLEAN") != "" {
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
}
@@ -99,7 +88,7 @@ func (sm *Manager) startARP(id string) error {
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
err = sm.startServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
@@ -134,7 +123,7 @@ func (sm *Manager) startARP(id string) error {
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
err = sm.servicesWatcher(ctx, sm.syncServices)
if err != nil {
log.Error("service watcher", "err", err)
panic("") // TODO: - emulating log.fatal here
@@ -142,10 +131,12 @@ func (sm *Manager) startARP(id string) error {
},
OnStoppedLeading: func() {
// we can do cleanup here
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "new leader", id)
sm.svcProcessor.Stop()
for _, instance := range sm.serviceInstances {
for _, cluster := range instance.clusters {
cluster.Stop()
}
}
log.Error("lost leadership, restarting kube-vip")
panic("") // TODO: - emulating log.fatal here

View File

@@ -3,13 +3,16 @@ package manager
import (
"context"
"fmt"
"os"
"syscall"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/equinixmetal"
api "github.com/osrg/gobgp/v3/api"
"github.com/packethost/packngo"
"github.com/prometheus/client_golang/prometheus"
)
@@ -19,15 +22,37 @@ func (sm *Manager) startBGP() error {
// var ns string
var err error
if sm.bgpServer == nil {
sm.bgpServer, err = bgp.NewBGPServer(sm.config.BGPConfig)
// If Equinix Metal is enabled then we can begin our preparation work
var packetClient *packngo.Client
if sm.config.EnableMetal {
if sm.config.ProviderConfig != "" {
key, project, err := equinixmetal.GetPacketConfig(sm.config.ProviderConfig)
if err != nil {
return err
}
// Set the environment variable with the key for the project
os.Setenv("PACKET_AUTH_TOKEN", key)
// Update the configuration with the project key
sm.config.MetalProjectID = project
}
packetClient, err = packngo.NewClient()
if err != nil {
return fmt.Errorf("creating BGP server: %w", err)
return err
}
// We're using Equinix Metal with BGP, populate the Peer information from the API
if sm.config.EnableBGP {
log.Info("Looking up the BGP configuration from Equinix Metal")
err = equinixmetal.BGPLookup(packetClient, sm.config)
if err != nil {
return err
}
}
}
log.Info("Starting the BGP server to advertise VIP routes to BGP peers")
if err := sm.bgpServer.Start(func(p *api.WatchEventResponse_PeerEvent) {
sm.bgpServer, err = bgp.NewBGPServer(&sm.config.BGPConfig, func(p *api.WatchEventResponse_PeerEvent) {
ipaddr := p.GetPeer().GetState().GetNeighborAddress()
port := uint64(179)
peerDescription := fmt.Sprintf("%s:%d", ipaddr, port)
@@ -43,8 +68,9 @@ func (sm *Manager) startBGP() error {
"peer": peerDescription,
}).Set(metricValue)
}
}); err != nil {
return fmt.Errorf("starting BGP server: %w", err)
})
if err != nil {
return err
}
// use a Go context so we can tell the leaderelection code when we
@@ -61,28 +87,19 @@ func (sm *Manager) startBGP() error {
// Shutdown function that will wait on this signal, unless we call it ourselves
go func() {
for {
sig := <-sm.signalChan
switch sig {
case syscall.SIGUSR1:
log.Info("Received SIGUSR1, dumping configuration")
sm.dumpConfiguration()
case syscall.SIGINT, syscall.SIGTERM:
log.Info("Received termination, signaling shutdown")
if sm.config.EnableControlPlane {
if cpCluster != nil {
cpCluster.Stop()
}
}
// Cancel the context, which will in turn cancel the leadership
cancel()
return
<-sm.signalChan
log.Info("Received termination, signaling shutdown")
if sm.config.EnableControlPlane {
if cpCluster != nil {
cpCluster.Stop()
}
}
// Cancel the context, which will in turn cancel the leadership
cancel()
}()
if sm.config.EnableControlPlane {
cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
cpCluster, err = cluster.InitCluster(sm.config, false)
if err != nil {
return err
}
@@ -96,7 +113,7 @@ func (sm *Manager) startBGP() error {
if sm.config.EnableLeaderElection {
err = cpCluster.StartCluster(sm.config, clusterManager, sm.bgpServer)
} else {
err = cpCluster.StartVipService(sm.config, clusterManager, sm.bgpServer)
err = cpCluster.StartVipService(sm.config, clusterManager, sm.bgpServer, packetClient)
}
if err != nil {
log.Error("Control Plane", "err", err)
@@ -114,18 +131,9 @@ func (sm *Manager) startBGP() error {
}
}
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
} else {
log.Info("beginning watching services without leader election")
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
return err
}
err = sm.servicesWatcher(ctx, sm.syncServices)
if err != nil {
return err
}
log.Info("Shutting down Kube-Vip")

View File

@@ -1,235 +0,0 @@
package manager
import (
"context"
"fmt"
"os"
"time"
"github.com/kube-vip/kube-vip/pkg/nftables"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// dumpConfiguration prints the current configuration to stdout when SIGUSR1 is received
func (sm *Manager) dumpConfiguration() {
sm.mutex.Lock()
defer sm.mutex.Unlock()
fmt.Printf("\n")
fmt.Printf("================================================================================\n")
fmt.Printf(" KUBE-VIP CONFIGURATION DUMP\n")
fmt.Printf("================================================================================\n")
fmt.Printf("Timestamp: %s\n", time.Now().Format(time.RFC3339))
fmt.Printf("Node Name: %s\n", sm.config.NodeName)
fmt.Printf("Process ID: %d\n", os.Getpid())
fmt.Printf("================================================================================\n")
fmt.Printf("\n")
sm.dumpConfigSection()
sm.dumpBGPSection()
sm.dumpARPSection()
sm.dumpServicesSection()
sm.dumpNetworkInterfacesSection()
sm.dumpLeaderElectionSection()
sm.dumpRuntimeSection()
sm.dumpNFTablesSection()
fmt.Printf("================================================================================\n")
fmt.Printf(" END OF CONFIGURATION DUMP\n")
fmt.Printf("================================================================================\n")
fmt.Printf("\n")
}
func (sm *Manager) dumpConfigSection() {
fmt.Printf("--- BASIC CONFIGURATION ---\n")
fmt.Printf("VIP: %s\n", sm.config.Address)
fmt.Printf("VIP Subnet: %s\n", sm.config.VIPSubnet)
fmt.Printf("Port: %d\n", sm.config.Port)
fmt.Printf("Namespace: %s\n", sm.config.Namespace)
fmt.Printf("Service Namespace: %s\n", sm.config.ServiceNamespace)
fmt.Printf("Interface: %s\n", sm.config.Interface)
fmt.Printf("Services Interface: %s\n", sm.config.ServicesInterface)
fmt.Printf("Single Node Mode: %t\n", sm.config.SingleNode)
fmt.Printf("Start As Leader: %t\n", sm.config.StartAsLeader)
fmt.Printf("\n")
}
func (sm *Manager) dumpBGPSection() {
fmt.Printf("--- BGP CONFIGURATION ---\n")
fmt.Printf("BGP Enabled: %t\n", sm.config.EnableBGP)
if sm.config.EnableBGP {
fmt.Printf("BGP AS: %d\n", sm.config.BGPConfig.AS)
fmt.Printf("BGP Router ID: %s\n", sm.config.BGPConfig.RouterID)
fmt.Printf("BGP Source IP: %s\n", sm.config.BGPConfig.SourceIP)
fmt.Printf("BGP Source Interface: %s\n", sm.config.BGPConfig.SourceIF)
fmt.Printf("BGP Hold Time: %d\n", sm.config.BGPConfig.HoldTime)
fmt.Printf("BGP Keepalive Interval: %d\n", sm.config.BGPConfig.KeepaliveInterval)
fmt.Printf("BGP Peers: %d\n", len(sm.config.BGPConfig.Peers))
for i, peer := range sm.config.BGPConfig.Peers {
fmt.Printf(" Peer %d: %s:%d (AS: %d, MultiHop: %t)\n",
i+1, peer.Address, peer.Port, peer.AS, peer.MultiHop)
}
}
fmt.Printf("\n")
}
func (sm *Manager) dumpARPSection() {
fmt.Printf("--- ARP/NDP CONFIGURATION ---\n")
fmt.Printf("ARP Enabled: %t\n", sm.config.EnableARP)
if sm.config.EnableARP {
fmt.Printf("ARP Broadcast Rate: %d\n", sm.config.ArpBroadcastRate)
}
fmt.Printf("Wireguard Enabled: %t\n", sm.config.EnableWireguard)
fmt.Printf("Routing Table Enabled: %t\n", sm.config.EnableRoutingTable)
if sm.config.EnableRoutingTable {
fmt.Printf("Routing Table ID: %d\n", sm.config.RoutingTableID)
fmt.Printf("Routing Protocol: %d\n", sm.config.RoutingProtocol)
fmt.Printf("Clean Routing Table: %t\n", sm.config.CleanRoutingTable)
}
fmt.Printf("\n")
}
func (sm *Manager) dumpServicesSection() {
fmt.Printf("--- SERVICES CONFIGURATION ---\n")
fmt.Printf("Services Enabled: %t\n", sm.config.EnableServices)
if sm.config.EnableServices {
fmt.Printf("Services Election: %t\n", sm.config.EnableServicesElection)
fmt.Printf("Load Balancer Class Only: %t\n", sm.config.LoadBalancerClassOnly)
fmt.Printf("Load Balancer Class Name: %s\n", sm.config.LoadBalancerClassName)
fmt.Printf("Disable Service Updates: %t\n", sm.config.DisableServiceUpdates)
fmt.Printf("Enable Endpoints: %t\n", sm.config.EnableEndpoints)
fmt.Printf("Service Security Enabled: %t\n", sm.config.EnableServiceSecurity)
if sm.svcProcessor != nil {
instances := sm.svcProcessor.ServiceInstances
fmt.Printf("Kube-vip Active Service Instances: %d\n", len(instances))
for i, inst := range instances {
if inst.ServiceSnapshot != nil {
svc := inst.ServiceSnapshot
vipConfigs := ""
for j, cfg := range svc.Status.LoadBalancer.Ingress {
if j > 0 {
vipConfigs += ", "
}
vipConfigs += cfg.IP
}
fmt.Printf(" Service %d: %s/%s (Type: %s, VIPs: %s)\n",
i+1, svc.Namespace, svc.Name, svc.Spec.Type, vipConfigs)
}
}
}
if sm.clientSet != nil {
fmt.Println()
// Kubernetes configuration
fmt.Println("--- KUBERNETES CONFIGURATION (SERVICES/ENDPOINTSLICES) ---")
fmt.Println("Service Configuration:")
svcList, err := sm.clientSet.CoreV1().Services(v1.NamespaceAll).List(context.TODO(), metav1.ListOptions{})
if err != nil {
fmt.Println("Unable to retrieve all Services")
} else {
for x := range svcList.Items {
// Build all addresses
vipConfigs := ""
for j, cfg := range svcList.Items[x].Status.LoadBalancer.Ingress {
if j > 0 {
vipConfigs += ", "
}
vipConfigs += cfg.IP
}
fmt.Printf("Name=%s, UUID=%s, Addresses=%s\n", svcList.Items[x].Name, string(svcList.Items[x].UID), vipConfigs)
}
fmt.Println()
}
fmt.Println("EndpointSlice Configuration (note endpoint names have -XXXXX prefixed):")
epList, err := sm.clientSet.DiscoveryV1().EndpointSlices(v1.NamespaceAll).List(context.TODO(), metav1.ListOptions{})
if err != nil {
fmt.Println("Unable to retrieve all EndpointSlices")
} else {
for x := range epList.Items {
// Build all addresses
fmt.Printf(" Endpoint Slice Name: %s\n", epList.Items[x].Name)
for _, ep := range epList.Items[x].Endpoints {
endpoints := ""
for i, addresses := range ep.Addresses {
if i > 0 {
endpoints += ", "
}
endpoints += addresses
}
nodeName := "Unknown"
targetPod := "Unknown"
if ep.NodeName != nil {
nodeName = *ep.NodeName
}
if ep.TargetRef != nil {
targetPod = ep.TargetRef.Name
}
fmt.Printf("\tNode: %s, Target Pod:%s, Addresses: %s\n", nodeName, targetPod, endpoints)
}
}
}
}
}
fmt.Printf("\n")
}
func (sm *Manager) dumpNetworkInterfacesSection() {
fmt.Printf("--- NETWORK INTERFACES ---\n")
fmt.Printf("Network Interface Manager: %t\n", sm.intfMgr != nil)
fmt.Printf("ARP Manager: %t\n", sm.arpMgr != nil)
fmt.Printf("\n")
}
func (sm *Manager) dumpLeaderElectionSection() {
fmt.Printf("--- LEADER ELECTION CONFIGURATION ---\n")
fmt.Printf("Control Plane Enabled: %t\n", sm.config.EnableControlPlane)
if sm.config.EnableControlPlane {
fmt.Printf("Detect Control Plane: %t\n", sm.config.DetectControlPlane)
}
fmt.Printf("Leader Election Type: %s\n", sm.config.LeaderElectionType)
fmt.Printf("Leader Election Enabled: %t\n", sm.config.EnableLeaderElection)
if sm.config.EnableLeaderElection {
fmt.Printf("Lease Name: %s\n", sm.config.LeaseName)
fmt.Printf("Lease Duration: %d seconds\n", sm.config.LeaseDuration)
fmt.Printf("Renew Deadline: %d seconds\n", sm.config.RenewDeadline)
fmt.Printf("Retry Period: %d seconds\n", sm.config.RetryPeriod)
}
fmt.Printf("Services Lease Name: %s\n", sm.config.ServicesLeaseName)
fmt.Printf("Node Labeling Enabled: %t\n", sm.config.EnableNodeLabeling)
fmt.Printf("\n")
}
func (sm *Manager) dumpRuntimeSection() {
fmt.Printf("--- RUNTIME STATISTICS ---\n")
fmt.Printf("Load Balancer Enabled: %t\n", sm.config.EnableLoadBalancer)
if sm.config.EnableLoadBalancer {
fmt.Printf("Load Balancer Port: %d\n", sm.config.LoadBalancerPort)
fmt.Printf("Load Balancer Forwarding Method: %s\n", sm.config.LoadBalancerForwardingMethod)
fmt.Printf("Load Balancers Configured: %d\n", len(sm.config.LoadBalancers))
}
fmt.Printf("Prometheus HTTP Server: %s\n", sm.config.PrometheusHTTPServer)
fmt.Printf("Health Check Port: %d\n", sm.config.HealthCheckPort)
fmt.Printf("UPNP Enabled: %t\n", sm.config.EnableUPNP)
fmt.Printf("Egress Clean Enabled: %t\n", sm.config.EgressClean)
if sm.config.EgressClean {
fmt.Printf("Egress with nftables: %t\n", sm.config.EgressWithNftables)
fmt.Printf("Egress Pod CIDR: %s\n", sm.config.EgressPodCidr)
fmt.Printf("Egress Service CIDR: %s\n", sm.config.EgressServiceCidr)
}
fmt.Printf("\n")
}
func (sm *Manager) dumpNFTablesSection() {
fmt.Printf("--- NFTABLES CONFIGURATION ---\n")
chains, err := nftables.ListChains()
if err != nil {
fmt.Printf("Unable to retrieve NFTables chains, error=%s", err)
}
for x := range chains {
fmt.Printf("Chain: %s\n", chains[x])
}
fmt.Println()
}

View File

@@ -1,238 +0,0 @@
package manager
import (
"bytes"
"io"
"os"
"testing"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/stretchr/testify/assert"
)
func TestDumpConfiguration(t *testing.T) {
config := &kubevip.Config{
Address: "192.168.1.100",
Interface: "eth0",
Port: 6443,
EnableARP: true,
EnableBGP: false,
EnableControlPlane: true,
EnableServices: true,
LeaderElectionType: "kubernetes",
Namespace: "kube-system",
NodeName: "test-node",
KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
EnableLeaderElection: true,
LeaseName: "test-lease",
},
}
mgr := &Manager{
config: config,
}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpConfiguration()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "KUBE-VIP CONFIGURATION DUMP", "should contain header")
assert.Contains(t, output, "Node Name: test-node", "should contain node name")
assert.Contains(t, output, "VIP: 192.168.1.100", "should contain VIP address")
assert.Contains(t, output, "Interface: eth0", "should contain interface")
assert.Contains(t, output, "Port: 6443", "should contain port")
assert.Contains(t, output, "ARP Enabled: true", "should contain ARP status")
assert.Contains(t, output, "BGP Enabled: false", "should contain BGP status")
}
func TestDumpConfigSection(t *testing.T) {
config := &kubevip.Config{
Address: "192.168.1.100",
Interface: "eth0",
Port: 6443,
VIPSubnet: "/24",
EnableARP: true,
EnableBGP: false,
EnableControlPlane: true,
EnableServices: true,
LeaderElectionType: "kubernetes",
Namespace: "kube-system",
KubernetesLeaderElection: kubevip.KubernetesLeaderElection{
EnableLeaderElection: true,
LeaseName: "test-lease",
},
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpConfigSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "--- BASIC CONFIGURATION ---")
assert.Contains(t, output, "VIP: 192.168.1.100")
assert.Contains(t, output, "VIP Subnet: /24")
assert.Contains(t, output, "Interface: eth0")
assert.Contains(t, output, "Port: 6443")
assert.Contains(t, output, "Namespace: kube-system")
assert.Contains(t, output, "Single Node Mode: false")
assert.Contains(t, output, "Start As Leader: false")
}
func TestDumpBGPSection(t *testing.T) {
t.Run("BGP disabled", func(t *testing.T) {
config := &kubevip.Config{
EnableBGP: false,
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpBGPSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "BGP Enabled: false")
})
t.Run("BGP enabled", func(t *testing.T) {
config := &kubevip.Config{
EnableBGP: true,
BGPConfig: kubevip.BGPConfig{
RouterID: "192.168.1.1",
AS: 65000,
Peers: []kubevip.BGPPeer{
{Address: "192.168.1.2", AS: 65001},
{Address: "192.168.1.3", AS: 65002},
},
},
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpBGPSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "BGP Enabled: true")
assert.Contains(t, output, "BGP Router ID: 192.168.1.1")
assert.Contains(t, output, "BGP AS: 65000")
assert.Contains(t, output, "BGP Peers: 2")
})
}
func TestDumpARPSection(t *testing.T) {
t.Run("ARP disabled", func(t *testing.T) {
config := &kubevip.Config{
EnableARP: false,
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpARPSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "ARP Enabled: false")
})
t.Run("ARP enabled", func(t *testing.T) {
config := &kubevip.Config{
EnableARP: true,
ArpBroadcastRate: 5,
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpARPSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "ARP Enabled: true")
assert.Contains(t, output, "ARP Broadcast Rate: 5")
})
}
func TestDumpRuntimeSection(t *testing.T) {
config := &kubevip.Config{
EnableLoadBalancer: false,
PrometheusHTTPServer: "",
HealthCheckPort: 0,
EnableUPNP: false,
EgressClean: false,
}
mgr := &Manager{config: config}
old := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
mgr.dumpRuntimeSection()
w.Close()
os.Stdout = old
var buf bytes.Buffer
_, err := io.Copy(&buf, r)
assert.NoError(t, err, "io.Copy should not return error")
output := buf.String()
assert.Contains(t, output, "--- RUNTIME STATISTICS ---", "should contain runtime section header")
assert.Contains(t, output, "Load Balancer Enabled: false", "should contain load balancer status")
assert.Contains(t, output, "UPNP Enabled: false", "should contain UPNP status")
}

View File

@@ -3,13 +3,14 @@ package manager
import (
"context"
"fmt"
"os"
"strconv"
"syscall"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
@@ -39,34 +40,48 @@ func (sm *Manager) startTableMode(id string) error {
}()
}
if sm.config.EgressClean {
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv6)
log.Debug("IPtables rules cleaned on startup")
egressCleanEnv := os.Getenv("EGRESS_CLEAN")
if egressCleanEnv != "" {
egressClean, err := strconv.ParseBool(egressCleanEnv)
if err != nil {
log.Warn("failed to parse EGRESS_CLEAN env value [%s]. Egress cleaning will not be performed: %s", egressCleanEnv, err.Error())
}
if egressClean {
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv6)
log.Debug("IPtables rules cleaned on startup")
}
}
// Shutdown function that will wait on this signal, unless we call it ourselves
go func() {
for {
sig := <-sm.signalChan
switch sig {
case syscall.SIGUSR1:
log.Info("Received SIGUSR1, dumping configuration")
sm.dumpConfiguration()
case syscall.SIGINT, syscall.SIGTERM:
log.Info("Received kube-vip termination, signaling shutdown")
if sm.config.EnableControlPlane {
cpCluster.Stop()
}
// Cancel the context, which will in turn cancel the leadership
cancel()
return
}
<-sm.signalChan
log.Info("Received kube-vip termination, signaling shutdown")
if sm.config.EnableControlPlane {
cpCluster.Stop()
}
// Cancel the context, which will in turn cancel the leadership
cancel()
}()
if sm.config.EnableServices {
log.Debug("starting Services")
if sm.config.EnableControlPlane {
cpCluster, err = cluster.InitCluster(sm.config, false)
if err != nil {
return fmt.Errorf("cluster initialization error: %w", err)
}
clusterManager, err := initClusterManager(sm)
if err != nil {
return fmt.Errorf("cluster manager initialization error: %w", err)
}
if err := cpCluster.StartVipService(sm.config, clusterManager, nil, nil); err != nil {
log.Error("Control Plane", "err", err)
// Trigger the shutdown of this manager instance
sm.signalChan <- syscall.SIGINT
}
} else {
ns, err := returnNameSpace()
if err != nil {
log.Warn("unable to auto-detect namespace", "dropping to", sm.config.Namespace)
@@ -77,7 +92,7 @@ func (sm *Manager) startTableMode(id string) error {
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
err = sm.startServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
@@ -111,7 +126,7 @@ func (sm *Manager) startTableMode(id string) error {
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
err = sm.servicesWatcher(ctx, sm.syncServices)
if err != nil {
log.Error(err.Error())
panic("")
@@ -119,10 +134,12 @@ func (sm *Manager) startTableMode(id string) error {
},
OnStoppedLeading: func() {
// we can do cleanup here
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "id", id)
sm.svcProcessor.Stop()
for _, instance := range sm.serviceInstances {
for _, cluster := range instance.clusters {
cluster.Stop()
}
}
log.Error("lost leadership, restarting kube-vip")
panic("")
@@ -139,44 +156,17 @@ func (sm *Manager) startTableMode(id string) error {
})
} else {
log.Info("beginning watching services without leader election")
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
err = sm.servicesWatcher(ctx, sm.syncServices)
if err != nil {
log.Error("Cannot watch services", "err", err)
} else {
log.Debug("watching services")
}
}
}
if sm.config.EnableControlPlane {
log.Debug("initCluster for ControlPlane")
cpCluster, err = cluster.InitCluster(sm.config, false, sm.intfMgr, sm.arpMgr)
if err != nil {
log.Debug("init of ControlPlane NOT successful")
return fmt.Errorf("cluster initialization error: %w", err)
}
log.Debug("init of ControlPlane successful")
log.Debug("init ClusterManager")
clusterManager, err := initClusterManager(sm)
if err != nil {
log.Debug("init cluster manager NOT successful")
return fmt.Errorf("cluster manager initialization error: %w", err)
}
log.Debug("init ClusterManager successful")
if err := cpCluster.StartVipService(sm.config, clusterManager, nil); err != nil {
log.Error("Control Plane", "err", err)
// Trigger the shutdown of this manager instance
sm.signalChan <- syscall.SIGINT
} else {
log.Debug("start VipServer for cluster manager successful")
}
}
return nil
}
func (sm *Manager) cleanRoutes() error {
sm.mutex.Lock()
defer sm.mutex.Unlock()
routes, err := vip.ListRoutes(sm.config.RoutingTableID, sm.config.RoutingProtocol)
if err != nil {
return fmt.Errorf("error getting routes: %w", err)
@@ -187,7 +177,7 @@ func (sm *Manager) cleanRoutes() error {
if sm.config.EnableControlPlane {
found = (routes[i].Dst.IP.String() == sm.config.Address)
} else {
found = endpoints.CountRouteReferences(&routes[i], &sm.svcProcessor.ServiceInstances) > 0
found = sm.countRouteReferences(&routes[i]) > 0
}
if !found {
@@ -201,3 +191,20 @@ func (sm *Manager) cleanRoutes() error {
}
return nil
}
func (sm *Manager) countRouteReferences(route *netlink.Route) int {
cnt := 0
for _, instance := range sm.serviceInstances {
if instance.HasEndpoints {
for _, cluster := range instance.clusters {
for n := range cluster.Network {
r := cluster.Network[n].PrepareRoute()
if r.Dst.String() == route.Dst.String() {
cnt++
}
}
}
}
}
return cnt
}

View File

@@ -2,7 +2,6 @@ package manager
import (
"context"
"syscall"
"time"
log "log/slog"
@@ -40,19 +39,11 @@ func (sm *Manager) startWireguard(id string) error {
// Shutdown function that will wait on this signal, unless we call it ourselves
go func() {
for {
sig := <-sm.signalChan
switch sig {
case syscall.SIGUSR1:
log.Info("Received SIGUSR1, dumping configuration")
sm.dumpConfiguration()
case syscall.SIGINT, syscall.SIGTERM:
log.Info("Received termination, signaling shutdown")
// Cancel the context, which will in turn cancel the leadership
cancel()
return
}
}
<-sm.signalChan
log.Info("Received termination, signaling shutdown")
// Cancel the context, which will in turn cancel the leadership
cancel()
}()
ns, err = returnNameSpace()
@@ -65,7 +56,7 @@ func (sm *Manager) startWireguard(id string) error {
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
err = sm.startServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
@@ -100,7 +91,7 @@ func (sm *Manager) startWireguard(id string) error {
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
err = sm.servicesWatcher(ctx, sm.syncServices)
if err != nil {
log.Error(err.Error())
panic("")
@@ -108,10 +99,12 @@ func (sm *Manager) startWireguard(id string) error {
},
OnStoppedLeading: func() {
// we can do cleanup here
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "id", id)
sm.svcProcessor.Stop()
for _, instance := range sm.serviceInstances {
for _, cluster := range instance.clusters {
cluster.Stop()
}
}
log.Error("lost leadership, restarting kube-vip")
panic("")

View File

@@ -7,12 +7,16 @@ import (
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
)
const (
nodeLabelIndex = "kube-vip.io/has-ip"
nodeLabelJSONPath = `kube-vip.io~1has-ip`
)
type patchStringLabel struct {
Op string `json:"op"`
Path string `json:"path"`
@@ -31,8 +35,8 @@ func applyNodeLabel(clientSet *kubernetes.Clientset, address, id, identity strin
log.Debug(fmt.Sprintf("node %s labels: %+v", id, node.Labels))
value, ok := node.Labels[kubevip.HasIP]
path := fmt.Sprintf("/metadata/labels/%s", kubevip.HasIPJSONPath)
value, ok := node.Labels[nodeLabelIndex]
path := fmt.Sprintf("/metadata/labels/%s", nodeLabelJSONPath)
log.Debug(fmt.Sprintf("Received identity: %s - id: %s", identity, id))
if ok && value == address {
log.Debug(fmt.Sprintf("removing node label `has-ip=%s` on %s", address, id))

View File

@@ -4,12 +4,5 @@ import "github.com/prometheus/client_golang/prometheus"
// PrometheusCollector defines a service watch event counter.
func (sm *Manager) PrometheusCollector() []prometheus.Collector {
collectors := []prometheus.Collector{}
if sm.svcProcessor != nil {
collectors = append(collectors, sm.svcProcessor.CountServiceWatchEvent)
}
if sm.bgpServer != nil {
collectors = append(collectors, sm.bgpServer.BGPSessionInfoGauge)
}
return collectors
return []prometheus.Collector{sm.countServiceWatchEvent, sm.bgpSessionInfoGauge}
}

View File

@@ -1,4 +1,4 @@
package services
package manager
import (
"bufio"
@@ -11,9 +11,6 @@ import (
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -24,7 +21,7 @@ const (
defaultServiceCIDR = "10.96.0.0/12"
)
func (p *Processor) iptablesCheck() error {
func (sm *Manager) iptablesCheck() error {
file, err := os.Open("/proc/modules")
if err != nil {
return err
@@ -51,39 +48,14 @@ func (p *Processor) iptablesCheck() error {
return nil
}
func (p *Processor) nftablesCheck() error {
file, err := os.Open("/proc/modules")
if err != nil {
return err
}
defer file.Close()
scanner := bufio.NewScanner(file)
scanner.Split(bufio.ScanLines)
var queue, ct bool
for scanner.Scan() {
line := strings.Fields(scanner.Text())
switch line[0] {
case "nft_queue":
queue = true
case "nft_ct":
ct = true
}
}
if !queue || !ct {
return fmt.Errorf("missing nftables modules -> nft_ct [%t] -> ntf_queue [%t]", ct, queue)
}
return nil
}
func getSameFamilyCidr(sourceCidrs, ip string) string { //Todo: not sure how this ever worked
cidrs := strings.Split(sourceCidrs, ",")
isV6 := utils.IsIPv6(ip)
isV6 := vip.IsIPv6(ip)
matchingFamily := []string{}
for _, cidr := range cidrs {
// Is the ip an IPv6 address
if isV6 {
if utils.IsIPv6CIDR(cidr) {
if vip.IsIPv6CIDR(cidr) {
matchingFamily = append(matchingFamily, cidr)
selectedCIDR, err := checkCIDR(ip, cidr)
if err != nil {
@@ -95,7 +67,7 @@ func getSameFamilyCidr(sourceCidrs, ip string) string { //Todo: not sure how thi
}
}
} else {
if utils.IsIPv4CIDR(cidr) {
if vip.IsIPv4CIDR(cidr) {
matchingFamily = append(matchingFamily, cidr)
selectedCidr, err := checkCIDR(ip, cidr)
if err != nil {
@@ -135,27 +107,26 @@ func checkCIDR(ip, cidr string) (string, error) {
return "", nil
}
func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string, annotations map[string]string) error {
func (sm *Manager) configureEgress(vipIP, podIP, namespace string, annotations map[string]string) error {
var podCidr, serviceCidr string
var autoServiceCIDR, autoPodCIDR string
var discoverErr error
// Look up the destination ports from the annotations on the service
destinationPorts := annotations[kubevip.EgressDestinationPorts]
deniedNetworks := annotations[kubevip.EgressDeniedNetworks]
allowedNetworks := annotations[kubevip.EgressAllowedNetworks]
internalEgress := annotations[kubevip.EgressInternal]
destinationPorts := annotations[egressDestinationPorts]
deniedNetworks := annotations[egressDeniedNetworks]
allowedNetworks := annotations[egressAllowedNetworks]
if p.config.EgressPodCidr == "" || p.config.EgressServiceCidr == "" {
autoServiceCIDR, autoPodCIDR, discoverErr = p.AutoDiscoverCIDRs()
if sm.config.EgressPodCidr == "" || sm.config.EgressServiceCidr == "" {
autoServiceCIDR, autoPodCIDR, discoverErr = sm.AutoDiscoverCIDRs()
}
if discoverErr != nil {
log.Warn("autodiscover CIDR", "err", discoverErr)
}
if p.config.EgressPodCidr != "" {
podCidr = getSameFamilyCidr(p.config.EgressPodCidr, podIP)
if sm.config.EgressPodCidr != "" {
podCidr = getSameFamilyCidr(sm.config.EgressPodCidr, podIP)
} else {
if discoverErr == nil {
podCidr = getSameFamilyCidr(autoPodCIDR, podIP)
@@ -164,14 +135,14 @@ func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string,
if podCidr == "" {
// There's no default IPv6 pod CIDR, therefore we silently back off if CIDR s not specified.
if !utils.IsIPv4(podIP) {
if !vip.IsIPv4(podIP) {
return fmt.Errorf("error with the CIDR [%s]", podIP)
}
podCidr = defaultPodCIDR
}
if p.config.EgressServiceCidr != "" {
serviceCidr = getSameFamilyCidr(p.config.EgressServiceCidr, vipIP)
if sm.config.EgressServiceCidr != "" {
serviceCidr = getSameFamilyCidr(sm.config.EgressServiceCidr, vipIP)
} else {
if discoverErr == nil {
serviceCidr = getSameFamilyCidr(autoServiceCIDR, vipIP)
@@ -180,7 +151,7 @@ func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string,
if serviceCidr == "" {
// There's no default IPv6 service CIDR, therefore we silently back off if CIDR s not specified.
if !utils.IsIPv4(vipIP) {
if !vip.IsIPv4(vipIP) {
return nil
}
serviceCidr = defaultServiceCIDR
@@ -189,50 +160,27 @@ func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string,
log.Info("[Egress]", "podCIDR", podCidr, "serviceCIDR", serviceCidr, "vip", serviceCidr, "pod", podIP)
// checking if all addresses are of the same IP family
if utils.IsIPv4(podIP) != utils.IsIPv4CIDR(podCidr) {
if vip.IsIPv4(podIP) != vip.IsIPv4CIDR(podCidr) {
log.Error("[Egress] family is not matching. Backing off...", "pod", podIP, "podCIDR", podCidr)
return nil
}
if utils.IsIPv4(vipIP) != utils.IsIPv4CIDR(serviceCidr) {
if vip.IsIPv4(vipIP) != vip.IsIPv4CIDR(serviceCidr) {
log.Error("[Egress] family is not matching. Backing off...", "pod", podIP, "serviceCIDR", serviceCidr)
return nil
}
if utils.IsIPv4(vipIP) != utils.IsIPv4(podIP) {
if vip.IsIPv4(vipIP) != vip.IsIPv4(podIP) {
log.Error("[Egress] family is not matching. Backing off...", "pod", podIP, "vipIP", vipIP)
return nil
}
protocol := iptables.ProtocolIPv4
if utils.IsIPv6(vipIP) {
if vip.IsIPv6(vipIP) {
protocol = iptables.ProtocolIPv6
}
// Use the internal egress implementation
if internalEgress != "" {
// Create an array of CIDRs that we wont SNAT to.
ignoreCIDRs := []string{
podCidr,
serviceCidr,
}
// Add any specifically denied networks
if deniedNetworks != "" {
networks := strings.Split(strings.TrimSpace(deniedNetworks), ",") //Remove whitespace characters and then create an array from the CIDRs
ignoreCIDRs = append(ignoreCIDRs, networks...)
}
// Apply the SNAT rules
err := nftables.ApplySNAT(podIP, vipIP, serviceUUID, destinationPorts, ignoreCIDRs, utils.IsIPv6(vipIP))
if err != nil {
return fmt.Errorf("error performing netlink nftables [%s]", err)
}
return nil
}
i, err := vip.CreateIptablesClient(p.config.EgressWithNftables, namespace, protocol)
i, err := vip.CreateIptablesClient(sm.config.EgressWithNftables, namespace, protocol)
if err != nil {
return fmt.Errorf("error Creating iptables client [%s]", err)
}
@@ -270,7 +218,7 @@ func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string,
}
mask := "/32"
if !utils.IsIPv4(podIP) {
if !vip.IsIPv4(podIP) {
mask = "/128"
}
@@ -331,12 +279,12 @@ func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string,
return nil
}
func (p *Processor) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error) {
func (sm *Manager) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error) {
log.Debug("Trying to automatically discover Service and Pod CIDRs")
options := v1.ListOptions{
LabelSelector: "component=kube-controller-manager",
}
podList, err := p.clientSet.CoreV1().Pods("kube-system").List(context.TODO(), options)
podList, err := sm.clientSet.CoreV1().Pods("kube-system").List(context.TODO(), options)
if err != nil {
return "", "", fmt.Errorf("[Egress] Unable to get kube-controller-manager pod: %w", err)
}
@@ -359,3 +307,83 @@ func (p *Processor) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error)
return
}
func (sm *Manager) TeardownEgress(podIP, vipIP, namespace string, annotations map[string]string) error {
// Look up the destination ports from the annotations on the service
destinationPorts := annotations[egressDestinationPorts]
deniedNetworks := annotations[egressDeniedNetworks]
allowedNetworks := annotations[egressAllowedNetworks]
protocol := iptables.ProtocolIPv4
if vip.IsIPv6(podIP) {
protocol = iptables.ProtocolIPv6
}
i, err := vip.CreateIptablesClient(sm.config.EgressWithNftables, namespace, protocol)
if err != nil {
return fmt.Errorf("error Creating iptables client [%s]", err)
}
if deniedNetworks != "" {
networks := strings.Split(deniedNetworks, ",")
for x := range networks {
err = i.DeleteMangleReturnForNetwork(vip.MangleChainName, networks[x])
if err != nil {
return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
}
if allowedNetworks != "" {
networks := strings.Split(allowedNetworks, ",")
for x := range networks {
err = i.DeleteMangleMarkingForNetwork(podIP, vip.MangleChainName, networks[x])
if err != nil {
return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
} else {
// Remove the marking of egress packets
err = i.DeleteMangleMarking(podIP, vip.MangleChainName)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
// Clear up SNAT rules
if destinationPorts != "" {
fixedPorts := strings.Split(destinationPorts, ",")
for _, fixedPort := range fixedPorts {
var proto, port string
data := strings.Split(fixedPort, ":")
if len(data) == 0 {
continue
} else if len(data) == 1 {
proto = "tcp"
port = data[0]
} else {
proto = data[0]
port = data[1]
}
err = i.DeleteSourceNatForDestinationPort(podIP, vipIP, port, proto)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
} else {
err = i.DeleteSourceNat(podIP, vipIP)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
err = vip.DeleteExistingSessions(podIP, false, destinationPorts, "")
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
return nil
}

View File

@@ -1,4 +1,4 @@
package services
package manager
import "testing"

497
pkg/manager/services.go Normal file
View File

@@ -0,0 +1,497 @@
package manager
import (
"context"
"fmt"
"net"
"slices"
"strings"
"time"
log "log/slog"
"github.com/google/go-cmp/cmp"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
"github.com/kube-vip/kube-vip/pkg/upnp"
"github.com/kube-vip/kube-vip/pkg/vip"
)
func (sm *Manager) syncServices(ctx context.Context, svc *v1.Service) error {
log.Debug("[STARTING] Service Sync")
// Iterate through the synchronising services
foundInstance := false
newServiceAddresses := fetchServiceAddresses(svc)
newServiceUID := svc.UID
ingressIPs := []string{}
for _, ingress := range svc.Status.LoadBalancer.Ingress {
ingressIPs = append(ingressIPs, ingress.IP)
}
shouldBreake := false
for x := range sm.serviceInstances {
if shouldBreake {
break
}
for _, newServiceAddress := range newServiceAddresses {
log.Debug("service", "isDHCP", sm.serviceInstances[x].isDHCP, "newServiceAddress", newServiceAddress)
if sm.serviceInstances[x].serviceSnapshot.UID == newServiceUID {
// If the found instance's DHCP configuration doesn't match the new service, delete it.
if (sm.serviceInstances[x].isDHCP && newServiceAddress != "0.0.0.0") ||
(!sm.serviceInstances[x].isDHCP && newServiceAddress == "0.0.0.0") ||
(!sm.serviceInstances[x].isDHCP && len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, newServiceAddress)) ||
(len(svc.Status.LoadBalancer.Ingress) > 0 && !comparePortsAndPortStatuses(svc)) ||
(sm.serviceInstances[x].isDHCP && len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(ingressIPs, sm.serviceInstances[x].dhcpInterfaceIP)) {
if err := sm.deleteService(newServiceUID); err != nil {
return err
}
shouldBreake = true
break
}
foundInstance = true
}
}
}
// This instance wasn't found, we need to add it to the manager
if !foundInstance && len(newServiceAddresses) > 0 {
if err := sm.addService(ctx, svc); err != nil {
return err
}
}
return nil
}
func comparePortsAndPortStatuses(svc *v1.Service) bool {
portsStatus := svc.Status.LoadBalancer.Ingress[0].Ports
if len(portsStatus) != len(svc.Spec.Ports) {
return false
}
for i, portSpec := range svc.Spec.Ports {
if portsStatus[i].Port != portSpec.Port || portsStatus[i].Protocol != portSpec.Protocol {
return false
}
}
return true
}
func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
startTime := time.Now()
newService, err := NewInstance(svc, sm.config)
if err != nil {
return err
}
for x := range newService.vipConfigs {
newService.clusters[x].StartLoadBalancerService(newService.vipConfigs[x], sm.bgpServer)
}
sm.upnpMap(ctx, newService)
if newService.isDHCP && len(newService.vipConfigs) == 1 {
go func() {
for ip := range newService.dhcpClient.IPChannel() {
log.Debug("IP changed", "ip", ip)
newService.vipConfigs[0].VIP = ip
newService.dhcpInterfaceIP = ip
if !sm.config.DisableServiceUpdates {
if err := sm.updateStatus(newService); err != nil {
log.Warn("updating svc", "err", err)
}
}
}
log.Debug("IP update channel closed, stopping")
}()
}
sm.serviceInstances = append(sm.serviceInstances, newService)
if !sm.config.DisableServiceUpdates {
log.Debug("service update", "namespace", newService.serviceSnapshot.Namespace, "name", newService.serviceSnapshot.Name)
if err := sm.updateStatus(newService); err != nil {
log.Error("updating status", "namespace", newService.serviceSnapshot.Namespace, "name", newService.serviceSnapshot.Name, "err", err)
}
}
serviceIPs := fetchServiceAddresses(svc)
// Check if we need to flush any conntrack connections (due to some dangling conntrack connections)
if svc.Annotations[flushContrack] == "true" {
log.Debug("Flushing conntrack rules", "service", svc.Name)
for _, serviceIP := range serviceIPs {
err = vip.DeleteExistingSessions(serviceIP, false, svc.Annotations[egressDestinationPorts], svc.Annotations[egressSourcePorts])
if err != nil {
log.Error("flushing any remaining egress connections", "err", err)
}
err = vip.DeleteExistingSessions(serviceIP, true, svc.Annotations[egressDestinationPorts], svc.Annotations[egressSourcePorts])
if err != nil {
log.Error("flushing any remaining ingress connections", "err", err)
}
}
}
// Check if egress is enabled on the service, if so we'll need to configure some rules
if svc.Annotations[egress] == "true" && len(serviceIPs) > 0 {
log.Debug("enabling egress", "service", svc.Name)
// We will need to modify the iptables rules
err = sm.iptablesCheck()
if err != nil {
log.Error("configuring egress", "service", svc.Name, "err", err)
}
var podIP string
errList := []error{}
// Should egress be IPv6
if svc.Annotations[egressIPv6] == "true" {
// Does the service have an active IPv6 endpoint
if svc.Annotations[activeEndpointIPv6] != "" {
for _, serviceIP := range serviceIPs {
if sm.config.EnableEndpointSlices && vip.IsIPv6(serviceIP) {
podIP = svc.Annotations[activeEndpointIPv6]
err = sm.configureEgress(serviceIP, podIP, svc.Namespace, svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Error("configuring egress", "service", svc.Name, "err", err)
}
}
}
}
} else if svc.Annotations[activeEndpoint] != "" { // Not expected to be IPv6, so should be an IPv4 address
for _, serviceIP := range serviceIPs {
podIPs := svc.Annotations[activeEndpoint]
if sm.config.EnableEndpointSlices && vip.IsIPv6(serviceIP) {
podIPs = svc.Annotations[activeEndpointIPv6]
}
err = sm.configureEgress(serviceIP, podIPs, svc.Namespace, svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Error("configuring egress", "service", svc.Name, "err", err)
}
}
}
if len(errList) == 0 {
var provider epProvider
if !sm.config.EnableEndpointSlices {
provider = &endpointsProvider{label: "endpoints"}
} else {
provider = &endpointslicesProvider{label: "endpointslices"}
}
err = provider.updateServiceAnnotation(svc.Annotations[activeEndpoint], svc.Annotations[activeEndpointIPv6], svc, sm)
if err != nil {
log.Error("configuring egress", "service", svc.Name, "err", err)
}
}
}
finishTime := time.Since(startTime)
log.Info("[service] synchronised", "in", fmt.Sprintf("%dms", finishTime.Milliseconds()))
return nil
}
func (sm *Manager) deleteService(uid types.UID) error {
// protect multiple calls
sm.mutex.Lock()
defer sm.mutex.Unlock()
var updatedInstances []*Instance
var serviceInstance *Instance
found := false
for x := range sm.serviceInstances {
log.Debug("service lookup", "target UID", uid, "found UID ", sm.serviceInstances[x].serviceSnapshot.UID)
// Add the running services to the new array
if sm.serviceInstances[x].serviceSnapshot.UID != uid {
updatedInstances = append(updatedInstances, sm.serviceInstances[x])
} else {
// Flip the found when we match
found = true
serviceInstance = sm.serviceInstances[x]
}
}
// If we've been through all services and not found the correct one then error
if !found {
// TODO: - fix UX
// return fmt.Errorf("unable to find/stop service [%s]", uid)
return nil
}
// Determine if this this VIP is shared with other loadbalancers
shared := false
vipSet := make(map[string]interface{})
for x := range updatedInstances {
for _, vip := range fetchServiceAddresses(updatedInstances[x].serviceSnapshot) { //updatedInstances[x].serviceSnapshot.Spec.LoadBalancerIP {
vipSet[vip] = nil
}
}
for _, vip := range fetchServiceAddresses(serviceInstance.serviceSnapshot) {
if _, found := vipSet[vip]; found {
shared = true
}
}
if !shared {
for x := range serviceInstance.clusters {
serviceInstance.clusters[x].Stop()
}
if serviceInstance.isDHCP {
serviceInstance.dhcpClient.Stop()
macvlan, err := netlink.LinkByName(serviceInstance.dhcpInterface)
if err != nil {
return fmt.Errorf("error finding VIP Interface: %v", err)
}
err = netlink.LinkDel(macvlan)
if err != nil {
return fmt.Errorf("error deleting DHCP Link : %v", err)
}
}
// TODO: Implement dual-stack loadbalancer support if BGP is enabled
for i := range serviceInstance.vipConfigs {
if serviceInstance.vipConfigs[i].EnableBGP {
cidrVip := fmt.Sprintf("%s/%s", serviceInstance.vipConfigs[i].VIP, serviceInstance.vipConfigs[i].VIPCIDR)
err := sm.bgpServer.DelHost(cidrVip)
if err != nil {
return fmt.Errorf("[BGP] error deleting BGP host: %v", err)
}
log.Debug("[BGP] delete", "host", cidrVip)
}
}
// We will need to tear down the egress
if serviceInstance.serviceSnapshot.Annotations[egress] == "true" {
if serviceInstance.serviceSnapshot.Annotations[activeEndpoint] != "" {
log.Info("egress re-write enabled", "service", serviceInstance.serviceSnapshot.Name)
err := sm.TeardownEgress(serviceInstance.serviceSnapshot.Annotations[activeEndpoint], serviceInstance.serviceSnapshot.Spec.LoadBalancerIP, serviceInstance.serviceSnapshot.Namespace, serviceInstance.serviceSnapshot.Annotations)
if err != nil {
log.Error("egress teardown", "err", err)
}
}
}
}
// Update the service array
sm.serviceInstances = updatedInstances
log.Info("Removed instance from manager", "uid", uid, "remaining advertised services", len(sm.serviceInstances))
return nil
}
// Set up UPNP forwards for a service
// We first try to use the more modern Pinhole API introduced in UPNPv2 and fall back to UPNPv2 Port Forwarding if no forward was successful
func (sm *Manager) upnpMap(ctx context.Context, s *Instance) {
if !isUPNPEnabled(s.serviceSnapshot) {
// Skip services missing the annotation
return
}
if !sm.upnp {
log.Warn("[UPNP] Found kube-vip.io/forwardUPNP on service while UPNP forwarding is disabled in the kube-vip config. Not forwarding", "service", s.serviceSnapshot.Name)
}
// If upnp is enabled then update the gateway/router with the address
// TODO - check if this implementation for dualstack is correct
gateways := upnp.GetGatewayClients(ctx)
// Reset Gateway IPs to remove stale addresses
s.upnpGatewayIPs = make([]string, 0)
for _, vip := range fetchServiceAddresses(s.serviceSnapshot) {
for _, port := range s.serviceSnapshot.Spec.Ports {
for _, gw := range gateways {
log.Info("[UPNP] Adding map", "vip", vip, "port", port.Port, "service", s.serviceSnapshot.Name, "gateway", gw.WANIPv6FirewallControlClient.Location)
forwardSucessful := false
if gw.WANIPv6FirewallControlClient != nil {
pinholeID, pinholeErr := gw.WANIPv6FirewallControlClient.AddPinholeCtx(ctx, "0.0.0.0", uint16(port.Port), vip, uint16(port.Port), upnp.MapProtocolToIANA(string(port.Protocol)), 3600) //nolint TODO
if pinholeErr == nil {
forwardSucessful = true
log.Info("[UPNP] Service should be accessible externally", "port", port.Port, "pinhold ID", pinholeID)
} else {
//TODO: Cleanup
log.Error("[UPNP] Unable to map port to gateway using Pinhole API", "err", pinholeErr.Error())
}
}
// Fallback to PortForward
if !forwardSucessful {
portMappingErr := gw.ConnectionClient.AddPortMapping("0.0.0.0", uint16(port.Port), strings.ToUpper(string(port.Protocol)), uint16(port.Port), vip, true, s.serviceSnapshot.Name, 3600) //nolint TODO
if portMappingErr == nil {
log.Info("[UPNP] Service should be accessible externally", "port", port.Port)
forwardSucessful = true
} else {
//TODO: Cleanup
log.Error("[UPNP] Unable to map port to gateway using PortForward API", "err", portMappingErr.Error())
}
}
if forwardSucessful {
ip, err := gw.ConnectionClient.GetExternalIPAddress()
if err == nil {
s.upnpGatewayIPs = append(s.upnpGatewayIPs, ip)
}
}
}
}
}
// Remove duplicate IPs
slices.Sort(s.upnpGatewayIPs)
s.upnpGatewayIPs = slices.Compact(s.upnpGatewayIPs)
}
func (sm *Manager) updateStatus(i *Instance) error {
// let's retry status update every 10ms for 30s
retryConfig := wait.Backoff{
Steps: 3000,
Duration: 10 * time.Millisecond,
Factor: 0,
Jitter: 0.1,
}
// will retry for every error encountered, TODO: should a list of errors that will trigger retry be specified?
err := retry.OnError(retryConfig, func(error) bool { return true }, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := sm.clientSet.CoreV1().Services(i.serviceSnapshot.Namespace).Get(context.TODO(), i.serviceSnapshot.Name, metav1.GetOptions{})
if err != nil {
return err
}
currentServiceCopy := currentService.DeepCopy()
if currentServiceCopy.Annotations == nil {
currentServiceCopy.Annotations = make(map[string]string)
}
// If we're using ARP then we can only broadcast the VIP from one place, add an annotation to the service
if sm.config.EnableARP {
// Add the current host
currentServiceCopy.Annotations[vipHost] = sm.config.NodeName
}
if i.dhcpInterfaceHwaddr != "" || i.dhcpInterfaceIP != "" {
currentServiceCopy.Annotations[hwAddrKey] = i.dhcpInterfaceHwaddr
currentServiceCopy.Annotations[requestedIP] = i.dhcpInterfaceIP
}
if currentService.Annotations["development.kube-vip.io/synthetic-api-server-error-on-update"] == "true" {
log.Error("(Synthetic error ) updating Spec", "service", i.serviceSnapshot.Name, "err", err)
return fmt.Errorf("(Synthetic) simulating api server errors")
}
if !cmp.Equal(currentService, currentServiceCopy) {
currentService, err = sm.clientSet.CoreV1().Services(currentServiceCopy.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("updating Spec", "service", i.serviceSnapshot.Name, "err", err)
return err
}
}
ports := make([]v1.PortStatus, 0, len(i.serviceSnapshot.Spec.Ports))
for _, port := range i.serviceSnapshot.Spec.Ports {
ports = append(ports, v1.PortStatus{
Port: port.Port,
Protocol: port.Protocol,
})
}
ingresses := []v1.LoadBalancerIngress{}
for _, c := range i.vipConfigs {
if !vip.IsIP(c.VIP) {
ips, err := vip.LookupHost(c.VIP, sm.config.DNSMode)
if err != nil {
return err
}
for _, ip := range ips {
i := v1.LoadBalancerIngress{
IP: ip,
Ports: ports,
}
ingresses = append(ingresses, i)
}
} else {
i := v1.LoadBalancerIngress{
IP: c.VIP,
Ports: ports,
}
ingresses = append(ingresses, i)
}
if isUPNPEnabled(currentService) {
for _, ip := range i.upnpGatewayIPs {
i := v1.LoadBalancerIngress{
IP: ip,
Ports: ports,
}
ingresses = append(ingresses, i)
}
}
}
if !cmp.Equal(currentService.Status.LoadBalancer.Ingress, ingresses) {
currentService.Status.LoadBalancer.Ingress = ingresses
_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).UpdateStatus(context.TODO(), currentService, metav1.UpdateOptions{})
if err != nil {
log.Error("updating Service", "namespace", i.serviceSnapshot.Namespace, "name", i.serviceSnapshot.Name, "err", err)
return err
}
}
return nil
})
return err
}
// fetchServiceAddresses tries to get the addresses from annotations
// kube-vip.io/loadbalancerIPs, then from spec.loadbalancerIP
func fetchServiceAddresses(s *v1.Service) []string {
annotationAvailable := false
if s.Annotations != nil {
if v, annotationAvailable := s.Annotations[loadbalancerIPAnnotation]; annotationAvailable {
ips := strings.Split(v, ",")
var trimmedIPs []string
for _, ip := range ips {
trimmedIPs = append(trimmedIPs, strings.TrimSpace(ip))
}
return trimmedIPs
}
}
lbStatusAddresses := []string{}
if !annotationAvailable {
if len(s.Status.LoadBalancer.Ingress) > 0 {
for _, ingress := range s.Status.LoadBalancer.Ingress {
lbStatusAddresses = append(lbStatusAddresses, ingress.IP)
}
}
}
lbIP := net.ParseIP(s.Spec.LoadBalancerIP)
isLbIPv4 := vip.IsIPv4(s.Spec.LoadBalancerIP)
if len(lbStatusAddresses) > 0 {
for _, a := range lbStatusAddresses {
if lbStatusIP := net.ParseIP(a); lbStatusIP != nil && lbIP != nil && vip.IsIPv4(a) == isLbIPv4 && !lbIP.Equal(lbStatusIP) {
return []string{s.Spec.LoadBalancerIP}
}
}
return lbStatusAddresses
}
if s.Spec.LoadBalancerIP != "" {
return []string{s.Spec.LoadBalancerIP}
}
return []string{}
}
func isUPNPEnabled(s *v1.Service) bool {
return metav1.HasAnnotation(s.ObjectMeta, upnpEnabled) && s.Annotations[upnpEnabled] == "true"
}

View File

@@ -0,0 +1,118 @@
package manager
import (
"context"
"fmt"
"sync"
"time"
log "log/slog"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
)
var (
svcLocks map[string]*sync.Mutex
)
func init() {
svcLocks = make(map[string]*sync.Mutex)
}
// The startServicesWatchForLeaderElection function will start a services watcher, the
func (sm *Manager) startServicesWatchForLeaderElection(ctx context.Context) error {
err := sm.servicesWatcher(ctx, sm.StartServicesLeaderElection)
if err != nil {
return err
}
for _, instance := range sm.serviceInstances {
for _, cluster := range instance.clusters {
for i := range cluster.Network {
_ = cluster.Network[i].DeleteRoute()
}
cluster.Stop()
}
}
log.Info("Shutting down kube-Vip")
return nil
}
// The startServicesWatchForLeaderElection function will start a services watcher, the
func (sm *Manager) StartServicesLeaderElection(ctx context.Context, service *v1.Service) error {
serviceLease := fmt.Sprintf("kubevip-%s", service.Name)
log.Info("new leader election", "service", service.Name, "namespace", service.Namespace, "lock_name", serviceLease, "host_id", sm.config.NodeName)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: serviceLease,
Namespace: service.Namespace,
},
Client: sm.clientSet.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: sm.config.NodeName,
},
}
childCtx, childCancel := context.WithCancel(ctx)
defer childCancel()
if _, ok := svcLocks[serviceLease]; !ok {
svcLocks[serviceLease] = new(sync.Mutex)
}
svcLocks[serviceLease].Lock()
defer svcLocks[serviceLease].Unlock()
activeService[string(service.UID)] = true
// start the leader election code loop
leaderelection.RunOrDie(childCtx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(sm.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(sm.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
// Mark this service as active (as we've started leading)
// we run this in background as it's blocking
if err := sm.syncServices(ctx, service); err != nil {
log.Error("service sync", "err", err)
childCancel()
}
},
OnStoppedLeading: func() {
// we can do cleanup here
log.Info("leadership lost", "service", service.Name, "leader", sm.config.NodeName)
if activeService[string(service.UID)] {
if err := sm.deleteService(service.UID); err != nil {
log.Error("service deletion", "err", err)
}
}
// Mark this service is inactive
activeService[string(service.UID)] = false
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if identity == sm.config.NodeName {
// I just got the lock
return
}
log.Info("new leader", "leader", identity)
},
},
})
log.Info("stopping leader election", "service", service.Name)
return nil
}

View File

@@ -10,7 +10,7 @@ import (
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/davecgh/go-spew/spew"
v1 "k8s.io/api/core/v1"
@@ -57,8 +57,7 @@ func (sm *Manager) annotationsWatcher() error {
// they're as needed
log.Warn(err.Error())
// TODO, will need refactoring as part of rikatz work
rw, err := watchtools.NewRetryWatcherWithContext(context.TODO(), node.ResourceVersion, &cache.ListWatch{
rw, err := watchtools.NewRetryWatcher(node.ResourceVersion, &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.rwClientSet.CoreV1().Nodes().Watch(context.Background(), listOptions)
},
@@ -151,8 +150,8 @@ func (sm *Manager) annotationsWatcher() error {
// * `<info>` is the relevant information, such as `node-asn` or `peer-ip`
// * `{{n}}` is the number of the peer, always starting with `0`
// * kube-vip is only designed to manage one peer, just look for {{n}} == 0
func parseBgpAnnotations(bgpConfig kubevip.BGPConfig, node *v1.Node, prefix string) (kubevip.BGPConfig, kubevip.BGPPeer, error) {
bgpPeer := kubevip.BGPPeer{}
func parseBgpAnnotations(bgpConfig bgp.Config, node *v1.Node, prefix string) (bgp.Config, bgp.Peer, error) {
bgpPeer := bgp.Peer{}
nodeASN := ""
for k, v := range node.Annotations {
@@ -216,7 +215,7 @@ func parseBgpAnnotations(bgpConfig kubevip.BGPConfig, node *v1.Node, prefix stri
peerIPs := strings.Split(peerIPString, ",")
bgpConfig.Peers = make([]kubevip.BGPPeer, 0, len(peerIPs))
bgpConfig.Peers = make([]bgp.Peer, 0, len(peerIPs))
for _, peerIP := range peerIPs {
ipAddr := strings.TrimSpace(peerIP)

View File

@@ -0,0 +1,501 @@
package manager
import (
"context"
"errors"
"fmt"
"net"
"strings"
"syscall"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/client-go/util/retry"
)
type epProvider interface {
createRetryWatcher(context.Context, *Manager,
*v1.Service) (*watchtools.RetryWatcher, error)
getAllEndpoints() ([]string, error)
getLocalEndpoints(string, *kubevip.Config) ([]string, error)
getLabel() string
updateServiceAnnotation(string, string, *v1.Service, *Manager) error
loadObject(runtime.Object, context.CancelFunc) error
getProtocol() string
}
type endpointsProvider struct {
label string
endpoints *v1.Endpoints
}
func (ep *endpointsProvider) createRetryWatcher(ctx context.Context, sm *Manager,
service *v1.Service) (*watchtools.RetryWatcher, error) {
opts := metav1.ListOptions{
FieldSelector: fields.OneTermEqualSelector("metadata.name", service.Name).String(),
}
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.rwClientSet.CoreV1().Endpoints(service.Namespace).Watch(ctx, opts)
},
})
if err != nil {
return nil, fmt.Errorf("error creating endpoint watcher: %s", err.Error())
}
return rw, nil
}
func (ep *endpointsProvider) loadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
eps, ok := endpoints.(*v1.Endpoints)
if !ok {
cancel()
return fmt.Errorf("[%s] unable to parse Kubernetes services from API watcher", ep.getLabel())
}
ep.endpoints = eps
return nil
}
func (ep *endpointsProvider) getAllEndpoints() ([]string, error) {
result := []string{}
for subset := range ep.endpoints.Subsets {
for address := range ep.endpoints.Subsets[subset].Addresses {
addr := strings.Split(ep.endpoints.Subsets[subset].Addresses[address].IP, "/")
result = append(result, addr[0])
}
}
return result, nil
}
func (ep *endpointsProvider) getLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
var localEndpoints []string
for _, subset := range ep.endpoints.Subsets {
for _, address := range subset.Addresses {
log.Debug("processing endpoint", "label", ep.label, "ip", address.IP)
// 1. Compare the Nodename
if address.NodeName != nil && id == *address.NodeName {
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname, "nodename", *address.NodeName)
localEndpoints = append(localEndpoints, address.IP)
continue
}
// 2. Compare the Hostname (only useful if address.NodeName is not available)
if id == address.Hostname {
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname)
localEndpoints = append(localEndpoints, address.IP)
continue
}
}
}
return localEndpoints, nil
}
func (ep *endpointsProvider) updateServiceAnnotation(endpoint string, _ string, service *v1.Service, sm *Manager) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := sm.clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
currentServiceCopy := currentService.DeepCopy()
if currentServiceCopy.Annotations == nil {
currentServiceCopy.Annotations = make(map[string]string)
}
currentServiceCopy.Annotations[activeEndpoint] = endpoint
_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("error updating Service Spec", "label", ep.getLabel(), "name", currentServiceCopy.Name, "err", err)
return err
}
return nil
})
if retryErr != nil {
log.Error("failed to set Services", "label", ep.getLabel(), "err", retryErr)
return retryErr
}
return nil
}
func (ep *endpointsProvider) getLabel() string {
return ep.label
}
func (ep *endpointsProvider) getProtocol() string {
return ""
}
func (sm *Manager) watchEndpoint(ctx context.Context, id string, service *v1.Service, provider epProvider) error {
log.Info("watching", "provider", provider.getLabel(), "service_name", service.Name, "namespace", service.Namespace)
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
leaderContext, cancel := context.WithCancel(ctx)
defer cancel()
var leaderElectionActive bool
rw, err := provider.createRetryWatcher(leaderContext, sm, service)
if err != nil {
cancel()
return fmt.Errorf("[%s] error watching endpoints: %w", provider.getLabel(), err)
}
exitFunction := make(chan struct{})
go func() {
select {
case <-ctx.Done():
log.Debug("context cancelled", "provider", provider.getLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-sm.shutdownChan:
log.Debug("shutdown called", "provider", provider.getLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-exitFunction:
log.Debug("function ending", "provider", provider.getLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
}
}()
ch := rw.ResultChan()
var lastKnownGoodEndpoint string
for event := range ch {
activeEndpointAnnotation := activeEndpoint
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
if err = provider.loadObject(event.Object, cancel); err != nil {
return fmt.Errorf("[%s] error loading k8s object: %w", provider.getLabel(), err)
}
if sm.config.EnableEndpointSlices && provider.getProtocol() == string(discoveryv1.AddressTypeIPv6) {
activeEndpointAnnotation = activeEndpointIPv6
}
// Build endpoints
var endpoints []string
if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
if endpoints, err = provider.getAllEndpoints(); err != nil {
return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
}
} else {
if endpoints, err = provider.getLocalEndpoints(id, sm.config); err != nil {
return fmt.Errorf("[%s] error getting local endpoints: %w", provider.getLabel(), err)
}
}
if sm.config.EnableRoutingTable {
instance := sm.findServiceInstance(service)
if instance == nil {
log.Error("error finding instance", "service", service.UID, "provider", provider.getLabel(), "err", err)
} else {
instance.HasEndpoints = len(endpoints) > 0
}
}
// Find out if we have any local endpoints
// if out endpoint is empty then populate it
// if not, go through the endpoints and see if ours still exists
// If we have a local endpoint then begin the leader Election, unless it's already running
//
// Check that we have local endpoints
if len(endpoints) != 0 {
// Ignore IPv4
if service.Annotations[egressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
continue
}
// if we haven't populated one, then do so
if lastKnownGoodEndpoint != "" {
// check out previous endpoint exists
stillExists := false
for x := range endpoints {
if endpoints[x] == lastKnownGoodEndpoint {
stillExists = true
}
}
// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
if !stillExists {
if sm.config.EnableRoutingTable {
if err := sm.TeardownEgress(lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
service.Namespace, service.Annotations); err != nil {
log.Warn("removing redundant egress rules", "err", err)
}
}
if leaderElectionActive && (sm.config.EnableServicesElection || sm.config.EnableLeaderElection) {
log.Warn(" existing endpoint has been removed, restarting leaderElection", "provider", provider.getLabel(), "endpoint", lastKnownGoodEndpoint)
// Stop the existing leaderElection
cancel()
// disable last leaderElection flag
leaderElectionActive = false
}
// Set our active endpoint to an existing one
lastKnownGoodEndpoint = endpoints[0]
}
} else {
lastKnownGoodEndpoint = endpoints[0]
}
if !leaderElectionActive && sm.config.EnableServicesElection {
go func() {
leaderContext, cancel = context.WithCancel(ctx)
// This is a blocking function, that will restart (in the event of failure)
for {
// if the context isn't cancelled restart
if leaderContext.Err() != context.Canceled {
leaderElectionActive = true
err := sm.StartServicesLeaderElection(leaderContext, service)
if err != nil {
log.Error(err.Error())
}
leaderElectionActive = false
} else {
leaderElectionActive = false
break
}
}
}()
}
isRouteConfigured, err := isRouteConfigured(service.UID)
if err != nil {
return fmt.Errorf("[%s] error while checking if route is configured: %w", provider.getLabel(), err)
}
// There are local endpoints available on the node
if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection && !isRouteConfigured {
// If routing table mode is enabled - routes should be added per node
if sm.config.EnableRoutingTable {
instance, err := sm.findServiceInstanceWithTimeout(ctx, service)
if err != nil {
log.Error("error finding instance", "service", service.UID, "provider", provider.getLabel(), "err", err)
}
if instance != nil {
for _, cluster := range instance.clusters {
for i := range cluster.Network {
err := cluster.Network[i].AddRoute(false)
if err != nil {
if errors.Is(err, syscall.EEXIST) {
// If route exists try to update it if necessary
isUpdated, err := cluster.Network[i].UpdateRoutes()
if err != nil {
return fmt.Errorf("[%s] error updating existing routes: %w", provider.getLabel(), err)
}
if isUpdated {
log.Debug("updated route", "provider", provider.getLabel(), "ip", cluster.Network[i].IP())
}
} else {
// If other error occurs, return error
return fmt.Errorf("[%s] error adding route: %s", provider.getLabel(), err.Error())
}
} else {
log.Info("added route", "provider",
provider.getLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", sm.config.RoutingTableID)
configuredLocalRoutes.Store(string(service.UID), true)
leaderElectionActive = true
}
}
}
}
}
// If BGP mode is enabled - hosts should be added per node
if sm.config.EnableBGP {
instance, err := sm.findServiceInstanceWithTimeout(ctx, service)
if err != nil {
log.Error("error finding instance", "service", service.UID, "provider", provider.getLabel(), "err", err)
}
if instance != nil {
for _, cluster := range instance.clusters {
for i := range cluster.Network {
address := fmt.Sprintf("%s/%s", cluster.Network[i].IP(), sm.config.VIPCIDR)
log.Debug("attempting to advertise BGP service", "provider", provider.getLabel(), "ip", address)
err := sm.bgpServer.AddHost(address)
if err != nil {
log.Error("error adding BGP host", "provider", provider.getLabel(), "err", err)
} else {
log.Info("added BGP host", "provider",
provider.getLabel(), "ip", address, "service name", service.Name, "namespace", service.Namespace)
configuredLocalRoutes.Store(string(service.UID), true)
leaderElectionActive = true
}
}
}
}
}
}
} else {
// There are no local endpoints
if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection {
// If routing table mode is enabled - routes should be deleted
if sm.config.EnableRoutingTable {
if errs := sm.clearRoutes(service); len(errs) == 0 {
configuredLocalRoutes.Store(string(service.UID), false)
} else {
for _, err := range errs {
log.Error("error while clearing routes", "err", err)
}
}
}
// If BGP mode is enabled - routes should be deleted
if sm.config.EnableBGP {
if instance := sm.findServiceInstance(service); instance != nil {
for _, cluster := range instance.clusters {
for i := range cluster.Network {
address := fmt.Sprintf("%s/%s", cluster.Network[i].IP(), sm.config.VIPCIDR)
err := sm.bgpServer.DelHost(address)
if err != nil {
log.Error("deleting BGP host", "provider", provider.getLabel(), "ip", address, "err", err)
} else {
log.Info("deleted BGP host", "provider",
provider.getLabel(), "ip", address, "service name", service.Name, "namespace", service.Namespace)
configuredLocalRoutes.Store(string(service.UID), false)
leaderElectionActive = false
}
}
}
}
}
}
// If there are no local endpoints, and we had one then remove it and stop the leaderElection
if lastKnownGoodEndpoint != "" {
log.Warn("existing endpoint has been removed, no remaining endpoints for leaderElection", "provider", provider.getLabel(), "endpoint", lastKnownGoodEndpoint)
if err := sm.TeardownEgress(lastKnownGoodEndpoint, service.Spec.LoadBalancerIP, service.Namespace, service.Annotations); err != nil {
log.Error("error removing redundant egress rules", "err", err)
}
lastKnownGoodEndpoint = "" // reset endpoint
if sm.config.EnableServicesElection || sm.config.EnableLeaderElection {
cancel() // stop services watcher
}
leaderElectionActive = false
}
}
// Set the service accordingly
if service.Annotations[egress] == "true" {
service.Annotations[activeEndpointAnnotation] = lastKnownGoodEndpoint
}
log.Debug("watcher", "provider",
provider.getLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", lastKnownGoodEndpoint, "active leader election", leaderElectionActive)
case watch.Deleted:
// When no-leader-elecition mode
if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection {
// find all existing local endpoints
var endpoints []string
if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
if endpoints, err = provider.getAllEndpoints(); err != nil {
return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
}
} else {
if endpoints, err = provider.getLocalEndpoints(id, sm.config); err != nil {
return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
}
}
// If there were local endpoints deleted
if len(endpoints) > 0 {
// Delete all routes in routing table mode
if sm.config.EnableRoutingTable {
sm.clearRoutes(service)
}
// Delete all hosts in BGP mode
if sm.config.EnableBGP {
sm.clearBGPHosts(service)
}
}
}
// Close the goroutine that will end the retry watcher, then exit the endpoint watcher function
close(exitFunction)
log.Info("stopping watching", "provider", provider.getLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil
case watch.Error:
errObject := apierrors.FromObject(event.Object)
statusErr, _ := errObject.(*apierrors.StatusError)
log.Error("watch error", "provider", provider.getLabel(), "err", statusErr)
}
}
close(exitFunction)
log.Info("stopping watching", "provider", provider.getLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil //nolint:govet
}
func (sm *Manager) clearRoutes(service *v1.Service) []error {
errs := []error{}
if instance := sm.findServiceInstance(service); instance != nil {
for _, cluster := range instance.clusters {
for i := range cluster.Network {
route := cluster.Network[i].PrepareRoute()
// check if route we are about to delete is not referenced by more than one service
if sm.countRouteReferences(route) <= 1 {
err := cluster.Network[i].DeleteRoute()
if err != nil && !errors.Is(err, syscall.ESRCH) {
log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
errs = append(errs, err)
}
log.Debug("deleted route", "ip",
cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", sm.config.RoutingTableID)
}
}
}
}
return errs
}
func (sm *Manager) clearBGPHosts(service *v1.Service) {
if instance := sm.findServiceInstance(service); instance != nil {
for _, cluster := range instance.clusters {
for i := range cluster.Network {
address := fmt.Sprintf("%s/%s", cluster.Network[i].IP(), sm.config.VIPCIDR)
err := sm.bgpServer.DelHost(address)
if err != nil {
log.Error("[endpoint] error deleting BGP host", "err", err)
} else {
log.Debug("[endpoint] deleted BGP host", "ip",
address, "service name", service.Name, "namespace", service.Namespace)
}
}
}
}
}

View File

@@ -1,4 +1,4 @@
package providers
package manager
import (
"context"
@@ -13,24 +13,17 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/client-go/util/retry"
)
type Endpointslices struct {
type endpointslicesProvider struct {
label string
endpoints *discoveryv1.EndpointSlice
}
func NewEndpointslices() Provider {
return &Endpointslices{
label: "endpointslices",
}
}
func (ep *Endpointslices) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
func (ep *endpointslicesProvider) createRetryWatcher(ctx context.Context, sm *Manager,
service *v1.Service) (*watchtools.RetryWatcher, error) {
labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/service-name": service.Name}}
@@ -38,9 +31,9 @@ func (ep *Endpointslices) CreateRetryWatcher(ctx context.Context, clientSet *kub
LabelSelector: labels.Set(labelSelector.MatchLabels).String(),
}
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return clientSet.DiscoveryV1().EndpointSlices(service.Namespace).Watch(ctx, opts)
return sm.rwClientSet.DiscoveryV1().EndpointSlices(service.Namespace).Watch(ctx, opts)
},
})
if err != nil {
@@ -50,7 +43,7 @@ func (ep *Endpointslices) CreateRetryWatcher(ctx context.Context, clientSet *kub
return rw, nil
}
func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
func (ep *endpointslicesProvider) loadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
eps, ok := endpoints.(*discoveryv1.EndpointSlice)
if !ok {
cancel()
@@ -60,7 +53,7 @@ func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.Ca
return nil
}
func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
func (ep *endpointslicesProvider) getAllEndpoints() ([]string, error) {
result := []string{}
for _, ep := range ep.endpoints.Endpoints {
result = append(result, ep.Addresses...)
@@ -68,10 +61,10 @@ func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
return result, nil
}
func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
func (ep *endpointslicesProvider) getLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
var localEndpoints []string
for _, endpoint := range ep.endpoints.Endpoints {
if endpoint.Conditions.Serving == nil || !*endpoint.Conditions.Serving {
if !*endpoint.Conditions.Serving {
continue
}
for _, address := range endpoint.Addresses {
@@ -98,11 +91,11 @@ func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]str
return localEndpoints, nil
}
func (ep *Endpointslices) UpdateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, clientSet *kubernetes.Clientset) error {
func (ep *endpointslicesProvider) updateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, sm *Manager) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
currentService, err := sm.clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
@@ -112,10 +105,10 @@ func (ep *Endpointslices) UpdateServiceAnnotation(endpoint, endpointIPv6 string,
currentServiceCopy.Annotations = make(map[string]string)
}
currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
currentServiceCopy.Annotations[kubevip.ActiveEndpointIPv6] = endpointIPv6
currentServiceCopy.Annotations[activeEndpoint] = endpoint
currentServiceCopy.Annotations[activeEndpointIPv6] = endpointIPv6
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("error updating Service Spec", "provider", ep.label, "service name", currentServiceCopy.Name, "err", err)
return err
@@ -130,10 +123,10 @@ func (ep *Endpointslices) UpdateServiceAnnotation(endpoint, endpointIPv6 string,
return nil
}
func (ep *Endpointslices) GetLabel() string {
func (ep *endpointslicesProvider) getLabel() string {
return ep.label
}
func (ep *Endpointslices) GetProtocol() string {
func (ep *endpointslicesProvider) getProtocol() string {
return string(ep.endpoints.AddressType)
}

View File

@@ -0,0 +1,411 @@
package manager
import (
"context"
"fmt"
"reflect"
"sync"
log "log/slog"
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/prometheus/client_golang/prometheus"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
)
// TODO: Fix the naming of these contexts
// activeServiceLoadBalancer keeps track of services that already have a leaderElection in place
var activeServiceLoadBalancer map[string]context.Context
// activeServiceLoadBalancer keeps track of services that already have a leaderElection in place
var activeServiceLoadBalancerCancel map[string]func()
// activeService keeps track of services that already have a leaderElection in place
var activeService map[string]bool
// watchedService keeps track of services that are already being watched
var watchedService map[string]bool
// watchedService keeps track of routes that has been configured on the node
var configuredLocalRoutes sync.Map
func init() {
// Set up the caches for monitoring existing active or watched services
activeServiceLoadBalancerCancel = make(map[string]func())
activeServiceLoadBalancer = make(map[string]context.Context)
activeService = make(map[string]bool)
watchedService = make(map[string]bool)
}
// This function handles the watching of a services endpoints and updates a load balancers endpoint configurations accordingly
func (sm *Manager) servicesWatcher(ctx context.Context, serviceFunc func(context.Context, *v1.Service) error) error {
// first start port mirroring if enabled
if err := sm.startTrafficMirroringIfEnabled(); err != nil {
return err
}
defer func() {
// clean up traffic mirror related config
err := sm.stopTrafficMirroringIfEnabled()
if err != nil {
log.Error("Stopping traffic mirroring", "err", err)
}
}()
if sm.config.ServiceNamespace == "" {
// v1.NamespaceAll is actually "", but we'll stay with the const in case things change upstream
sm.config.ServiceNamespace = v1.NamespaceAll
log.Info("(svcs) starting services watcher for all namespaces")
} else {
log.Info("(svcs) starting services watcher", "namespace", sm.config.ServiceNamespace)
}
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.rwClientSet.CoreV1().Services(sm.config.ServiceNamespace).Watch(ctx, metav1.ListOptions{})
},
})
if err != nil {
return fmt.Errorf("error creating services watcher: %s", err.Error())
}
exitFunction := make(chan struct{})
go func() {
select {
case <-sm.shutdownChan:
log.Debug("(svcs) shutdown called")
// Stop the retry watcher
rw.Stop()
return
case <-exitFunction:
log.Debug("(svcs) function ending")
// Stop the retry watcher
rw.Stop()
return
}
}()
ch := rw.ResultChan()
// Used for tracking an active endpoint / pod
for event := range ch {
sm.countServiceWatchEvent.With(prometheus.Labels{"type": string(event.Type)}).Add(1)
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
svc, ok := event.Object.(*v1.Service)
if !ok {
return fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
break
}
// Check if we ignore this service
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Info("ignore annotation for kube-vip", "service name", svc.Name)
break
}
// Select loadbalancer class filtering function
lbClassFilterFunc := sm.lbClassFilter
if sm.config.LoadBalancerClassLegacyHandling {
lbClassFilterFunc = sm.lbClassFilterLegacy
}
// Check the loadBalancer class
if lbClassFilterFunc(svc) {
break
}
svcAddresses := fetchServiceAddresses(svc)
// We only care about LoadBalancer services that have been allocated an address
if len(svcAddresses) <= 0 {
break
}
// The modified event should only be triggered if the service has been modified (i.e. moved somewhere else)
if event.Type == watch.Modified {
for _, addr := range svcAddresses {
// log.Debugf("(svcs) Retreiving local addresses, to ensure that this modified address doesn't exist: %s", addr)
f, err := vip.GarbageCollect(sm.config.Interface, addr)
if err != nil {
log.Error("(svcs) cleaning existing address error", "err", err)
}
if f {
log.Warn("(svcs) already found existing config", "address", addr, "adapter", sm.config.Interface)
}
}
// This service has been modified, but it was also active..
if activeService[string(svc.UID)] {
i := sm.findServiceInstance(svc)
if i != nil {
originalService := fetchServiceAddresses(i.serviceSnapshot)
newService := fetchServiceAddresses(svc)
if !reflect.DeepEqual(originalService, newService) {
// Calls the cancel function of the context
if activeServiceLoadBalancerCancel[string(svc.UID)] != nil {
log.Warn("(svcs) The load balancer has changed, cancelling original load balancer")
activeServiceLoadBalancerCancel[string(svc.UID)]()
<-activeServiceLoadBalancer[string(svc.UID)].Done()
log.Warn("(svcs) waiting for load balancer to finish")
}
err = sm.deleteService(svc.UID)
if err != nil {
log.Error("(svc) unable to remove", "service", svc.UID)
}
activeService[string(svc.UID)] = false
watchedService[string(svc.UID)] = false
delete(activeServiceLoadBalancer, string(svc.UID))
configuredLocalRoutes.Store(string(svc.UID), false)
}
// in theory this should never fail
}
}
}
// Architecture walkthrough: (Had to do this as this code path is making my head hurt)
// Is the service active (bool), if not then process this new service
// Does this service use an election per service?
//
if !activeService[string(svc.UID)] {
log.Debug("(svcs) has been added/modified with addresses", "service name", svc.Name, "ip", fetchServiceAddresses(svc))
activeServiceLoadBalancer[string(svc.UID)], activeServiceLoadBalancerCancel[string(svc.UID)] = context.WithCancel(ctx)
if sm.config.EnableServicesElection || // Service Election
((sm.config.EnableRoutingTable || sm.config.EnableBGP) && // Routing table mode or BGP
(!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection)) { // No leaderelection or services election
// If this load balancer Traffic Policy is "local"
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Start an endpoint watcher if we're not watching it already
if !watchedService[string(svc.UID)] {
// background the endpoint watcher
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Add Endpoint or EndpointSlices watcher
var provider epProvider
if !sm.config.EnableEndpointSlices {
provider = &endpointsProvider{label: "endpoints"}
} else {
provider = &endpointslicesProvider{label: "endpointslices"}
}
if err = sm.watchEndpoint(activeServiceLoadBalancer[string(svc.UID)], sm.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
}
}()
if (sm.config.EnableRoutingTable || sm.config.EnableBGP) && (!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection) {
go func() {
err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc)
if err != nil {
log.Error(err.Error())
}
}()
}
// We're now watching this service
watchedService[string(svc.UID)] = true
}
} else if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && (!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection) {
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
// Add Endpoint watcher
var provider epProvider
if !sm.config.EnableEndpointSlices {
provider = &endpointsProvider{label: "endpoints"}
} else {
provider = &endpointslicesProvider{label: "endpointslices"}
}
if err = sm.watchEndpoint(activeServiceLoadBalancer[string(svc.UID)], sm.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
}
}()
go func() {
err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc)
if err != nil {
log.Error(err.Error())
}
}()
} else {
go func() {
for {
select {
case <-activeServiceLoadBalancer[string(svc.UID)].Done():
log.Warn("(svcs) restartable service watcher ending", "uid", svc.UID)
return
default:
log.Info("(svcs) restartable service watcher starting", "uid", svc.UID)
err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc)
if err != nil {
log.Error(err.Error())
}
}
}
}()
}
} else {
// Increment the waitGroup before the service Func is called (Done is completed in there)
err = serviceFunc(activeServiceLoadBalancer[string(svc.UID)], svc)
if err != nil {
log.Error(err.Error())
}
}
activeService[string(svc.UID)] = true
}
case watch.Deleted:
svc, ok := event.Object.(*v1.Service)
if !ok {
return fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
if activeService[string(svc.UID)] {
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
break
}
// We can ignore this service
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Info("(svcs)ignore annotation for kube-vip", "service name", svc.Name)
break
}
isRouteConfigured, err := isRouteConfigured(svc.UID)
if err != nil {
return fmt.Errorf("error while checkig if route is configured: %w", err)
}
// If no leader election is enabled, delete routes here
if !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
sm.config.EnableRoutingTable && isRouteConfigured {
if errs := sm.clearRoutes(svc); len(errs) == 0 {
configuredLocalRoutes.Store(string(svc.UID), false)
}
}
// If this is an active service then and additional leaderElection will handle stopping
err = sm.deleteService(svc.UID)
if err != nil {
log.Error(err.Error())
}
// Calls the cancel function of the context
if activeServiceLoadBalancerCancel[string(svc.UID)] != nil {
activeServiceLoadBalancerCancel[string(svc.UID)]()
}
activeService[string(svc.UID)] = false
watchedService[string(svc.UID)] = false
}
if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && sm.config.EnableLeaderElection && !sm.config.EnableServicesElection {
if sm.config.EnableBGP {
instance := sm.findServiceInstance(svc)
for _, vip := range instance.vipConfigs {
vipCidr := fmt.Sprintf("%s/%s", vip.VIP, vip.VIPCIDR)
err = sm.bgpServer.DelHost(vipCidr)
if err != nil {
log.Error("error deleting host", "ip", vipCidr, "err", err)
}
}
} else {
sm.clearRoutes(svc)
}
}
log.Info("(svcs) deleted", "service name", svc.Name, "namespace", svc.Namespace)
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes services")
// This round trip allows us to handle unstructured status
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Error("services", "err", status)
default:
}
}
close(exitFunction)
log.Warn("Stopping watching services for type: LoadBalancer in all namespaces")
return nil
}
func (sm *Manager) lbClassFilterLegacy(svc *v1.Service) bool {
if svc == nil {
log.Info("(svcs) service is nil, ignoring")
return true
}
if svc.Spec.LoadBalancerClass != nil {
// if this isn't nil then it has been configured, check if it the kube-vip loadBalancer class
if *svc.Spec.LoadBalancerClass != sm.config.LoadBalancerClassName {
log.Info("(svcs) specified the wrong loadBalancer class", "service name", svc.Name, "lbClass", *svc.Spec.LoadBalancerClass)
return true
}
} else if sm.config.LoadBalancerClassOnly {
// if kube-vip is configured to only recognize services with kube-vip's lb class, then ignore the services without any lb class
log.Info("(svcs) kube-vip configured to only recognize services with kube-vip's lb class but the service didn't specify any loadBalancer class, ignoring", "service name", svc.Name)
return true
}
return false
}
func (sm *Manager) lbClassFilter(svc *v1.Service) bool {
if svc == nil {
log.Info("(svcs) service is nil, ignoring")
return true
}
if svc.Spec.LoadBalancerClass == nil && sm.config.LoadBalancerClassName != "" {
log.Info("(svcs) no loadBalancer class, ignoring", "service name", svc.Name, "expected lbClass", sm.config.LoadBalancerClassName)
return true
}
if svc.Spec.LoadBalancerClass == nil && sm.config.LoadBalancerClassName == "" {
return false
}
if *svc.Spec.LoadBalancerClass != sm.config.LoadBalancerClassName {
log.Info("(svcs) specified wrong loadBalancer class, ignoring", "service name", svc.Name, "wrong lbClass", *svc.Spec.LoadBalancerClass, "expected lbClass", sm.config.LoadBalancerClassName)
return true
}
return false
}
func isRouteConfigured(serviceUID types.UID) (bool, error) {
isConfigured := false
value, ok := configuredLocalRoutes.Load(string(serviceUID))
if ok {
isConfigured, ok = value.(bool)
if !ok {
return false, fmt.Errorf("error converting configuredLocalRoute item to boolean value")
}
}
return isConfigured, nil
}

View File

@@ -4,7 +4,7 @@ import (
"reflect"
"testing"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -15,7 +15,7 @@ func TestParseBgpAnnotations(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{Name: "test", Annotations: map[string]string{}},
}
bgpConfigBase := kubevip.BGPConfig{
bgpConfigBase := bgp.Config{
HoldTime: 15,
KeepaliveInterval: 5,
}
@@ -54,7 +54,7 @@ func TestParseBgpAnnotations(t *testing.T) {
t.Fatal("Parsing BGP annotations should return nil when minimum config is met")
}
bgpPeers := []kubevip.BGPPeer{
bgpPeers := []bgp.Peer{
{Address: "10.0.0.1", AS: uint32(64000), Password: "password"},
{Address: "10.0.0.2", AS: uint32(64000), Password: "password"},
{Address: "10.0.0.3", AS: uint32(64000), Password: "password"},
@@ -77,7 +77,7 @@ func TestParseNewBgpAnnotations(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{Name: "test", Annotations: map[string]string{}},
}
bgpConfigBase := kubevip.BGPConfig{
bgpConfigBase := bgp.Config{
HoldTime: 15,
KeepaliveInterval: 5,
}
@@ -99,7 +99,7 @@ func TestParseNewBgpAnnotations(t *testing.T) {
t.Fatalf("Parsing BGP annotations should return nil when minimum config is met [%v]", err)
}
bgpPeers := []kubevip.BGPPeer{
bgpPeers := []bgp.Peer{
{Address: "10.0.0.1", AS: uint32(64000), Password: "password"},
{Address: "10.0.0.2", AS: uint32(64000), Password: "password"},
{Address: "10.0.0.3", AS: uint32(64000), Password: "password"},
@@ -121,15 +121,15 @@ func Test_parseBgpAnnotations(t *testing.T) {
tests := []struct {
name string
args args
want kubevip.BGPConfig
want1 kubevip.BGPPeer
want bgp.Config
want1 bgp.Peer
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, got1, err := parseBgpAnnotations(kubevip.BGPConfig{}, tt.args.node, tt.args.prefix)
got, got1, err := parseBgpAnnotations(bgp.Config{}, tt.args.node, tt.args.prefix)
if (err != nil) != tt.wantErr {
t.Errorf("parseBgpAnnotations() error = %v, wantErr %v", err, tt.wantErr)
return

View File

@@ -1,34 +0,0 @@
package networkinterface
import (
"sync"
"github.com/vishvananda/netlink"
)
type Manager struct {
interfaces map[string]*Link
}
type Link struct {
Lock sync.Mutex
Intf netlink.Link
}
func NewManager() *Manager {
return &Manager{
interfaces: make(map[string]*Link),
}
}
func (m *Manager) Get(intf netlink.Link) *Link {
if l, ok := m.interfaces[intf.Attrs().Name]; ok {
return l
}
result := &Link{
Intf: intf,
}
m.interfaces[intf.Attrs().Name] = result
return result
}

View File

@@ -1,486 +0,0 @@
package nftables
import (
"errors"
"fmt"
"log/slog"
"net"
"strconv"
"strings"
"github.com/google/nftables"
"github.com/google/nftables/binaryutil"
"github.com/google/nftables/expr"
"github.com/kube-vip/kube-vip/pkg/utils"
"golang.org/x/sys/unix"
)
const (
NatTable = "kube_vip_%s"
SNatChain = "kube_vip_snat_%s"
)
func ApplySNAT(podIP, vipIP, service, destinationPorts string, ignoreCIDR []string, IPv6 bool) error {
conn, err := nftables.New()
if err != nil {
return err
}
var tableName string
if IPv6 {
tableName = fmt.Sprintf(NatTable, "v6")
} else {
tableName = fmt.Sprintf(NatTable, "v4")
}
// Look up the table
if t, err := FilterTable(conn, tableName, IPv6); err != nil {
if t == nil {
// If it doesn't exist then create it
slog.Debug("[egress]", "Creating Table", tableName)
conn.AddTable(GetTable(IPv6))
}
}
slog.Debug("[egress]", "Creating Chain for service", service, utils.IPv6Family, IPv6)
conn.AddChain(GetSNatChain(IPv6, service))
err = conn.Flush()
if err != nil {
return err
}
// Create our nftables rule
rule, err := CreateRule(podIP, vipIP, service, destinationPorts, ignoreCIDR, conn, IPv6)
if err != nil {
return err
}
slog.Debug("[egress]", "table", rule.Table.Name, "chain", rule.Chain.Name, "expr", rule.Exprs)
conn.AddRule(rule) // Add the rule
err = conn.Flush() // Commit the rule to nftables
if err != nil {
return err
}
return conn.CloseLasting() // Close out any remaining netlink communication
}
func DeleteSNAT(IPv6 bool, service string) error {
conn, err := nftables.New()
if err != nil {
return err
}
var chainName = fmt.Sprintf(SNatChain, service)
slog.Info("[egress]", "Looking for", chainName)
chain, err := conn.ListChain(GetTable(IPv6), chainName)
if err != nil {
return err
}
if chain != nil {
slog.Info("[egress]", "Deleting chain", chainName)
conn.DelChain(chain)
return conn.Flush()
}
return fmt.Errorf("unable to find chain [%s]", chainName)
}
func GetTable(IPv6 bool) *nftables.Table {
var tableName string
if IPv6 {
tableName = fmt.Sprintf(NatTable, "v6")
} else {
tableName = fmt.Sprintf(NatTable, "v4")
}
// Default to IPv4
table := &nftables.Table{
Family: nftables.TableFamilyIPv4,
Name: tableName,
}
// Move to IPv6 if needed
if IPv6 {
table.Family = nftables.TableFamilyIPv6
}
return table
}
func GetSNatChain(IPv6 bool, service string) *nftables.Chain {
var chainName = fmt.Sprintf(SNatChain, service)
policy := nftables.ChainPolicyAccept
return &nftables.Chain{
Name: chainName,
Table: GetTable(IPv6),
Type: nftables.ChainTypeNAT,
Hooknum: nftables.ChainHookPostrouting,
Priority: nftables.ChainPriorityNATSource,
Policy: &policy,
}
}
func FilterTable(conn *nftables.Conn, tableName string, IPv6 bool) (*nftables.Table, error) {
if IPv6 {
return conn.ListTableOfFamily(tableName, nftables.TableFamilyIPv6)
}
return conn.ListTableOfFamily(tableName, nftables.TableFamilyIPv4)
}
// ClearTable will remove the original tables and create new empty ones
func ClearTable(conn *nftables.Conn) error {
tableName := fmt.Sprintf(NatTable, "v6")
if t, err := FilterTable(conn, tableName, false); err != nil {
return err
} else if t != nil {
conn.DelTable(t)
}
// These don't return errors, so not 100% sure how to guarantee things were created
conn.AddTable(GetTable(true))
tableName = fmt.Sprintf(NatTable, "v4")
if t, err := FilterTable(conn, tableName, true); err != nil {
return err
} else if t != nil {
conn.DelTable(t)
}
// These don't return errors, so not 100% sure how to guarantee things were created
conn.AddTable(GetTable(false))
return nil
}
// Create our nftables rule
func CreateRule(podIP, vipIP, service, destinationPorts string, ignoreCIDR []string, conn *nftables.Conn, IPv6 bool) (*nftables.Rule, error) {
// Validate pod IP
if net.ParseIP(podIP) == nil {
return nil, errors.New("ip is invalid")
}
// Validate vip IP
if net.ParseIP(vipIP) == nil {
return nil, errors.New("output_ip is not a valid ip")
}
// Get the kube-vip table
table := GetTable(IPv6)
// Create our rule
rule := &nftables.Rule{
Table: table,
Exprs: []expr.Any{},
}
// Set the correct chain
rule.Chain = GetSNatChain(IPv6, service)
// Create a set for our original/source address
set := &nftables.Set{
Table: table,
Anonymous: true,
Constant: true,
KeyType: nftables.TypeIPAddr,
Interval: false,
}
if IPv6 {
set.KeyType = nftables.TypeIP6Addr
} else {
set.KeyType = nftables.TypeIPAddr
}
// Create an element using our pod IP
elements := []nftables.SetElement{}
if IPv6 {
elements = append(elements, nftables.SetElement{Key: net.ParseIP(podIP).To16()})
} else {
elements = append(elements, nftables.SetElement{Key: net.ParseIP(podIP).To4()})
}
// Add the elements to the set
err := conn.AddSet(set, elements)
if err != nil {
return nil, err
}
// Create the expression using the set
expression := []expr.Any{}
payload := &expr.Payload{
OperationType: expr.PayloadLoad,
Base: expr.PayloadBaseNetworkHeader,
DestRegister: 1,
SourceRegister: 0,
}
// Set the length of the data based upon the type of IP version being used
if IPv6 {
payload.Offset = 8
payload.Len = 16
} else {
payload.Offset = 12
payload.Len = 4
}
lookup := &expr.Lookup{
SourceRegister: 1,
DestRegister: 0,
SetID: set.ID,
}
// Add expressions
expression = append(expression, payload)
expression = append(expression, lookup)
// Add expression to the rule
rule.Exprs = append(rule.Exprs, expression...)
// If we filter on ports protocols then parse them
if destinationPorts != "" {
fixedPorts := strings.Split(destinationPorts, ",")
// Create an element using our pod IP
tcpElements := []nftables.SetElement{}
udpElements := []nftables.SetElement{}
sctpElements := []nftables.SetElement{}
tcpSet := &nftables.Set{
Anonymous: true,
Constant: true,
Table: table,
KeyType: nftables.TypeInetService,
}
udpSet := &nftables.Set{
Anonymous: true,
Constant: true,
Table: table,
KeyType: nftables.TypeInetService,
}
sctpSet := &nftables.Set{
Anonymous: true,
Constant: true,
Table: table,
KeyType: nftables.TypeInetService,
}
for _, fixedPort := range fixedPorts {
data := strings.Split(fixedPort, ":")
if len(data) == 0 {
continue
} else if len(data) == 2 { // Ensure we have two elements { proto:port }
// parse the port to a number
port, err := strconv.Atoi(data[1])
if err != nil {
slog.Error("[egress]", "unable to process port", data[1])
continue
}
// Ensure the port is within the valid range for uint16
if port < 0 || port > 65535 {
slog.Error("[egress]", "port out of range for uint16", data[1])
continue
}
switch data[0] {
case "tcp":
//nolint:gosec
tcpElements = append(tcpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
case "udp":
//nolint:gosec
udpElements = append(udpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
case "sctp":
//nolint:gosec
sctpElements = append(sctpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
default:
slog.Error("[egress]", "unknown protocol", data[0])
}
}
}
// Add TCP Ports
if len(tcpElements) != 0 {
err = conn.AddSet(tcpSet, tcpElements)
if err != nil {
return nil, err
}
expression := []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
// [ cmp eq reg 1 0x00000006 ]
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte{unix.IPPROTO_TCP},
},
// [ payload load 2b @ transport header + 2 => reg 1 ]
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseTransportHeader,
Offset: 2,
Len: 2,
},
// [ lookup reg 1 set __set%d ]
&expr.Lookup{
SourceRegister: 1,
SetName: tcpSet.Name,
SetID: tcpSet.ID,
},
}
rule.Exprs = append(rule.Exprs, expression...)
}
// Add UDP ports
if len(udpElements) != 0 {
err = conn.AddSet(udpSet, udpElements)
if err != nil {
return nil, err
}
expression := []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
// [ cmp eq reg 1 0x00000006 ]
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte{unix.IPPROTO_UDP},
},
// [ payload load 2b @ transport header + 2 => reg 1 ]
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseTransportHeader,
Offset: 2,
Len: 2,
},
// [ lookup reg 1 set __set%d ]
&expr.Lookup{
SourceRegister: 1,
SetName: udpSet.Name,
SetID: udpSet.ID,
},
}
rule.Exprs = append(rule.Exprs, expression...)
}
// Add SCTP Ports
if len(sctpElements) != 0 {
err = conn.AddSet(sctpSet, sctpElements)
if err != nil {
return nil, err
}
expression := []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
// [ cmp eq reg 1 0x00000006 ]
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte{unix.IPPROTO_SCTP},
},
// [ payload load 2b @ transport header + 2 => reg 1 ]
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseTransportHeader,
Offset: 2,
Len: 2,
},
// [ lookup reg 1 set __set%d ]
&expr.Lookup{
SourceRegister: 1,
SetName: sctpSet.Name,
SetID: sctpSet.ID,
},
}
rule.Exprs = append(rule.Exprs, expression...)
}
}
// Parse which CIDRs we will not SNAT for
for _, cidr := range ignoreCIDR {
start, end, err := nftables.NetFirstAndLastIP(cidr)
if err != nil {
return nil, err
}
expression = []expr.Any{}
payload := &expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
}
notEqualRange := &expr.Range{
Op: expr.CmpOpNeq,
Register: 1,
}
if IPv6 {
payload.Len = 16
payload.Offset = 24
notEqualRange.FromData = start.To16()
notEqualRange.ToData = end.To16()
} else {
payload.Offset = 16
payload.Len = 4
notEqualRange.FromData = start.To4()
notEqualRange.ToData = end.To4()
}
// Add expressions
expression = append(expression, payload)
expression = append(expression, notEqualRange)
// // Add expression to the rule
rule.Exprs = append(rule.Exprs, expression...)
}
// Final expression to the rule is the SNAT to the VIP address
expression = []expr.Any{}
immediate := &expr.Immediate{
Register: 1,
}
nat := &expr.NAT{
Type: expr.NATTypeSourceNAT,
RegAddrMin: 1,
RegAddrMax: 1,
RegProtoMin: 0,
RegProtoMax: 0,
Random: false,
FullyRandom: false,
Persistent: false,
Prefix: false,
}
if IPv6 {
immediate.Data = net.ParseIP(vipIP).To16()
nat.Family = unix.NFPROTO_IPV6
} else {
immediate.Data = net.ParseIP(vipIP).To4()
nat.Family = unix.NFPROTO_IPV4
}
// https://github.com/google/nftables/blob/main/nftables_test.go#L5375
// Add expressions
expression = append(expression, immediate)
expression = append(expression, nat)
rule.Exprs = append(rule.Exprs, expression...)
return rule, nil
}
// Returns a list of all chains IPv4/IPv6 in nftables
func ListChains() ([]string, error) {
chains := []string{}
conn, err := nftables.New()
if err != nil {
return nil, err
}
ipv4, err := conn.ListChainsOfTableFamily(nftables.TableFamilyIPv4)
if err != nil {
return nil, err
}
ipv6, err := conn.ListChainsOfTableFamily(nftables.TableFamilyIPv6)
if err != nil {
return nil, err
}
for x := range ipv4 {
chains = append(chains, fmt.Sprintf("Table=%s, Chain=%s", ipv4[x].Table.Name, ipv4[x].Name))
}
for x := range ipv6 {
chains = append(chains, fmt.Sprintf("Table=%s, Chain=%s", ipv6[x].Table.Name, ipv6[x].Name))
}
_ = conn.CloseLasting() // TODO: Should we ignore this error, we're not actually doing any actions with nftables
return chains, nil
}

View File

@@ -1,164 +0,0 @@
package labeler
import (
"context"
"encoding/hex"
"encoding/json"
"fmt"
"net"
"strings"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
)
const (
// labelName is the name of the label that will be added to the node
// it is prefix for the label key before "/"
labelName = "service-provided.kube-vip.io"
)
// labelOperation is the operation to perform on the node labels
type labelOperation string
// labelOperation constants
const (
labelOperationRemove labelOperation = "remove"
labelOperationAdd labelOperation = "add"
)
// NewManager creates a new Label Manager for the given node
func NewManager(nodeName string, clientSet *kubernetes.Clientset) *Manager {
return &Manager{
nodeName: nodeName,
clientSet: clientSet,
}
}
// Manager is the label Manager for the node
type Manager struct {
// nodeName is the name of the node to manage
nodeName string
// clientSet is the Kubernetes client set to use
clientSet *kubernetes.Clientset
}
// AddLabel a new label to the node
func (m *Manager) AddLabel(ctx context.Context, svc *corev1.Service) error {
log.Debug("[service] add label to node", "namespace", svc.Namespace, "name", svc.Name)
labelKey, labelValue := generateNodeLabelKeyValue(svc)
return m.patchNode(ctx, labelOperationAdd, map[string]string{labelKey: labelValue})
}
// RemoveLabel a label from the node
func (m *Manager) RemoveLabel(ctx context.Context, svc *corev1.Service) error {
log.Debug("[service] delete label from node", "namespace", svc.Namespace, "name", svc.Name)
labelKey, _ := generateNodeLabelKeyValue(svc)
return m.patchNode(ctx, labelOperationRemove, map[string]string{labelKey: ""})
}
// clean up the node labels
func (m *Manager) CleanUpLabels(timeout time.Duration) error {
log.Debug("cleaning up labels for node", "node", m.nodeName, "timeout", timeout)
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
// get the node
node, err := m.clientSet.CoreV1().Nodes().Get(ctx, m.nodeName, metav1.GetOptions{})
if err != nil {
return errors.Wrapf(err, "failed to get node %s", m.nodeName)
}
// collect all labels with the prefix to remove
labels := map[string]string{}
for k := range node.Labels {
if strings.HasPrefix(k, labelName) {
labels[k] = ""
}
}
if len(labels) == 0 {
log.Debug("no labels to remove for node", "node", m.nodeName)
return nil
}
// patch the node with the labels to remove
return m.patchNode(ctx, labelOperationRemove, labels)
}
// generateNodeLabelKeyValue generates a label key and value for the given service
func generateNodeLabelKeyValue(svc *corev1.Service) (string, string) {
addresses, _ := instance.FetchServiceAddresses(svc)
sanitized := make([]string, len(addresses))
for i, addr := range addresses {
sanitized[i] = sanitizeIPForLabel(addr)
}
return fmt.Sprintf("%s/%s.%s", labelName, svc.Name, svc.Namespace), strings.Join(sanitized, ",")
}
// Helper function to convert IPv6 hex address without colons
func sanitizeIPForLabel(addr string) string {
ip := net.ParseIP(addr)
if ip == nil || ip.To4() != nil {
return addr
}
return hex.EncodeToString(ip.To16())
}
// patchNode patches the node with the given labels
func (m *Manager) patchNode(ctx context.Context, operation labelOperation, labels map[string]string) error {
type patchStringLabel struct {
Op string `json:"op"`
Path string `json:"path"`
Value string `json:"value"`
}
patchLabels := []patchStringLabel{}
// generate the patch
for k, v := range labels {
patchLabels = append(patchLabels, patchStringLabel{
Op: string(operation),
// replace all slashes with ~1
Path: fmt.Sprintf("/metadata/labels/%s", strings.ReplaceAll(k, "/", "~1")),
Value: v,
})
}
patchData, err := json.Marshal(patchLabels)
if err != nil {
log.Debug("node patch marshaling failed", "err", err, "labels", labels, "patch", patchLabels)
return errors.Wrapf(err, "node patch marshaling failed for labels %v", labels)
}
log.Debug("patching node",
"node", m.nodeName,
"patch", string(patchData),
"operation", operation,
"labels", labels,
"clientSetNil", m.clientSet == nil)
if m.clientSet == nil {
return errors.New("kubernetes client is not initialized")
}
// patch node
node, err := m.clientSet.CoreV1().Nodes().Patch(ctx, m.nodeName, types.JSONPatchType, patchData, metav1.PatchOptions{})
if err != nil {
log.Debug("node patching failed", "err", err, "patchData", patchData)
return errors.Wrapf(err, "node patching failed with patch %s", string(patchData))
}
log.Debug("updated", "node", m.nodeName, "labels", node.Labels)
return nil
}

View File

@@ -1,27 +0,0 @@
package noop
import (
"context"
"time"
corev1 "k8s.io/api/core/v1"
)
// NewManager creates a new NoOp label manager
func NewManager() *Manager {
return &Manager{}
}
type Manager struct{}
func (m *Manager) AddLabel(_ context.Context, _ *corev1.Service) error {
return nil
}
func (m *Manager) RemoveLabel(_ context.Context, _ *corev1.Service) error {
return nil
}
func (m *Manager) CleanUpLabels(_ time.Duration) error {
return nil
}

View File

@@ -1,49 +0,0 @@
package node
import (
"context"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/node/labeler"
"github.com/kube-vip/kube-vip/pkg/node/noop"
v1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
)
// LabelManager is the interface for the node label manager
type LabelManager interface {
// AddLabel adds a label to the node for the given service
AddLabel(ctx context.Context, svc *v1.Service) error
// RemoveLabel removes the label from the node for the given service
RemoveLabel(ctx context.Context, svc *v1.Service) error
// CleanUpLabels removes all labels from the node
CleanUpLabels(timeout time.Duration) error
}
// NewManager creates a new Label Manager for the given node
// NoOp implementation is returned if node labeling is disabled, or if
// running in control plane mode, or if the client is not ready
func NewManager(config *kubevip.Config, clientSet *kubernetes.Clientset) LabelManager {
if !config.EnableNodeLabeling {
return noop.NewManager()
}
if config.EnableControlPlane {
log.Debug("Skip node labeling, control plane mode enabled")
return noop.NewManager()
}
if clientSet == nil {
log.Debug("Skip node labeling, client is not ready")
return noop.NewManager()
}
log.Debug("Node labeling enabled")
return labeler.NewManager(config.NodeName, clientSet)
}

View File

@@ -1,36 +0,0 @@
package servicecontext
import (
"context"
"sync"
)
type Context struct {
Ctx context.Context
Cancel context.CancelFunc
IsActive bool
IsWatched bool
ConfiguredNetworks sync.Map
}
func New(ctx context.Context) *Context {
svcCtx, svcCancel := context.WithCancel(ctx)
return &Context{
Ctx: svcCtx,
Cancel: svcCancel,
}
}
func (ctx *Context) HasConfiguredNetworks() bool {
cnt := 0
ctx.ConfiguredNetworks.Range(func(_ any, _ any) bool {
cnt++
return cnt < 1
})
return cnt > 0
}
func (ctx *Context) IsNetworkConfigured(ip string) bool {
_, exists := ctx.ConfiguredNetworks.Load(ip)
return exists
}

View File

@@ -1,151 +0,0 @@
package services
import (
"context"
"fmt"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/lease"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
)
// The StartServicesWatchForLeaderElection function will start a services watcher, the
func (p *Processor) StartServicesWatchForLeaderElection(ctx context.Context) error {
err := p.ServicesWatcher(ctx, p.StartServicesLeaderElection)
if err != nil {
return err
}
if p.config.EnableRoutingTable {
for _, instance := range p.ServiceInstances {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
_ = cluster.Network[i].DeleteRoute()
}
cluster.Stop()
}
}
}
log.Info("Shutting down kube-Vip")
return nil
}
// The startServicesWatchForLeaderElection function will start a services watcher, the
func (p *Processor) StartServicesLeaderElection(ctx context.Context, service *v1.Service) error {
serviceLease, _ := lease.GetName(service)
log.Info("new leader election", "service", service.Name, "namespace", service.Namespace, "lock_name", serviceLease, "host_id", p.config.NodeName)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{
Name: serviceLease,
Namespace: service.Namespace,
},
Client: p.clientSet.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: p.config.NodeName,
},
}
go func() {
// wait for the service context to end and delete the lease then
<-ctx.Done()
p.leaseMgr.Delete(service)
}()
svcCtx, err := p.getServiceContext(service.UID)
if err != nil {
return fmt.Errorf("failed to get context for service %q with UID %q: %w", service.Name, service.UID, err)
}
if svcCtx == nil {
return fmt.Errorf("failed to get context for service %q with UID %q: nil context", service.Name, service.UID)
}
svcCtx.IsActive = true
svcLease, isNew := p.leaseMgr.Add(service)
// this service is sharing lease
if !isNew {
// wait for leader election to start or context to be done
select {
case <-svcLease.Started:
case <-svcLease.Ctx.Done():
svcCtx.IsActive = false
return nil
}
<-svcLease.Started
if lease.UsesCommon(service) {
if err := p.SyncServices(ctx, service); err != nil {
log.Error("service sync", "err", err)
svcLease.Cancel()
}
// just block until context is cancelled
<-ctx.Done()
if svcCtx.IsActive {
if err := p.deleteService(service.UID); err != nil {
log.Error("service deletion", "err", err)
}
}
}
// wait for leaderelection to be finished
<-svcLease.Ctx.Done()
// Mark this service is inactive
svcCtx.IsActive = false
return nil
}
// start the leader election code loop
leaderelection.RunOrDie(svcLease.Ctx, leaderelection.LeaderElectionConfig{
Lock: lock,
// IMPORTANT: you MUST ensure that any code you have that
// is protected by the lease must terminate **before**
// you call cancel. Otherwise, you could have a background
// loop still running and another process could
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(p.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(p.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(p.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
// Mark this service as active (as we've started leading)
// we run this in background as it's blocking
if err := p.SyncServices(ctx, service); err != nil {
log.Error("service sync", "err", err)
svcLease.Cancel()
}
close(svcLease.Started)
},
OnStoppedLeading: func() {
// we can do cleanup here
log.Info("leadership lost", "service", service.Name, "leader", p.config.NodeName)
if svcCtx.IsActive {
if err := p.deleteService(service.UID); err != nil {
log.Error("service deletion", "err", err)
}
}
// Mark this service is inactive
svcCtx.IsActive = false
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if identity == p.config.NodeName {
// I just got the lock
return
}
log.Info("new leader", "leader", identity)
},
},
})
log.Info("stopping leader election", "service", service.Name)
return nil
}

View File

@@ -1,357 +0,0 @@
package services
import (
"context"
"fmt"
log "log/slog"
"reflect"
"sync"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/lease"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/prometheus/client_golang/prometheus"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
)
type Processor struct {
config *kubevip.Config
lbClassFilter func(svc *v1.Service, config *kubevip.Config) bool
svcMap sync.Map
// Keeps track of all running instances
ServiceInstances []*instance.Instance
mutex sync.Mutex
bgpServer *bgp.Server
clientSet *kubernetes.Clientset
rwClientSet *kubernetes.Clientset
shutdownChan chan struct{}
// This is a prometheus counter used to count the number of events received
// from the service watcher
CountServiceWatchEvent *prometheus.CounterVec
intfMgr *networkinterface.Manager
arpMgr *arp.Manager
leaseMgr *lease.Manager
// nodeLabelManager is the manager for the node labels
nodeLabelManager labelManager
}
// labelManager is the interface for the node label manager to add/remove labels
type labelManager interface {
AddLabel(ctx context.Context, svc *v1.Service) error
RemoveLabel(ctx context.Context, svc *v1.Service) error
}
func NewServicesProcessor(config *kubevip.Config, bgpServer *bgp.Server,
clientSet *kubernetes.Clientset, rwClientSet *kubernetes.Clientset, shutdownChan chan struct{},
intfMgr *networkinterface.Manager, arpMgr *arp.Manager, nodeLabelManager labelManager) *Processor {
lbClassFilterFunc := lbClassFilter
if config.LoadBalancerClassLegacyHandling {
lbClassFilterFunc = lbClassFilterLegacy
}
return &Processor{
config: config,
lbClassFilter: lbClassFilterFunc,
ServiceInstances: []*instance.Instance{},
bgpServer: bgpServer,
clientSet: clientSet,
rwClientSet: rwClientSet,
shutdownChan: shutdownChan,
CountServiceWatchEvent: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "kube_vip",
Subsystem: "manager",
Name: "all_services_events",
Help: "Count all events fired by the service watcher categorised by event type",
}, []string{"type"}),
intfMgr: intfMgr,
arpMgr: arpMgr,
leaseMgr: lease.NewManager(),
nodeLabelManager: nodeLabelManager,
}
}
func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceFunc func(context.Context, *v1.Service) error) (bool, error) {
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
svc, ok := event.Object.(*v1.Service)
if !ok {
return false, fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
return true, nil
}
// Check if we ignore this service
if svc.Annotations[kubevip.LoadbalancerIgnore] == "true" {
log.Info("ignore annotation for kube-vip", "service name", svc.Name)
return true, nil
}
// Check the loadBalancer class
if p.lbClassFilter(svc, p.config) {
return true, nil
}
svcAddresses, svcHostnames := instance.FetchServiceAddresses(svc)
// We only care about LoadBalancer services that have been allocated an address
if len(svcAddresses) <= 0 {
return true, nil
}
_, usesCommonLease := svc.Annotations[kubevip.ServiceLease]
if usesCommonLease && svc.Spec.ExternalTrafficPolicy != v1.ServiceExternalTrafficPolicyTypeCluster {
return false, fmt.Errorf("annotation %q cannot be used with service traffic policy other than %q",
kubevip.ServiceLease, v1.ServiceExternalTrafficPolicyTypeCluster)
}
svcCtx, err := p.getServiceContext(svc.UID)
if err != nil {
return false, fmt.Errorf("failed to get service context: %w", err)
}
// The modified event should only be triggered if the service has been modified (i.e. moved somewhere else)
if event.Type == watch.Modified {
i := instance.FindServiceInstance(svc, p.ServiceInstances)
shouldGarbageCollect := true
if i != nil {
originalServiceAddresses, originalServiceHostnames := instance.FetchServiceAddresses(i.ServiceSnapshot)
shouldGarbageCollect =
// Service addresses changed
!reflect.DeepEqual(originalServiceAddresses, svcAddresses) ||
// Service hostnames changed
!reflect.DeepEqual(originalServiceHostnames, svcHostnames) ||
// ExternalTrafficPolicy changed
svc.Spec.ExternalTrafficPolicy != i.ServiceSnapshot.Spec.ExternalTrafficPolicy
}
if shouldGarbageCollect {
for _, addr := range svcAddresses {
// log.Debugf("(svcs) Retrieving local addresses, to ensure that this modified address doesn't exist: %s", addr)
f, err := vip.GarbageCollect(p.config.Interface, addr, p.intfMgr)
if err != nil {
log.Error("(svcs) cleaning existing address error", "err", err)
}
if f {
log.Warn("(svcs) already found existing config", "address", addr, "adapter", p.config.Interface)
}
}
// This service has been modified, but it was also active.
if svcCtx != nil && svcCtx.IsActive {
log.Warn("(svcs) The load balancer has changed, cancelling original load balancer")
//Set it to inactive
svcCtx.IsActive = false
svcCtx.Cancel()
log.Warn("(svcs) waiting for load balancer to finish")
<-svcCtx.Ctx.Done()
if err := p.deleteService(svc.UID); err != nil {
log.Error("(svc) unable to remove", "service", svc.UID)
}
// in theory this should never fail
p.svcMap.Delete(svc.UID)
// Reset the the svcCtx when it was garbage collected
// As the next function will create a new context when nil
svcCtx = nil
}
}
}
// Architecture walkthrough: (Had to do this as this code path is making my head hurt)
// Is the service active (bool), if not then process this new service
// Does this service use an election per service?
//
if svcCtx == nil || svcCtx != nil && !svcCtx.IsActive {
ips, hostnames := instance.FetchServiceAddresses(svc)
log.Debug("(svcs) has been added/modified with addresses", "service name", svc.Name, "ips", ips, "hostnames", hostnames)
if svcCtx == nil {
svcCtx = servicecontext.New(ctx)
p.svcMap.Store(svc.UID, svcCtx)
}
if p.config.EnableServicesElection || // Service Election
((p.config.EnableRoutingTable || p.config.EnableBGP) && // Routing table mode or BGP
(!p.config.EnableLeaderElection && !p.config.EnableServicesElection)) { // No leaderelection or services election
// If this load balancer Traffic Policy is "local"
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Start an endpoint watcher if we're not watching it already
if !svcCtx.IsWatched {
// background the endpoint watcher
if (p.config.EnableRoutingTable || p.config.EnableBGP) && (!p.config.EnableLeaderElection && !p.config.EnableServicesElection) {
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Add Endpoint or EndpointSlices watcher
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
} else {
provider = providers.NewEndpointslices()
}
if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
}
}()
// We're now watching this service
svcCtx.IsWatched = true
}
} else if (p.config.EnableBGP || p.config.EnableRoutingTable) && (!p.config.EnableLeaderElection && !p.config.EnableServicesElection) {
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
// Add Endpoint watcher
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
} else {
provider = providers.NewEndpointslices()
}
if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
}
}()
// We're now watching this service
svcCtx.IsWatched = true
} else {
go func() {
for {
select {
case <-svcCtx.Ctx.Done():
log.Warn("(svcs) restartable service watcher ending", "uid", svc.UID)
return
default:
log.Info("(svcs) restartable service watcher starting", "uid", svc.UID)
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
}
}()
}
} else {
// Increment the waitGroup before the service Func is called (Done is completed in there)
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
svcCtx.IsActive = true
}
return false, nil
}
func (p *Processor) Delete(event watch.Event) (bool, error) {
svc, ok := event.Object.(*v1.Service)
if !ok {
return false, fmt.Errorf("(svcs) unable to parse Kubernetes services from API watcher")
}
svcCtx, err := p.getServiceContext(svc.UID)
if err != nil {
return false, fmt.Errorf("(svcs) unable to get context: %w", err)
}
if svcCtx != nil {
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
return true, nil
}
// We can ignore this service
if svc.Annotations[kubevip.LoadbalancerIgnore] == "true" {
log.Info("(svcs) ignore annotation for kube-vip", "service name", svc.Name)
return true, nil
}
// If no leader election is enabled, delete routes here
if !p.config.EnableLeaderElection && !p.config.EnableServicesElection &&
p.config.EnableRoutingTable && svcCtx.HasConfiguredNetworks() {
if errs := endpoints.ClearRoutes(svc, &p.ServiceInstances); len(errs) == 0 {
svcCtx.ConfiguredNetworks.Clear()
}
}
// If this is an active service then and additional leaderElection will handle stopping
err = p.deleteService(svc.UID)
if err != nil {
log.Error(err.Error())
}
// Calls the cancel function of the context
log.Warn("(svcs) The load balancer was deleted, cancelling context")
svcCtx.IsActive = false
svcCtx.Cancel()
log.Warn("(svcs) waiting for load balancer to finish")
<-svcCtx.Ctx.Done()
p.svcMap.Delete(svc.UID)
}
log.Info("(svcs) deleted", "service name", svc.Name, "namespace", svc.Namespace)
return true, nil
}
func (p *Processor) Stop() {
for _, instance := range p.ServiceInstances {
for _, cluster := range instance.Clusters {
cluster.Stop()
}
}
}
func (p *Processor) getServiceContext(uid types.UID) (*servicecontext.Context, error) {
svcCtx, ok := p.svcMap.Load(uid)
if !ok {
return nil, nil
}
ctx, ok := svcCtx.(*servicecontext.Context)
if !ok {
return nil, fmt.Errorf("failed to cast service context pointer - UID: %s", uid)
}
return ctx, nil
}
func (p *Processor) CountRouteReferences(route *netlink.Route) int {
return endpoints.CountRouteReferences(route, &p.ServiceInstances)
}

View File

@@ -1,550 +0,0 @@
package services
import (
"context"
"fmt"
"slices"
"strings"
"time"
log "log/slog"
"github.com/google/go-cmp/cmp"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
"github.com/kube-vip/kube-vip/pkg/egress"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/upnp"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/vip"
)
type ServiceInstanceAction string
const (
ActionDelete ServiceInstanceAction = "delete"
ActionAdd ServiceInstanceAction = "add"
ActionNone ServiceInstanceAction = "none"
)
func (p *Processor) SyncServices(ctx context.Context, svc *v1.Service) error {
log.Debug("[STARTING] Service Sync", "namespace", svc.Namespace, "name", svc.Name)
// Iterate through the synchronising services
action := p.getServiceInstanceAction(svc)
switch action {
case ActionDelete:
// remove the label from the node before deleting the service
if err := p.nodeLabelManager.RemoveLabel(ctx, svc); err != nil {
return fmt.Errorf("error removing label from node: %w", err)
}
log.Debug("[service] delete", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
if err := p.deleteService(svc.UID); err != nil {
return fmt.Errorf("error deleting service %s/%s: %w", svc.Namespace, svc.Name, err)
}
case ActionAdd:
log.Debug("[service] add", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
if err := p.addService(ctx, svc); err != nil {
return fmt.Errorf("error adding service %s/%s: %w", svc.Namespace, svc.Name, err)
}
// add the label to the node after adding the service
if err := p.nodeLabelManager.AddLabel(ctx, svc); err != nil {
return fmt.Errorf("error adding label to node: %w", err)
}
case ActionNone:
log.Debug("[service] no action", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
}
log.Debug("[FINISHED] Service Sync", "namespace", svc.Namespace, "name", svc.Name, "uid", svc.UID)
return nil
}
func (p *Processor) getServiceInstanceAction(svc *v1.Service) ServiceInstanceAction {
// protect against multiple calls
// get the annotations or legacy values from manual configuration
addresses, hostnames := instance.FetchServiceAddresses(svc)
// get the status information of the LB Service
statusAddresses, _ := instance.FetchLoadBalancerIngress(svc)
p.mutex.Lock()
defer p.mutex.Unlock()
for _, instance := range p.ServiceInstances {
if instance != nil && instance.ServiceSnapshot.UID == svc.UID {
for _, address := range addresses {
// handle the case where the service instance needs to be deleted
if instance.IsDHCP {
if address != "0.0.0.0" && address != "::" {
return ActionDelete
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(statusAddresses, instance.DHCPInterfaceIP) {
return ActionDelete
}
}
if !instance.IsDHCP {
if address == "0.0.0.0" || address == "::" {
return ActionDelete
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !slices.Contains(statusAddresses, address) {
return ActionDelete
}
}
if len(svc.Status.LoadBalancer.Ingress) > 0 && !comparePortsAndPortStatuses(svc) {
return ActionDelete
}
}
// If we reach here, it means the service instance matches the service UID and is not a DHCP service, so we can return "no action"
return ActionNone
}
}
if len(addresses) > 0 || len(hostnames) > 0 {
log.Debug("no matching service instance found", "service", svc.Name, "namespace", svc.Namespace, "addresses", addresses, "hostnames", hostnames)
return ActionAdd // If no matching instance is found, we need to add a new service instance
}
return ActionNone
}
func comparePortsAndPortStatuses(svc *v1.Service) bool {
if len(svc.Status.LoadBalancer.Ingress) == 0 {
return false
}
portsStatus := svc.Status.LoadBalancer.Ingress[0].Ports
if len(portsStatus) != len(svc.Spec.Ports) {
return false
}
for i, portSpec := range svc.Spec.Ports {
if portsStatus[i].Port != portSpec.Port || portsStatus[i].Protocol != portSpec.Protocol {
return false
}
}
return true
}
func (p *Processor) addService(ctx context.Context, svc *v1.Service) error {
// protect against addService while reading
p.mutex.Lock()
defer p.mutex.Unlock()
startTime := time.Now()
newService, err := instance.NewInstance(svc, p.config, p.intfMgr, p.arpMgr)
if err != nil {
return err
}
for x := range newService.VIPConfigs {
log.Debug("starting loadbalancer for service", "name", svc.Name, "namespace", svc.Namespace)
newService.Clusters[x].StartLoadBalancerService(ctx, newService.VIPConfigs[x], p.bgpServer, svc.Name, p.CountRouteReferences)
}
p.upnpMap(ctx, newService)
if newService.IsDHCP && len(newService.VIPConfigs) == 1 {
go func() {
for ip := range newService.DHCPClient.IPChannel() {
log.Debug("IP changed", "ip", ip)
newService.VIPConfigs[0].VIP = ip
newService.DHCPInterfaceIP = ip
if !p.config.DisableServiceUpdates {
if err := p.updateStatus(newService); err != nil {
log.Warn("updating svc", "err", err)
}
}
}
log.Debug("IP update channel closed, stopping")
}()
}
p.ServiceInstances = append(p.ServiceInstances, newService)
if !p.config.DisableServiceUpdates {
log.Debug("[service] update", "namespace", newService.ServiceSnapshot.Namespace, "name", newService.ServiceSnapshot.Name)
if err := p.updateStatus(newService); err != nil {
log.Error("[service] updating status", "namespace", newService.ServiceSnapshot.Namespace, "name", newService.ServiceSnapshot.Name, "err", err)
}
}
serviceIPs, _ := instance.FetchServiceAddresses(svc)
// Check if we need to flush any conntrack connections (due to some dangling conntrack connections)
if svc.Annotations[kubevip.FlushContrack] == "true" {
log.Debug("[service] Flushing conntrack rules", "service", svc.Name, "namespace", svc.Namespace)
for _, serviceIP := range serviceIPs {
err = vip.DeleteExistingSessions(serviceIP, false, svc.Annotations[kubevip.EgressDestinationPorts], svc.Annotations[kubevip.EgressSourcePorts])
if err != nil {
log.Error("[service] flushing any remaining egress connections", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
err = vip.DeleteExistingSessions(serviceIP, true, svc.Annotations[kubevip.EgressDestinationPorts], svc.Annotations[kubevip.EgressSourcePorts])
if err != nil {
log.Error("[service] flushing any remaining ingress connections", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
// Check if egress is enabled on the service, if so we'll need to configure some rules
if svc.Annotations[kubevip.Egress] == "true" && len(serviceIPs) > 0 {
log.Debug("[service] enabling egress", "service", svc.Name, "namespace", svc.Namespace)
// If we'er not using NFtables, then ensure that the correct iptables modules are loaded
if p.config.EgressWithNftables {
// Ensure that kernel modules are loaded and report back missing modules.
err = p.nftablesCheck()
if err != nil {
log.Warn("[service] configuring nft egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
} else {
// Ensure that kernel modules are loaded and report back missing modules.
err = p.iptablesCheck()
if err != nil {
log.Warn("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
var podIP string
errList := []error{}
// Should egress be IPv6
if svc.Annotations[kubevip.EgressIPv6] == "true" {
// Does the service have an active IPv6 endpoint
if svc.Annotations[kubevip.ActiveEndpointIPv6] != "" {
for _, serviceIP := range serviceIPs {
if !p.config.EnableEndpoints && utils.IsIPv6(serviceIP) {
podIP = svc.Annotations[kubevip.ActiveEndpointIPv6]
err = p.configureEgress(serviceIP, podIP, svc.Namespace, string(svc.UID), svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Warn("[service] configuring egress IPv6", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
}
} else if svc.Annotations[kubevip.ActiveEndpoint] != "" { // Not expected to be IPv6, so should be an IPv4 address
for _, serviceIP := range serviceIPs {
podIPs := svc.Annotations[kubevip.ActiveEndpoint]
if !p.config.EnableEndpoints && utils.IsIPv6(serviceIP) {
podIPs = svc.Annotations[kubevip.ActiveEndpointIPv6]
}
err = p.configureEgress(serviceIP, podIPs, svc.Namespace, string(svc.UID), svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Warn("[service] configuring egress IPv4", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
if len(errList) == 0 {
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
} else {
provider = providers.NewEndpointslices()
}
err = provider.UpdateServiceAnnotation(svc.Annotations[kubevip.ActiveEndpoint], svc.Annotations[kubevip.ActiveEndpointIPv6], svc, p.clientSet)
if err != nil {
log.Warn("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
}
}
finishTime := time.Since(startTime)
log.Info("[service]", "service", svc.Name, "namespace", svc.Namespace, "synchronised in", fmt.Sprintf("%dms", finishTime.Milliseconds()))
return nil
}
func (p *Processor) deleteService(uid types.UID) error {
// protect multiple calls
p.mutex.Lock()
defer p.mutex.Unlock()
var updatedInstances []*instance.Instance
var serviceInstance *instance.Instance
found := false
for x := range p.ServiceInstances {
log.Debug("[service] lookup", "target UID", uid, "found UID ", p.ServiceInstances[x].ServiceSnapshot.UID, "name", p.ServiceInstances[x].ServiceSnapshot.Name, "namespace", p.ServiceInstances[x].ServiceSnapshot.Namespace)
// Add the running services to the new array
if p.ServiceInstances[x].ServiceSnapshot.UID != uid {
updatedInstances = append(updatedInstances, p.ServiceInstances[x])
} else {
// Flip the found when we match
found = true
serviceInstance = p.ServiceInstances[x]
}
}
// If we've been through all services and not found the correct one then error
if !found {
// TODO: - fix UX
// return fmt.Errorf("unable to find/stop service [%s]", uid)
log.Warn("unable to find/stop service", "uid", uid)
return nil
}
for _, c := range serviceInstance.Clusters {
for n := range c.Network {
c.Network[n].SetHasEndpoints(false)
}
}
// Determine if this this VIP is shared with other loadbalancers
shared := false
vipSet := make(map[string]interface{})
for x := range updatedInstances {
vips, _ := instance.FetchServiceAddresses(updatedInstances[x].ServiceSnapshot)
for _, vip := range vips { //updatedInstances[x].ServiceSnapshot.Spec.LoadBalancerIP {
vipSet[vip] = nil
}
}
vips, _ := instance.FetchServiceAddresses(serviceInstance.ServiceSnapshot)
for _, vip := range vips {
if _, found := vipSet[vip]; found {
shared = true
}
}
if !shared {
for x := range serviceInstance.Clusters {
serviceInstance.Clusters[x].Stop()
}
if serviceInstance.IsDHCP {
serviceInstance.DHCPClient.Stop()
macvlan, err := netlink.LinkByName(serviceInstance.DHCPInterface)
if err != nil {
return fmt.Errorf("[service] error finding VIP Interface: %v", err)
}
err = netlink.LinkDel(macvlan)
if err != nil {
return fmt.Errorf("[service] error deleting DHCP Link : %v", err)
}
}
if p.config.EnableBGP {
endpoints.ClearBGPHostsByInstance(serviceInstance, p.bgpServer)
}
if p.config.EnableRoutingTable && (p.config.EnableLeaderElection || p.config.EnableServicesElection) {
if errs := endpoints.ClearRoutesByInstance(serviceInstance.ServiceSnapshot, serviceInstance, &p.ServiceInstances); len(errs) > 0 {
for _, err := range errs {
log.Error("unable to clear routes", "err", err)
}
}
}
// We will need to tear down the egress
if serviceInstance.ServiceSnapshot.Annotations[kubevip.Egress] == "true" {
if serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint] != "" {
log.Info("[service] egress re-write enabled", "service", serviceInstance.ServiceSnapshot.Name)
err := egress.Teardown(serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint], serviceInstance.ServiceSnapshot.Spec.LoadBalancerIP, serviceInstance.ServiceSnapshot.Namespace, string(serviceInstance.ServiceSnapshot.UID), serviceInstance.ServiceSnapshot.Annotations, p.config.EgressWithNftables)
if err != nil {
log.Error("[service] egress teardown", "err", err)
}
}
}
}
// Update the service array
p.ServiceInstances = updatedInstances
log.Info("Removed instance from manager", "uid", uid, "name", serviceInstance.ServiceSnapshot.Name, "remaining advertised services", len(p.ServiceInstances))
return nil
}
// Set up UPNP forwards for a service
// We first try to use the more modern Pinhole API introduced in UPNPv2 and fall back to UPNPv2 Port Forwarding if no forward was successful
func (p *Processor) upnpMap(ctx context.Context, s *instance.Instance) {
if !isUPNPEnabled(s.ServiceSnapshot) {
// Skip services missing the annotation
return
}
if !p.config.EnableUPNP {
log.Warn("[UPNP] Found kube-vip.io/forwardUPNP on service while UPNP forwarding is disabled in the kube-vip config. Not forwarding", "service", s.ServiceSnapshot.Name)
return
}
// If upnp is enabled then update the gateway/router with the address
// TODO - check if this implementation for dualstack is correct
gateways := upnp.GetGatewayClients(ctx)
// Reset Gateway IPs to remove stale addresses
s.UPNPGatewayIPs = make([]string, 0)
vips, _ := instance.FetchServiceAddresses(s.ServiceSnapshot)
for _, vip := range vips {
for _, port := range s.ServiceSnapshot.Spec.Ports {
for _, gw := range gateways {
forwardSucessful := false
if gw.WANIPv6FirewallControlClient != nil {
log.Info("[UPNP] Adding map", "vip", vip, "port", port.Port, "service", s.ServiceSnapshot.Name, "gateway", gw.WANIPv6FirewallControlClient.Location)
pinholeID, pinholeErr := gw.WANIPv6FirewallControlClient.AddPinholeCtx(ctx, "0.0.0.0", uint16(port.Port), vip, uint16(port.Port), upnp.MapProtocolToIANA(string(port.Protocol)), 3600) //nolint TODO
if pinholeErr == nil {
forwardSucessful = true
log.Info("[UPNP] Service should be accessible externally", "port", port.Port, "pinhold ID", pinholeID)
} else {
//TODO: Cleanup
log.Error("[UPNP] Unable to map port to gateway using Pinhole API", "err", pinholeErr.Error())
}
}
// Fallback to PortForward
if !forwardSucessful {
log.Info("[UPNP] Adding map", "vip", vip, "port", port.Port, "service", s.ServiceSnapshot.Name)
portMappingErr := gw.ConnectionClient.AddPortMapping("0.0.0.0", uint16(port.Port), strings.ToUpper(string(port.Protocol)), uint16(port.Port), vip, true, s.ServiceSnapshot.Name, 3600) //nolint TODO
if portMappingErr == nil {
ip, err := gw.ConnectionClient.GetExternalIPAddress()
if err != nil {
// Log the error but continue on the off chance the mapping was successful
log.Error("[UPNP] Unable to get external IP address from gateway", "port", port.Port, "err", err)
} else {
log.Info("[UPNP] Service should be accessible externally", "port", port.Port, "externalip", ip)
}
forwardSucessful = true
} else {
//TODO: Cleanup
log.Error("[UPNP] Unable to map port to gateway using PortForward API", "err", portMappingErr.Error())
}
}
if forwardSucessful {
ip, err := gw.ConnectionClient.GetExternalIPAddress()
if err == nil {
s.UPNPGatewayIPs = append(s.UPNPGatewayIPs, ip)
}
}
}
}
}
// Remove duplicate IPs
slices.Sort(s.UPNPGatewayIPs)
s.UPNPGatewayIPs = slices.Compact(s.UPNPGatewayIPs)
}
func (p *Processor) updateStatus(i *instance.Instance) error {
// let's retry status update every 10ms for 30s
retryConfig := wait.Backoff{
Steps: 3000,
Duration: 10 * time.Millisecond,
Factor: 0,
Jitter: 0.1,
}
// will retry for every error encountered, TODO: should a list of errors that will trigger retry be specified?
err := retry.OnError(retryConfig, func(error) bool { return true }, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := p.clientSet.CoreV1().Services(i.ServiceSnapshot.Namespace).Get(context.TODO(), i.ServiceSnapshot.Name, metav1.GetOptions{})
if err != nil {
return err
}
currentServiceCopy := currentService.DeepCopy()
if currentServiceCopy.Annotations == nil {
currentServiceCopy.Annotations = make(map[string]string)
}
// If we're using ARP then we can only broadcast the VIP from one place, also useful for other software when running BGP, add an annotation to the service
if p.config.EnableARP || p.config.EnableBGP {
// Add the current host
currentServiceCopy.Annotations[kubevip.VipHost] = p.config.NodeName
}
if i.DHCPInterfaceHwaddr != "" || i.DHCPInterfaceIP != "" {
currentServiceCopy.Annotations[kubevip.HwAddrKey] = i.DHCPInterfaceHwaddr
currentServiceCopy.Annotations[kubevip.RequestedIP] = i.DHCPInterfaceIP
}
if currentService.Annotations["development.kube-vip.io/synthetic-api-server-error-on-update"] == "true" {
log.Error("(Synthetic error ) updating Spec", "service", i.ServiceSnapshot.Name, "err", err)
return fmt.Errorf("(Synthetic) simulating api server errors")
}
if !cmp.Equal(currentService, currentServiceCopy) {
currentService, err = p.clientSet.CoreV1().Services(currentServiceCopy.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("updating Spec", "service", i.ServiceSnapshot.Name, "err", err)
return err
}
}
ports := make([]v1.PortStatus, 0, len(i.ServiceSnapshot.Spec.Ports))
for _, port := range i.ServiceSnapshot.Spec.Ports {
ports = append(ports, v1.PortStatus{
Port: port.Port,
Protocol: port.Protocol,
})
}
ingresses := []v1.LoadBalancerIngress{}
for _, c := range i.VIPConfigs {
if !utils.IsIP(c.VIP) {
ips, err := utils.LookupHost(c.VIP, p.config.DNSMode)
if err != nil {
return err
}
for _, ip := range ips {
i := v1.LoadBalancerIngress{
IP: ip,
Ports: ports,
}
ingresses = append(ingresses, i)
}
} else {
i := v1.LoadBalancerIngress{
IP: c.VIP,
Ports: ports,
}
ingresses = append(ingresses, i)
}
if isUPNPEnabled(currentService) {
for _, ip := range i.UPNPGatewayIPs {
i := v1.LoadBalancerIngress{
IP: ip,
Ports: ports,
}
ingresses = append(ingresses, i)
}
}
}
if !cmp.Equal(currentService.Status.LoadBalancer.Ingress, ingresses) {
currentService.Status.LoadBalancer.Ingress = ingresses
_, err = p.clientSet.CoreV1().Services(currentService.Namespace).UpdateStatus(context.TODO(), currentService, metav1.UpdateOptions{})
if err != nil && !apierrors.IsInvalid(err) {
log.Error("updating Service", "namespace", i.ServiceSnapshot.Namespace, "name", i.ServiceSnapshot.Name, "err", err)
return err
}
}
return nil
})
return err
}
func isUPNPEnabled(s *v1.Service) bool {
return metav1.HasAnnotation(s.ObjectMeta, kubevip.UpnpEnabled) && s.Annotations[kubevip.UpnpEnabled] == "true"
}
// Refresh UPNP Port Forwards for all Service Instances registered in the processor
func (p *Processor) RefreshUPNPForwards() {
log.Info("Starting UPNP Port Refresher")
for {
time.Sleep(300 * time.Second)
log.Info("[UPNP] Refreshing Instances", "number of instances", len(p.ServiceInstances))
for i := range p.ServiceInstances {
p.upnpMap(context.TODO(), p.ServiceInstances[i])
if err := p.updateStatus(p.ServiceInstances[i]); err != nil {
log.Warn("[UPNP] Error updating service", "ip", p.ServiceInstances[i].ServiceSnapshot.Name, "err", err)
}
}
}
}

View File

@@ -1,94 +0,0 @@
package services
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/watch"
)
func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, service *v1.Service, provider providers.Provider) error {
log.Info("watching", "provider", provider.GetLabel(), "service_name", service.Name, "namespace", service.Namespace)
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
leaderCtx, cancel := context.WithCancel(svcCtx.Ctx)
defer cancel()
var leaderElectionActive bool
rw, err := provider.CreateRetryWatcher(leaderCtx, p.rwClientSet, service)
if err != nil {
return fmt.Errorf("[%s] error watching endpoints: %w", provider.GetLabel(), err)
}
exitFunction := make(chan struct{})
go func() {
select {
case <-svcCtx.Ctx.Done():
log.Debug("context cancelled", "provider", provider.GetLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-p.shutdownChan:
log.Debug("shutdown called", "provider", provider.GetLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-exitFunction:
log.Debug("function ending", "provider", provider.GetLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
}
}()
ch := rw.ResultChan()
epProcessor := endpoints.NewEndpointProcessor(p.config, provider, p.bgpServer, &p.ServiceInstances)
var lastKnownGoodEndpoint string
for event := range ch {
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
restart, err := epProcessor.AddOrModify(svcCtx, event, &lastKnownGoodEndpoint, service, id, &leaderElectionActive, p.StartServicesLeaderElection, &leaderCtx, &cancel)
if restart {
continue
} else if err != nil {
return fmt.Errorf("[%s] error while processing add/modify event: %w", provider.GetLabel(), err)
}
case watch.Deleted:
if err := epProcessor.Delete(service, id); err != nil {
return fmt.Errorf("[%s] error while processing delete event: %w", provider.GetLabel(), err)
}
// Close the goroutine that will end the retry watcher, then exit the endpoint watcher function
close(exitFunction)
log.Info("stopping watching", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil
case watch.Error:
errObject := apierrors.FromObject(event.Object)
statusErr, _ := errObject.(*apierrors.StatusError)
log.Error("watch error", "provider", provider.GetLabel(), "err", statusErr)
}
}
close(exitFunction)
log.Info("stopping watching", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil //nolint:govet
}

View File

@@ -1,183 +0,0 @@
package services
import (
"context"
"fmt"
log "log/slog"
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/trafficmirror"
"github.com/prometheus/client_golang/prometheus"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
)
// This function handles the watching of a services endpoints and updates a load balancers endpoint configurations accordingly
func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc func(context.Context, *v1.Service) error) error {
// first start port mirroring if enabled
if err := p.startTrafficMirroringIfEnabled(); err != nil {
return err
}
defer func() {
// clean up traffic mirror related config
err := p.stopTrafficMirroringIfEnabled()
if err != nil {
log.Error("Stopping traffic mirroring", "err", err)
}
}()
if p.config.ServiceNamespace == "" {
// v1.NamespaceAll is actually "", but we'll stay with the const in case things change upstream
p.config.ServiceNamespace = v1.NamespaceAll
log.Info("(svcs) starting services watcher for all namespaces")
} else {
log.Info("(svcs) starting services watcher", "namespace", p.config.ServiceNamespace)
}
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return p.rwClientSet.CoreV1().Services(p.config.ServiceNamespace).Watch(ctx, metav1.ListOptions{})
},
})
if err != nil {
return fmt.Errorf("error creating services watcher: %s", err.Error())
}
exitFunction := make(chan struct{})
go func() {
select {
case <-p.shutdownChan:
log.Debug("(svcs) shutdown called")
// Stop the retry watcher
rw.Stop()
return
case <-exitFunction:
log.Debug("(svcs) function ending")
// Stop the retry watcher
rw.Stop()
return
}
}()
ch := rw.ResultChan()
// Used for tracking an active endpoint / pod
for event := range ch {
p.CountServiceWatchEvent.With(prometheus.Labels{"type": string(event.Type)}).Add(1)
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
restart, err := p.AddOrModify(ctx, event, serviceFunc)
if restart {
break
}
if err != nil {
return fmt.Errorf("add/modify service error: %w", err)
}
case watch.Deleted:
restart, err := p.Delete(event)
if restart {
break
}
if err != nil {
return fmt.Errorf("delete service error: %w", err)
}
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes services")
// This round trip allows us to handle unstructured status
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Error("services", "err", status)
default:
}
}
close(exitFunction)
log.Warn("Stopping watching services for type: LoadBalancer in all namespaces")
return nil
}
func lbClassFilterLegacy(svc *v1.Service, config *kubevip.Config) bool {
if svc == nil {
log.Info("(svcs) service is nil, ignoring")
return true
}
if svc.Spec.LoadBalancerClass != nil {
// if this isn't nil then it has been configured, check if it the kube-vip loadBalancer class
if *svc.Spec.LoadBalancerClass != config.LoadBalancerClassName {
log.Info("(svcs) specified the wrong loadBalancer class", "service name", svc.Name, "lbClass", *svc.Spec.LoadBalancerClass)
return true
}
} else if config.LoadBalancerClassOnly {
// if kube-vip is configured to only recognize services with kube-vip's lb class, then ignore the services without any lb class
log.Info("(svcs) kube-vip configured to only recognize services with kube-vip's lb class but the service didn't specify any loadBalancer class, ignoring", "service name", svc.Name)
return true
}
return false
}
func lbClassFilter(svc *v1.Service, config *kubevip.Config) bool {
if svc == nil {
log.Info("(svcs) service is nil, ignoring")
return true
}
if svc.Spec.LoadBalancerClass == nil && config.LoadBalancerClassName != "" {
log.Info("(svcs) no loadBalancer class, ignoring", "service name", svc.Name, "expected lbClass", config.LoadBalancerClassName)
return true
}
if svc.Spec.LoadBalancerClass == nil && config.LoadBalancerClassName == "" {
return false
}
if *svc.Spec.LoadBalancerClass != config.LoadBalancerClassName {
log.Info("(svcs) specified wrong loadBalancer class, ignoring", "service name", svc.Name, "wrong lbClass", *svc.Spec.LoadBalancerClass, "expected lbClass", config.LoadBalancerClassName)
return true
}
return false
}
func (p *Processor) serviceInterface() string {
svcIf := p.config.Interface
if p.config.ServicesInterface != "" {
svcIf = p.config.ServicesInterface
}
return svcIf
}
func (p *Processor) startTrafficMirroringIfEnabled() error {
if p.config.MirrorDestInterface != "" {
svcIf := p.serviceInterface()
log.Info("mirroring traffic", "src", svcIf, "dest", p.config.MirrorDestInterface)
if err := trafficmirror.MirrorTrafficFromNIC(svcIf, p.config.MirrorDestInterface); err != nil {
return err
}
} else {
log.Debug("skip starting traffic mirroring since it's not enabled.")
}
return nil
}
func (p *Processor) stopTrafficMirroringIfEnabled() error {
if p.config.MirrorDestInterface != "" {
svcIf := p.serviceInterface()
log.Info("clean up qdisc config", "interface", svcIf)
if err := trafficmirror.CleanupQDSICFromNIC(svcIf); err != nil {
return err
}
} else {
log.Debug("skip stopping traffic mirroring since it's not enabled.")
}
return nil
}

View File

@@ -1,72 +0,0 @@
package utils
import (
"fmt"
"net"
"strings"
"github.com/pkg/errors"
)
const (
IPv4Family = "IPv4"
IPv6Family = "IPv6"
DualFamily = "dual"
)
// LookupHost resolves dnsName and return an IP or an error
func LookupHost(dnsName, dnsMode string) ([]string, error) {
result, err := net.LookupHost(dnsName)
if err != nil {
return nil, err
}
if len(result) == 0 {
return nil, errors.Errorf("empty address for %s", dnsName)
}
addrs := []string{}
switch dnsMode {
case strings.ToLower(IPv4Family), strings.ToLower(IPv6Family), DualFamily:
a, err := getIPbyFamily(result, dnsMode)
if err != nil {
return nil, err
}
addrs = append(addrs, a...)
default:
addrs = append(addrs, result[0])
}
return addrs, nil
}
func getIPbyFamily(addresses []string, family string) ([]string, error) {
var checkers []func(string) bool
families := []string{}
if family == DualFamily || family == strings.ToLower(IPv4Family) {
checkers = append(checkers, IsIPv4)
families = append(families, IPv4Family)
}
if family == DualFamily || family == strings.ToLower(IPv6Family) {
checkers = append(checkers, IsIPv6)
families = append(families, IPv6Family)
}
addrs := []string{}
for i, c := range checkers {
addr, err := getIPbyChecker(addresses, c)
if err != nil {
return nil, fmt.Errorf("error getting %s address: %w", families[i], err)
}
addrs = append(addrs, addr)
}
return addrs, nil
}
func getIPbyChecker(addresses []string, checker func(string) bool) (string, error) {
for _, addr := range addresses {
if checker(addr) {
return addr, nil
}
}
return "", fmt.Errorf("address not found")
}

View File

@@ -1,58 +0,0 @@
package utils
import (
"fmt"
"net"
)
// FormatIPWithSubnetMask takes a raw IP address and a subnet mask, and returns a formatted string in CIDR notation.
func FormatIPWithSubnetMask(rawIP string, subnetMask string) (string, error) {
addr := fmt.Sprintf("%s/%s", rawIP, subnetMask)
// Check if the input is valid
_, _, err := net.ParseCIDR(addr)
if err != nil {
return "", fmt.Errorf("invalid CIDR: %q, %w", addr, err)
}
return addr, nil
}
// IsIP returns if address is an IP or not
func IsIP(address string) bool {
ip := net.ParseIP(address)
return ip != nil
}
// IsIPv4 returns true only if address is a valid IPv4 address
func IsIPv4(address string) bool {
ip := net.ParseIP(address)
if ip == nil {
return false
}
return ip.To4() != nil
}
// IsIPv6 returns true only if address is a valid IPv6 address
func IsIPv6(address string) bool {
ip := net.ParseIP(address)
if ip == nil {
return false
}
return ip.To4() == nil
}
func IsIPv4CIDR(cidr string) bool {
ip, _, _ := net.ParseCIDR(cidr)
if ip == nil {
return false
}
return ip.To4() != nil
}
func IsIPv6CIDR(cidr string) bool {
ip, _, _ := net.ParseCIDR(cidr)
if ip == nil {
return false
}
return ip.To4() == nil
}

View File

@@ -1,55 +0,0 @@
package utils
import (
"fmt"
"net"
"github.com/vishvananda/netlink"
)
// GetInterfaceByIP returns the network interface that has the specified IP address assigned.
func GetInterfaceByIP(ipAddr string) (*netlink.Link, error) {
ip := net.ParseIP(ipAddr)
if ip == nil {
return nil, fmt.Errorf("invalid IP address: %s", ipAddr)
}
links, err := netlink.LinkList()
if err != nil {
return nil, fmt.Errorf("failed to list network interfaces: %v", err)
}
for i := range links {
addrs, err := netlink.AddrList(links[i], netlink.FAMILY_ALL)
if err != nil {
return nil, fmt.Errorf("failed to list addresses for interface %s: %v", links[i].Attrs().Name, err)
}
for _, addr := range addrs {
if addr.IP.Equal(ip) {
return &links[i], nil
}
}
}
return nil, fmt.Errorf("no interface found with IP address: %s", ipAddr)
}
// GetNonLinkLocalIP returns the first non link-local IPv4/IPv6 address on the given interface.
func GetNonLinkLocalIP(iface *netlink.Link, family int) (string, error) {
a, err := netlink.AddrList(*iface, family)
if err != nil {
return "", fmt.Errorf("failed to list addresses for interface %s: %v", (*iface).Attrs().Name, err)
}
for _, addr := range a {
if addr.IPNet != nil {
ip := addr.IPNet.IP
if !ip.IsLinkLocalUnicast() {
return ip.String(), nil
}
}
}
return "", fmt.Errorf("failed to find non-local IP on interface: %s", (*iface).Attrs().Name)
}

View File

@@ -1,8 +1,6 @@
package utils
import (
"os"
)
import "os"
func FileExists(filename string) bool {
info, err := os.Stat(filename)

View File

@@ -2,9 +2,8 @@ package vip
import (
"fmt"
"math"
"net"
"slices"
"os"
"strconv"
"strings"
"sync"
@@ -18,30 +17,24 @@ import (
v1 "k8s.io/api/core/v1"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
)
const (
defaultValidLft = 60
iptablesComment = "%s kube-vip load balancer IP"
iptablesCommentMarkRule = "kube-vip load balancer IP set mark for masquerade"
defaultMaskIPv6 = 128
defaultMaskIPv4 = 32
defaultValidLft = 60
iptablesComment = "%s kube-vip load balancer IP"
iptablesCommentMarkRule = "kube-vip load balancer IP set mark for masquerade"
ignoreServiceSecurityAnnotation = "kube-vip.io/ignore-service-security"
)
// Network is an interface that enable managing operations for a given IP
type Network interface {
AddIP(precheck bool, skipDAD bool) (bool, error)
AddIP(precheck bool) error
AddRoute(precheck bool) error
DeleteIP() (bool, error)
DeleteIP() error
DeleteRoute() error
UpdateRoutes() (bool, error)
IsSet() (bool, error)
IP() string
CIDR() string
IPisLinkLocal() bool
PrepareRoute() *netlink.Route
SetIP(ip string) error
@@ -53,9 +46,6 @@ type Network interface {
DDNSHostName() string
DNSName() string
SetMask(mask string) error
SetHasEndpoints(value bool)
HasEndpoints() bool
ARPName() string
}
// network - This allows network configuration
@@ -63,10 +53,9 @@ type network struct {
mu sync.Mutex
address *netlink.Addr
link *networkinterface.Link
link netlink.Link
ports []v1.ServicePort
serviceName string
enableSecurity bool
ignoreSecurity bool
dnsName string
@@ -80,14 +69,18 @@ type network struct {
routingProtocol int
ipvsEnabled bool
}
hasEndpoints bool
func netlinkParse(addr string) (*netlink.Addr, error) {
mask, err := GetFullMask(addr)
if err != nil {
return nil, err
}
return netlink.ParseAddr(addr + mask)
}
// NewConfig will attempt to provide an interface to the kernel network configuration
func NewConfig(address string, iface string, loGlobalScope bool, subnet string, isDDNS bool, tableID int, tableType int,
routingProtocol int, dnsMode, forwardMethod, iptablesBackend string,
ipvsEnabled, enableSecurity bool, intfMgr *networkinterface.Manager) ([]Network, error) {
func NewConfig(address string, iface string, loGlobalScope bool, subnet string, isDDNS bool, tableID int, tableType int, routingProtocol int, dnsMode, forwardMethod, iptablesBackend string, ipvsEnabled bool) ([]Network, error) {
networks := []Network{}
link, err := netlink.LinkByName(iface)
@@ -95,11 +88,9 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
return networks, errors.Wrapf(err, "could not get link for interface '%s'", iface)
}
networkLink := intfMgr.Get(link)
if utils.IsIP(address) {
if IsIP(address) {
result := &network{
link: networkLink,
link: link,
routeTable: tableID,
routingTableType: tableType,
routingProtocol: routingProtocol,
@@ -108,26 +99,18 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
ipvsEnabled: ipvsEnabled,
}
subnet, err = SelectSubnet(address, subnet)
if err != nil {
return networks, fmt.Errorf("unable to select subnet for IP %q from %q: %w", address, subnet, err)
}
// Check if the subnet needs overriding
cidr, err := utils.FormatIPWithSubnetMask(address, subnet)
if err != nil {
return networks, errors.Wrapf(err, "could not format address '%s' with subnetMask '%s'", address, subnet)
if subnet != "" {
result.address, err = netlink.ParseAddr(address + subnet)
if err != nil {
return networks, errors.Wrapf(err, "could not parse address '%s'", address)
}
} else {
result.address, err = netlinkParse(address)
if err != nil {
return networks, errors.Wrapf(err, "could not parse address '%s'", address)
}
}
result.address, err = netlink.ParseAddr(cidr)
if err != nil {
return networks, errors.Wrapf(err, "could not parse address '%s'", address)
}
// set address as deprecated so it isn't used as source address according to RFC 3484
result.address.PreferedLft = 0
// Also set ValidLft so the netlink library actually sets them
result.address.ValidLft = math.MaxInt
if iface == "lo" && !loGlobalScope {
// set host scope on loopback, otherwise global scope will be used by default
@@ -137,14 +120,13 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
networks = append(networks, result)
} else {
// try to resolve the address
log.Debug("looking up host", "address", address, "dnsMode", dnsMode)
ips, err := utils.LookupHost(address, dnsMode)
ips, err := LookupHost(address, dnsMode)
if err != nil {
// return early for ddns if no IP is allocated for the domain
// when leader starts, should do get IP from DHCP for the domain
if isDDNS {
result := &network{
link: networkLink,
link: link,
routeTable: tableID,
routingTableType: tableType,
routingProtocol: routingProtocol,
@@ -152,20 +134,8 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
iptablesBackend: iptablesBackend,
isDDNS: isDDNS,
dnsName: address,
ipvsEnabled: ipvsEnabled,
enableSecurity: enableSecurity,
address: &netlink.Addr{ // create placeholder for the address
IPNet: &net.IPNet{}, // that will be added later in the process
Peer: &net.IPNet{},
},
}
// set address as deprecated so it isn't used as source address according to RFC 3484
result.address.PreferedLft = 0
// Also set ValidLft so the netlink library actually sets them
result.address.ValidLft = math.MaxInt
networks = append(networks, result)
return networks, nil
}
@@ -174,7 +144,7 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
for _, ip := range ips {
result := &network{
link: networkLink,
link: link,
routeTable: tableID,
routingTableType: tableType,
routingProtocol: routingProtocol,
@@ -182,28 +152,15 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
iptablesBackend: iptablesBackend,
isDDNS: isDDNS,
dnsName: address,
ipvsEnabled: ipvsEnabled,
enableSecurity: enableSecurity,
}
// we're able to resolve store this as the initial IP
subnets := Split(subnet)
s := subnet
if len(subnets) > 1 {
s = selectSubnet(ip, subnets)
}
if result.address, err = netlink.ParseAddr(fmt.Sprintf("%s/%s", ip, s)); err != nil {
if result.address, err = netlinkParse(ip); err != nil {
return networks, err
}
// set ValidLft so that the VIP expires if the DNS entry is updated, otherwise it'll be refreshed by the DNS prober
result.address.ValidLft = defaultValidLft
// set address as deprecated so it isn't used as source address according to RFC 3484
result.address.PreferedLft = 0
networks = append(networks, result)
}
@@ -246,7 +203,7 @@ func (configurator *network) PrepareRoute() *netlink.Route {
route := &netlink.Route{
Scope: routeScope,
Dst: configurator.address.IPNet,
LinkIndex: configurator.link.Intf.Attrs().Index,
LinkIndex: configurator.link.Attrs().Index,
Table: configurator.routeTable,
Type: configurator.routingTableType,
Protocol: netlink.RouteProtocol(configurator.routingProtocol),
@@ -256,8 +213,6 @@ func (configurator *network) PrepareRoute() *netlink.Route {
// AddRoute - Add an IP address to a route table
func (configurator *network) AddRoute(precheck bool) error {
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
route := configurator.PrepareRoute()
exists := false
@@ -279,7 +234,7 @@ func (configurator *network) AddRoute(precheck bool) error {
}
func (configurator *network) routeExists(route *netlink.Route) (bool, error) {
routes, err := netlink.RouteList(configurator.link.Intf, netlink.FAMILY_ALL)
routes, err := netlink.RouteList(configurator.link, netlink.FAMILY_ALL)
if err != nil {
return false, errors.Wrap(err, "failed to list routes")
}
@@ -329,45 +284,30 @@ func (configurator *network) UpdateRoutes() (bool, error) {
}
// AddIP - Add an IP address to the interface
// precheck: if true, check if the IP already exists before adding
// skipDAD: if true, set IFA_F_NODAD flag for IPv6 addresses to skip Duplicate Address Detection
func (configurator *network) AddIP(precheck bool, skipDAD bool) (bool, error) {
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
func (configurator *network) AddIP(precheck bool) error {
exists := false
var err error
if precheck {
if exists, err = configurator.IsSet(); err != nil {
return false, errors.Wrap(err, "could not check if address exists")
if exists, err = configurator.addressExists(); err != nil {
return errors.Wrap(err, "could not check if address exists")
}
}
if exists {
return false, nil
if !exists {
if err := netlink.AddrReplace(configurator.link, configurator.address); err != nil {
return errors.Wrap(err, "could not add ip")
}
if err := configurator.configureIPTables(); err != nil {
return errors.Wrap(err, "could not configure IPTables")
}
}
// For IPv6 addresses, optionally set NODAD flag to skip Duplicate Address Detection (DAD)
// This prevents DADFAILED loops when recovering from a previous DADFAILED state
// The flag tells the kernel to skip DAD, which is safe when we're re-adding
// an address that we know should be ours (e.g., after DADFAILED recovery)
if skipDAD && utils.IsIPv6(configurator.address.IP.String()) {
configurator.address.Flags |= unix.IFA_F_NODAD
log.Debug("Setting IFA_F_NODAD flag for IPv6 address to skip DAD", "ip", configurator.address.IP.String())
}
if err := netlink.AddrReplace(configurator.link.Intf, configurator.address); err != nil {
return false, errors.Wrap(err, "could not add ip")
}
if err := configurator.configureIPTables(); err != nil {
return true, errors.Wrap(err, "could not configure IPTables")
}
return true, nil
return nil
}
func (configurator *network) configureIPTables() error {
if configurator.enableSecurity && !configurator.ignoreSecurity {
if os.Getenv("enable_service_security") == "true" && !configurator.ignoreSecurity {
if err := configurator.addIptablesRulesToLimitTrafficPorts(); err != nil {
return errors.Wrap(err, "could not add iptables rules to limit traffic ports")
}
@@ -383,6 +323,21 @@ func (configurator *network) configureIPTables() error {
return nil
}
func (configurator *network) addressExists() (bool, error) {
addrs, err := netlink.AddrList(configurator.link, netlink.FAMILY_ALL)
if err != nil {
return false, errors.Wrap(err, "could not list addresses")
}
for _, addr := range addrs {
if addr.Equal(*configurator.address) {
return true, nil
}
}
return false, nil
}
func (configurator *network) addIptablesRulesToLimitTrafficPorts() error {
ipt, err := iptables.New()
if err != nil {
@@ -512,37 +467,34 @@ func (configurator *network) removeIptablesRuleToLimitTrafficPorts() error {
}
// DeleteIP - Remove an IP address from the interface
func (configurator *network) DeleteIP() (bool, error) {
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
func (configurator *network) DeleteIP() error {
result, err := configurator.IsSet()
if err != nil {
return false, errors.Wrap(err, "ip check in DeleteIP failed")
return errors.Wrap(err, "ip check in DeleteIP failed")
}
// Nothing to delete
if !result {
return false, nil
return nil
}
if err = netlink.AddrDel(configurator.link.Intf, configurator.address); err != nil {
return false, errors.Wrap(err, "could not delete ip")
if err = netlink.AddrDel(configurator.link, configurator.address); err != nil {
return errors.Wrap(err, "could not delete ip")
}
if configurator.enableSecurity && !configurator.ignoreSecurity {
if os.Getenv("enable_service_security") == "true" && !configurator.ignoreSecurity {
if err := configurator.removeIptablesRuleToLimitTrafficPorts(); err != nil {
return true, errors.Wrap(err, "could not remove iptables rules to limit traffic ports")
return errors.Wrap(err, "could not remove iptables rules to limit traffic ports")
}
}
if configurator.ipvsEnabled && configurator.forwardMethod == "masquerade" && configurator.address.IP.To4() != nil {
if err := configurator.removeIptablesRulesForMasquerade(); err != nil {
return true, errors.Wrap(err, "could not remove iptables masquerade rules ")
return errors.Wrap(err, "could not remove iptables masquerade rules ")
}
}
return true, nil
return nil
}
func (configurator *network) addIptablesRulesForMasquerade() error {
@@ -609,15 +561,12 @@ func delMasqueradeRuleForVIP(ipt *iptables.IPTables, vip, comment string) error
// IsDADFAIL - Returns true if the address is IPv6 and has DADFAILED flag
func (configurator *network) IsDADFAIL() bool {
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
if configurator.address == nil || !utils.IsIPv6(configurator.address.IP.String()) {
if configurator.address == nil || !IsIPv6(configurator.address.IP.String()) {
return false
}
// Get all the address
addresses, err := netlink.AddrList(configurator.link.Intf, netlink.FAMILY_V6)
addresses, err := netlink.AddrList(configurator.link, netlink.FAMILY_V6)
if err != nil {
return false
}
@@ -636,7 +585,7 @@ func addressHasDADFAILEDFlag(address netlink.Addr) bool {
return address.Flags&unix.IFA_F_DADFAILED != 0
}
// isSet - Check to see if VIP is set
// IsSet - Check to see if VIP is set
func (configurator *network) IsSet() (result bool, err error) {
var addresses []netlink.Addr
@@ -644,15 +593,11 @@ func (configurator *network) IsSet() (result bool, err error) {
return false, nil
}
if configurator.address.Mask == nil {
return false, nil
}
addresses, err = netlink.AddrList(configurator.link.Intf, 0)
addresses, err = netlink.AddrList(configurator.link, 0)
if err != nil {
err = errors.Wrap(err, "could not list addresses")
return false, err
return
}
for _, address := range addresses {
@@ -669,30 +614,13 @@ func (configurator *network) SetIP(ip string) error {
configurator.mu.Lock()
defer configurator.mu.Unlock()
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
if strings.Contains("/", ip) {
return fmt.Errorf("ip should not contain CIDR notation got: %s", ip)
}
ones, _ := configurator.address.Mask.Size()
cidr, err := utils.FormatIPWithSubnetMask(ip, strconv.Itoa(ones))
if err != nil {
return fmt.Errorf("could not format address '%s' with subnetMask '%s'", ip, strconv.Itoa(ones))
}
addr, err := netlink.ParseAddr(cidr)
addr, err := netlinkParse(ip)
if err != nil {
return err
}
if configurator.address != nil && configurator.IsDNS() {
addr.ValidLft = defaultValidLft
} else {
addr.ValidLft = math.MaxInt
}
// set address as deprecated so it isn't used as source address according to RFC 3484
addr.PreferedLft = 0
configurator.address = addr
return nil
}
@@ -705,7 +633,7 @@ func (configurator *network) SetServicePorts(service *v1.Service) {
configurator.ports = service.Spec.Ports
configurator.serviceName = service.Namespace + "/" + service.Name
configurator.ignoreSecurity = service.Annotations[kubevip.ServiceSecurityIgnore] == "true"
configurator.ignoreSecurity = service.Annotations[ignoreServiceSecurityAnnotation] == "true"
}
// IP - return the IP Address
@@ -713,24 +641,9 @@ func (configurator *network) IP() string {
configurator.mu.Lock()
defer configurator.mu.Unlock()
if configurator.address == nil || configurator.address.IP == nil {
return ""
}
return configurator.address.IP.String()
}
func (configurator *network) CIDR() string {
configurator.mu.Lock()
defer configurator.mu.Unlock()
if configurator.address == nil || configurator.address.IPNet == nil {
return ""
}
return configurator.address.IPNet.String()
}
// IP - return the IP Address
func (configurator *network) IPisLinkLocal() bool {
configurator.mu.Lock()
@@ -764,23 +677,19 @@ func (configurator *network) DDNSHostName() string {
// Interface - return the Interface name
func (configurator *network) Interface() string {
return configurator.link.Intf.Attrs().Name
return configurator.link.Attrs().Name
}
func GarbageCollect(adapter, address string, intfMgr *networkinterface.Manager) (found bool, err error) {
func GarbageCollect(adapter, address string) (found bool, err error) {
// Get adapter
link, err := netlink.LinkByName(adapter)
if err != nil {
return true, errors.Wrapf(err, "could not get link for interface '%s'", adapter)
}
l := intfMgr.Get(link)
l.Lock.Lock()
defer l.Lock.Unlock()
// Get addresses on adapter
addrs, err := netlink.AddrList(l.Intf, netlink.FAMILY_ALL)
addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL)
if err != nil {
return false, err
}
@@ -792,7 +701,7 @@ func GarbageCollect(adapter, address string, intfMgr *networkinterface.Manager)
found = true
// linting issue
existing := existing
if err = netlink.AddrDel(l.Intf, &existing); err != nil {
if err = netlink.AddrDel(link, &existing); err != nil {
return true, errors.Wrap(err, "could not delete ip")
}
}
@@ -801,34 +710,21 @@ func GarbageCollect(adapter, address string, intfMgr *networkinterface.Manager)
}
func (configurator *network) SetMask(mask string) error {
selectedMask := mask
var err error
if configurator.IP() != "" {
selectedMask, err = SelectSubnet(configurator.IP(), mask)
if err != nil {
return fmt.Errorf("failed to select mask %q: %w", mask, err)
}
}
m, err := strconv.Atoi(selectedMask)
m, err := strconv.Atoi(mask)
if err != nil {
return err
}
size := defaultMaskIPv4
family := utils.IPv4Family
size := 32
family := "IPv4"
if configurator.IP() != "" {
if utils.IsIPv6(configurator.IP()) {
size = defaultMaskIPv6
family = utils.IPv6Family
}
if m > size {
return fmt.Errorf("provided CIDR mask '%d' is greater than the highest mask value for the %s family (%d)", m, family, size)
}
if IsIPv6(configurator.IP()) {
size = 128
family = "IPv6"
}
if m > size {
return fmt.Errorf("provided CIDR mask '%d' is greater than the highest mask value for the %s family (%d)", m, family, size)
}
toSet := net.CIDRMask(m, size)
@@ -842,50 +738,3 @@ func (configurator *network) SetMask(mask string) error {
configurator.address.Mask = toSet
return nil
}
func (configurator *network) SetHasEndpoints(value bool) {
log.Debug("setting HasEndpoints", "ip", configurator.IP(), "value", value)
configurator.hasEndpoints = value
}
func (configurator *network) HasEndpoints() bool {
log.Debug("getting HasEndpoints", "ip", configurator.IP(), "value", configurator.hasEndpoints)
return configurator.hasEndpoints
}
func (configurator *network) ARPName() string {
return fmt.Sprintf("%s-%s", configurator.CIDR(), configurator.Interface())
}
// SelectSubnet formats an IP address with the appropriate CIDR based on the input.
// The input SubnetMasks can be "32,128" (dual-stack), "32", "128" (SingleStack).
func SelectSubnet(rawIP string, subnetMasks string) (string, error) {
// Split the SubnetMasks input into DualStack or SingleStack
// If the input is "32,128", it will be split into ["32", "128"]
subnetMasksParts := strings.Split(subnetMasks, ",")
if len(subnetMasksParts) == 0 {
return "", fmt.Errorf("no subnetMasks provided got: %q", subnetMasks)
} else if len(subnetMasksParts) > 2 {
return "", fmt.Errorf("invalid subnetMasks provided got: %q", subnetMasks)
}
if slices.Contains(subnetMasksParts, "auto") {
return "", fmt.Errorf("auto subnet discovery only works for services: %q", subnetMasks)
}
// Parse the raw IP address
ip := net.ParseIP(rawIP)
if ip == nil {
return "", fmt.Errorf("invalid IP address: %s", rawIP)
}
if ip.To4() != nil {
return subnetMasksParts[0], nil
}
if ip.To16() != nil {
subnetMask := subnetMasksParts[0]
if len(subnetMasksParts) == 2 {
subnetMask = subnetMasksParts[1]
}
return subnetMask, nil
}
return "", fmt.Errorf("unable to select subnet mask for: IP %q and masks %q", rawIP, subnetMasks)
}

View File

@@ -135,7 +135,9 @@ func sendARP(iface *net.Interface, m *arpMessage) error {
Halen: m.hardwareAddressLength,
}
target := ethernetBroadcast
copy(ll.Addr[:], target)
for i := 0; i < len(target); i++ { //nolint
ll.Addr[i] = target[i]
}
b, err := m.bytes()
if err != nil {

View File

@@ -5,8 +5,6 @@ import (
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/utils"
)
// IPUpdater is the interface to plug dns updaters
@@ -35,24 +33,23 @@ func (d *ipUpdater) Run(ctx context.Context) {
return
default:
mode := "ipv4"
if utils.IsIPv6(d.vip.IP()) {
if IsIPv6(d.vip.IP()) {
mode = "ipv6"
}
ip, err := utils.LookupHost(d.vip.DNSName(), mode)
ip, err := LookupHost(d.vip.DNSName(), mode)
if err != nil {
log.Warn("cannot lookup", "name", d.vip.DNSName(), "err", err)
// fallback to renewing the existing IP
ip = []string{d.vip.IP()}
}
log.Debug("setting IP", "address", ip)
log.Info("setting IP", "address", ip)
if err := d.vip.SetIP(ip[0]); err != nil {
log.Error("setting IP", "address", ip, "err", err)
}
// Normal VIP addition for DNS, use skipDAD=false for normal DAD process
if _, err := d.vip.AddIP(false, false); err != nil {
if err := d.vip.AddIP(false); err != nil {
log.Error("error adding virtual IP", "err", err)
}

View File

@@ -8,7 +8,6 @@ import (
log "log/slog"
iptables "github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/utils"
ct "github.com/florianl/go-conntrack"
)
@@ -37,9 +36,9 @@ type Egress struct {
}
func CreateIptablesClient(nftables bool, namespace string, protocol iptables.Protocol) (*Egress, error) {
proto := utils.IPv4Family
proto := "IPv4"
if protocol == iptables.ProtocolIPv6 {
proto = utils.IPv6Family
proto = "IPv6"
}
log.Info("[egress] Creating an iptables client", "nftables", nftables, "protocol", proto)
e := new(Egress)

View File

@@ -5,17 +5,78 @@ import (
"crypto/rand"
"fmt"
"net"
"strconv"
"strings"
"syscall"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/pkg/errors"
"github.com/vishvananda/netlink"
)
// LookupHost resolves dnsName and return an IP or an error
func LookupHost(dnsName, dnsMode string) ([]string, error) {
result, err := net.LookupHost(dnsName)
if err != nil {
return nil, err
}
if len(result) == 0 {
return nil, errors.Errorf("empty address for %s", dnsName)
}
addrs := []string{}
switch dnsMode {
case "ipv4", "ipv6", "dual":
a, err := getIPbyFamily(result, dnsMode)
if err != nil {
return nil, err
}
addrs = append(addrs, a...)
default:
addrs = append(addrs, result[0])
}
return addrs, nil
}
func getIPbyFamily(addresses []string, family string) ([]string, error) {
var checkers []func(string) bool
families := []string{}
if family == "dual" || family == "ipv4" {
checkers = append(checkers, IsIPv4)
families = append(families, "IPv4")
}
if family == "dual" || family == "ipv6" {
checkers = append(checkers, IsIPv6)
families = append(families, "IPv6")
}
addrs := []string{}
for i, c := range checkers {
addr, err := getIPbyChecker(addresses, c)
if err != nil {
return nil, fmt.Errorf("error getting %s address: %w", families[i], err)
}
addrs = append(addrs, addr)
}
return addrs, nil
}
func getIPbyChecker(addresses []string, checker func(string) bool) (string, error) {
for _, addr := range addresses {
if checker(addr) {
return addr, nil
}
}
return "", fmt.Errorf("address not found")
}
// IsIP returns if address is an IP or not
func IsIP(address string) bool {
ip := net.ParseIP(address)
return ip != nil
}
// getHostName return the hostname from the fqdn
func getHostName(dnsName string) string {
if dnsName == "" {
@@ -26,6 +87,51 @@ func getHostName(dnsName string) string {
return fields[0]
}
// IsIPv4 returns true only if address is a valid IPv4 address
func IsIPv4(address string) bool {
ip := net.ParseIP(address)
if ip == nil {
return false
}
return ip.To4() != nil
}
// IsIPv6 returns true only if address is a valid IPv6 address
func IsIPv6(address string) bool {
ip := net.ParseIP(address)
if ip == nil {
return false
}
return ip.To4() == nil
}
func IsIPv4CIDR(cidr string) bool {
ip, _, _ := net.ParseCIDR(cidr)
if ip == nil {
return false
}
return ip.To4() != nil
}
func IsIPv6CIDR(cidr string) bool {
ip, _, _ := net.ParseCIDR(cidr)
if ip == nil {
return false
}
return ip.To4() == nil
}
// GetFullMask returns /32 for an IPv4 address and /128 for an IPv6 address
func GetFullMask(address string) (string, error) {
if IsIPv4(address) {
return "/32", nil
}
if IsIPv6(address) {
return "/128", nil
}
return "", fmt.Errorf("failed to parse %s as either IPv4 or IPv6", address)
}
// GetDefaultGatewayInterface return default gateway interface link
func GetDefaultGatewayInterface() (*net.Interface, error) {
routes, err := netlink.RouteList(nil, syscall.AF_INET)
@@ -144,22 +250,3 @@ func GetNonLinkLocalIP(iface *netlink.Link, family int) (string, error) {
return "", fmt.Errorf("failed to find non-local IP on interface: %s", (*iface).Attrs().Name)
}
func selectSubnet(address string, subnets []string) string {
subnet := ""
if utils.IsIPv4(address) {
if subnets[0] != "" {
subnet = subnets[0]
} else {
subnet = strconv.Itoa(defaultMaskIPv4)
}
} else {
if len(subnets) > 1 && subnets[1] != "" {
subnet = subnets[1]
} else {
subnet = strconv.Itoa(defaultMaskIPv6)
}
}
return subnet
}

View File

@@ -1,3 +0,0 @@
[global.config]
as = {{ .AS }}
router-id = "1.1.1.1"

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,97 +0,0 @@
//go:build e2e
// +build e2e
package e2e_test
import (
"context"
"os"
"os/exec"
"syscall"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gbytes"
"github.com/onsi/gomega/gexec"
)
var _ = Describe("SIGUSR1 Signal Handler", func() {
var (
session *gexec.Session
ctx context.Context
cancel context.CancelFunc
)
BeforeEach(func() {
ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
})
AfterEach(func() {
if session != nil {
session.Terminate()
Eventually(session).Should(gexec.Exit())
}
cancel()
})
When("kube-vip receives SIGUSR1", func() {
It("should dump configuration without terminating", func() {
Skip("Skipping until signal handler is implemented")
kvPath := os.Getenv("KUBE_VIP_PATH")
if kvPath == "" {
kvPath = "../../kube-vip"
}
cmd := exec.CommandContext(ctx, kvPath, "manager",
"--address", "192.168.1.100",
"--interface", "lo",
"--port", "6443",
)
var err error
session, err = gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
Expect(err).NotTo(HaveOccurred())
time.Sleep(2 * time.Second)
err = session.Command.Process.Signal(syscall.SIGUSR1)
Expect(err).NotTo(HaveOccurred())
Eventually(session.Out, "5s").Should(gbytes.Say("KUBE-VIP CONFIGURATION DUMP"))
Eventually(session.Out).Should(gbytes.Say("VIP: 192.168.1.100"))
Consistently(session, "2s").ShouldNot(gexec.Exit())
})
It("should handle multiple SIGUSR1 signals", func() {
Skip("Skipping until signal handler is implemented")
kvPath := os.Getenv("KUBE_VIP_PATH")
if kvPath == "" {
kvPath = "../../kube-vip"
}
cmd := exec.CommandContext(ctx, kvPath, "manager",
"--address", "192.168.1.100",
"--interface", "lo",
)
var err error
session, err = gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
Expect(err).NotTo(HaveOccurred())
time.Sleep(2 * time.Second)
for i := 0; i < 3; i++ {
err = session.Command.Process.Signal(syscall.SIGUSR1)
Expect(err).NotTo(HaveOccurred())
time.Sleep(1 * time.Second)
}
Eventually(session.Out).Should(gbytes.Say("KUBE-VIP CONFIGURATION DUMP"))
Consistently(session, "2s").ShouldNot(gexec.Exit())
})
})
})

View File

@@ -4,41 +4,15 @@
package e2e_test
import (
"log"
"os"
"sync"
"testing"
"github.com/kube-vip/kube-vip/testing/e2e"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
const (
ModeEnv = "TEST_MODE"
ModeARP = "arp"
ModeRT = "rt"
ModeBGP = "bgp"
)
var (
SOffset *e2e.SecureOffset
ConfigMtx *sync.Mutex
Mode string
"github.com/kube-vip/kube-vip/testing/e2e"
)
func TestE2E(t *testing.T) {
mode := os.Getenv(ModeEnv)
if mode == "" {
Mode = ModeARP
} else if mode != ModeARP && mode != ModeRT && mode != ModeBGP {
log.Fatal("invalid", "mode", mode)
os.Exit(1)
} else {
Mode = mode
}
SOffset = e2e.NewOffset(5)
ConfigMtx = &sync.Mutex{}
RegisterFailHandler(Fail)
RunSpecs(t, "E2E Suite")
}

File diff suppressed because it is too large Load Diff

View File

@@ -268,6 +268,15 @@ func (c *Cluster) Delete() {
Expect(c.provider.Delete(c.Name, "")).To(Succeed())
}
func startKubeletForEtcd(node nodes.Node) {
e2e.RunInNode(node,
"kubeadm", "init", "phase", "kubeconfig", "admin", "--config", "/kind/kubeadm.conf",
)
e2e.RunInNode(node,
"kubeadm", "init", "phase", "kubelet-start", "--config", "/kind/kubeadm.conf",
)
}
func createCerts(node nodes.Node) {
e2e.RunInNode(node,
"kubeadm",

View File

@@ -15,7 +15,6 @@ import (
"github.com/onsi/gomega/format"
"k8s.io/klog/v2"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/kube-vip/kube-vip/testing/e2e"
"github.com/kube-vip/kube-vip/testing/e2e/etcd"
)
@@ -61,7 +60,7 @@ var _ = Describe("kube-vip with etcd leader election", func() {
test.kubeVipImage = os.Getenv("E2E_IMAGE_PATH")
test.vip = e2e.GenerateVIP(utils.IPv4Family, 5)
test.vip = e2e.GenerateIPv4VIP()
test.logger.Printf("Selected VIP %s", test.vip)
test.currentDir, err = os.Getwd()

Some files were not shown because too many files have changed in this diff Show More