Compare commits

...

45 Commits

Author SHA1 Message Date
Dan Finneran
296c9c1580 Unintentionally created duplicate functions that can't be used
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-08-12 12:17:20 +00:00
Daniel Finneran
d8877072d4 Merge pull request #1219 from kube-vip/v1-release
Version ONE DOT ZERO
2025-07-31 16:42:23 +01:00
Dan Finneran
89a8dc7de1 default setting to 0
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-31 16:09:10 +02:00
Dan Finneran
704c346f5e naming convention lint fix
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-31 16:09:10 +02:00
Dan Finneran
65061c5cd9 Adds an annotation to set the return path filter for a macvlan nic
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-31 16:09:10 +02:00
Daniel Finneran
32233918b4 Version ONE DOT ZERO 2025-07-31 12:34:37 +01:00
Daniel Finneran
76169da60f Merge pull request #1216 from kube-vip/dependabot/go_modules/github.com/docker/docker-28.3.3incompatible
Bump github.com/docker/docker from 28.3.0+incompatible to 28.3.3+incompatible
2025-07-31 12:33:44 +01:00
Daniel Finneran
9bcf1413f0 Merge pull request #1211 from kube-vip/dependabot/github_actions/anchore/sbom-action-0.20.4
Bump anchore/sbom-action from 0.20.1 to 0.20.4
2025-07-31 12:33:28 +01:00
Daniel Finneran
8e428e875f Merge pull request #1203 from kube-vip/dependabot/docker/alpine-3.22.1
Bump alpine from 3.22.0 to 3.22.1
2025-07-31 12:33:14 +01:00
dependabot[bot]
2fbecc25e5 Bump alpine from 3.22.0 to 3.22.1
Bumps alpine from 3.22.0 to 3.22.1.

---
updated-dependencies:
- dependency-name: alpine
  dependency-version: 3.22.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-07-30 15:17:21 +00:00
Daniel Finneran
02e77271d0 Merge pull request #1202 from kube-vip/dependabot/docker/golang-1.24.5-alpine3.22 2025-07-30 16:16:18 +01:00
Daniel Finneran
3d61888e58 Merge pull request #1192 from kube-vip/dependabot/go_modules/sigs.k8s.io/kind-0.29.0 2025-07-30 16:15:52 +01:00
dependabot[bot]
efe75f491b Bump sigs.k8s.io/kind from 0.27.0 to 0.29.0
Bumps [sigs.k8s.io/kind](https://github.com/kubernetes-sigs/kind) from 0.27.0 to 0.29.0.
- [Release notes](https://github.com/kubernetes-sigs/kind/releases)
- [Commits](https://github.com/kubernetes-sigs/kind/compare/v0.27.0...v0.29.0)

---
updated-dependencies:
- dependency-name: sigs.k8s.io/kind
  dependency-version: 0.29.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-07-30 13:55:46 +00:00
Daniel Finneran
000c139004 Merge pull request #1194 from kube-vip/dependabot/go_modules/github.com/vishvananda/netlink-1.3.1 2025-07-30 14:50:38 +01:00
Daniel Finneran
c39b84f0a9 Merge pull request #1193 from kube-vip/dependabot/go_modules/github.com/osrg/gobgp/v3-3.37.0 2025-07-30 14:50:25 +01:00
dependabot[bot]
ee958addaa Bump github.com/docker/docker
Bumps [github.com/docker/docker](https://github.com/docker/docker) from 28.3.0+incompatible to 28.3.3+incompatible.
- [Release notes](https://github.com/docker/docker/releases)
- [Commits](https://github.com/docker/docker/compare/v28.3.0...v28.3.3)

---
updated-dependencies:
- dependency-name: github.com/docker/docker
  dependency-version: 28.3.3+incompatible
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-07-30 13:48:44 +00:00
Daniel Finneran
8fe53351f8 Merge pull request #1215 from kube-vip/endpointslice 2025-07-30 14:47:26 +01:00
Daniel Finneran
bc9d860d83 Merge pull request #1213 from kube-vip/egress_internal 2025-07-30 14:47:07 +01:00
Dan Finneran
332a23e543 Bump k8s api and endpointslices as default
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-29 13:36:32 +00:00
Dan Finneran
b20713b50f linting issue with port out of bounds.
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-28 16:18:47 +00:00
Dan Finneran
94e96581ef typo and add SCTP
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-28 16:13:20 +00:00
Dan Finneran
61be6d0b6a fighting with merge conflicts
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-28 16:08:53 +00:00
Dan Finneran
be22805a7d sigh
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-28 16:08:22 +00:00
Dan Finneran
25f6253286 Adds the internal egress functionality
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-28 16:01:00 +00:00
dependabot[bot]
f3e9fb6ea9 Bump github.com/osrg/gobgp/v3 from 3.36.0 to 3.37.0
Bumps [github.com/osrg/gobgp/v3](https://github.com/osrg/gobgp) from 3.36.0 to 3.37.0.
- [Release notes](https://github.com/osrg/gobgp/releases)
- [Changelog](https://github.com/osrg/gobgp/blob/master/.goreleaser.yml)
- [Commits](https://github.com/osrg/gobgp/compare/v3.36.0...v3.37.0)

---
updated-dependencies:
- dependency-name: github.com/osrg/gobgp/v3
  dependency-version: 3.37.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-07-28 15:17:29 +00:00
Patryk Strusiewicz-Surmacki
0f3dda02c4 Refactoring services-watcher code
Co-authored-by: Cellebyte <marcel.fest@live.de>
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk-pawel.strusiewicz-surmacki@external.telekom.de>
2025-07-28 17:14:09 +02:00
Dan Finneran
a2873b5465 Adds the internal egress functionality
Signed-off-by: Dan Finneran <dan@thebsdbox.co.uk>
2025-07-28 13:57:50 +00:00
dependabot[bot]
202d45e5ab Bump anchore/sbom-action from 0.20.1 to 0.20.4
Bumps [anchore/sbom-action](https://github.com/anchore/sbom-action) from 0.20.1 to 0.20.4.
- [Release notes](https://github.com/anchore/sbom-action/releases)
- [Changelog](https://github.com/anchore/sbom-action/blob/main/RELEASE.md)
- [Commits](https://github.com/anchore/sbom-action/compare/v0.20.1...v0.20.4)

---
updated-dependencies:
- dependency-name: anchore/sbom-action
  dependency-version: 0.20.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-07-28 03:11:16 +00:00
Daniel Finneran
f5e4612c03 Merge pull request #1210 from kube-vip/fix/ci 2025-07-27 22:14:39 +01:00
Cellebyte
de888c501c split ci into two files
Signed-off-by: Cellebyte <marcel.fest@live.de>
2025-07-27 23:02:09 +02:00
Cellebyte
47bc83c248 Don't execute the e2e-tests in parallel
Signed-off-by: Cellebyte <marcel.fest@live.de>
2025-07-27 23:02:09 +02:00
Patryk Strusiewicz-Surmacki
1cf637c569 Moved endpoint-related code from pkg/manager to pkg/endpoints
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk-pawel.strusiewicz-surmacki@external.telekom.de>
2025-07-24 08:07:23 +02:00
Patryk Strusiewicz-Surmacki
ae2571e241 Refactoring endpoint-watcher code
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk-pawel.strusiewicz-surmacki@external.telekom.de>
2025-07-24 08:07:23 +02:00
Patryk Strusiewicz-Surmacki
86f5e9b8b2 Removed insecure client use in E2E tests
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
2025-07-24 08:07:00 +02:00
Patryk Strusiewicz-Surmacki
ce61ff085a Use insecure client only in E2E tests
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
2025-07-21 17:54:12 +02:00
dependabot[bot]
b816e154cf Bump golang from 1.24.4-alpine3.22 to 1.24.5-alpine3.22
Bumps golang from 1.24.4-alpine3.22 to 1.24.5-alpine3.22.

---
updated-dependencies:
- dependency-name: golang
  dependency-version: 1.24.5-alpine3.22
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-07-14 03:15:14 +00:00
Patryk Strusiewicz-Surmacki
ac1238c337 Rebase
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
2025-07-10 17:50:15 +02:00
Patryk Strusiewicz-Surmacki
10dbf2c0ef Refactoring endpoint-watcher code
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk-pawel.strusiewicz-surmacki@external.telekom.de>
2025-07-10 17:50:15 +02:00
Jan-Philipp Litza
cf68f8639c Add VIPs as deprecated
This causes IPv6 source address selection according to RFC 3484 to
ignore VIPs when choosing which address the host should use for egress
communication.

Signed-off-by: Jan-Philipp Litza <janphilipp@litza.de>
2025-07-09 10:54:28 +02:00
Daniel Finneran
b114c11b0f Merge pull request #1191 from kube-vip/thebsdbox-patch-1
Update README.md
2025-07-03 09:59:49 +01:00
Patryk Strusiewicz-Surmacki
42b7a8152b Basic e2e tests for BGP mode
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>
2025-07-01 12:36:33 +02:00
mushrushu
f7821c7fb3 fixed vars formatting & log statement
Signed-off-by: mushrushu <mushrushu@outlook.com>
2025-07-01 12:34:43 +02:00
mushrushu
42478905d0 added zebra integration and its configuration
Signed-off-by: mushrushu <mushrushu@outlook.com>
2025-07-01 12:34:43 +02:00
dependabot[bot]
8d55bd3b63 Bump github.com/vishvananda/netlink from 1.3.0 to 1.3.1
Bumps [github.com/vishvananda/netlink](https://github.com/vishvananda/netlink) from 1.3.0 to 1.3.1.
- [Release notes](https://github.com/vishvananda/netlink/releases)
- [Commits](https://github.com/vishvananda/netlink/compare/v1.3.0...v1.3.1)

---
updated-dependencies:
- dependency-name: github.com/vishvananda/netlink
  dependency-version: 1.3.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-06-30 03:20:14 +00:00
Daniel Finneran
16247fc3a3 Update README.md
Adds health of project badges! (looking healthy) 😄
2025-06-27 14:20:14 +01:00
65 changed files with 3747 additions and 1637 deletions

View File

@@ -26,6 +26,6 @@ jobs:
with:
ref: ${{ github.ref_name }}
- name: Anchore SBOM Action
uses: anchore/sbom-action@v0.20.1
uses: anchore/sbom-action@v0.20.4
with:
format: cyclonedx-json

51
.github/workflows/ci-pull-request.yaml vendored Normal file
View File

@@ -0,0 +1,51 @@
name: For each PR
on:
pull_request:
jobs:
e2e-tests:
runs-on: ubuntu-latest
name: E2E tests
strategy:
matrix:
mode: ["arp", "rt", "bgp"]
fail-fast: true
max-parallel: 1
steps:
- name: Ensure fs wont cause issues
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Checkout code
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image locally
run: make dockerx86Local
- name: Run Manifest generation tests
run: make manifest-test
- name: Run ARP mode tests v1.29.0 onwards
run: make e2e-tests129-arp
if: matrix.mode== 'arp'
- name: Run RT mode tests v1.29.0 onwards
run: make e2e-tests129-rt
if: matrix.mode== 'rt'
- name: Get GoBGP binaries
run: make get-gobgp
if: matrix.mode== 'bgp'
- name: Run BGP mode tests v1.29.0 onwards
run: sudo -E PATH=$PATH DOCKER_API_VERSION=1.48 make e2e-tests129-bgp
if: matrix.mode== 'bgp'
service-e2e-tests:
runs-on: ubuntu-latest
name: E2E service tests
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image with iptables
run: make dockerx86ActionIPTables
- name: Run tests
run: DOCKERTAG=action make service-tests

View File

@@ -1,4 +1,4 @@
name: For each commit and PR
name: For each commit
on:
push:
pull_request:
@@ -43,45 +43,6 @@ jobs:
go-version-file: go.mod
- name: Run tests
run: make integration-tests
e2e-tests:
runs-on: ubuntu-latest
name: E2E tests
strategy:
matrix:
mode: ["arp", "rt"]
steps:
- name: Ensure fs wont cause issues
run: sudo sysctl fs.inotify.max_user_instances=8192 && sudo sysctl fs.inotify.max_user_watches=524288
- name: Checkout code
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image locally
run: make dockerx86Local
- name: Run Manifest generation tests
run: make manifest-test
- name: Run ARP mode tests v1.29.0 onwards
run: make e2e-tests129-arp
if: matrix.mode== 'arp'
- name: Run RT mode tests v1.29.0 onwards
run: make e2e-tests129-rt
if: matrix.mode== 'rt'
service-e2e-tests:
runs-on: ubuntu-latest
name: E2E service tests
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build image with iptables
run: make dockerx86ActionIPTables
- name: Run tests
run: DOCKERTAG=action make service-tests
image-vul-check:
runs-on: ubuntu-latest
name: Image vulnerability scan

View File

@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:experimental
FROM golang:1.24.4-alpine3.22 as dev
FROM golang:1.24.5-alpine3.22 as dev
RUN apk add --no-cache git ca-certificates make
RUN adduser -D appuser
COPY . /src/

View File

@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:experimental
FROM golang:1.24.4-alpine3.22 as dev
FROM golang:1.24.5-alpine3.22 as dev
RUN apk add --no-cache git make
RUN adduser -D appuser
COPY . /src/
@@ -11,7 +11,7 @@ RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \
--mount=type=cache,sharing=locked,id=goroot,target=/root/.cache/go-build \
CGO_ENABLED=0 GOOS=linux make build
FROM alpine:3.22.0
FROM alpine:3.22.1
# Update pkgs and add iptables
RUN apk upgrade && \
apk add --no-cache iptables iptables-legacy

View File

@@ -5,7 +5,7 @@ TARGET := kube-vip
.DEFAULT_GOAL := $(TARGET)
# These will be provided to the target
VERSION := v0.9.2
VERSION := v1.0.0
BUILD := `git rev-parse HEAD`
@@ -141,7 +141,10 @@ e2e-tests129-arp:
e2e-tests129-rt:
GOMAXPROCS=4 TEST_MODE=rt V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129: e2e-tests129-arp e2e-tests129-rt
e2e-tests129-bgp:
GOMAXPROCS=4 TEST_MODE=bgp V129=true K8S_IMAGE_PATH=kindest/node:v1.29.0 E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run github.com/onsi/ginkgo/v2/ginkgo --tags=e2e -v -p ./testing/e2e
e2e-tests129: e2e-tests129-arp e2e-tests129-rt e2e-tests129-bgp
service-tests:
E2E_IMAGE_PATH=$(REPOSITORY)/$(TARGET):$(DOCKERTAG) go run ./testing/services -Services -simple -deployments -leaderActive -leaderFailover -localDeploy -egress -egressIPv6 -dualStack
@@ -168,3 +171,9 @@ kind-quick:
kind-reload:
kind load docker-image $(REPOSITORY)/$(TARGET):$(DOCKERTAG) --name kube-vip
kubectl rollout restart -n kube-system daemonset/kube-vip-ds
get-gobgp:
mkdir -p bin
wget -nc --directory-prefix=bin https://github.com/osrg/gobgp/releases/download/v3.37.0/gobgp_3.37.0_linux_amd64.tar.gz
tar -xvzf bin/gobgp_3.37.0_linux_amd64.tar.gz -C bin

View File

@@ -4,7 +4,7 @@ High Availability and Load-Balancing
![](https://github.com/kube-vip/kube-vip/raw/main/kube-vip.png)
[![Build and publish main image regularly](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml/badge.svg)](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml)
[![Build and publish main image regularly](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml/badge.svg)](https://github.com/kube-vip/kube-vip/actions/workflows/main.yaml) [![LFX Health Score](https://img.shields.io/static/v1?label=Health%20Score&message=Healthy&color=A7F3D0&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip) [![LFX Active Contributors](https://img.shields.io/static/v1?label=Active%20contributors%20(1Y)&message=212&color=0094FF&logo=linuxfoundation&logoColor=white&style=flat)](https://insights.linuxfoundation.org/project/kube-vip)
## Overview
Kubernetes Virtual IP and Load-Balancer for both control plane and Kubernetes services

View File

@@ -93,6 +93,11 @@ func init() {
kubeVipCmd.PersistentFlags().StringSliceVar(&initConfig.BGPPeers, "bgppeers", []string{}, "Comma separated BGP Peer, format: address:as:password:multihop")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.Annotations, "annotations", "", "Set Node annotations prefix for parsing")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.BGPConfig.Zebra.Enabled, "zebra", false, "This will enable Zebra support within kube-vip")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.URL, "zebraUrl", "unix:/var/run/frr/zserv.api", "Path to the unix domain socket for connecting to Zebra daemon")
kubeVipCmd.PersistentFlags().Uint32Var(&initConfig.BGPConfig.Zebra.Version, "zebraVersion", 6, "Zebra API Version")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.BGPConfig.Zebra.SoftwareName, "zebraSoftwareName", "frr8.3", "Software Name for Zebra")
// Namespace for kube-vip
kubeVipCmd.PersistentFlags().StringVarP(&initConfig.Namespace, "namespace", "n", "kube-system", "The namespace for the configmap defined within the cluster")
@@ -123,7 +128,7 @@ func init() {
kubeVipCmd.PersistentFlags().StringVar(&initConfig.ServicesLeaseName, "servicesLeaseName", "plndr-svcs-lock", "Name of the lease that is used for leader election for services (in arp mode)")
kubeVipCmd.PersistentFlags().StringVar(&initConfig.DNSMode, "dnsMode", "first", "Name of the mode that DNS lookup will be performed (first, ipv4, ipv6, dual)")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.DisableServiceUpdates, "disableServiceUpdates", false, "If true, kube-vip will process services as usual, but will not update service's Status.LoadBalancer.Ingress slice")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpointSlices, "enableEndpointSlices", false, "If enabled, kube-vip will only advertise services, but will use EndpointSlices instead of endpoints to get IPs of Pods")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.EnableEndpoints, "enableEndpoints", false, "If enabled, kube-vip will only advertise services, but will use the (deprecated since v1.33) endpoints for IP addresses")
kubeVipCmd.PersistentFlags().BoolVar(&initConfig.LoInterfaceGlobalScope, "loInterfaceGlobalScope", false, "If true, kube-vip will set global scope when using the lo interface, otherwise a host scope will be used by default")
kubeVipCmd.PersistentFlags().IntVar(&initConfig.HealthCheckPort, "healthCheckPort", 0, "If set to non-zero (> 1024), then this is the port that the healthcheck will listen on")

74
go.mod
View File

@@ -5,9 +5,10 @@ go 1.24.4
require (
github.com/cloudflare/ipvs v0.11.0
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
github.com/docker/docker v28.3.3+incompatible
github.com/florianl/go-conntrack v0.4.0
github.com/golang/protobuf v1.5.4
github.com/google/go-cmp v0.7.0
github.com/google/nftables v0.3.0
github.com/gookit/slog v0.5.8
github.com/huin/goupnp v1.3.0
github.com/insomniacslk/dhcp v0.0.0-20241224095048-b56fa0d5f25d
@@ -15,60 +16,69 @@ require (
github.com/mdlayher/ndp v1.1.0
github.com/onsi/ginkgo/v2 v2.23.4
github.com/onsi/gomega v1.37.0
github.com/osrg/gobgp/v3 v3.36.0
github.com/osrg/gobgp/v3 v3.37.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.22.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.9.1
github.com/stretchr/testify v1.10.0
github.com/vishvananda/netlink v1.3.0
github.com/vishvananda/netlink v1.3.1
go.etcd.io/etcd/api/v3 v3.6.1
go.etcd.io/etcd/client/pkg/v3 v3.6.1
go.etcd.io/etcd/client/v3 v3.6.1
go.uber.org/zap v1.27.0
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
golang.org/x/sync v0.13.0
golang.org/x/sync v0.15.0
golang.org/x/sys v0.33.0
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10
google.golang.org/grpc v1.73.0
google.golang.org/protobuf v1.36.6
k8s.io/api v0.32.3
k8s.io/apimachinery v0.32.3
k8s.io/client-go v0.32.3
k8s.io/api v0.33.3
k8s.io/apimachinery v0.33.3
k8s.io/client-go v0.33.3
k8s.io/klog/v2 v2.130.1
sigs.k8s.io/kind v0.27.0
sigs.k8s.io/kind v0.29.0
sigs.k8s.io/yaml v1.4.0
)
require (
al.essio.dev/pkg/shellescape v1.5.1 // indirect
github.com/BurntSushi/toml v1.4.0 // indirect
github.com/Microsoft/go-winio v0.4.14 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/containerd/errdefs v1.0.0 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/coreos/go-semver v0.3.1 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/dlclark/regexp2 v1.11.4 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/eapache/channels v1.1.0 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.8.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.21.0 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.6.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
github.com/google/safetext v0.0.0-20240722112252-5a72de7e7962 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gookit/color v1.5.4 // indirect
github.com/gookit/goutil v0.6.18 // indirect
github.com/gookit/gsr v0.1.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
@@ -79,14 +89,19 @@ require (
github.com/mailru/easyjson v0.9.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mdlayher/genetlink v1.3.2 // indirect
github.com/mdlayher/netlink v1.7.2 // indirect
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect
github.com/mdlayher/packet v1.1.2 // indirect
github.com/mdlayher/socket v0.5.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/sys/atomicwriter v0.1.0 // indirect
github.com/moby/term v0.5.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pborman/uuid v1.2.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/pierrec/lz4/v4 v4.1.22 // indirect
@@ -111,26 +126,32 @@ require (
github.com/xlab/c-for-go v1.3.0 // indirect
github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.37.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 // indirect
go.opentelemetry.io/otel/metric v1.37.0 // indirect
go.opentelemetry.io/otel/sdk v1.37.0 // indirect
go.opentelemetry.io/otel/trace v1.37.0 // indirect
go.uber.org/automaxprocs v1.6.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.36.0 // indirect
golang.org/x/mod v0.24.0 // indirect
golang.org/x/net v0.38.0 // indirect
golang.org/x/oauth2 v0.27.0 // indirect
golang.org/x/term v0.30.0 // indirect
golang.org/x/text v0.23.0 // indirect
golang.org/x/crypto v0.39.0 // indirect
golang.org/x/mod v0.25.0 // indirect
golang.org/x/net v0.41.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/term v0.32.0 // indirect
golang.org/x/text v0.26.0 // indirect
golang.org/x/time v0.9.0 // indirect
golang.org/x/tools v0.31.0 // indirect
golang.org/x/tools v0.33.0 // indirect
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect
google.golang.org/grpc v1.71.1 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 // indirect
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect
modernc.org/cc/v4 v4.24.4 // indirect
modernc.org/mathutil v1.7.1 // indirect
@@ -139,5 +160,6 @@ require (
modernc.org/strutil v1.2.1 // indirect
modernc.org/token v1.1.0 // indirect
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.5.0 // indirect
sigs.k8s.io/randfill v1.0.0 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
)

175
go.sum
View File

@@ -1,17 +1,29 @@
al.essio.dev/pkg/shellescape v1.5.1 h1:86HrALUujYS/h+GtqoB26SBEdkWfmMI6FubjXlsXyho=
al.essio.dev/pkg/shellescape v1.5.1/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8=
github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
github.com/cloudflare/ipvs v0.11.0 h1:niLcbqfv8+RSYk+yI+jhiCIHobEdEhO++mcwCX0Znsw=
github.com/cloudflare/ipvs v0.11.0/go.mod h1:XsbuKcQpqb3rpjYPtsFsM8BNZlgLw+Z0iC6RhocqDd8=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
@@ -23,8 +35,16 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k=
@@ -35,6 +55,8 @@ github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtz
github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/florianl/go-conntrack v0.4.0 h1:TlYkxytdwgVayfU0cKwkHurQA0Rd1ZSEBRckRYDUu18=
github.com/florianl/go-conntrack v0.4.0/go.mod h1:iPDx4oIats2T7X7Jm3PFyRCJM1GfZhJaSHOWROYOrE8=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
@@ -44,8 +66,9 @@ github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/
github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
@@ -76,15 +99,12 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/nftables v0.3.0 h1:bkyZ0cbpVeMHXOrtlFc8ISmfVqq5gPJukoYieyVmITg=
github.com/google/nftables v0.3.0/go.mod h1:BCp9FsrbF1Fn/Yu6CLUc9GGZFw/+hsxfluNXXmxBfRM=
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
github.com/google/safetext v0.0.0-20240722112252-5a72de7e7962 h1:+9C/TgFfcCmZBV7Fjb3kQCGlkpFrhtvFDgbdQHB9RaA=
github.com/google/safetext v0.0.0-20240722112252-5a72de7e7962/go.mod h1:H3K1Iu/utuCfa10JO+GsmKUYSWi7ug57Rk6GaDRHaaQ=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
@@ -95,8 +115,8 @@ github.com/gookit/gsr v0.1.0 h1:0gadWaYGU4phMs0bma38t+Do5OZowRMEVlHv31p0Zig=
github.com/gookit/gsr v0.1.0/go.mod h1:7wv4Y4WCnil8+DlDYHBjidzrEzfHhXEoFjEA0pPPWpI=
github.com/gookit/slog v0.5.8 h1:XZCeHLQvvOZWcSUDZcqxXITsL9+d1ESsKZoASBmK1lI=
github.com/gookit/slog v0.5.8/go.mod h1:s0ViFOY/IgUuT4MDPF0l9x5/npcciy8pL4xwWZadnoc=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714 h1:/jC7qQFrv8CrSJVmaolDVOxTfS9kc36uB6H40kdbQq8=
@@ -131,6 +151,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -164,8 +185,8 @@ github.com/mdlayher/netlink v1.3.0/go.mod h1:xK/BssKuwcRXHrtN04UBkwQ6dY9VviGGuri
github.com/mdlayher/netlink v1.4.0/go.mod h1:dRJi5IABcZpBD2A3D0Mv/AiX8I9uDEu5oGkAVrekmf8=
github.com/mdlayher/netlink v1.4.1/go.mod h1:e4/KuJ+s8UhfUpO9z00/fDZZmhSrs+oxyqAS9cNgn6Q=
github.com/mdlayher/netlink v1.5.0/go.mod h1:1Kr8BBFxGyUyNmztC9WLOayqYVAd2wsgOZm18nqGuzQ=
github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g=
github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 h1:A1Cq6Ysb0GM0tpKMbdCXCIfBclan4oHk1Jb+Hrejirg=
github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42/go.mod h1:BB4YCPDOzfy7FniQ/lxuYQ3dgmM2cZumHbK8RpTjN2o=
github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY=
github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4=
github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00/go.mod h1:GAFlyu4/XV68LkQKYzKhIo/WW7j3Zi0YRAz/BOoanUc=
@@ -177,29 +198,42 @@ github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE9
github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw=
github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs=
github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus=
github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8=
github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y=
github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0=
github.com/osrg/gobgp/v3 v3.36.0 h1:6KbNDyvSbN2GAIiVMykAgLUsvcSldNPiCCP5KzV0VP4=
github.com/osrg/gobgp/v3 v3.36.0/go.mod h1:ldZ/eydK80FuAmTGYBV23tWhTRJtIk2tw5NSYhPeqVk=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
github.com/osrg/gobgp/v3 v3.37.0 h1:+ObuOdvj7G7nxrT0fKFta+EAupdWf/q1WzbXydr8IOY=
github.com/osrg/gobgp/v3 v3.37.0/go.mod h1:kVHVFy1/fyZHJ8P32+ctvPeJogn9qKwa1YCeMRXXrP0=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -224,6 +258,7 @@ github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3
github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
@@ -239,6 +274,10 @@ github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
@@ -251,9 +290,8 @@ github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 h1:pyC9PaHYZFgEKFdlp3G8
github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk=
github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs=
github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
@@ -275,16 +313,24 @@ go.etcd.io/etcd/client/v3 v3.6.1 h1:KelkcizJGsskUXlsxjVrSmINvMMga0VWwFF0tSPGEP0=
go.etcd.io/etcd/client/v3 v3.6.1/go.mod h1:fCbPUdjWNLfx1A6ATo9syUmFVxqHH9bCnPLBZmnLmMY=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY=
go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=
go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ=
go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE=
go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A=
go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU=
go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk=
go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w=
go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k=
go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0 h1:bDMKF3RUSxshZ5OjOTi8rsHGaPKsAt76FaqgvIUySLc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.37.0/go.mod h1:dDT67G/IkA46Mr2l9Uj7HsQVwsjASyV9SjGofsiUZDA=
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
@@ -296,16 +342,16 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329 h1:9kj3STMvgqy3YA4VQXBrN7925ICMxD5wzMRcgA30588=
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -326,20 +372,22 @@ golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M=
golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -369,13 +417,13 @@ golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=
golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -384,8 +432,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU=
golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ=
golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -394,12 +442,12 @@ golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uI
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA=
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10 h1:3GDAcqdIg1ozBNLgPy4SLT84nfcBjr6rhGtXYtrkWLU=
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10/go.mod h1:T97yPqesLiNrOYxkwmhMI0ZIlJDm+p0PMR8eRVeR5tQ=
google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950=
google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I=
google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI=
google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec=
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY=
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok=
google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc=
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -420,16 +468,16 @@ gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
k8s.io/api v0.32.3 h1:Hw7KqxRusq+6QSplE3NYG4MBxZw1BZnq4aP4cJVINls=
k8s.io/api v0.32.3/go.mod h1:2wEDTXADtm/HA7CCMD8D8bK4yuBUptzaRhYcYEEYA3k=
k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U=
k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE=
k8s.io/client-go v0.32.3 h1:RKPVltzopkSgHS7aS98QdscAgtgah/+zmpAogooIqVU=
k8s.io/client-go v0.32.3/go.mod h1:3v0+3k4IcT9bXTc4V2rt+d2ZPPG700Xy6Oi0Gdl2PaY=
k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8=
k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE=
k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA=
k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM=
k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA=
k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg=
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 h1:hcha5B1kVACrLujCKLbr8XWMxCxzQx42DY8QKYJrDLg=
k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7/go.mod h1:GewRfANuJ70iYzvn+i4lezLDAFzvjxZYK1gn1lWcfas=
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4=
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8=
k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0=
k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
modernc.org/cc/v4 v4.24.4 h1:TFkx1s6dCkQpd6dKurBNmpo+G8Zl4Sq/ztJ+2+DEsh0=
@@ -450,9 +498,12 @@ pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
sigs.k8s.io/kind v0.27.0 h1:PQ3f0iAWNIj66LYkZ1ivhEg/+Zb6UPMbO+qVei/INZA=
sigs.k8s.io/kind v0.27.0/go.mod h1:RZVFmy6qcwlSWwp6xeIUv7kXCPF3i8MXsEXxW/J+gJY=
sigs.k8s.io/structured-merge-diff/v4 v4.5.0 h1:nbCitCK2hfnhyiKo6uf2HxUPTCodY6Qaf85SbDIaMBk=
sigs.k8s.io/structured-merge-diff/v4 v4.5.0/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4=
sigs.k8s.io/kind v0.29.0 h1:3TpCsyh908IkXXpcSnsMjWdwdWjIl7o9IMZImZCWFnI=
sigs.k8s.io/kind v0.29.0/go.mod h1:ldWQisw2NYyM6k64o/tkZng/1qQW7OlzcN5a8geJX3o=
sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc=
sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps=
sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

View File

@@ -40,7 +40,6 @@ func (e *Entry) Check() bool {
switch {
case utils.FileExists(adminConfigPath):
config, err = k8s.NewRestConfig(adminConfigPath, false, k8sAddr)
// client, err = k8s.NewClientset(adminConfigPath, false, k8sAddr)
if err != nil {
log.Error("create k8s REST config", "path", adminConfigPath, "err", err)
return false
@@ -53,10 +52,6 @@ func (e *Entry) Check() bool {
}
}
// TODO: check if using insecure client for this check is OK.
// This was changed for more straightforward e2e-test implementation.
config.Insecure = true
config.CAData = []byte{}
client, err = k8s.NewClientset(config)
if err != nil {
log.Error("create k8s client", "err", err)

View File

@@ -8,7 +8,6 @@ import (
"strings"
//nolint
"github.com/golang/protobuf/ptypes/any"
"github.com/kube-vip/kube-vip/pkg/vip"
api "github.com/osrg/gobgp/v3/api"
"github.com/osrg/gobgp/v3/pkg/server"
@@ -142,7 +141,7 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
Safi: api.Family_SAFI_UNICAST,
},
Nlri: nlri,
Pattrs: []*any.Any{originAttr, nhAttr},
Pattrs: []*anypb.Any{originAttr, nhAttr},
}
} else {
//nolint
@@ -160,13 +159,13 @@ func (b *Server) getPath(ip net.IP) (path *api.Path) {
mpAttr, _ := anypb.New(&api.MpReachNLRIAttribute{
Family: v6Family,
NextHops: []string{"::"}, // gobgp will fill this
Nlris: []*any.Any{nlri},
Nlris: []*anypb.Any{nlri},
})
path = &api.Path{
Family: v6Family,
Nlri: nlri,
Pattrs: []*any.Any{originAttr, mpAttr},
Pattrs: []*anypb.Any{originAttr, mpAttr},
}
}
return

View File

@@ -9,10 +9,11 @@ import (
api "github.com/osrg/gobgp/v3/api"
gobgp "github.com/osrg/gobgp/v3/pkg/server"
"github.com/prometheus/client_golang/prometheus"
)
// NewBGPServer takes a configuration and returns a running BGP server instance
func NewBGPServer(c *Config, peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (b *Server, err error) {
func NewBGPServer(c *Config) (b *Server, err error) {
if c.AS == 0 {
return nil, fmt.Errorf("you need to provide AS")
}
@@ -28,13 +29,25 @@ func NewBGPServer(c *Config, peerStateChangeCallback func(*api.WatchEventRespons
b = &Server{
s: gobgp.NewBgpServer(),
c: c,
BGPSessionInfoGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "kube_vip",
Subsystem: "manager",
Name: "bgp_session_info",
Help: "Display state of session by setting metric for label value with current state to 1",
}, []string{"state", "peer"}),
}
return
}
// Start starts the BGP server
func (b *Server) Start(peerStateChangeCallback func(*api.WatchEventResponse_PeerEvent)) (err error) {
go b.s.Serve()
if err = b.s.StartBgp(context.Background(), &api.StartBgpRequest{
Global: &api.Global{
Asn: c.AS,
RouterId: c.RouterID,
Asn: b.c.AS,
RouterId: b.c.RouterID,
ListenPort: -1,
},
}); err != nil {
@@ -52,12 +65,23 @@ func NewBGPServer(c *Config, peerStateChangeCallback func(*api.WatchEventRespons
return
}
for _, p := range c.Peers {
for _, p := range b.c.Peers {
if err = b.AddPeer(p); err != nil {
return
}
}
if b.c.Zebra.Enabled {
if err = b.s.EnableZebra(context.Background(), &api.EnableZebraRequest{
Url: b.c.Zebra.URL,
Version: b.c.Zebra.Version,
SoftwareName: b.c.Zebra.SoftwareName,
}); err != nil {
log.Error(err.Error())
return
}
}
return
}

View File

@@ -7,6 +7,7 @@ import (
"github.com/kube-vip/kube-vip/pkg/vip"
api "github.com/osrg/gobgp/v3/api"
gobgp "github.com/osrg/gobgp/v3/pkg/server"
"github.com/prometheus/client_golang/prometheus"
"github.com/vishvananda/netlink"
)
@@ -121,10 +122,24 @@ type Config struct {
KeepaliveInterval uint64
Peers []Peer
Zebra ZebraConfig
}
// Defines Zebra connection configuration. More on the topic - https://github.com/osrg/gobgp/blob/master/docs/sources/zebra.md#configuration
type ZebraConfig struct {
Enabled bool
URL string
Version uint32
SoftwareName string
}
// Server manages a server object
type Server struct {
s *gobgp.BgpServer
c *Config
// This is a prometheus gauge indicating the state of the sessions.
// 1 means "ESTABLISHED", 0 means "NOT ESTABLISHED"
BGPSessionInfoGauge *prometheus.GaugeVec
}

View File

@@ -55,7 +55,8 @@ func startNetworking(c *kubevip.Config, intfMgr *networkinterface.Manager) ([]vi
networks := []vip.Network{}
for _, addr := range addresses {
network, err := vip.NewConfig(addr, c.Interface, c.LoInterfaceGlobalScope, c.VIPSubnet, c.DDNS, c.RoutingTableID,
c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend, c.EnableLoadBalancer, intfMgr)
c.RoutingTableType, c.RoutingProtocol, c.DNSMode, c.LoadBalancerForwardingMethod, c.IptablesBackend,
c.EnableLoadBalancer, c.EnableServiceSecurity, intfMgr)
if err != nil {
return nil, err
}

View File

@@ -151,10 +151,13 @@ func (cluster *Cluster) StartCluster(c *kubevip.Config, sm *Manager, bgpServer *
if c.EnableBGP && bgpServer == nil {
// Lets start BGP
log.Info("Starting the BGP server to advertise VIP routes to VGP peers")
bgpServer, err = bgp.NewBGPServer(&c.BGPConfig, nil)
bgpServer, err = bgp.NewBGPServer(&c.BGPConfig)
if err != nil {
log.Error("new BGP server", "err", err)
}
if err := bgpServer.Start(nil); err != nil {
log.Error("starting BGP server", "err", err)
}
}
run := &runConfig{
@@ -281,7 +284,7 @@ func (cluster *Cluster) runEtcdLeaderElectionOrDie(ctx context.Context, run *run
})
}
func (sm *Manager) NodeWatcher(lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
func (sm *Manager) NodeWatcher(ctxArp context.Context, lb *loadbalancer.IPVSLoadBalancer, port uint16) error {
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
log.Info("Kube-Vip is watching nodes for control-plane labels")
@@ -289,7 +292,7 @@ func (sm *Manager) NodeWatcher(lb *loadbalancer.IPVSLoadBalancer, port uint16) e
LabelSelector: "node-role.kubernetes.io/control-plane",
}
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
rw, err := watchtools.NewRetryWatcherWithContext(ctxArp, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.RetryWatcherClient.CoreV1().Nodes().Watch(context.Background(), listOptions)
},

View File

@@ -84,7 +84,7 @@ func (cluster *Cluster) vipService(ctxArp, ctxDNS context.Context, c *kubevip.Co
}
go func() {
err = sm.NodeWatcher(lb, c.Port)
err = sm.NodeWatcher(ctxArp, lb, c.Port) //TODO: We're using the ctxARP as the context this will change when rkatz finishes his change
if err != nil {
log.Error("Error watching node labels", "err", err)
}
@@ -257,11 +257,11 @@ func getNodeIPs(ctx context.Context, nodename string, client *kubernetes.Clients
}
// StartLoadBalancerService will start a VIP instance and leave it for kube-proxy to handle
func (cluster *Cluster) StartLoadBalancerService(c *kubevip.Config, bgp *bgp.Server, name string, serviceInstances *[]*Instance) {
func (cluster *Cluster) StartLoadBalancerService(ctx context.Context, c *kubevip.Config, bgp *bgp.Server, name string, CountRouteReferences func(*netlink.Route) int) {
// use a Go context so we can tell the arp loop code when we
// want to step down
//nolint
ctxArp, cancelArp := context.WithCancel(context.Background())
ctxArp, cancelArp := context.WithCancel(ctx)
cluster.stop = make(chan bool, 1)
cluster.completed = make(chan bool, 1)
@@ -318,7 +318,7 @@ func (cluster *Cluster) StartLoadBalancerService(c *kubevip.Config, bgp *bgp.Ser
for i := range cluster.Network {
// chek if route is not referenced by another service
r := cluster.Network[i].PrepareRoute()
if CountRouteReferences(serviceInstances, r) < 1 {
if CountRouteReferences(r) < 1 {
log.Info("[VIP] Deleting Route for VIP", "IP", cluster.Network[i].IP())
if err := cluster.Network[i].DeleteRoute(); err != nil {
log.Warn(err.Error())
@@ -379,22 +379,3 @@ func (cluster *Cluster) layer2Update(ctx context.Context, network vip.Network, c
log.Debug("ending layer 2 update", "ip", ipString, "interface", network.Interface(), "ms", c.ArpBroadcastRate)
cluster.arpMgr.Remove(arpInstance)
}
func CountRouteReferences(serviceInstances *[]*Instance, route *netlink.Route) int {
// Count how many service instances have the same route
// This function is not thread-safe, it should be called only from the main thread with a lock held
cnt := 0
for _, instance := range *serviceInstances {
for _, cluster := range instance.Clusters {
for n := range cluster.Network {
if cluster.Network[n].HasEndpoints() {
r := cluster.Network[n].PrepareRoute()
if r.Dst.String() == route.Dst.String() {
cnt++
}
}
}
}
}
return cnt
}

99
pkg/egress/egress.go Normal file
View File

@@ -0,0 +1,99 @@
package egress
import (
"fmt"
"strings"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/vip"
)
func Teardown(podIP, vipIP, namespace, serviceUUID string, annotations map[string]string, useNftables bool) error {
// Look up the destination ports from the annotations on the service
destinationPorts := annotations[kubevip.EgressDestinationPorts]
deniedNetworks := annotations[kubevip.EgressDeniedNetworks]
allowedNetworks := annotations[kubevip.EgressAllowedNetworks]
internalEgress := annotations[kubevip.EgressInternal]
protocol := iptables.ProtocolIPv4
IPv6 := false
if vip.IsIPv6(podIP) {
protocol = iptables.ProtocolIPv6
IPv6 = true
}
// Use the internal egress implementation
if internalEgress != "" {
return nftables.DeleteSNAT(IPv6, serviceUUID)
}
i, err := vip.CreateIptablesClient(useNftables, namespace, protocol)
if err != nil {
return fmt.Errorf("error Creating iptables client [%s]", err)
}
if deniedNetworks != "" {
networks := strings.Split(deniedNetworks, ",")
for x := range networks {
err = i.DeleteMangleReturnForNetwork(vip.MangleChainName, networks[x])
if err != nil {
return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
}
if allowedNetworks != "" {
networks := strings.Split(allowedNetworks, ",")
for x := range networks {
err = i.DeleteMangleMarkingForNetwork(podIP, vip.MangleChainName, networks[x])
if err != nil {
return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
} else {
// Remove the marking of egress packets
err = i.DeleteMangleMarking(podIP, vip.MangleChainName)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
// Clear up SNAT rules
if destinationPorts != "" {
fixedPorts := strings.Split(destinationPorts, ",")
for _, fixedPort := range fixedPorts {
var proto, port string
data := strings.Split(fixedPort, ":")
if len(data) == 0 {
continue
} else if len(data) == 1 {
proto = "tcp"
port = data[0]
} else {
proto = data[0]
port = data[1]
}
err = i.DeleteSourceNatForDestinationPort(podIP, vipIP, port, proto)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
} else {
err = i.DeleteSourceNat(podIP, vipIP)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
err = vip.DeleteExistingSessions(podIP, false, destinationPorts, "")
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
return nil
}

165
pkg/endpoints/endpoints.go Normal file
View File

@@ -0,0 +1,165 @@
package endpoints
import (
"context"
"fmt"
"net"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
"k8s.io/apimachinery/pkg/watch"
)
type Processor struct {
config *kubevip.Config
provider providers.Provider
bgpServer *bgp.Server
worker endpointWorker
instances *[]*instance.Instance
}
func NewEndpointProcessor(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server,
instances *[]*instance.Instance) *Processor {
return &Processor{
config: config,
provider: provider,
bgpServer: bgpServer,
instances: instances,
worker: newEndpointWorker(config, provider, bgpServer, instances),
}
}
func (p *Processor) AddOrModify(ctx *servicecontext.Context, event watch.Event,
lastKnownGoodEndpoint *string, service *v1.Service, id string, leaderElectionActive *bool,
serviceFunc func(context.Context, *v1.Service) error,
leaderCtx *context.Context, cancel *context.CancelFunc) (bool, error) {
var err error
if err = p.provider.LoadObject(event.Object, *cancel); err != nil {
return false, fmt.Errorf("[%s] error loading k8s object: %w", p.provider.GetLabel(), err)
}
endpoints, err := p.worker.getEndpoints(service, id)
if err != nil {
return false, err
}
if err := p.worker.setInstanceEndpointsStatus(service, endpoints); err != nil {
log.Error("updating instance", "err", err)
}
// Find out if we have any local endpoints
// if out endpoint is empty then populate it
// if not, go through the endpoints and see if ours still exists
// If we have a local endpoint then begin the leader Election, unless it's already running
//
// Check that we have local endpoints
if len(endpoints) != 0 {
// Ignore IPv4
if service.Annotations[kubevip.EgressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
return true, nil
}
p.updateLastKnownGoodEndpoint(lastKnownGoodEndpoint, endpoints, service, leaderElectionActive, *cancel)
// start leader election if it's enabled and not already started
if !*leaderElectionActive && p.config.EnableServicesElection {
go func() {
*leaderCtx, *cancel = context.WithCancel(ctx.Ctx)
startLeaderElection(*leaderCtx, leaderElectionActive, service, serviceFunc)
}()
}
// There are local endpoints available on the node
if !p.config.EnableServicesElection && !p.config.EnableLeaderElection {
if err := p.worker.processInstance(ctx, service, leaderElectionActive); err != nil {
return false, fmt.Errorf("failed to process non-empty instance: %w", err)
}
}
} else {
// There are no local endpoints
p.worker.clear(ctx, lastKnownGoodEndpoint, service, *cancel, leaderElectionActive)
}
// Set the service accordingly
p.updateAnnotations(service, lastKnownGoodEndpoint)
log.Debug("watcher", "provider",
p.provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", *lastKnownGoodEndpoint, "active leader election", *leaderElectionActive)
return false, nil
}
func (p *Processor) Delete(service *v1.Service, id string) error {
if err := p.worker.delete(service, id); err != nil {
return fmt.Errorf("[%s] error deleting service: %w", p.provider.GetLabel(), err)
}
return nil
}
func (p *Processor) updateLastKnownGoodEndpoint(lastKnownGoodEndpoint *string, endpoints []string, service *v1.Service, leaderElectionActive *bool, cancel context.CancelFunc) {
// if we haven't populated one, then do so
if *lastKnownGoodEndpoint == "" {
*lastKnownGoodEndpoint = endpoints[0]
return
}
// check out previous endpoint exists
stillExists := false
for x := range endpoints {
if endpoints[x] == *lastKnownGoodEndpoint {
stillExists = true
}
}
// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
if !stillExists {
p.worker.removeEgress(service, lastKnownGoodEndpoint)
if *leaderElectionActive && (p.config.EnableServicesElection || p.config.EnableLeaderElection) {
log.Warn("existing endpoint has been removed, restarting leaderElection", "provider", p.provider.GetLabel(), "endpoint", *lastKnownGoodEndpoint)
// Stop the existing leaderElection
cancel()
// disable last leaderElection flag
*leaderElectionActive = false
}
// Set our active endpoint to an existing one
*lastKnownGoodEndpoint = endpoints[0]
}
}
func (p *Processor) updateAnnotations(service *v1.Service, lastKnownGoodEndpoint *string) {
// Set the service accordingly
if service.Annotations[kubevip.Egress] == "true" {
activeEndpointAnnotation := kubevip.ActiveEndpoint
if !p.config.EnableEndpoints && p.provider.GetProtocol() == string(discoveryv1.AddressTypeIPv6) {
activeEndpointAnnotation = kubevip.ActiveEndpointIPv6
}
service.Annotations[activeEndpointAnnotation] = *lastKnownGoodEndpoint
}
}
func startLeaderElection(ctx context.Context, leaderElectionActive *bool, service *v1.Service, serviceFunc func(context.Context, *v1.Service) error) {
// This is a blocking function, that will restart (in the event of failure)
for {
// if the context isn't cancelled restart
if ctx.Err() != context.Canceled {
*leaderElectionActive = true
err := serviceFunc(ctx, service)
if err != nil {
log.Error(err.Error())
}
*leaderElectionActive = false
} else {
*leaderElectionActive = false
break
}
}
}

View File

@@ -0,0 +1,125 @@
package endpoints
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
)
type BGP struct {
generic
bgpServer *bgp.Server
}
func newBGP(generic generic, bgpServer *bgp.Server) endpointWorker {
return &BGP{
generic: generic,
bgpServer: bgpServer,
}
}
func (b *BGP) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) {
log.Debug("attempting to advertise BGP service", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP())
err := b.bgpServer.AddHost(cluster.Network[i].CIDR())
if err != nil {
log.Error("error adding BGP host", "provider", b.provider.GetLabel(), "err", err)
} else {
log.Info("added BGP host", "provider",
b.provider.GetLabel(), "ip", cluster.Network[i].CIDR(), "service name", service.Name, "namespace", service.Namespace)
ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
*leaderElectionActive = true
}
}
}
}
}
return nil
}
func (b *BGP) clear(ctx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
// If BGP mode is enabled - routes should be deleted
if instance := instance.FindServiceInstance(service, *b.instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
err := b.bgpServer.DelHost(cluster.Network[i].CIDR())
if err != nil {
log.Error("deleting BGP host", "provider", b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "err", err)
} else {
log.Info("deleted BGP host", "provider",
b.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace)
ctx.ConfiguredNetworks.Delete(cluster.Network[i])
*leaderElectionActive = false
}
}
}
}
}
b.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (b *BGP) getEndpoints(service *v1.Service, id string) ([]string, error) {
return b.getAllEndpoints(service, id)
}
func (b *BGP) delete(service *v1.Service, id string) error {
// When no-leader-elecition mode
if !b.config.EnableServicesElection && !b.config.EnableLeaderElection {
// find all existing local endpoints
endpoints, err := b.getEndpoints(service, id)
if err != nil {
return fmt.Errorf("[%s] error getting endpoints: %w", b.provider.GetLabel(), err)
}
// If there were local endpoints deleted
if len(endpoints) > 0 {
b.deleteAction(service)
}
}
return nil
}
func (b *BGP) deleteAction(service *v1.Service) {
b.clearBGPHosts(service)
}
func (b *BGP) clearBGPHosts(service *v1.Service) {
ClearBGPHosts(service, b.instances, b.bgpServer)
}
func (b *BGP) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
return nil
}
func ClearBGPHosts(service *v1.Service, instances *[]*instance.Instance, bgpServer *bgp.Server) {
if instance := instance.FindServiceInstance(service, *instances); instance != nil {
ClearBGPHostsByInstance(instance, bgpServer)
}
}
func ClearBGPHostsByInstance(instance *instance.Instance, bgpServer *bgp.Server) {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
network := cluster.Network[i]
err := bgpServer.DelHost(network.CIDR())
if err != nil {
log.Error("[endpoint] error deleting BGP host", "err", err)
} else {
log.Debug("[endpoint] deleted BGP host", "ip",
network.CIDR(), "service name", instance.ServiceSnapshot.Name, "namespace", instance.ServiceSnapshot.Namespace)
}
}
}
}

View File

@@ -0,0 +1,118 @@
package endpoints
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/egress"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
)
type endpointWorker interface {
processInstance(svcCtx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error
clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool)
getEndpoints(service *v1.Service, id string) ([]string, error)
removeEgress(service *v1.Service, lastKnownGoodEndpoint *string)
delete(service *v1.Service, id string) error
setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error
}
func newEndpointWorker(config *kubevip.Config, provider providers.Provider, bgpServer *bgp.Server, instances *[]*instance.Instance) endpointWorker {
generic := newGeneric(config, provider, instances)
if config.EnableRoutingTable {
return newRoutingTable(generic)
}
if config.EnableBGP {
return newBGP(generic, bgpServer)
}
return &generic
}
type generic struct {
config *kubevip.Config
provider providers.Provider
instances *[]*instance.Instance
}
func newGeneric(config *kubevip.Config, provider providers.Provider, instances *[]*instance.Instance) generic {
return generic{
config: config,
provider: provider,
instances: instances,
}
}
func (g *generic) processInstance(_ *servicecontext.Context, _ *v1.Service, _ *bool) error {
return nil
}
func (g *generic) clear(_ *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
g.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (g *generic) clearEgress(lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if *lastKnownGoodEndpoint != "" {
log.Warn("existing endpoint has been removed, no remaining endpoints for leaderElection", "provider", g.provider.GetLabel(), "endpoint", lastKnownGoodEndpoint)
if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP, service.Namespace, string(service.UID), service.Annotations, g.config.EgressWithNftables); err != nil {
log.Error("error removing redundant egress rules", "err", err)
}
*lastKnownGoodEndpoint = "" // reset endpoint
if g.config.EnableServicesElection || g.config.EnableLeaderElection {
cancel() // stop services watcher
}
*leaderElectionActive = false
}
}
func (g *generic) getEndpoints(_ *v1.Service, id string) ([]string, error) {
return g.getLocalEndpoints(id)
}
func (g *generic) getLocalEndpoints(id string) ([]string, error) {
// Build endpoints
var endpoints []string
var err error
if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
}
return endpoints, nil
}
func (g *generic) getAllEndpoints(service *v1.Service, id string) ([]string, error) {
// Build endpoints
var err error
var endpoints []string
if !g.config.EnableLeaderElection && !g.config.EnableServicesElection &&
service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
if endpoints, err = g.provider.GetAllEndpoints(); err != nil {
return nil, fmt.Errorf("[%s] error getting all endpoints: %w", g.provider.GetLabel(), err)
}
} else {
if endpoints, err = g.provider.GetLocalEndpoints(id, g.config); err != nil {
return nil, fmt.Errorf("[%s] error getting local endpoints: %w", g.provider.GetLabel(), err)
}
}
return endpoints, nil
}
func (g *generic) removeEgress(_ *v1.Service, _ *string) {
}
func (g *generic) delete(_ *v1.Service, _ string) error {
return nil
}
func (g *generic) setInstanceEndpointsStatus(_ *v1.Service, _ []string) error {
return nil
}

View File

@@ -0,0 +1,178 @@
package endpoints
import (
"context"
"errors"
"fmt"
"net"
"syscall"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/egress"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
)
type RoutingTable struct {
generic
}
func newRoutingTable(generic generic) endpointWorker {
return &RoutingTable{
generic: generic,
}
}
func (rt *RoutingTable) processInstance(ctx *servicecontext.Context, service *v1.Service, leaderElectionActive *bool) error {
instance := instance.FindServiceInstance(service, *rt.instances)
if instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
if !ctx.IsNetworkConfigured(cluster.Network[i].IP()) && cluster.Network[i].HasEndpoints() {
err := cluster.Network[i].AddRoute(false)
if err != nil {
if errors.Is(err, syscall.EEXIST) {
// If route exists, but protocol is not set (e.g. the route was created by the older version
// of kube-vip) try to update it if necessary
isUpdated, err := cluster.Network[i].UpdateRoutes()
if err != nil {
return fmt.Errorf("[%s] error updating existing routes: %w", rt.provider.GetLabel(), err)
}
if isUpdated {
log.Info("updated route", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
} else {
log.Info("route already present", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
}
} else {
// If other error occurs, return error
return fmt.Errorf("[%s] error adding route: %s", rt.provider.GetLabel(), err.Error())
}
} else {
log.Info("added route", "provider",
rt.provider.GetLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace",
service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", rt.config.RoutingTableID)
ctx.ConfiguredNetworks.Store(cluster.Network[i].IP(), true)
*leaderElectionActive = true
}
}
}
}
}
return nil
}
func (rt *RoutingTable) clear(svcCtx *servicecontext.Context, lastKnownGoodEndpoint *string, service *v1.Service, cancel context.CancelFunc, leaderElectionActive *bool) {
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
if errs := ClearRoutes(service, rt.instances); len(errs) == 0 {
svcCtx.ConfiguredNetworks.Clear()
} else {
for _, err := range errs {
log.Error("error while clearing routes", "err", err)
}
}
}
rt.clearEgress(lastKnownGoodEndpoint, service, cancel, leaderElectionActive)
}
func (rt *RoutingTable) getEndpoints(service *v1.Service, id string) ([]string, error) {
return rt.getAllEndpoints(service, id)
}
func (rt *RoutingTable) removeEgress(service *v1.Service, lastKnownGoodEndpoint *string) {
if err := egress.Teardown(*lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
service.Namespace, string(service.UID), service.Annotations, rt.config.EgressWithNftables); err != nil {
log.Warn("removing redundant egress rules", "err", err)
}
}
func (rt *RoutingTable) delete(service *v1.Service, id string) error {
// When no-leader-elecition mode
if !rt.config.EnableServicesElection && !rt.config.EnableLeaderElection {
// find all existing local endpoints
endpoints, err := rt.getEndpoints(service, id)
if err != nil {
return fmt.Errorf("[%s] error getting endpoints: %w", rt.provider.GetLabel(), err)
}
// If there were local endpoints deleted
if len(endpoints) > 0 {
rt.deleteAction(service)
}
}
return nil
}
func (rt *RoutingTable) deleteAction(service *v1.Service) {
ClearRoutes(service, rt.instances)
}
func (rt *RoutingTable) setInstanceEndpointsStatus(service *v1.Service, endpoints []string) error {
instance := instance.FindServiceInstance(service, *rt.instances)
if instance == nil {
log.Error("failed to find the instance", "service", service.UID, "provider", rt.provider.GetLabel())
} else {
for _, c := range instance.Clusters {
for n := range c.Network {
// if there are no endpoints set HasEndpoints false just in case
if len(endpoints) < 1 {
c.Network[n].SetHasEndpoints(false)
}
// check if endpoint are available and are of same IP family as service
if len(endpoints) > 0 && ((net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(endpoints[0]).To4() == nil)) {
c.Network[n].SetHasEndpoints(true)
}
}
}
}
return nil
}
func ClearRoutes(service *v1.Service, instances *[]*instance.Instance) []error {
errs := []error{}
if instance := instance.FindServiceInstance(service, *instances); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
route := cluster.Network[i].PrepareRoute()
// check if route we are about to delete is not referenced by more than one service
if CountRouteReferences(route, instances) <= 1 {
err := cluster.Network[i].DeleteRoute()
if err != nil && !errors.Is(err, syscall.ESRCH) {
log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
errs = append(errs, err)
}
log.Debug("deleted route", "ip",
cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface())
}
}
}
}
return errs
}
func CountRouteReferences(route *netlink.Route, instances *[]*instance.Instance) int {
cnt := 0
for _, instance := range *instances {
for _, cluster := range instance.Clusters {
for n := range cluster.Network {
if cluster.Network[n].HasEndpoints() {
r := cluster.Network[n].PrepareRoute()
if r.Dst.String() == route.Dst.String() {
cnt++
}
}
}
}
}
return cnt
}

View File

@@ -0,0 +1,137 @@
package providers
import (
"context"
"fmt"
"strings"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/client-go/util/retry"
log "log/slog"
)
type Endpoints struct {
label string
//nolint:staticcheck // SA1019 endpoints are moving to an opt-in only
endpoints *v1.Endpoints
}
func NewEndpoints() Provider {
return &Endpoints{
label: "endpoints",
}
}
func (ep *Endpoints) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
service *v1.Service) (*watchtools.RetryWatcher, error) {
opts := metav1.ListOptions{
FieldSelector: fields.OneTermEqualSelector("metadata.name", service.Name).String(),
}
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return clientSet.CoreV1().Endpoints(service.Namespace).Watch(ctx, opts)
},
})
if err != nil {
return nil, fmt.Errorf("error creating endpoint watcher: %s", err.Error())
}
return rw, nil
}
func (ep *Endpoints) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
//nolint:staticcheck // SA1019 endpoints have to be explicitly requested now
eps, ok := endpoints.(*v1.Endpoints)
if !ok {
cancel()
return fmt.Errorf("[%s] unable to parse Kubernetes services from API watcher", ep.GetLabel())
}
ep.endpoints = eps
return nil
}
func (ep *Endpoints) GetAllEndpoints() ([]string, error) {
result := []string{}
for subset := range ep.endpoints.Subsets {
for address := range ep.endpoints.Subsets[subset].Addresses {
addr := strings.Split(ep.endpoints.Subsets[subset].Addresses[address].IP, "/")
result = append(result, addr[0])
}
}
return result, nil
}
func (ep *Endpoints) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
var localEndpoints []string
for _, subset := range ep.endpoints.Subsets {
for _, address := range subset.Addresses {
log.Debug("processing endpoint", "label", ep.label, "ip", address.IP)
// 1. Compare the Nodename
if address.NodeName != nil && id == *address.NodeName {
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname, "nodename", *address.NodeName)
localEndpoints = append(localEndpoints, address.IP)
continue
}
// 2. Compare the Hostname (only useful if address.NodeName is not available)
if id == address.Hostname {
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname)
localEndpoints = append(localEndpoints, address.IP)
continue
}
}
}
return localEndpoints, nil
}
func (ep *Endpoints) UpdateServiceAnnotation(endpoint string, _ string, service *v1.Service, clientSet *kubernetes.Clientset) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
currentServiceCopy := currentService.DeepCopy()
if currentServiceCopy.Annotations == nil {
currentServiceCopy.Annotations = make(map[string]string)
}
currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("error updating Service Spec", "label", ep.GetLabel(), "name", currentServiceCopy.Name, "err", err)
return err
}
return nil
})
if retryErr != nil {
log.Error("failed to set Services", "label", ep.GetLabel(), "err", retryErr)
return retryErr
}
return nil
}
func (ep *Endpoints) GetLabel() string {
return ep.label
}
func (ep *Endpoints) GetProtocol() string {
return ""
}

View File

@@ -1,4 +1,4 @@
package manager
package providers
import (
"context"
@@ -13,17 +13,24 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/client-go/util/retry"
)
type endpointslicesProvider struct {
type Endpointslices struct {
label string
endpoints *discoveryv1.EndpointSlice
}
func (ep *endpointslicesProvider) createRetryWatcher(ctx context.Context, sm *Manager,
func NewEndpointslices() Provider {
return &Endpointslices{
label: "endpointslices",
}
}
func (ep *Endpointslices) CreateRetryWatcher(ctx context.Context, clientSet *kubernetes.Clientset,
service *v1.Service) (*watchtools.RetryWatcher, error) {
labelSelector := metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/service-name": service.Name}}
@@ -31,9 +38,9 @@ func (ep *endpointslicesProvider) createRetryWatcher(ctx context.Context, sm *Ma
LabelSelector: labels.Set(labelSelector.MatchLabels).String(),
}
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.rwClientSet.DiscoveryV1().EndpointSlices(service.Namespace).Watch(ctx, opts)
return clientSet.DiscoveryV1().EndpointSlices(service.Namespace).Watch(ctx, opts)
},
})
if err != nil {
@@ -43,7 +50,7 @@ func (ep *endpointslicesProvider) createRetryWatcher(ctx context.Context, sm *Ma
return rw, nil
}
func (ep *endpointslicesProvider) loadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
func (ep *Endpointslices) LoadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
eps, ok := endpoints.(*discoveryv1.EndpointSlice)
if !ok {
cancel()
@@ -53,7 +60,7 @@ func (ep *endpointslicesProvider) loadObject(endpoints runtime.Object, cancel co
return nil
}
func (ep *endpointslicesProvider) getAllEndpoints() ([]string, error) {
func (ep *Endpointslices) GetAllEndpoints() ([]string, error) {
result := []string{}
for _, ep := range ep.endpoints.Endpoints {
result = append(result, ep.Addresses...)
@@ -61,7 +68,7 @@ func (ep *endpointslicesProvider) getAllEndpoints() ([]string, error) {
return result, nil
}
func (ep *endpointslicesProvider) getLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
func (ep *Endpointslices) GetLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
var localEndpoints []string
for _, endpoint := range ep.endpoints.Endpoints {
if !*endpoint.Conditions.Serving {
@@ -91,11 +98,11 @@ func (ep *endpointslicesProvider) getLocalEndpoints(id string, _ *kubevip.Config
return localEndpoints, nil
}
func (ep *endpointslicesProvider) updateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, sm *Manager) error {
func (ep *Endpointslices) UpdateServiceAnnotation(endpoint, endpointIPv6 string, service *v1.Service, clientSet *kubernetes.Clientset) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := sm.clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
currentService, err := clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
@@ -105,10 +112,10 @@ func (ep *endpointslicesProvider) updateServiceAnnotation(endpoint, endpointIPv6
currentServiceCopy.Annotations = make(map[string]string)
}
currentServiceCopy.Annotations[activeEndpoint] = endpoint
currentServiceCopy.Annotations[activeEndpointIPv6] = endpointIPv6
currentServiceCopy.Annotations[kubevip.ActiveEndpoint] = endpoint
currentServiceCopy.Annotations[kubevip.ActiveEndpointIPv6] = endpointIPv6
_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
_, err = clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("error updating Service Spec", "provider", ep.label, "service name", currentServiceCopy.Name, "err", err)
return err
@@ -123,10 +130,10 @@ func (ep *endpointslicesProvider) updateServiceAnnotation(endpoint, endpointIPv6
return nil
}
func (ep *endpointslicesProvider) getLabel() string {
func (ep *Endpointslices) GetLabel() string {
return ep.label
}
func (ep *endpointslicesProvider) getProtocol() string {
func (ep *Endpointslices) GetProtocol() string {
return string(ep.endpoints.AddressType)
}

View File

@@ -0,0 +1,22 @@
package providers
import (
"context"
"github.com/kube-vip/kube-vip/pkg/kubevip"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
watchtools "k8s.io/client-go/tools/watch"
)
type Provider interface {
CreateRetryWatcher(context.Context, *kubernetes.Clientset,
*v1.Service) (*watchtools.RetryWatcher, error)
GetAllEndpoints() ([]string, error)
GetLocalEndpoints(string, *kubevip.Config) ([]string, error)
GetLabel() string
UpdateServiceAnnotation(string, string, *v1.Service, *kubernetes.Clientset) error
LoadObject(runtime.Object, context.CancelFunc) error
GetProtocol() string
}

View File

@@ -1,4 +1,4 @@
package cluster
package instance
import (
"fmt"
@@ -6,36 +6,27 @@ import (
"strconv"
"strings"
"log/slog"
log "log/slog"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/sysctl"
"github.com/kube-vip/kube-vip/pkg/vip"
)
const (
// Hardware address of the host that has the VIP
HWAddrKey = "kube-vip.io/hwaddr"
// The IP address that is requested
RequestedIP = "kube-vip.io/requestedIP"
LoadbalancerHostname = "kube-vip.io/loadbalancerHostname"
ServiceInterface = "kube-vip.io/serviceInterface"
LoadbalancerIPAnnotation = "kube-vip.io/loadbalancerIPs"
)
// Instance defines an instance of everything needed to manage vips
type Instance struct {
// Virtual IP / Load Balancer configuration
VIPConfigs []*kubevip.Config
// cluster instances
Clusters []*Cluster
Clusters []*cluster.Cluster
// Service uses DHCP
IsDHCP bool
@@ -68,7 +59,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
for _, address := range instanceAddresses {
// Detect if we're using a specific interface for services
var svcInterface string
svcInterface = svc.Annotations[ServiceInterface] // If the service has a specific interface defined, then use it
svcInterface = svc.Annotations[kubevip.ServiceInterface] // If the service has a specific interface defined, then use it
if svcInterface == kubevip.Auto {
link, err = autoFindInterface(address)
if err != nil {
@@ -157,8 +148,6 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
}
}
//log.Info("new instance", "svc", *svc, "interface", svcInterface)
// Generate new Virtual IP configuration
newVips = append(newVips, &kubevip.Config{
VIP: address,
@@ -196,9 +185,9 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
// }
if svc.Annotations != nil {
instance.DHCPInterfaceHwaddr = svc.Annotations[HWAddrKey]
instance.DHCPInterfaceIP = svc.Annotations[RequestedIP]
instance.DHCPHostname = svc.Annotations[LoadbalancerHostname]
instance.DHCPInterfaceHwaddr = svc.Annotations[kubevip.HwAddrKey]
instance.DHCPInterfaceIP = svc.Annotations[kubevip.RequestedIP]
instance.DHCPHostname = svc.Annotations[kubevip.LoadbalancerHostname]
}
configPorts := make([]kubevip.Port, 0)
@@ -241,7 +230,7 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
}
for _, vipConfig := range instance.VIPConfigs {
c, err := InitCluster(vipConfig, false, intfMgr, arpMgr)
c, err := cluster.InitCluster(vipConfig, false, intfMgr, arpMgr)
if err != nil {
log.Error("Failed to add Service %s/%s", svc.Namespace, svc.Name)
return nil, err
@@ -253,7 +242,6 @@ func NewInstance(svc *v1.Service, config *kubevip.Config, intfMgr *networkinterf
instance.Clusters = append(instance.Clusters, c)
log.Info("(svcs) adding VIP", "ip", vipConfig.VIP, "interface", vipConfig.Interface, "namespace", svc.Namespace, "name", svc.Name)
}
return instance, nil
@@ -371,6 +359,28 @@ func (i *Instance) startDHCP() error {
log.Info("Using existing macvlan interface for DHCP", "interface", interfaceName)
}
// Default rp_filter setting (https://github.com/kube-vip/kube-vip/issues/1170)
rpfilterSetting := "0"
// Check if we need to set an override rp_filter value for the interface
if i.ServiceSnapshot.Annotations[kubevip.RPFilter] != "" {
// Check the rp_filter value
rpFilter, err := strconv.Atoi(i.ServiceSnapshot.Annotations[kubevip.RPFilter])
if err != nil {
slog.Error("[DHCP] unable to process rp_filter", "value", rpFilter)
} else {
if rpFilter >= 0 && rpFilter < 3 { // Ensure the value is 0,1,2
rpfilterSetting = i.ServiceSnapshot.Annotations[kubevip.RPFilter]
} else {
slog.Error("[DHCP] rp_filter value not within range 0-2", "value", rpFilter)
}
}
}
err = sysctl.WriteProcSys("/proc/sys/net/ipv4/conf/"+interfaceName+"/rp_filter", rpfilterSetting)
if err != nil {
slog.Error("[DHCP] unable to write rp_filter", "value", rpfilterSetting, "err", err)
}
var initRebootFlag bool
if i.DHCPInterfaceIP != "" {
initRebootFlag = true
@@ -421,7 +431,8 @@ func FetchLoadBalancerIngressAddresses(s *v1.Service) []string {
func FetchServiceAddresses(s *v1.Service) []string {
annotationAvailable := false
if s.Annotations != nil {
if v, annotationAvailable := s.Annotations[LoadbalancerIPAnnotation]; annotationAvailable {
if v, annotationAvailable := s.Annotations[kubevip.LoadbalancerIPAnnotation]; annotationAvailable {
ips := strings.Split(v, ",")
var trimmedIPs []string
for _, ip := range ips {
@@ -458,3 +469,14 @@ func FetchServiceAddresses(s *v1.Service) []string {
return []string{}
}
func FindServiceInstance(svc *v1.Service, instances []*Instance) *Instance {
log.Debug("finding service", "UID", svc.UID)
for i := range instances {
log.Debug("saved service", "instance", i, "UID", instances[i].ServiceSnapshot.UID)
if instances[i].ServiceSnapshot.UID == svc.UID {
return instances[i]
}
}
return nil
}

View File

@@ -0,0 +1,49 @@
package kubevip
const (
// Hardware address of the host that has the VIP
HwAddrKey = "kube-vip.io/hwaddr"
// The IP address that is requested
RequestedIP = "kube-vip.io/requestedIP"
// The host that has the VIP
VipHost = "kube-vip.io/vipHost"
// Enable Egress on a service
Egress = "kube-vip.io/egress"
// Enable internal Egress
EgressInternal = "kube-vip.io/egress-internal"
// Egress should be IPv6
EgressIPv6 = "kube-vip.io/egress-ipv6"
// Ports that traffic is allowed to access from the egress VIP
EgressDestinationPorts = "kube-vip.io/egress-destination-ports"
// Allowed incoming ports to the VIP
EgressSourcePorts = "kube-vip.io/egress-source-ports"
// Allowed networks for the Egress to be enabled for
EgressAllowedNetworks = "kube-vip.io/egress-allowed-networks"
// Networks that we wont Egress for
EgressDeniedNetworks = "kube-vip.io/egress-denied-networks"
// The current active endpoint(pod) for the Egress VIP
ActiveEndpoint = "kube-vip.io/active-endpoint"
// The current active endpoint(pod) for the Egress VIP (v6)
ActiveEndpointIPv6 = "kube-vip.io/active-endpoint-ipv6"
// Flush the conntrack rules (remove existing sessions) once Egress is configured
FlushContrack = "kube-vip.io/flush-conntrack"
LoadbalancerIPAnnotation = "kube-vip.io/loadbalancerIPs"
LoadbalancerHostname = "kube-vip.io/loadbalancerHostname"
ServiceInterface = "kube-vip.io/serviceInterface"
UpnpEnabled = "kube-vip.io/forwardUPNP"
RPFilter = "kube-vip.io/rp_filter" // Set the return path filter for a specific service interface
)

View File

@@ -7,6 +7,7 @@ import (
"math/bits"
"os"
"strconv"
"strings"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/detector"
@@ -500,6 +501,34 @@ func ParseEnvironment(c *Config) error {
c.BGPConfig.KeepaliveInterval = u64
}
env = os.Getenv(zebraEnable)
if env != "" {
result, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.BGPConfig.Zebra.Enabled = result
}
env = os.Getenv(zebraURL)
if env != "" {
c.BGPConfig.Zebra.URL = env
}
env = os.Getenv(zebraVersion)
if env != "" {
u64, err := strconv.ParseUint(env, 10, 32)
if err != nil {
return err
}
c.BGPConfig.Zebra.Version = uint32(u64)
}
env = os.Getenv(zebraSoftwareName)
if env != "" {
c.BGPConfig.Zebra.SoftwareName = env
}
// Enable the load-balancer
env = os.Getenv(lbEnable)
if env != "" {
@@ -578,13 +607,13 @@ func ParseEnvironment(c *Config) error {
c.K8sConfigFile = env
}
env = os.Getenv(enableEndpointSlices)
env = os.Getenv(enableEndpoints)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.EnableEndpointSlices = b
c.EnableEndpoints = b
}
env = os.Getenv(mirrorDestInterface)
@@ -618,5 +647,25 @@ func ParseEnvironment(c *Config) error {
c.HealthCheckPort = int(i)
}
env = os.Getenv(enableUPNP)
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.EnableUPNP = b
}
if env = os.Getenv(egressClean); env == "" {
env = os.Getenv(strings.ToUpper(egressClean))
}
if env != "" {
b, err := strconv.ParseBool(env)
if err != nil {
return err
}
c.EgressClean = b
}
return nil
}

View File

@@ -110,6 +110,15 @@ const (
// bgpKeepaliveInterval defines bgp timers keepalive interval
bgpKeepaliveInterval = "bgp_keepalive_interval"
// zebraEnable defines if Zebra integraton should be enabled
zebraEnable = "zebra_enable"
// zebraUrl specifies path to the unix domain socket for connecting to Zebra daemon
zebraURL = "zebra_url"
// zebraVersion specifies Zebra API Version
zebraVersion = "zebra_version"
// zebraSoftwareName specifies Software Name for Zebra
zebraSoftwareName = "zebra_software_name"
// mpbgpNexthop defines MPBGP mode
mpbgpNexthop = "mpbgp_nexthop"
// mpbgpIPv4 defines fixed IPv4 to be used with MPBGP
@@ -203,8 +212,8 @@ const (
// disableServiceUpdates disables service updating
disableServiceUpdates = "disable_service_updates"
// enableEndpointSlices enables use of EndpointSlices instead of Endpoints
enableEndpointSlices = "enable_endpointslices"
// enableEndpoints enables use of Endpoints instead of EndpointSlices
enableEndpoints = "enable_endpoints"
// mirrorDestInterface is the network interface where all traffics that go through service interface
// will be mirrored to. The source interface is ServicesInterface by default, fall back to Interface if not set.
@@ -219,4 +228,10 @@ const (
// healthCheckPort, if set to non-zero will be the port the health check will listen on
healthCheckPort = "health_check_port"
// enableUPNP enables UPNP functions
enableUPNP = "enable_upnp"
// egressClean enables egress cleaning on kube-vip's start
egressClean = "egress_clean"
)

View File

@@ -521,10 +521,10 @@ func generatePodSpec(c *Config, imageVersion string, inCluster bool) *corev1.Pod
}
newEnvironment = append(newEnvironment, prometheus...)
if c.EnableEndpointSlices {
if c.EnableEndpoints {
newEnvironment = append(newEnvironment, corev1.EnvVar{
Name: enableEndpointSlices,
Value: strconv.FormatBool(c.EnableEndpointSlices),
Name: enableEndpoints,
Value: strconv.FormatBool(c.EnableEndpoints),
})
}

View File

@@ -161,8 +161,8 @@ type Config struct {
// DisableServiceUpdates, if true, kube-vip will only advertise service, but it will not update service's Status.LoadBalancer.Ingress slice
DisableServiceUpdates bool `yaml:"disableServiceUpdates"`
// EnableEndpointSlices, if enabled, EndpointSlices will be used instead of Endpoints
EnableEndpointSlices bool `yaml:"enableEndpointSlices"`
// EnableEndpoints, if enabled, Endpoints will be used instead of EndpointSlices
EnableEndpoints bool `yaml:"enableEndpoints"`
// MirrorDestInterface is the network interface where all traffics that go through service interface
// will be mirrored to. If ServicesInterface is not set, fall back to Interface.
@@ -180,6 +180,12 @@ type Config struct {
// HealthCheckPort, if non-zero then will enable the healthcheck to return ok on this port
HealthCheckPort int `yaml:"healthCheckPort"`
// EnableUPNP, enables UPNP functions
EnableUPNP bool `yaml:"enableUPNP"`
// EgressClean, enables egress cleaning on Kube-vip's start
EgressClean bool `yaml:"egressClean"`
}
// KubernetesLeaderElection defines all of the settings for Kubernetes KubernetesLeaderElection

View File

@@ -1,36 +0,0 @@
package manager
const (
// The host that has the VIP
vipHost = "kube-vip.io/vipHost"
// Enable egress on a service
egress = "kube-vip.io/egress"
// Egress should be IPv6
egressIPv6 = "kube-vip.io/egress-ipv6"
// Ports that traffic is allowed to access from the egress VIP
egressDestinationPorts = "kube-vip.io/egress-destination-ports"
// Allowed incoming ports to the VIP
egressSourcePorts = "kube-vip.io/egress-source-ports"
// Allowed networks for the Egress to be enabled for
egressAllowedNetworks = "kube-vip.io/egress-allowed-networks"
// Networks that we wont Egress for
egressDeniedNetworks = "kube-vip.io/egress-denied-networks"
// The current active endpoint(pod) for the Egress VIP
activeEndpoint = "kube-vip.io/active-endpoint"
// The current active endpoint(pod) for the Egress VIP (v6)
activeEndpointIPv6 = "kube-vip.io/active-endpoint-ipv6"
// Flush the conntrack rules (remove existing sessions) once Egress is configured
flushContrack = "kube-vip.io/flush-conntrack"
upnpEnabled = "kube-vip.io/forwardUPNP"
)

View File

@@ -7,7 +7,6 @@ import (
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
@@ -17,15 +16,13 @@ import (
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/k8s"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/trafficmirror"
"github.com/kube-vip/kube-vip/pkg/services"
"github.com/kube-vip/kube-vip/pkg/upnp"
"github.com/kube-vip/kube-vip/pkg/utils"
"github.com/prometheus/client_golang/prometheus"
v1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
@@ -42,11 +39,6 @@ type Manager struct {
// Manager services
// service bool
// Keeps track of all running instances
serviceInstances []*cluster.Instance
// UPNP functionality
upnp bool
// BGP Manager, this is a singleton that manages all BGP advertisements
bgpServer *bgp.Server
@@ -56,6 +48,8 @@ type Manager struct {
// This channel is used to signal a shutdown
shutdownChan chan struct{}
svcProcessor *services.Processor
// This is a prometheus counter used to count the number of events received
// from the service watcher
countServiceWatchEvent *prometheus.CounterVec
@@ -177,6 +171,32 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
// }
// }
// listen for interrupts or the Linux SIGTERM signal and cancel
// our context, which the leader election code will observe and
// step down
signalChan := make(chan os.Signal, 1)
// Add Notification for Userland interrupt
signal.Notify(signalChan, syscall.SIGINT)
// Add Notification for SIGTERM (sent from Kubernetes)
signal.Notify(signalChan, syscall.SIGTERM)
// All watchers and other goroutines should have an additional goroutine that blocks on this, to shut things down
shutdownChan := make(chan struct{})
intfMgr := networkinterface.NewManager()
arpMgr := arp.NewManager(config)
var bgpServer *bgp.Server
if config.EnableBGP {
bgpServer, err = bgp.NewBGPServer(&config.BGPConfig)
if err != nil {
return nil, fmt.Errorf("creating BGP server: %w", err)
}
}
svcProcessor := services.NewServicesProcessor(config, bgpServer, clientset, rwClientSet, shutdownChan, intfMgr, arpMgr)
return &Manager{
clientSet: clientset,
rwClientSet: rwClientSet,
@@ -194,8 +214,12 @@ func New(configMap string, config *kubevip.Config) (*Manager, error) {
Name: "bgp_session_info",
Help: "Display state of session by setting metric for label value with current state to 1",
}, []string{"state", "peer"}),
intfMgr: networkinterface.NewManager(),
arpMgr: arp.NewManager(config),
signalChan: signalChan,
shutdownChan: shutdownChan,
svcProcessor: svcProcessor,
intfMgr: intfMgr,
arpMgr: arpMgr,
bgpServer: bgpServer,
}, nil
}
@@ -248,16 +272,12 @@ func (sm *Manager) Start() error {
}
if sm.config.EnableARP || sm.config.EnableWireguard {
// Before starting the leader Election enable any additional functionality
upnpEnabled, _ := strconv.ParseBool(os.Getenv("enableUPNP"))
if upnpEnabled {
sm.upnp = true
if sm.config.EnableUPNP {
clients := upnp.GetConnectionClients(context.TODO())
if len(clients) == 0 {
log.Error("Error Enabling UPNP. No Clients found")
// Set the struct to false so nothing should use it in future
sm.upnp = false
sm.config.EnableUPNP = false
} else {
for _, c := range clients {
ip, err := c.GetExternalIPAddress()
@@ -265,12 +285,11 @@ func (sm *Manager) Start() error {
log.Error("unable to find IGD2 Gateway address", "err", err)
}
log.Info("Found UPNP IGD2 Gateway address", "ip", ip)
}
}
}
// TODO: It would be nice to run the UPNP refresh only on the leader.
go sm.refreshUPNPForwards()
go sm.svcProcessor.RefreshUPNPForwards()
}
// If ARP is enabled then we start a LeaderElection that will use ARP to advertise VIPs
@@ -315,76 +334,3 @@ func (sm *Manager) parseAnnotations() error {
}
return nil
}
func (sm *Manager) serviceInterface() string {
svcIf := sm.config.Interface
if sm.config.ServicesInterface != "" {
svcIf = sm.config.ServicesInterface
}
return svcIf
}
func (sm *Manager) startTrafficMirroringIfEnabled() error {
if sm.config.MirrorDestInterface != "" {
svcIf := sm.serviceInterface()
log.Info("mirroring traffic", "src", svcIf, "dest", sm.config.MirrorDestInterface)
if err := trafficmirror.MirrorTrafficFromNIC(svcIf, sm.config.MirrorDestInterface); err != nil {
return err
}
} else {
log.Debug("skip starting traffic mirroring since it's not enabled.")
}
return nil
}
func (sm *Manager) stopTrafficMirroringIfEnabled() error {
if sm.config.MirrorDestInterface != "" {
svcIf := sm.serviceInterface()
log.Info("clean up qdisc config", "interface", svcIf)
if err := trafficmirror.CleanupQDSICFromNIC(svcIf); err != nil {
return err
}
} else {
log.Debug("skip stopping traffic mirroring since it's not enabled.")
}
return nil
}
func (sm *Manager) findServiceInstance(svc *v1.Service) *cluster.Instance {
sm.mutex.Lock()
defer sm.mutex.Unlock()
if svc == nil {
return nil // If the service is nil then we cannot find it
}
log.Debug("finding service", "UID", svc.UID, "name", svc.Name, "namespace", svc.Namespace)
for i := range sm.serviceInstances {
log.Debug("saved service", "UID", svc.UID, "name", svc.Name, "namespace", svc.Namespace)
if sm.serviceInstances[i].ServiceSnapshot.UID == svc.UID {
log.Debug("found service instance", "UID", svc.UID, "name", svc.Name, "namespace", svc.Namespace)
return sm.serviceInstances[i]
}
}
log.Debug("service instance not found", "UID", svc.UID)
return nil
}
// Refresh UPNP Port Forwards for all Service Instances registered in the SM
func (sm *Manager) refreshUPNPForwards() {
log.Info("Starting UPNP Port Refresher")
for {
time.Sleep(300 * time.Second)
log.Info("[UPNP] Refreshing Instances", "number of instances", len(sm.serviceInstances))
func() {
sm.mutex.Lock()
defer sm.mutex.Unlock()
for i := range sm.serviceInstances {
sm.upnpMap(context.TODO(), sm.serviceInstances[i])
if err := sm.updateStatus(sm.serviceInstances[i]); err != nil {
log.Warn("[UPNP] Error updating service", "ip", sm.serviceInstances[i].ServiceSnapshot.Name, "err", err)
}
}
}()
}
}

View File

@@ -2,7 +2,6 @@ package manager
import (
"context"
"os"
"syscall"
"time"
@@ -83,7 +82,7 @@ func (sm *Manager) startARP(id string) error {
}
// This will tidy any dangling kube-vip iptables rules
if os.Getenv("EGRESS_CLEAN") != "" {
if sm.config.EgressClean {
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
}
@@ -91,7 +90,7 @@ func (sm *Manager) startARP(id string) error {
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.startServicesWatchForLeaderElection(ctx)
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
@@ -126,7 +125,7 @@ func (sm *Manager) startARP(id string) error {
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.servicesWatcher(ctx, sm.syncServices)
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
log.Error("service watcher", "err", err)
panic("") // TODO: - emulating log.fatal here
@@ -137,11 +136,7 @@ func (sm *Manager) startARP(id string) error {
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "new leader", id)
for _, instance := range sm.serviceInstances {
for _, cluster := range instance.Clusters {
cluster.Stop()
}
}
sm.svcProcessor.Stop()
log.Error("lost leadership, restarting kube-vip")
panic("") // TODO: - emulating log.fatal here

View File

@@ -19,8 +19,15 @@ func (sm *Manager) startBGP() error {
// var ns string
var err error
if sm.bgpServer == nil {
sm.bgpServer, err = bgp.NewBGPServer(&sm.config.BGPConfig)
if err != nil {
return fmt.Errorf("creating BGP server: %w", err)
}
}
log.Info("Starting the BGP server to advertise VIP routes to BGP peers")
sm.bgpServer, err = bgp.NewBGPServer(&sm.config.BGPConfig, func(p *api.WatchEventResponse_PeerEvent) {
if err := sm.bgpServer.Start(func(p *api.WatchEventResponse_PeerEvent) {
ipaddr := p.GetPeer().GetState().GetNeighborAddress()
port := uint64(179)
peerDescription := fmt.Sprintf("%s:%d", ipaddr, port)
@@ -36,9 +43,8 @@ func (sm *Manager) startBGP() error {
"peer": peerDescription,
}).Set(metricValue)
}
})
if err != nil {
return err
}); err != nil {
return fmt.Errorf("starting BGP server: %w", err)
}
// use a Go context so we can tell the leaderelection code when we
@@ -99,7 +105,7 @@ func (sm *Manager) startBGP() error {
}
}
err = sm.servicesWatcher(ctx, sm.syncServices)
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
return err
}

View File

@@ -3,14 +3,13 @@ package manager
import (
"context"
"fmt"
"os"
"strconv"
"syscall"
"time"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/vishvananda/netlink"
@@ -40,17 +39,10 @@ func (sm *Manager) startTableMode(id string) error {
}()
}
egressCleanEnv := os.Getenv("EGRESS_CLEAN")
if egressCleanEnv != "" {
egressClean, err := strconv.ParseBool(egressCleanEnv)
if err != nil {
log.Warn("failed to parse EGRESS_CLEAN env value [%s]. Egress cleaning will not be performed: %s", egressCleanEnv, err.Error())
}
if egressClean {
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv6)
log.Debug("IPtables rules cleaned on startup")
}
if sm.config.EgressClean {
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv4)
vip.ClearIPTables(sm.config.EgressWithNftables, sm.config.ServiceNamespace, iptables.ProtocolIPv6)
log.Debug("IPtables rules cleaned on startup")
}
// Shutdown function that will wait on this signal, unless we call it ourselves
@@ -92,7 +84,7 @@ func (sm *Manager) startTableMode(id string) error {
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.startServicesWatchForLeaderElection(ctx)
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
@@ -126,7 +118,7 @@ func (sm *Manager) startTableMode(id string) error {
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.servicesWatcher(ctx, sm.syncServices)
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
log.Error(err.Error())
panic("")
@@ -137,11 +129,7 @@ func (sm *Manager) startTableMode(id string) error {
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "id", id)
for _, instance := range sm.serviceInstances {
for _, cluster := range instance.Clusters {
cluster.Stop()
}
}
sm.svcProcessor.Stop()
log.Error("lost leadership, restarting kube-vip")
panic("")
@@ -158,7 +146,7 @@ func (sm *Manager) startTableMode(id string) error {
})
} else {
log.Info("beginning watching services without leader election")
err = sm.servicesWatcher(ctx, sm.syncServices)
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
log.Error("Cannot watch services", "err", err)
}
@@ -181,21 +169,17 @@ func (sm *Manager) cleanRoutes() error {
if sm.config.EnableControlPlane {
found = (routes[i].Dst.IP.String() == sm.config.Address)
} else {
found = sm.countRouteReferences(&routes[i]) > 0
found = endpoints.CountRouteReferences(&routes[i], &sm.svcProcessor.ServiceInstances) > 0
}
if !found {
err = netlink.RouteDel(&(routes[i]))
if err != nil {
log.Error("[route] failed to delete", "route", routes[i], "err", err)
} else {
log.Debug("[route] deleted", "route", routes[i])
log.Error("[route] deletion", "route", routes[i], "err", err)
}
log.Debug("[route] deletion", "route", routes[i])
}
}
return nil
}
func (sm *Manager) countRouteReferences(route *netlink.Route) int {
return cluster.CountRouteReferences(&sm.serviceInstances, route)
}

View File

@@ -56,7 +56,7 @@ func (sm *Manager) startWireguard(id string) error {
// a lock based upon that service is created that they will all leaderElection on
if sm.config.EnableServicesElection {
log.Info("beginning watching services, leaderelection will happen for every service")
err = sm.startServicesWatchForLeaderElection(ctx)
err = sm.svcProcessor.StartServicesWatchForLeaderElection(ctx)
if err != nil {
return err
}
@@ -91,7 +91,7 @@ func (sm *Manager) startWireguard(id string) error {
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
err = sm.servicesWatcher(ctx, sm.syncServices)
err = sm.svcProcessor.ServicesWatcher(ctx, sm.svcProcessor.SyncServices)
if err != nil {
log.Error(err.Error())
panic("")
@@ -102,11 +102,7 @@ func (sm *Manager) startWireguard(id string) error {
sm.mutex.Lock()
defer sm.mutex.Unlock()
log.Info("leader lost", "id", id)
for _, instance := range sm.serviceInstances {
for _, cluster := range instance.Clusters {
cluster.Stop()
}
}
sm.svcProcessor.Stop()
log.Error("lost leadership, restarting kube-vip")
panic("")

View File

@@ -4,5 +4,12 @@ import "github.com/prometheus/client_golang/prometheus"
// PrometheusCollector defines a service watch event counter.
func (sm *Manager) PrometheusCollector() []prometheus.Collector {
return []prometheus.Collector{sm.countServiceWatchEvent, sm.bgpSessionInfoGauge}
collectors := []prometheus.Collector{}
if sm.svcProcessor != nil {
collectors = append(collectors, sm.svcProcessor.CountServiceWatchEvent)
}
if sm.bgpServer != nil {
collectors = append(collectors, sm.bgpServer.BGPSessionInfoGauge)
}
return collectors
}

View File

@@ -57,7 +57,8 @@ func (sm *Manager) annotationsWatcher() error {
// they're as needed
log.Warn(err.Error())
rw, err := watchtools.NewRetryWatcher(node.ResourceVersion, &cache.ListWatch{
// TODO, will need refactoring as part of rikatz work
rw, err := watchtools.NewRetryWatcherWithContext(context.TODO(), node.ResourceVersion, &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.rwClientSet.CoreV1().Nodes().Watch(context.Background(), listOptions)
},

View File

@@ -1,527 +0,0 @@
package manager
import (
"context"
"fmt"
"net"
"strings"
"syscall"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/pkg/errors"
v1 "k8s.io/api/core/v1"
discoveryv1 "k8s.io/api/discovery/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/client-go/util/retry"
)
type epProvider interface {
createRetryWatcher(context.Context, *Manager,
*v1.Service) (*watchtools.RetryWatcher, error)
getAllEndpoints() ([]string, error)
getLocalEndpoints(string, *kubevip.Config) ([]string, error)
getLabel() string
updateServiceAnnotation(string, string, *v1.Service, *Manager) error
loadObject(runtime.Object, context.CancelFunc) error
getProtocol() string
}
type endpointsProvider struct {
label string
endpoints *v1.Endpoints
}
func (ep *endpointsProvider) createRetryWatcher(ctx context.Context, sm *Manager,
service *v1.Service) (*watchtools.RetryWatcher, error) {
opts := metav1.ListOptions{
FieldSelector: fields.OneTermEqualSelector("metadata.name", service.Name).String(),
}
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.rwClientSet.CoreV1().Endpoints(service.Namespace).Watch(ctx, opts)
},
})
if err != nil {
return nil, fmt.Errorf("error creating endpoint watcher: %s", err.Error())
}
return rw, nil
}
func (ep *endpointsProvider) loadObject(endpoints runtime.Object, cancel context.CancelFunc) error {
eps, ok := endpoints.(*v1.Endpoints)
if !ok {
cancel()
return fmt.Errorf("[%s] unable to parse Kubernetes services from API watcher", ep.getLabel())
}
ep.endpoints = eps
return nil
}
func (ep *endpointsProvider) getAllEndpoints() ([]string, error) {
result := []string{}
for subset := range ep.endpoints.Subsets {
for address := range ep.endpoints.Subsets[subset].Addresses {
addr := strings.Split(ep.endpoints.Subsets[subset].Addresses[address].IP, "/")
result = append(result, addr[0])
}
}
return result, nil
}
func (ep *endpointsProvider) getLocalEndpoints(id string, _ *kubevip.Config) ([]string, error) {
var localEndpoints []string
for _, subset := range ep.endpoints.Subsets {
for _, address := range subset.Addresses {
log.Debug("processing endpoint", "label", ep.label, "ip", address.IP)
// 1. Compare the Nodename
if address.NodeName != nil && id == *address.NodeName {
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname, "nodename", *address.NodeName)
localEndpoints = append(localEndpoints, address.IP)
continue
}
// 2. Compare the Hostname (only useful if address.NodeName is not available)
if id == address.Hostname {
log.Debug("found local endpoint", "label", ep.label, "ip", address.IP, "hostname", address.Hostname)
localEndpoints = append(localEndpoints, address.IP)
continue
}
}
}
return localEndpoints, nil
}
func (ep *endpointsProvider) updateServiceAnnotation(endpoint string, _ string, service *v1.Service, sm *Manager) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := sm.clientSet.CoreV1().Services(service.Namespace).Get(context.TODO(), service.Name, metav1.GetOptions{})
if err != nil {
return err
}
currentServiceCopy := currentService.DeepCopy()
if currentServiceCopy.Annotations == nil {
currentServiceCopy.Annotations = make(map[string]string)
}
currentServiceCopy.Annotations[activeEndpoint] = endpoint
_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("error updating Service Spec", "label", ep.getLabel(), "name", currentServiceCopy.Name, "err", err)
return err
}
return nil
})
if retryErr != nil {
log.Error("failed to set Services", "label", ep.getLabel(), "err", retryErr)
return retryErr
}
return nil
}
func (ep *endpointsProvider) getLabel() string {
return ep.label
}
func (ep *endpointsProvider) getProtocol() string {
return ""
}
func (sm *Manager) watchEndpoint(svcCtx *serviceContext, id string, service *v1.Service, provider epProvider) error {
log.Info("watching", "provider", provider.getLabel(), "service_name", service.Name, "namespace", service.Namespace)
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
leaderContext, cancel := context.WithCancel(svcCtx.ctx)
defer cancel()
var leaderElectionActive bool
rw, err := provider.createRetryWatcher(leaderContext, sm, service)
if err != nil {
cancel()
return fmt.Errorf("[%s] error watching endpoints: %w", provider.getLabel(), err)
}
exitFunction := make(chan struct{})
go func() {
select {
case <-svcCtx.ctx.Done():
log.Debug("context cancelled", "provider", provider.getLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-sm.shutdownChan:
log.Debug("shutdown called", "provider", provider.getLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-exitFunction:
log.Debug("function ending", "provider", provider.getLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
}
}()
ch := rw.ResultChan()
var lastKnownGoodEndpoint string
for event := range ch {
activeEndpointAnnotation := activeEndpoint
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
if err = provider.loadObject(event.Object, cancel); err != nil {
return fmt.Errorf("[%s] error loading k8s object: %w", provider.getLabel(), err)
}
if sm.config.EnableEndpointSlices && provider.getProtocol() == string(discoveryv1.AddressTypeIPv6) {
activeEndpointAnnotation = activeEndpointIPv6
}
// Build endpoints
var endpoints []string
if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
if endpoints, err = provider.getAllEndpoints(); err != nil {
return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
}
} else {
if endpoints, err = provider.getLocalEndpoints(id, sm.config); err != nil {
return fmt.Errorf("[%s] error getting local endpoints: %w", provider.getLabel(), err)
}
}
if sm.config.EnableRoutingTable {
instance := sm.findServiceInstance(service)
if err != nil {
log.Error("failed to find the instance", "service", service.UID, "provider", provider.getLabel(), "err", err)
}
if instance == nil {
log.Error("failed to find the instance", "service", service.UID, "provider", provider.getLabel())
} else {
for _, c := range instance.Clusters {
for n := range c.Network {
// if there are no endpoints set HasEndpoints false just in case
if len(endpoints) < 1 {
c.Network[n].SetHasEndpoints(false)
}
// check if endpoint are available and are of same IP family as service
if len(endpoints) > 0 && ((net.ParseIP(c.Network[n].IP()).To4() == nil) == (net.ParseIP(endpoints[0]).To4() == nil)) {
c.Network[n].SetHasEndpoints(true)
}
}
}
}
}
// Find out if we have any local endpoints
// if out endpoint is empty then populate it
// if not, go through the endpoints and see if ours still exists
// If we have a local endpoint then begin the leader Election, unless it's already running
//
// Check that we have local endpoints
if len(endpoints) != 0 {
// Ignore IPv4
if service.Annotations[egressIPv6] == "true" && net.ParseIP(endpoints[0]).To4() != nil {
continue
}
// if we haven't populated one, then do so
if lastKnownGoodEndpoint != "" {
// check out previous endpoint exists
stillExists := false
for x := range endpoints {
if endpoints[x] == lastKnownGoodEndpoint {
stillExists = true
}
}
// If the last endpoint no longer exists, we cancel our leader Election, and set another endpoint as last known good
if !stillExists {
if sm.config.EnableRoutingTable {
if err := sm.TeardownEgress(lastKnownGoodEndpoint, service.Spec.LoadBalancerIP,
service.Namespace, service.Annotations); err != nil {
log.Warn("removing redundant egress rules", "err", err)
}
}
if leaderElectionActive && (sm.config.EnableServicesElection || sm.config.EnableLeaderElection) {
log.Warn(" existing endpoint has been removed, restarting leaderElection", "provider", provider.getLabel(), "endpoint", lastKnownGoodEndpoint)
// Stop the existing leaderElection
cancel()
// disable last leaderElection flag
leaderElectionActive = false
}
// Set our active endpoint to an existing one
lastKnownGoodEndpoint = endpoints[0]
}
} else {
lastKnownGoodEndpoint = endpoints[0]
}
if !leaderElectionActive && sm.config.EnableServicesElection {
go func() {
leaderContext, cancel = context.WithCancel(svcCtx.ctx)
// This is a blocking function, that will restart (in the event of failure)
for {
// if the context isn't cancelled restart
if leaderContext.Err() != context.Canceled {
leaderElectionActive = true
err := sm.StartServicesLeaderElection(leaderContext, service)
if err != nil {
log.Error(err.Error())
}
leaderElectionActive = false
} else {
leaderElectionActive = false
break
}
}
}()
}
// There are local endpoints available on the node
if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection {
// If routing table mode is enabled - routes should be added per node
if sm.config.EnableRoutingTable {
instance := sm.findServiceInstance(service)
if instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
if !svcCtx.isNetworkConfigured(cluster.Network[i].IP()) && cluster.Network[i].HasEndpoints() {
err := cluster.Network[i].AddRoute(false)
if err != nil {
if errors.Is(err, syscall.EEXIST) {
// If route exists, but protocol is not set (e.g. the route was created by the older version
// of kube-vip) try to update it if necessary
isUpdated, err := cluster.Network[i].UpdateRoutes()
if err != nil {
return fmt.Errorf("[%s] error updating existing routes: %w", provider.getLabel(), err)
}
if isUpdated {
log.Info("updated route", "provider",
provider.getLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", sm.config.RoutingTableID)
} else {
log.Info("route already present", "provider",
provider.getLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", sm.config.RoutingTableID)
}
} else {
// If other error occurs, return error
return fmt.Errorf("[%s] error adding route: %s", provider.getLabel(), err.Error())
}
} else {
log.Info("added route", "provider",
provider.getLabel(), "ip", cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", sm.config.RoutingTableID)
svcCtx.configuredNetworks.Store(cluster.Network[i].IP(), cluster.Network[i])
leaderElectionActive = true
}
}
}
}
}
}
// If BGP mode is enabled - hosts should be added per node
if sm.config.EnableBGP {
if instance := sm.findServiceInstance(service); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
if !svcCtx.isNetworkConfigured(cluster.Network[i].IP()) {
network := cluster.Network[i]
if err != nil {
log.Error("error formatting address with subnet mask", "err", err)
}
log.Debug("attempting to advertise BGP service", "provider", provider.getLabel(), "ip", network.CIDR())
err = sm.bgpServer.AddHost(network.CIDR())
if err != nil {
log.Error("error adding BGP host", "provider", provider.getLabel(), "err", err)
} else {
log.Info("added BGP host", "provider",
provider.getLabel(), "ip", network.CIDR(), "service name", service.Name, "namespace", service.Namespace)
svcCtx.configuredNetworks.Store(cluster.Network[i].IP(), cluster.Network[i])
leaderElectionActive = true
}
}
}
}
}
}
}
} else {
// There are no local endpoints
if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection {
// If routing table mode is enabled - routes should be deleted
if sm.config.EnableRoutingTable {
if errs := sm.clearRoutes(service); len(errs) == 0 {
svcCtx.configuredNetworks.Clear()
} else {
for _, err := range errs {
log.Error("error while clearing routes", "err", err)
}
}
}
// If BGP mode is enabled - routes should be deleted
if sm.config.EnableBGP {
if instance := sm.findServiceInstance(service); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
network := cluster.Network[i]
err = sm.bgpServer.DelHost(network.CIDR())
if err != nil {
log.Error("[endpoint] deleting BGP host", "provider", provider.getLabel(), "ip", network.CIDR(), "err", err)
} else {
log.Info("[endpoint] deleted BGP host", "provider",
provider.getLabel(), "ip", network.CIDR(), "service name", service.Name, "namespace", service.Namespace)
svcCtx.configuredNetworks.Delete(cluster.Network[i].IP())
leaderElectionActive = false
}
}
}
}
}
}
// If there are no local endpoints, and we had one then remove it and stop the leaderElection
if lastKnownGoodEndpoint != "" {
log.Warn("existing endpoint has been removed, no remaining endpoints for leaderElection", "provider", provider.getLabel(), "endpoint", lastKnownGoodEndpoint)
if err := sm.TeardownEgress(lastKnownGoodEndpoint, service.Spec.LoadBalancerIP, service.Namespace, service.Annotations); err != nil {
log.Error("error removing redundant egress rules", "err", err)
}
lastKnownGoodEndpoint = "" // reset endpoint
if sm.config.EnableServicesElection || sm.config.EnableLeaderElection {
cancel() // stop services watcher
}
leaderElectionActive = false
}
}
// Set the service accordingly
if service.Annotations[egress] == "true" {
service.Annotations[activeEndpointAnnotation] = lastKnownGoodEndpoint
}
log.Debug("watcher", "provider",
provider.getLabel(), "service name", service.Name, "namespace", service.Namespace, "endpoints", len(endpoints), "last endpoint", lastKnownGoodEndpoint, "active leader election", leaderElectionActive)
case watch.Deleted:
// When no-leader-elecition mode
if !sm.config.EnableServicesElection && !sm.config.EnableLeaderElection {
// find all existing local endpoints
var endpoints []string
if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
if endpoints, err = provider.getAllEndpoints(); err != nil {
return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
}
} else {
if endpoints, err = provider.getLocalEndpoints(id, sm.config); err != nil {
return fmt.Errorf("[%s] error getting all endpoints: %w", provider.getLabel(), err)
}
}
// If there were local endpoints deleted
if len(endpoints) > 0 {
// Delete all routes in routing table mode
if sm.config.EnableRoutingTable {
sm.clearRoutes(service)
}
// Delete all hosts in BGP mode
if sm.config.EnableBGP {
sm.clearBGPHosts(service)
}
}
}
// Close the goroutine that will end the retry watcher, then exit the endpoint watcher function
close(exitFunction)
log.Info("stopping watching", "provider", provider.getLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil
case watch.Error:
errObject := apierrors.FromObject(event.Object)
statusErr, _ := errObject.(*apierrors.StatusError)
log.Error("watch error", "provider", provider.getLabel(), "err", statusErr)
}
}
close(exitFunction)
log.Info("stopping watching", "provider", provider.getLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil //nolint:govet
}
func (sm *Manager) clearRoutes(service *v1.Service) []error {
errs := []error{}
if instance := sm.findServiceInstance(service); instance != nil {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
route := cluster.Network[i].PrepareRoute()
// check if route we are about to delete is not referenced by more than one service
if sm.countRouteReferences(route) <= 1 {
err := cluster.Network[i].DeleteRoute()
if err != nil && !errors.Is(err, syscall.ESRCH) {
log.Error("failed to delete route", "ip", cluster.Network[i].IP(), "err", err)
errs = append(errs, err)
}
log.Debug("deleted route", "ip",
cluster.Network[i].IP(), "service name", service.Name, "namespace", service.Namespace, "interface", cluster.Network[i].Interface(), "tableID", sm.config.RoutingTableID)
}
}
}
}
return errs
}
func (sm *Manager) clearBGPHosts(service *v1.Service) {
if instance := sm.findServiceInstance(service); instance != nil {
sm.clearBGPHostsByInstance(instance)
}
}
func (sm *Manager) clearBGPHostsByInstance(instance *cluster.Instance) {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
network := cluster.Network[i]
err := sm.bgpServer.DelHost(network.CIDR())
if err != nil {
log.Error("[endpoint] error deleting BGP host", "err", err)
} else {
log.Debug("[endpoint] deleted BGP host", "ip",
network.CIDR(), "service name", instance.ServiceSnapshot.Name, "namespace", instance.ServiceSnapshot.Namespace)
}
}
}
}
func (svcCtx *serviceContext) isNetworkConfigured(ip string) bool {
_, exists := svcCtx.configuredNetworks.Load(ip)
return exists
}

View File

@@ -1,415 +0,0 @@
package manager
import (
"context"
"fmt"
"reflect"
"sync"
log "log/slog"
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/prometheus/client_golang/prometheus"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
)
// services keeps track of services that already were processed
var services sync.Map
type serviceContext struct {
ctx context.Context
cancel context.CancelFunc
isActive bool
isWatched bool
configuredNetworks sync.Map
}
func newServiceContext(ctx context.Context) *serviceContext {
svcCtx, svcCancel := context.WithCancel(ctx)
return &serviceContext{
ctx: svcCtx,
cancel: svcCancel,
}
}
// This function handles the watching of a services endpoints and updates a load balancers endpoint configurations accordingly
func (sm *Manager) servicesWatcher(ctx context.Context, serviceFunc func(context.Context, *v1.Service) error) error {
// first start port mirroring if enabled
if err := sm.startTrafficMirroringIfEnabled(); err != nil {
return err
}
defer func() {
// clean up traffic mirror related config
err := sm.stopTrafficMirroringIfEnabled()
if err != nil {
log.Error("Stopping traffic mirroring", "err", err)
}
}()
if sm.config.ServiceNamespace == "" {
// v1.NamespaceAll is actually "", but we'll stay with the const in case things change upstream
sm.config.ServiceNamespace = v1.NamespaceAll
log.Info("(svcs) starting services watcher for all namespaces")
} else {
log.Info("(svcs) starting services watcher", "namespace", sm.config.ServiceNamespace)
}
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return sm.rwClientSet.CoreV1().Services(sm.config.ServiceNamespace).Watch(ctx, metav1.ListOptions{})
},
})
if err != nil {
return fmt.Errorf("error creating services watcher: %s", err.Error())
}
exitFunction := make(chan struct{})
go func() {
select {
case <-sm.shutdownChan:
log.Debug("(svcs) shutdown called")
// Stop the retry watcher
rw.Stop()
return
case <-exitFunction:
log.Debug("(svcs) function ending")
// Stop the retry watcher
rw.Stop()
return
}
}()
ch := rw.ResultChan()
// Used for tracking an active endpoint / pod
for event := range ch {
sm.countServiceWatchEvent.With(prometheus.Labels{"type": string(event.Type)}).Add(1)
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
svc, ok := event.Object.(*v1.Service)
if !ok {
return fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
break
}
// Check if we ignore this service
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Info("ignore annotation for kube-vip", "service name", svc.Name)
break
}
// Select loadbalancer class filtering function
lbClassFilterFunc := sm.lbClassFilter
if sm.config.LoadBalancerClassLegacyHandling {
lbClassFilterFunc = sm.lbClassFilterLegacy
}
// Check the loadBalancer class
if lbClassFilterFunc(svc) {
break
}
svcAddresses := cluster.FetchServiceAddresses(svc)
// We only care about LoadBalancer services that have been allocated an address
if len(svcAddresses) <= 0 {
break
}
svcCtx, err := getServiceContext(svc.UID)
if err != nil {
return fmt.Errorf("failed to get service context: %w", err)
}
// The modified event should only be triggered if the service has been modified (i.e. moved somewhere else)
if event.Type == watch.Modified {
i := sm.findServiceInstance(svc)
originalService := []string{}
shouldGarbageCollect := true
if i != nil {
originalService = cluster.FetchServiceAddresses(i.ServiceSnapshot)
shouldGarbageCollect = !reflect.DeepEqual(originalService, svcAddresses)
}
if shouldGarbageCollect {
for _, addr := range svcAddresses {
// log.Debugf("(svcs) Retreiving local addresses, to ensure that this modified address doesn't exist: %s", addr)
f, err := vip.GarbageCollect(sm.config.Interface, addr, sm.intfMgr)
if err != nil {
log.Error("(svcs) cleaning existing address error", "err", err)
}
if f {
log.Warn("(svcs) already found existing config", "address", addr, "adapter", sm.config.Interface)
}
}
}
// This service has been modified, but it was also active.
if svcCtx != nil && svcCtx.isActive {
if i != nil {
if !reflect.DeepEqual(originalService, svcAddresses) {
// Calls the cancel function of the context
if svcCtx != nil {
log.Warn("(svcs) The load balancer has changed, cancelling original load balancer")
svcCtx.cancel()
log.Warn("(svcs) waiting for load balancer to finish")
<-svcCtx.ctx.Done()
}
err = sm.deleteService(svc.UID)
if err != nil {
log.Error("(svc) unable to remove", "service", svc.UID)
}
services.Delete(svc.UID)
}
// in theory this should never fail
}
}
}
// Architecture walkthrough: (Had to do this as this code path is making my head hurt)
// Is the service active (bool), if not then process this new service
// Does this service use an election per service?
//
if svcCtx == nil || svcCtx != nil && !svcCtx.isActive {
log.Debug("(svcs) has been added/modified with addresses", "service name", svc.Name, "ip", cluster.FetchServiceAddresses(svc))
if svcCtx == nil {
svcCtx = newServiceContext(ctx)
services.Store(svc.UID, svcCtx)
}
if sm.config.EnableServicesElection || // Service Election
((sm.config.EnableRoutingTable || sm.config.EnableBGP) && // Routing table mode or BGP
(!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection)) { // No leaderelection or services election
// If this load balancer Traffic Policy is "local"
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Start an endpoint watcher if we're not watching it already
if !svcCtx.isWatched {
// background the endpoint watcher
if (sm.config.EnableRoutingTable || sm.config.EnableBGP) && (!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection) {
err = serviceFunc(svcCtx.ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Add Endpoint or EndpointSlices watcher
var provider epProvider
if !sm.config.EnableEndpointSlices {
provider = &endpointsProvider{label: "endpoints"}
} else {
provider = &endpointslicesProvider{label: "endpointslices"}
}
if err = sm.watchEndpoint(svcCtx, sm.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
}
}()
// We're now watching this service
svcCtx.isWatched = true
}
} else if (sm.config.EnableBGP || sm.config.EnableRoutingTable) && (!sm.config.EnableLeaderElection && !sm.config.EnableServicesElection) {
err = serviceFunc(svcCtx.ctx, svc)
if err != nil {
log.Error(err.Error())
}
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
// Add Endpoint watcher
var provider epProvider
if !sm.config.EnableEndpointSlices {
provider = &endpointsProvider{label: "endpoints"}
} else {
provider = &endpointslicesProvider{label: "endpointslices"}
}
if err = sm.watchEndpoint(svcCtx, sm.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
}
}()
// We're now watching this service
svcCtx.isWatched = true
} else {
go func() {
for {
select {
case <-svcCtx.ctx.Done():
log.Warn("(svcs) restartable service watcher ending", "uid", svc.UID)
return
default:
log.Info("(svcs) restartable service watcher starting", "uid", svc.UID)
err = serviceFunc(svcCtx.ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
}
}()
}
} else {
// Increment the waitGroup before the service Func is called (Done is completed in there)
err = serviceFunc(svcCtx.ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
svcCtx.isActive = true
}
case watch.Deleted:
svc, ok := event.Object.(*v1.Service)
if !ok {
return fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
svcCtx, err := getServiceContext(svc.UID)
if err != nil {
return fmt.Errorf("(svcs) unable to get context: %w", err)
}
if svcCtx != nil && svcCtx.isActive {
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
break
}
// We can ignore this service
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Info("(svcs)ignore annotation for kube-vip", "service name", svc.Name)
break
}
// If no leader election is enabled, delete routes here
if !sm.config.EnableLeaderElection && !sm.config.EnableServicesElection &&
sm.config.EnableRoutingTable && svcCtx.hasConfiguredNetworks() {
if errs := sm.clearRoutes(svc); len(errs) == 0 {
svcCtx.configuredNetworks.Clear()
}
}
// If this is an active service then and additional leaderElection will handle stopping
err = sm.deleteService(svc.UID)
if err != nil {
log.Error(err.Error())
}
// Calls the cancel function of the context
log.Warn("(svcs) The load balancer was deleted, cancelling context")
svcCtx.cancel()
log.Warn("(svcs) waiting for load balancer to finish")
<-svcCtx.ctx.Done()
services.Delete(svc.UID)
}
if sm.config.EnableLeaderElection && !sm.config.EnableServicesElection {
if sm.config.EnableBGP {
sm.clearBGPHosts(svc)
} else if sm.config.EnableRoutingTable {
sm.clearRoutes(svc)
}
}
log.Info("(svcs) deleted", "service name", svc.Name, "namespace", svc.Namespace)
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes services")
// This round trip allows us to handle unstructured status
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Error("services", "err", status)
default:
}
}
close(exitFunction)
log.Warn("Stopping watching services for type: LoadBalancer in all namespaces")
return nil
}
func (sm *Manager) lbClassFilterLegacy(svc *v1.Service) bool {
if svc == nil {
log.Info("(svcs) service is nil, ignoring")
return true
}
if svc.Spec.LoadBalancerClass != nil {
// if this isn't nil then it has been configured, check if it the kube-vip loadBalancer class
if *svc.Spec.LoadBalancerClass != sm.config.LoadBalancerClassName {
log.Info("(svcs) specified the wrong loadBalancer class", "service name", svc.Name, "lbClass", *svc.Spec.LoadBalancerClass)
return true
}
} else if sm.config.LoadBalancerClassOnly {
// if kube-vip is configured to only recognize services with kube-vip's lb class, then ignore the services without any lb class
log.Info("(svcs) kube-vip configured to only recognize services with kube-vip's lb class but the service didn't specify any loadBalancer class, ignoring", "service name", svc.Name)
return true
}
return false
}
func (sm *Manager) lbClassFilter(svc *v1.Service) bool {
if svc == nil {
log.Info("(svcs) service is nil, ignoring")
return true
}
if svc.Spec.LoadBalancerClass == nil && sm.config.LoadBalancerClassName != "" {
log.Info("(svcs) no loadBalancer class, ignoring", "service name", svc.Name, "expected lbClass", sm.config.LoadBalancerClassName)
return true
}
if svc.Spec.LoadBalancerClass == nil && sm.config.LoadBalancerClassName == "" {
return false
}
if *svc.Spec.LoadBalancerClass != sm.config.LoadBalancerClassName {
log.Info("(svcs) specified wrong loadBalancer class, ignoring", "service name", svc.Name, "wrong lbClass", *svc.Spec.LoadBalancerClass, "expected lbClass", sm.config.LoadBalancerClassName)
return true
}
return false
}
func (svcCtx *serviceContext) hasConfiguredNetworks() bool {
cnt := 0
svcCtx.configuredNetworks.Range(func(_ any, _ any) bool {
cnt++
return cnt < 1
})
return cnt > 0
}
func getServiceContext(uid types.UID) (*serviceContext, error) {
svcCtx, ok := services.Load(uid)
if !ok {
return nil, nil
}
ctx, ok := svcCtx.(*serviceContext)
if !ok {
return nil, fmt.Errorf("failed to cast service context pointer - UID: %s", uid)
}
return ctx, nil
}

458
pkg/nftables/nftables.go Normal file
View File

@@ -0,0 +1,458 @@
package nftables
import (
"errors"
"fmt"
"log/slog"
"net"
"strconv"
"strings"
"github.com/google/nftables"
"github.com/google/nftables/binaryutil"
"github.com/google/nftables/expr"
"golang.org/x/sys/unix"
)
const (
NatTable = "kube_vip_%s"
SNatChain = "kube_vip_snat_%s"
)
func ApplySNAT(podIP, vipIP, service, destinationPorts string, ignoreCIDR []string, IPv6 bool) error {
conn, err := nftables.New()
if err != nil {
return err
}
var tableName string
if IPv6 {
tableName = fmt.Sprintf(NatTable, "v6")
} else {
tableName = fmt.Sprintf(NatTable, "v4")
}
// Look up the table
if t, err := FilterTable(conn, tableName, IPv6); err != nil {
if t == nil {
// If it doesn't exist then create it
slog.Debug("[egress]", "Creating Table", tableName)
conn.AddTable(GetTable(IPv6))
}
}
slog.Debug("[egress]", "Creating Chain for service", service, "IPv6", IPv6)
// These don't return errors, so not 100% sure how to guarantee things were created
conn.AddChain(GetSNatChain(IPv6, service))
conn.Flush()
// Create our nftables rule
rule, err := CreateRule(podIP, vipIP, service, destinationPorts, ignoreCIDR, conn, IPv6)
if err != nil {
return err
}
slog.Debug("[egress]", "table", rule.Table.Name, "chain", rule.Chain.Name, "expr", rule.Exprs)
conn.AddRule(rule) // Add the rule
err = conn.Flush() // Commit the rule to nftables
if err != nil {
return err
}
return conn.CloseLasting() // Close out any remaining netlink communication
}
func DeleteSNAT(IPv6 bool, service string) error {
conn, err := nftables.New()
if err != nil {
return err
}
var chainName = fmt.Sprintf(SNatChain, service)
slog.Info("[egress]", "Looking for", chainName)
chain, err := conn.ListChain(GetTable(IPv6), chainName)
if err != nil {
return err
}
if chain != nil {
slog.Info("[egress]", "Deleting chain", chainName)
conn.DelChain(chain)
return conn.Flush()
}
return fmt.Errorf("unable to find chain [%s]", chainName)
}
func GetTable(IPv6 bool) *nftables.Table {
var tableName string
if IPv6 {
tableName = fmt.Sprintf(NatTable, "v6")
} else {
tableName = fmt.Sprintf(NatTable, "v4")
}
// Default to IPv4
table := &nftables.Table{
Family: nftables.TableFamilyIPv4,
Name: tableName,
}
// Move to IPv6 if needed
if IPv6 {
table.Family = nftables.TableFamilyIPv6
}
return table
}
func GetSNatChain(IPv6 bool, service string) *nftables.Chain {
var chainName = fmt.Sprintf(SNatChain, service)
policy := nftables.ChainPolicyAccept
return &nftables.Chain{
Name: chainName,
Table: GetTable(IPv6),
Type: nftables.ChainTypeNAT,
Hooknum: nftables.ChainHookPostrouting,
Priority: nftables.ChainPriorityNATSource,
Policy: &policy,
}
}
func FilterTable(conn *nftables.Conn, tableName string, IPv6 bool) (*nftables.Table, error) {
if IPv6 {
return conn.ListTableOfFamily(tableName, nftables.TableFamilyIPv6)
}
return conn.ListTableOfFamily(tableName, nftables.TableFamilyIPv4)
}
// ClearTable will remove the original tables and create new empty ones
func ClearTable(conn *nftables.Conn) error {
tableName := fmt.Sprintf(NatTable, "v6")
if t, err := FilterTable(conn, tableName, false); err != nil {
return err
} else if t != nil {
conn.DelTable(t)
}
// These don't return errors, so not 100% sure how to guarantee things were created
conn.AddTable(GetTable(true))
tableName = fmt.Sprintf(NatTable, "v4")
if t, err := FilterTable(conn, tableName, true); err != nil {
return err
} else if t != nil {
conn.DelTable(t)
}
// These don't return errors, so not 100% sure how to guarantee things were created
conn.AddTable(GetTable(false))
return nil
}
// Create our nftables rule
func CreateRule(podIP, vipIP, service, destinationPorts string, ignoreCIDR []string, conn *nftables.Conn, IPv6 bool) (*nftables.Rule, error) {
// Validate pod IP
if net.ParseIP(podIP) == nil {
return nil, errors.New("ip is invalid")
}
// Validate vip IP
if net.ParseIP(vipIP) == nil {
return nil, errors.New("output_ip is not a valid ip")
}
// Get the kube-vip table
table := GetTable(IPv6)
// Create our rule
rule := &nftables.Rule{
Table: table,
Exprs: []expr.Any{},
}
// Set the correct chain
rule.Chain = GetSNatChain(IPv6, service)
// Create a set for our original/source address
set := &nftables.Set{
Table: table,
Anonymous: true,
Constant: true,
KeyType: nftables.TypeIPAddr,
Interval: false,
}
if IPv6 {
set.KeyType = nftables.TypeIP6Addr
} else {
set.KeyType = nftables.TypeIPAddr
}
// Create an element using our pod IP
elements := []nftables.SetElement{}
if IPv6 {
elements = append(elements, nftables.SetElement{Key: net.ParseIP(podIP).To16()})
} else {
elements = append(elements, nftables.SetElement{Key: net.ParseIP(podIP).To4()})
}
// Add the elements to the set
err := conn.AddSet(set, elements)
if err != nil {
return nil, err
}
// Create the expression using the set
expression := []expr.Any{}
payload := &expr.Payload{
OperationType: expr.PayloadLoad,
Base: expr.PayloadBaseNetworkHeader,
DestRegister: 1,
SourceRegister: 0,
}
// Set the length of the data based upon the type of IP version being used
if IPv6 {
payload.Offset = 8
payload.Len = 16
} else {
payload.Offset = 12
payload.Len = 4
}
lookup := &expr.Lookup{
SourceRegister: 1,
DestRegister: 0,
SetID: set.ID,
}
// Add expressions
expression = append(expression, payload)
expression = append(expression, lookup)
// Add expression to the rule
rule.Exprs = append(rule.Exprs, expression...)
// If we filter on ports protocols then parse them
if destinationPorts != "" {
fixedPorts := strings.Split(destinationPorts, ",")
// Create an element using our pod IP
tcpElements := []nftables.SetElement{}
udpElements := []nftables.SetElement{}
sctpElements := []nftables.SetElement{}
tcpSet := &nftables.Set{
Anonymous: true,
Constant: true,
Table: table,
KeyType: nftables.TypeInetService,
}
udpSet := &nftables.Set{
Anonymous: true,
Constant: true,
Table: table,
KeyType: nftables.TypeInetService,
}
sctpSet := &nftables.Set{
Anonymous: true,
Constant: true,
Table: table,
KeyType: nftables.TypeInetService,
}
for _, fixedPort := range fixedPorts {
data := strings.Split(fixedPort, ":")
if len(data) == 0 {
continue
} else if len(data) == 2 { // Ensure we have two elements { proto:port }
// parse the port to a number
port, err := strconv.Atoi(data[1])
if err != nil {
slog.Error("[egress]", "unable to process port", data[1])
continue
}
// Ensure the port is within the valid range for uint16
if port < 0 || port > 65535 {
slog.Error("[egress]", "port out of range for uint16", data[1])
continue
}
switch data[0] {
case "tcp":
//nolint:gosec
tcpElements = append(tcpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
case "udp":
//nolint:gosec
udpElements = append(udpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
case "sctp":
//nolint:gosec
sctpElements = append(sctpElements, nftables.SetElement{Key: binaryutil.BigEndian.PutUint16(uint16(port))})
default:
slog.Error("[egress]", "unknown protocol", data[0])
}
}
}
// Add TCP Ports
if len(tcpElements) != 0 {
err = conn.AddSet(tcpSet, tcpElements)
if err != nil {
return nil, err
}
expression := []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
// [ cmp eq reg 1 0x00000006 ]
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte{unix.IPPROTO_TCP},
},
// [ payload load 2b @ transport header + 2 => reg 1 ]
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseTransportHeader,
Offset: 2,
Len: 2,
},
// [ lookup reg 1 set __set%d ]
&expr.Lookup{
SourceRegister: 1,
SetName: tcpSet.Name,
SetID: tcpSet.ID,
},
}
rule.Exprs = append(rule.Exprs, expression...)
}
// Add UDP ports
if len(udpElements) != 0 {
err = conn.AddSet(udpSet, udpElements)
if err != nil {
return nil, err
}
expression := []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
// [ cmp eq reg 1 0x00000006 ]
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte{unix.IPPROTO_UDP},
},
// [ payload load 2b @ transport header + 2 => reg 1 ]
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseTransportHeader,
Offset: 2,
Len: 2,
},
// [ lookup reg 1 set __set%d ]
&expr.Lookup{
SourceRegister: 1,
SetName: udpSet.Name,
SetID: udpSet.ID,
},
}
rule.Exprs = append(rule.Exprs, expression...)
}
// Add SCTP Ports
if len(sctpElements) != 0 {
err = conn.AddSet(sctpSet, sctpElements)
if err != nil {
return nil, err
}
expression := []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
// [ cmp eq reg 1 0x00000006 ]
&expr.Cmp{
Op: expr.CmpOpEq,
Register: 1,
Data: []byte{unix.IPPROTO_SCTP},
},
// [ payload load 2b @ transport header + 2 => reg 1 ]
&expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseTransportHeader,
Offset: 2,
Len: 2,
},
// [ lookup reg 1 set __set%d ]
&expr.Lookup{
SourceRegister: 1,
SetName: sctpSet.Name,
SetID: sctpSet.ID,
},
}
rule.Exprs = append(rule.Exprs, expression...)
}
}
// Parse which CIDRs we will not SNAT for
for _, cidr := range ignoreCIDR {
start, end, err := nftables.NetFirstAndLastIP(cidr)
if err != nil {
return nil, err
}
expression = []expr.Any{}
payload := &expr.Payload{
DestRegister: 1,
Base: expr.PayloadBaseNetworkHeader,
}
notEqualRange := &expr.Range{
Op: expr.CmpOpNeq,
Register: 1,
}
if IPv6 {
payload.Len = 16
payload.Offset = 24
notEqualRange.FromData = start.To16()
notEqualRange.ToData = end.To16()
} else {
payload.Offset = 16
payload.Len = 4
notEqualRange.FromData = start.To4()
notEqualRange.ToData = end.To4()
}
// Add expressions
expression = append(expression, payload)
expression = append(expression, notEqualRange)
// // Add expression to the rule
rule.Exprs = append(rule.Exprs, expression...)
}
// Final expression to the rule is the SNAT to the VIP address
expression = []expr.Any{}
immediate := &expr.Immediate{
Register: 1,
}
nat := &expr.NAT{
Type: expr.NATTypeSourceNAT,
RegAddrMin: 1,
RegAddrMax: 1,
RegProtoMin: 0,
RegProtoMax: 0,
Random: false,
FullyRandom: false,
Persistent: false,
Prefix: false,
}
if IPv6 {
immediate.Data = net.ParseIP(vipIP).To16()
nat.Family = unix.NFPROTO_IPV6
} else {
immediate.Data = net.ParseIP(vipIP).To4()
nat.Family = unix.NFPROTO_IPV4
}
// https://github.com/google/nftables/blob/main/nftables_test.go#L5375
// Add expressions
expression = append(expression, immediate)
expression = append(expression, nat)
rule.Exprs = append(rule.Exprs, expression...)
return rule, nil
}

View File

@@ -0,0 +1,36 @@
package servicecontext
import (
"context"
"sync"
)
type Context struct {
Ctx context.Context
Cancel context.CancelFunc
IsActive bool
IsWatched bool
ConfiguredNetworks sync.Map
}
func New(ctx context.Context) *Context {
svcCtx, svcCancel := context.WithCancel(ctx)
return &Context{
Ctx: svcCtx,
Cancel: svcCancel,
}
}
func (ctx *Context) HasConfiguredNetworks() bool {
cnt := 0
ctx.ConfiguredNetworks.Range(func(_ any, _ any) bool {
cnt++
return cnt < 1
})
return cnt > 0
}
func (ctx *Context) IsNetworkConfigured(ip string) bool {
_, exists := ctx.ConfiguredNetworks.Load(ip)
return exists
}

36
pkg/services/context.go Normal file
View File

@@ -0,0 +1,36 @@
package services
import (
"context"
"sync"
)
type Context struct {
Ctx context.Context
Cancel context.CancelFunc
IsActive bool
IsWatched bool
ConfiguredNetworks sync.Map
}
func NewContext(ctx context.Context) *Context {
svcCtx, svcCancel := context.WithCancel(ctx)
return &Context{
Ctx: svcCtx,
Cancel: svcCancel,
}
}
func (ctx *Context) HasConfiguredNetworks() bool {
cnt := 0
ctx.ConfiguredNetworks.Range(func(_ any, _ any) bool {
cnt++
return cnt < 1
})
return cnt > 0
}
func (ctx *Context) IsNetworkConfigured(ip string) bool {
_, exists := ctx.ConfiguredNetworks.Load(ip)
return exists
}

View File

@@ -1,4 +1,4 @@
package manager
package services
import (
"bufio"
@@ -11,6 +11,8 @@ import (
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/iptables"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/nftables"
"github.com/kube-vip/kube-vip/pkg/vip"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -21,7 +23,7 @@ const (
defaultServiceCIDR = "10.96.0.0/12"
)
func (sm *Manager) iptablesCheck() error {
func (p *Processor) iptablesCheck() error {
file, err := os.Open("/proc/modules")
if err != nil {
return err
@@ -48,7 +50,7 @@ func (sm *Manager) iptablesCheck() error {
return nil
}
func (sm *Manager) nftablesCheck() error {
func (p *Processor) nftablesCheck() error {
file, err := os.Open("/proc/modules")
if err != nil {
return err
@@ -132,26 +134,27 @@ func checkCIDR(ip, cidr string) (string, error) {
return "", nil
}
func (sm *Manager) configureEgress(vipIP, podIP, namespace string, annotations map[string]string) error {
func (p *Processor) configureEgress(vipIP, podIP, namespace, serviceUUID string, annotations map[string]string) error {
var podCidr, serviceCidr string
var autoServiceCIDR, autoPodCIDR string
var discoverErr error
// Look up the destination ports from the annotations on the service
destinationPorts := annotations[egressDestinationPorts]
deniedNetworks := annotations[egressDeniedNetworks]
allowedNetworks := annotations[egressAllowedNetworks]
destinationPorts := annotations[kubevip.EgressDestinationPorts]
deniedNetworks := annotations[kubevip.EgressDeniedNetworks]
allowedNetworks := annotations[kubevip.EgressAllowedNetworks]
internalEgress := annotations[kubevip.EgressInternal]
if sm.config.EgressPodCidr == "" || sm.config.EgressServiceCidr == "" {
autoServiceCIDR, autoPodCIDR, discoverErr = sm.AutoDiscoverCIDRs()
if p.config.EgressPodCidr == "" || p.config.EgressServiceCidr == "" {
autoServiceCIDR, autoPodCIDR, discoverErr = p.AutoDiscoverCIDRs()
}
if discoverErr != nil {
log.Warn("autodiscover CIDR", "err", discoverErr)
}
if sm.config.EgressPodCidr != "" {
podCidr = getSameFamilyCidr(sm.config.EgressPodCidr, podIP)
if p.config.EgressPodCidr != "" {
podCidr = getSameFamilyCidr(p.config.EgressPodCidr, podIP)
} else {
if discoverErr == nil {
podCidr = getSameFamilyCidr(autoPodCIDR, podIP)
@@ -166,8 +169,8 @@ func (sm *Manager) configureEgress(vipIP, podIP, namespace string, annotations m
podCidr = defaultPodCIDR
}
if sm.config.EgressServiceCidr != "" {
serviceCidr = getSameFamilyCidr(sm.config.EgressServiceCidr, vipIP)
if p.config.EgressServiceCidr != "" {
serviceCidr = getSameFamilyCidr(p.config.EgressServiceCidr, vipIP)
} else {
if discoverErr == nil {
serviceCidr = getSameFamilyCidr(autoServiceCIDR, vipIP)
@@ -205,7 +208,30 @@ func (sm *Manager) configureEgress(vipIP, podIP, namespace string, annotations m
protocol = iptables.ProtocolIPv6
}
i, err := vip.CreateIptablesClient(sm.config.EgressWithNftables, namespace, protocol)
// Use the internal egress implementation
if internalEgress != "" {
// Create an array of CIDRs that we wont SNAT to.
ignoreCIDRs := []string{
podCidr,
serviceCidr,
}
// Add any specifically denied networks
if deniedNetworks != "" {
networks := strings.Split(strings.TrimSpace(deniedNetworks), ",") //Remove whitespace characters and then create an array from the CIDRs
ignoreCIDRs = append(ignoreCIDRs, networks...)
}
// Apply the SNAT rules
err := nftables.ApplySNAT(podIP, vipIP, serviceUUID, destinationPorts, ignoreCIDRs, vip.IsIPv6(vipIP))
if err != nil {
return fmt.Errorf("error performing netlink nftables [%s]", err)
}
return nil
}
i, err := vip.CreateIptablesClient(p.config.EgressWithNftables, namespace, protocol)
if err != nil {
return fmt.Errorf("error Creating iptables client [%s]", err)
}
@@ -304,12 +330,12 @@ func (sm *Manager) configureEgress(vipIP, podIP, namespace string, annotations m
return nil
}
func (sm *Manager) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error) {
func (p *Processor) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error) {
log.Debug("Trying to automatically discover Service and Pod CIDRs")
options := v1.ListOptions{
LabelSelector: "component=kube-controller-manager",
}
podList, err := sm.clientSet.CoreV1().Pods("kube-system").List(context.TODO(), options)
podList, err := p.clientSet.CoreV1().Pods("kube-system").List(context.TODO(), options)
if err != nil {
return "", "", fmt.Errorf("[Egress] Unable to get kube-controller-manager pod: %w", err)
}
@@ -332,83 +358,3 @@ func (sm *Manager) AutoDiscoverCIDRs() (serviceCIDR, podCIDR string, err error)
return
}
func (sm *Manager) TeardownEgress(podIP, vipIP, namespace string, annotations map[string]string) error {
// Look up the destination ports from the annotations on the service
destinationPorts := annotations[egressDestinationPorts]
deniedNetworks := annotations[egressDeniedNetworks]
allowedNetworks := annotations[egressAllowedNetworks]
protocol := iptables.ProtocolIPv4
if vip.IsIPv6(podIP) {
protocol = iptables.ProtocolIPv6
}
i, err := vip.CreateIptablesClient(sm.config.EgressWithNftables, namespace, protocol)
if err != nil {
return fmt.Errorf("error Creating iptables client [%s]", err)
}
if deniedNetworks != "" {
networks := strings.Split(deniedNetworks, ",")
for x := range networks {
err = i.DeleteMangleReturnForNetwork(vip.MangleChainName, networks[x])
if err != nil {
return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
}
if allowedNetworks != "" {
networks := strings.Split(allowedNetworks, ",")
for x := range networks {
err = i.DeleteMangleMarkingForNetwork(podIP, vip.MangleChainName, networks[x])
if err != nil {
return fmt.Errorf("error deleting rules in mangle chain [%s], error [%s]", vip.MangleChainName, err)
}
}
} else {
// Remove the marking of egress packets
err = i.DeleteMangleMarking(podIP, vip.MangleChainName)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
// Clear up SNAT rules
if destinationPorts != "" {
fixedPorts := strings.Split(destinationPorts, ",")
for _, fixedPort := range fixedPorts {
var proto, port string
data := strings.Split(fixedPort, ":")
if len(data) == 0 {
continue
} else if len(data) == 1 {
proto = "tcp"
port = data[0]
} else {
proto = data[0]
port = data[1]
}
err = i.DeleteSourceNatForDestinationPort(podIP, vipIP, port, proto)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
} else {
err = i.DeleteSourceNat(podIP, vipIP)
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
}
err = vip.DeleteExistingSessions(podIP, false, destinationPorts, "")
if err != nil {
return fmt.Errorf("error changing iptables rules for egress [%s]", err)
}
return nil
}

View File

@@ -1,4 +1,4 @@
package manager
package services
import "testing"

View File

@@ -1,4 +1,4 @@
package manager
package services
import (
"context"
@@ -22,14 +22,14 @@ func init() {
svcLocks = make(map[string]*sync.Mutex)
}
// The startServicesWatchForLeaderElection function will start a services watcher, the
func (sm *Manager) startServicesWatchForLeaderElection(ctx context.Context) error {
err := sm.servicesWatcher(ctx, sm.StartServicesLeaderElection)
// The StartServicesWatchForLeaderElection function will start a services watcher, the
func (p *Processor) StartServicesWatchForLeaderElection(ctx context.Context) error {
err := p.ServicesWatcher(ctx, p.StartServicesLeaderElection)
if err != nil {
return err
}
for _, instance := range sm.serviceInstances {
for _, instance := range p.ServiceInstances {
for _, cluster := range instance.Clusters {
for i := range cluster.Network {
_ = cluster.Network[i].DeleteRoute()
@@ -44,9 +44,9 @@ func (sm *Manager) startServicesWatchForLeaderElection(ctx context.Context) erro
}
// The startServicesWatchForLeaderElection function will start a services watcher, the
func (sm *Manager) StartServicesLeaderElection(ctx context.Context, service *v1.Service) error {
func (p *Processor) StartServicesLeaderElection(ctx context.Context, service *v1.Service) error {
serviceLease := fmt.Sprintf("kubevip-%s", service.Name)
log.Info("new leader election", "service", service.Name, "namespace", service.Namespace, "lock_name", serviceLease, "host_id", sm.config.NodeName)
log.Info("new leader election", "service", service.Name, "namespace", service.Namespace, "lock_name", serviceLease, "host_id", p.config.NodeName)
// we use the Lease lock type since edits to Leases are less common
// and fewer objects in the cluster watch "all Leases".
lock := &resourcelock.LeaseLock{
@@ -54,9 +54,9 @@ func (sm *Manager) StartServicesLeaderElection(ctx context.Context, service *v1.
Name: serviceLease,
Namespace: service.Namespace,
},
Client: sm.clientSet.CoordinationV1(),
Client: p.clientSet.CoordinationV1(),
LockConfig: resourcelock.ResourceLockConfig{
Identity: sm.config.NodeName,
Identity: p.config.NodeName,
},
}
childCtx, childCancel := context.WithCancel(ctx)
@@ -69,7 +69,7 @@ func (sm *Manager) StartServicesLeaderElection(ctx context.Context, service *v1.
svcLocks[serviceLease].Lock()
defer svcLocks[serviceLease].Unlock()
svcCtx, err := getServiceContext(service.UID)
svcCtx, err := p.getServiceContext(service.UID)
if err != nil {
return fmt.Errorf("failed to get context for service %q with UID %q: %w", service.Name, service.UID, err)
}
@@ -77,7 +77,7 @@ func (sm *Manager) StartServicesLeaderElection(ctx context.Context, service *v1.
return fmt.Errorf("failed to get context for service %q with UID %q: nil context", service.Name, service.UID)
}
svcCtx.isActive = true
svcCtx.IsActive = true
// start the leader election code loop
leaderelection.RunOrDie(childCtx, leaderelection.LeaderElectionConfig{
@@ -89,32 +89,32 @@ func (sm *Manager) StartServicesLeaderElection(ctx context.Context, service *v1.
// get elected before your background loop finished, violating
// the stated goal of the lease.
ReleaseOnCancel: true,
LeaseDuration: time.Duration(sm.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(sm.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(sm.config.RetryPeriod) * time.Second,
LeaseDuration: time.Duration(p.config.LeaseDuration) * time.Second,
RenewDeadline: time.Duration(p.config.RenewDeadline) * time.Second,
RetryPeriod: time.Duration(p.config.RetryPeriod) * time.Second,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(ctx context.Context) {
// Mark this service as active (as we've started leading)
// we run this in background as it's blocking
if err := sm.syncServices(ctx, service); err != nil {
if err := p.SyncServices(ctx, service); err != nil {
log.Error("service sync", "err", err)
childCancel()
}
},
OnStoppedLeading: func() {
// we can do cleanup here
log.Info("leadership lost", "service", service.Name, "leader", sm.config.NodeName)
if svcCtx.isActive {
if err := sm.deleteService(service.UID); err != nil {
log.Info("leadership lost", "service", service.Name, "leader", p.config.NodeName)
if svcCtx.IsActive {
if err := p.deleteService(service.UID); err != nil {
log.Error("service deletion", "err", err)
}
}
// Mark this service is inactive
svcCtx.isActive = false
svcCtx.IsActive = false
},
OnNewLeader: func(identity string) {
// we're notified when new leader elected
if identity == sm.config.NodeName {
if identity == p.config.NodeName {
// I just got the lock
return
}

344
pkg/services/processor.go Normal file
View File

@@ -0,0 +1,344 @@
package services
import (
"context"
"fmt"
log "log/slog"
"reflect"
"sync"
"github.com/kube-vip/kube-vip/pkg/arp"
"github.com/kube-vip/kube-vip/pkg/bgp"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/networkinterface"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/prometheus/client_golang/prometheus"
"github.com/vishvananda/netlink"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
)
type Processor struct {
config *kubevip.Config
lbClassFilter func(svc *v1.Service, config *kubevip.Config) bool
svcMap sync.Map
// Keeps track of all running instances
ServiceInstances []*instance.Instance
mutex sync.Mutex
bgpServer *bgp.Server
clientSet *kubernetes.Clientset
rwClientSet *kubernetes.Clientset
shutdownChan chan struct{}
// This is a prometheus counter used to count the number of events received
// from the service watcher
CountServiceWatchEvent *prometheus.CounterVec
intfMgr *networkinterface.Manager
arpMgr *arp.Manager
}
func NewServicesProcessor(config *kubevip.Config, bgpServer *bgp.Server,
clientSet *kubernetes.Clientset, rwClientSet *kubernetes.Clientset, shutdownChan chan struct{},
intfMgr *networkinterface.Manager, arpMgr *arp.Manager) *Processor {
lbClassFilterFunc := lbClassFilter
if config.LoadBalancerClassLegacyHandling {
lbClassFilterFunc = lbClassFilterLegacy
}
return &Processor{
config: config,
lbClassFilter: lbClassFilterFunc,
ServiceInstances: []*instance.Instance{},
bgpServer: bgpServer,
clientSet: clientSet,
rwClientSet: rwClientSet,
shutdownChan: shutdownChan,
CountServiceWatchEvent: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "kube_vip",
Subsystem: "manager",
Name: "all_services_events",
Help: "Count all events fired by the service watcher categorised by event type",
}, []string{"type"}),
intfMgr: intfMgr,
arpMgr: arpMgr,
}
}
func (p *Processor) AddOrModify(ctx context.Context, event watch.Event, serviceFunc func(context.Context, *v1.Service) error) (bool, error) {
// log.Debugf("Endpoints for service [%s] have been Created or modified", s.service.ServiceName)
svc, ok := event.Object.(*v1.Service)
if !ok {
return false, fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
return true, nil
}
// Check if we ignore this service
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Info("ignore annotation for kube-vip", "service name", svc.Name)
return true, nil
}
// Check the loadBalancer class
if p.lbClassFilter(svc, p.config) {
return true, nil
}
svcAddresses := instance.FetchServiceAddresses(svc)
// We only care about LoadBalancer services that have been allocated an address
if len(svcAddresses) <= 0 {
return true, nil
}
svcCtx, err := p.getServiceContext(svc.UID)
if err != nil {
return false, fmt.Errorf("failed to get service context: %w", err)
}
// The modified event should only be triggered if the service has been modified (i.e. moved somewhere else)
if event.Type == watch.Modified {
i := instance.FindServiceInstance(svc, p.ServiceInstances)
var originalService []string
shouldGarbageCollect := true
if i != nil {
originalService = instance.FetchServiceAddresses(i.ServiceSnapshot)
shouldGarbageCollect = !reflect.DeepEqual(originalService, svcAddresses)
}
if shouldGarbageCollect {
for _, addr := range svcAddresses {
// log.Debugf("(svcs) Retreiving local addresses, to ensure that this modified address doesn't exist: %s", addr)
f, err := vip.GarbageCollect(p.config.Interface, addr, p.intfMgr)
if err != nil {
log.Error("(svcs) cleaning existing address error", "err", err)
}
if f {
log.Warn("(svcs) already found existing config", "address", addr, "adapter", p.config.Interface)
}
}
}
// This service has been modified, but it was also active.
if svcCtx != nil && svcCtx.IsActive {
if i != nil {
originalService := instance.FetchServiceAddresses(i.ServiceSnapshot)
newService := instance.FetchServiceAddresses(svc)
if !reflect.DeepEqual(originalService, newService) {
// Calls the cancel function of the context
if svcCtx != nil {
log.Warn("(svcs) The load balancer has changed, cancelling original load balancer")
svcCtx.Cancel()
log.Warn("(svcs) waiting for load balancer to finish")
<-svcCtx.Ctx.Done()
}
err = p.deleteService(svc.UID)
if err != nil {
log.Error("(svc) unable to remove", "service", svc.UID)
}
p.svcMap.Delete(svc.UID)
}
// in theory this should never fail
}
}
}
// Architecture walkthrough: (Had to do this as this code path is making my head hurt)
// Is the service active (bool), if not then process this new service
// Does this service use an election per service?
//
if svcCtx == nil || svcCtx != nil && !svcCtx.IsActive {
log.Debug("(svcs) has been added/modified with addresses", "service name", svc.Name, "ip", instance.FetchServiceAddresses(svc))
if svcCtx == nil {
svcCtx = servicecontext.New(ctx)
p.svcMap.Store(svc.UID, svcCtx)
}
if p.config.EnableServicesElection || // Service Election
((p.config.EnableRoutingTable || p.config.EnableBGP) && // Routing table mode or BGP
(!p.config.EnableLeaderElection && !p.config.EnableServicesElection)) { // No leaderelection or services election
// If this load balancer Traffic Policy is "local"
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Start an endpoint watcher if we're not watching it already
if !svcCtx.IsWatched {
// background the endpoint watcher
if (p.config.EnableRoutingTable || p.config.EnableBGP) && (!p.config.EnableLeaderElection && !p.config.EnableServicesElection) {
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal {
// Add Endpoint or EndpointSlices watcher
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
} else {
provider = providers.NewEndpointslices()
}
if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
}
}()
// We're now watching this service
svcCtx.IsWatched = true
}
} else if (p.config.EnableBGP || p.config.EnableRoutingTable) && (!p.config.EnableLeaderElection && !p.config.EnableServicesElection) {
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
go func() {
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster {
// Add Endpoint watcher
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
} else {
provider = providers.NewEndpointslices()
}
if err = p.watchEndpoint(svcCtx, p.config.NodeName, svc, provider); err != nil {
log.Error(err.Error())
}
}
}()
// We're now watching this service
svcCtx.IsWatched = true
} else {
go func() {
for {
select {
case <-svcCtx.Ctx.Done():
log.Warn("(svcs) restartable service watcher ending", "uid", svc.UID)
return
default:
log.Info("(svcs) restartable service watcher starting", "uid", svc.UID)
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
}
}()
}
} else {
// Increment the waitGroup before the service Func is called (Done is completed in there)
err = serviceFunc(svcCtx.Ctx, svc)
if err != nil {
log.Error(err.Error())
}
}
svcCtx.IsActive = true
}
return false, nil
}
func (p *Processor) Delete(event watch.Event) (bool, error) {
svc, ok := event.Object.(*v1.Service)
if !ok {
return false, fmt.Errorf("unable to parse Kubernetes services from API watcher")
}
svcCtx, err := p.getServiceContext(svc.UID)
if err != nil {
return false, fmt.Errorf("(svcs) unable to get context: %w", err)
}
if svcCtx != nil && svcCtx.IsActive {
// We only care about LoadBalancer services
if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
return true, nil
}
// We can ignore this service
if svc.Annotations["kube-vip.io/ignore"] == "true" {
log.Info("(svcs)ignore annotation for kube-vip", "service name", svc.Name)
return true, nil
}
// If no leader election is enabled, delete routes here
if !p.config.EnableLeaderElection && !p.config.EnableServicesElection &&
p.config.EnableRoutingTable && svcCtx.HasConfiguredNetworks() {
if errs := endpoints.ClearRoutes(svc, &p.ServiceInstances); len(errs) == 0 {
svcCtx.ConfiguredNetworks.Clear()
}
}
// If this is an active service then and additional leaderElection will handle stopping
err = p.deleteService(svc.UID)
if err != nil {
log.Error(err.Error())
}
// Calls the cancel function of the context
log.Warn("(svcs) The load balancer was deleted, cancelling context")
svcCtx.Cancel()
log.Warn("(svcs) waiting for load balancer to finish")
<-svcCtx.Ctx.Done()
p.svcMap.Delete(svc.UID)
}
if p.config.EnableLeaderElection && !p.config.EnableServicesElection {
if p.config.EnableBGP {
endpoints.ClearBGPHosts(svc, &p.ServiceInstances, p.bgpServer)
} else if p.config.EnableRoutingTable {
endpoints.ClearRoutes(svc, &p.ServiceInstances)
}
}
log.Info("(svcs) deleted", "service name", svc.Name, "namespace", svc.Namespace)
return true, nil
}
func (p *Processor) Stop() {
for _, instance := range p.ServiceInstances {
for _, cluster := range instance.Clusters {
cluster.Stop()
}
}
}
func (p *Processor) getServiceContext(uid types.UID) (*servicecontext.Context, error) {
svcCtx, ok := p.svcMap.Load(uid)
if !ok {
return nil, nil
}
ctx, ok := svcCtx.(*servicecontext.Context)
if !ok {
return nil, fmt.Errorf("failed to cast service context pointer - UID: %s", uid)
}
return ctx, nil
}
func (p *Processor) CountRouteReferences(route *netlink.Route) int {
return endpoints.CountRouteReferences(route, &p.ServiceInstances)
}

View File

@@ -1,4 +1,4 @@
package manager
package services
import (
"context"
@@ -17,7 +17,11 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
"github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/egress"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/instance"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/upnp"
"github.com/kube-vip/kube-vip/pkg/vip"
)
@@ -30,21 +34,21 @@ const (
ActionNone ServiceInstanceAction = "none"
)
func (sm *Manager) syncServices(ctx context.Context, svc *v1.Service) error {
func (p *Processor) SyncServices(ctx context.Context, svc *v1.Service) error {
log.Debug("[STARTING] Service Sync", "namespace", svc.Namespace, "name", svc.Name)
// Iterate through the synchronising services
action := sm.getServiceInstanceAction(svc)
action := p.getServiceInstanceAction(svc)
switch action {
case ActionDelete:
log.Debug("[service] delete", "namespace", svc.Namespace, "name", svc.Name)
if err := sm.deleteService(svc.UID); err != nil {
if err := p.deleteService(svc.UID); err != nil {
return fmt.Errorf("error deleting service %s/%s: %w", svc.Namespace, svc.Name, err)
}
case ActionAdd:
log.Debug("[service] add", "namespace", svc.Namespace, "name", svc.Name)
if err := sm.addService(ctx, svc); err != nil {
if err := p.addService(ctx, svc); err != nil {
return fmt.Errorf("error adding service %s/%s: %w", svc.Namespace, svc.Name, err)
}
case ActionNone:
@@ -54,14 +58,14 @@ func (sm *Manager) syncServices(ctx context.Context, svc *v1.Service) error {
return nil
}
func (sm *Manager) getServiceInstanceAction(svc *v1.Service) ServiceInstanceAction {
func (p *Processor) getServiceInstanceAction(svc *v1.Service) ServiceInstanceAction {
// protect against multiple calls
addresses := cluster.FetchServiceAddresses(svc)
ingressIPs := cluster.FetchLoadBalancerIngressAddresses(svc)
sm.mutex.Lock()
defer sm.mutex.Unlock()
addresses := instance.FetchServiceAddresses(svc)
ingressIPs := instance.FetchLoadBalancerIngressAddresses(svc)
p.mutex.Lock()
defer p.mutex.Unlock()
for _, instance := range sm.serviceInstances {
for _, instance := range p.ServiceInstances {
if instance != nil && instance.ServiceSnapshot.UID == svc.UID {
for _, address := range addresses {
// handle the case where the service instance needs to be deleted
@@ -112,24 +116,24 @@ func comparePortsAndPortStatuses(svc *v1.Service) bool {
return true
}
func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
func (p *Processor) addService(ctx context.Context, svc *v1.Service) error {
// protect against addService while reading
sm.mutex.Lock()
defer sm.mutex.Unlock()
p.mutex.Lock()
defer p.mutex.Unlock()
startTime := time.Now()
newService, err := cluster.NewInstance(svc, sm.config, sm.intfMgr, sm.arpMgr)
newService, err := instance.NewInstance(svc, p.config, p.intfMgr, p.arpMgr)
if err != nil {
return err
}
for x := range newService.VIPConfigs {
log.Debug("starting loadbalancer for service", "name", svc.Name, "namespace", svc.Namespace)
newService.Clusters[x].StartLoadBalancerService(newService.VIPConfigs[x], sm.bgpServer, svc.Name, &sm.serviceInstances)
newService.Clusters[x].StartLoadBalancerService(ctx, newService.VIPConfigs[x], p.bgpServer, svc.Name, p.CountRouteReferences)
}
sm.upnpMap(ctx, newService)
p.upnpMap(ctx, newService)
if newService.IsDHCP && len(newService.VIPConfigs) == 1 {
go func() {
@@ -137,8 +141,8 @@ func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
log.Debug("IP changed", "ip", ip)
newService.VIPConfigs[0].VIP = ip
newService.DHCPInterfaceIP = ip
if !sm.config.DisableServiceUpdates {
if err := sm.updateStatus(newService); err != nil {
if !p.config.DisableServiceUpdates {
if err := p.updateStatus(newService); err != nil {
log.Warn("updating svc", "err", err)
}
}
@@ -147,26 +151,26 @@ func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
}()
}
sm.serviceInstances = append(sm.serviceInstances, newService)
p.ServiceInstances = append(p.ServiceInstances, newService)
if !sm.config.DisableServiceUpdates {
if !p.config.DisableServiceUpdates {
log.Debug("[service] update", "namespace", newService.ServiceSnapshot.Namespace, "name", newService.ServiceSnapshot.Name)
if err := sm.updateStatus(newService); err != nil {
if err := p.updateStatus(newService); err != nil {
log.Error("[service] updating status", "namespace", newService.ServiceSnapshot.Namespace, "name", newService.ServiceSnapshot.Name, "err", err)
}
}
serviceIPs := cluster.FetchServiceAddresses(svc)
serviceIPs := instance.FetchServiceAddresses(svc)
// Check if we need to flush any conntrack connections (due to some dangling conntrack connections)
if svc.Annotations[flushContrack] == "true" {
if svc.Annotations[kubevip.FlushContrack] == "true" {
log.Debug("[service] Flushing conntrack rules", "service", svc.Name, "namespace", svc.Namespace)
for _, serviceIP := range serviceIPs {
err = vip.DeleteExistingSessions(serviceIP, false, svc.Annotations[egressDestinationPorts], svc.Annotations[egressSourcePorts])
err = vip.DeleteExistingSessions(serviceIP, false, svc.Annotations[kubevip.EgressDestinationPorts], svc.Annotations[kubevip.EgressSourcePorts])
if err != nil {
log.Error("[service] flushing any remaining egress connections", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
err = vip.DeleteExistingSessions(serviceIP, true, svc.Annotations[egressDestinationPorts], svc.Annotations[egressSourcePorts])
err = vip.DeleteExistingSessions(serviceIP, true, svc.Annotations[kubevip.EgressDestinationPorts], svc.Annotations[kubevip.EgressSourcePorts])
if err != nil {
log.Error("[service] flushing any remaining ingress connections", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
@@ -174,18 +178,18 @@ func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
}
// Check if egress is enabled on the service, if so we'll need to configure some rules
if svc.Annotations[egress] == "true" && len(serviceIPs) > 0 {
if svc.Annotations[kubevip.Egress] == "true" && len(serviceIPs) > 0 {
log.Debug("[service] enabling egress", "service", svc.Name, "namespace", svc.Namespace)
// If we'er not using NFtables, then ensure that the correct iptables modules are loaded
if sm.config.EgressWithNftables {
if p.config.EgressWithNftables {
// Ensure that kernel modules are loaded and report back missing modules.
err = sm.nftablesCheck()
err = p.nftablesCheck()
if err != nil {
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
} else {
// Ensure that kernel modules are loaded and report back missing modules.
err = sm.iptablesCheck()
err = p.iptablesCheck()
if err != nil {
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
@@ -194,15 +198,15 @@ func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
errList := []error{}
// Should egress be IPv6
if svc.Annotations[egressIPv6] == "true" {
if svc.Annotations[kubevip.EgressIPv6] == "true" {
// Does the service have an active IPv6 endpoint
if svc.Annotations[activeEndpointIPv6] != "" {
if svc.Annotations[kubevip.ActiveEndpointIPv6] != "" {
for _, serviceIP := range serviceIPs {
if sm.config.EnableEndpointSlices && vip.IsIPv6(serviceIP) {
if !p.config.EnableEndpoints && vip.IsIPv6(serviceIP) {
podIP = svc.Annotations[activeEndpointIPv6]
podIP = svc.Annotations[kubevip.ActiveEndpointIPv6]
err = sm.configureEgress(serviceIP, podIP, svc.Namespace, svc.Annotations)
err = p.configureEgress(serviceIP, podIP, svc.Namespace, string(svc.UID), svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Error("[service] configuring egress IPv6", "service", svc.Name, "namespace", svc.Namespace, "err", err)
@@ -210,13 +214,13 @@ func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
}
}
}
} else if svc.Annotations[activeEndpoint] != "" { // Not expected to be IPv6, so should be an IPv4 address
} else if svc.Annotations[kubevip.ActiveEndpoint] != "" { // Not expected to be IPv6, so should be an IPv4 address
for _, serviceIP := range serviceIPs {
podIPs := svc.Annotations[activeEndpoint]
if sm.config.EnableEndpointSlices && vip.IsIPv6(serviceIP) {
podIPs = svc.Annotations[activeEndpointIPv6]
podIPs := svc.Annotations[kubevip.ActiveEndpoint]
if !p.config.EnableEndpoints && vip.IsIPv6(serviceIP) {
podIPs = svc.Annotations[kubevip.ActiveEndpointIPv6]
}
err = sm.configureEgress(serviceIP, podIPs, svc.Namespace, svc.Annotations)
err = p.configureEgress(serviceIP, podIPs, svc.Namespace, string(svc.UID), svc.Annotations)
if err != nil {
errList = append(errList, err)
log.Error("[service] configuring egress IPv4", "service", svc.Name, "namespace", svc.Namespace, "err", err)
@@ -224,13 +228,13 @@ func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
}
}
if len(errList) == 0 {
var provider epProvider
if !sm.config.EnableEndpointSlices {
provider = &endpointsProvider{label: "endpoints"}
var provider providers.Provider
if p.config.EnableEndpoints {
provider = providers.NewEndpoints()
} else {
provider = &endpointslicesProvider{label: "endpointslices"}
provider = providers.NewEndpointslices()
}
err = provider.updateServiceAnnotation(svc.Annotations[activeEndpoint], svc.Annotations[activeEndpointIPv6], svc, sm)
err = provider.UpdateServiceAnnotation(svc.Annotations[kubevip.ActiveEndpoint], svc.Annotations[kubevip.ActiveEndpointIPv6], svc, p.clientSet)
if err != nil {
log.Error("[service] configuring egress", "service", svc.Name, "namespace", svc.Namespace, "err", err)
}
@@ -243,23 +247,23 @@ func (sm *Manager) addService(ctx context.Context, svc *v1.Service) error {
return nil
}
func (sm *Manager) deleteService(uid types.UID) error {
func (p *Processor) deleteService(uid types.UID) error {
// protect multiple calls
sm.mutex.Lock()
defer sm.mutex.Unlock()
p.mutex.Lock()
defer p.mutex.Unlock()
var updatedInstances []*cluster.Instance
var serviceInstance *cluster.Instance
var updatedInstances []*instance.Instance
var serviceInstance *instance.Instance
found := false
for x := range sm.serviceInstances {
log.Debug("service lookup", "target UID", uid, "found UID ", sm.serviceInstances[x].ServiceSnapshot.UID, "name", sm.serviceInstances[x].ServiceSnapshot.Name, "namespace", sm.serviceInstances[x].ServiceSnapshot.Namespace)
for x := range p.ServiceInstances {
log.Debug("[service] lookup", "target UID", uid, "found UID ", p.ServiceInstances[x].ServiceSnapshot.UID, "name", p.ServiceInstances[x].ServiceSnapshot.Name, "namespace", p.ServiceInstances[x].ServiceSnapshot.Namespace)
// Add the running services to the new array
if sm.serviceInstances[x].ServiceSnapshot.UID != uid {
updatedInstances = append(updatedInstances, sm.serviceInstances[x])
if p.ServiceInstances[x].ServiceSnapshot.UID != uid {
updatedInstances = append(updatedInstances, p.ServiceInstances[x])
} else {
// Flip the found when we match
found = true
serviceInstance = sm.serviceInstances[x]
serviceInstance = p.ServiceInstances[x]
}
}
// If we've been through all services and not found the correct one then error
@@ -279,66 +283,67 @@ func (sm *Manager) deleteService(uid types.UID) error {
shared := false
vipSet := make(map[string]interface{})
for x := range updatedInstances {
for _, vip := range cluster.FetchServiceAddresses(updatedInstances[x].ServiceSnapshot) { //updatedInstances[x].ServiceSnapshot.Spec.LoadBalancerIP {
for _, vip := range instance.FetchServiceAddresses(updatedInstances[x].ServiceSnapshot) { //updatedInstances[x].ServiceSnapshot.Spec.LoadBalancerIP {
vipSet[vip] = nil
}
}
for _, vip := range cluster.FetchServiceAddresses(serviceInstance.ServiceSnapshot) {
for _, vip := range instance.FetchServiceAddresses(serviceInstance.ServiceSnapshot) {
if _, found := vipSet[vip]; found {
shared = true
}
}
for x := range serviceInstance.Clusters {
serviceInstance.Clusters[x].Stop()
}
if !shared {
for x := range serviceInstance.Clusters {
serviceInstance.Clusters[x].Stop()
}
if serviceInstance.IsDHCP {
serviceInstance.DHCPClient.Stop()
macvlan, err := netlink.LinkByName(serviceInstance.DHCPInterface)
if err != nil {
return fmt.Errorf("error finding VIP Interface: %v", err)
return fmt.Errorf("[service] error finding VIP Interface: %v", err)
}
err = netlink.LinkDel(macvlan)
if err != nil {
return fmt.Errorf("error deleting DHCP Link : %v", err)
return fmt.Errorf("[service] error deleting DHCP Link : %v", err)
}
}
for i := range serviceInstance.VIPConfigs {
if serviceInstance.VIPConfigs[i].EnableBGP {
sm.clearBGPHostsByInstance(serviceInstance)
endpoints.ClearBGPHostsByInstance(serviceInstance, p.bgpServer)
}
}
// We will need to tear down the egress
if serviceInstance.ServiceSnapshot.Annotations[egress] == "true" {
if serviceInstance.ServiceSnapshot.Annotations[activeEndpoint] != "" {
log.Info("egress re-write enabled", "service", serviceInstance.ServiceSnapshot.Name)
err := sm.TeardownEgress(serviceInstance.ServiceSnapshot.Annotations[activeEndpoint], serviceInstance.ServiceSnapshot.Spec.LoadBalancerIP, serviceInstance.ServiceSnapshot.Namespace, serviceInstance.ServiceSnapshot.Annotations)
if serviceInstance.ServiceSnapshot.Annotations[kubevip.Egress] == "true" {
if serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint] != "" {
log.Info("[service] egress re-write enabled", "service", serviceInstance.ServiceSnapshot.Name)
err := egress.Teardown(serviceInstance.ServiceSnapshot.Annotations[kubevip.ActiveEndpoint], serviceInstance.ServiceSnapshot.Spec.LoadBalancerIP, serviceInstance.ServiceSnapshot.Namespace, string(serviceInstance.ServiceSnapshot.UID), serviceInstance.ServiceSnapshot.Annotations, p.config.EgressWithNftables)
if err != nil {
log.Error("egress teardown", "err", err)
log.Error("[service] egress teardown", "err", err)
}
}
}
}
// Update the service array
sm.serviceInstances = updatedInstances
p.ServiceInstances = updatedInstances
log.Info("Removed instance from manager", "uid", uid, "remaining advertised services", len(sm.serviceInstances))
log.Info("Removed instance from manager", "uid", uid, "remaining advertised services", len(p.ServiceInstances))
return nil
}
// Set up UPNP forwards for a service
// We first try to use the more modern Pinhole API introduced in UPNPv2 and fall back to UPNPv2 Port Forwarding if no forward was successful
func (sm *Manager) upnpMap(ctx context.Context, s *cluster.Instance) {
func (p *Processor) upnpMap(ctx context.Context, s *instance.Instance) {
if !isUPNPEnabled(s.ServiceSnapshot) {
// Skip services missing the annotation
return
}
if !sm.upnp {
if !p.config.EnableUPNP {
log.Warn("[UPNP] Found kube-vip.io/forwardUPNP on service while UPNP forwarding is disabled in the kube-vip config. Not forwarding", "service", s.ServiceSnapshot.Name)
return
}
// If upnp is enabled then update the gateway/router with the address
// TODO - check if this implementation for dualstack is correct
@@ -348,10 +353,10 @@ func (sm *Manager) upnpMap(ctx context.Context, s *cluster.Instance) {
// Reset Gateway IPs to remove stale addresses
s.UPNPGatewayIPs = make([]string, 0)
for _, vip := range cluster.FetchServiceAddresses(s.ServiceSnapshot) {
for _, vip := range instance.FetchServiceAddresses(s.ServiceSnapshot) {
for _, port := range s.ServiceSnapshot.Spec.Ports {
for _, gw := range gateways {
log.Info("[UPNP] Adding map", "vip", vip, "port", port.Port, "service", s.ServiceSnapshot.Name, "gateway", gw.ConnectionClient.GetServiceClient().Location)
log.Info("[UPNP] Adding map", "vip", vip, "port", port.Port, "service", s.ServiceSnapshot.Name, "gateway", gw.WANIPv6FirewallControlClient.Location)
forwardSucessful := false
if gw.WANIPv6FirewallControlClient != nil {
@@ -391,7 +396,7 @@ func (sm *Manager) upnpMap(ctx context.Context, s *cluster.Instance) {
s.UPNPGatewayIPs = slices.Compact(s.UPNPGatewayIPs)
}
func (sm *Manager) updateStatus(i *cluster.Instance) error {
func (p *Processor) updateStatus(i *instance.Instance) error {
// let's retry status update every 10ms for 30s
retryConfig := wait.Backoff{
Steps: 3000,
@@ -403,7 +408,7 @@ func (sm *Manager) updateStatus(i *cluster.Instance) error {
err := retry.OnError(retryConfig, func(error) bool { return true }, func() error {
// Retrieve the latest version of Deployment before attempting update
// RetryOnConflict uses exponential backoff to avoid exhausting the apiserver
currentService, err := sm.clientSet.CoreV1().Services(i.ServiceSnapshot.Namespace).Get(context.TODO(), i.ServiceSnapshot.Name, metav1.GetOptions{})
currentService, err := p.clientSet.CoreV1().Services(i.ServiceSnapshot.Namespace).Get(context.TODO(), i.ServiceSnapshot.Name, metav1.GetOptions{})
if err != nil {
return err
}
@@ -414,13 +419,13 @@ func (sm *Manager) updateStatus(i *cluster.Instance) error {
}
// If we're using ARP then we can only broadcast the VIP from one place, add an annotation to the service
if sm.config.EnableARP {
if p.config.EnableARP {
// Add the current host
currentServiceCopy.Annotations[vipHost] = sm.config.NodeName
currentServiceCopy.Annotations[kubevip.VipHost] = p.config.NodeName
}
if i.DHCPInterfaceHwaddr != "" || i.DHCPInterfaceIP != "" {
currentServiceCopy.Annotations[cluster.HWAddrKey] = i.DHCPInterfaceHwaddr
currentServiceCopy.Annotations[cluster.RequestedIP] = i.DHCPInterfaceIP
currentServiceCopy.Annotations[kubevip.HwAddrKey] = i.DHCPInterfaceHwaddr
currentServiceCopy.Annotations[kubevip.RequestedIP] = i.DHCPInterfaceIP
}
if currentService.Annotations["development.kube-vip.io/synthetic-api-server-error-on-update"] == "true" {
@@ -429,7 +434,7 @@ func (sm *Manager) updateStatus(i *cluster.Instance) error {
}
if !cmp.Equal(currentService, currentServiceCopy) {
currentService, err = sm.clientSet.CoreV1().Services(currentServiceCopy.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
currentService, err = p.clientSet.CoreV1().Services(currentServiceCopy.Namespace).Update(context.TODO(), currentServiceCopy, metav1.UpdateOptions{})
if err != nil {
log.Error("updating Spec", "service", i.ServiceSnapshot.Name, "err", err)
return err
@@ -448,7 +453,7 @@ func (sm *Manager) updateStatus(i *cluster.Instance) error {
for _, c := range i.VIPConfigs {
if !vip.IsIP(c.VIP) {
ips, err := vip.LookupHost(c.VIP, sm.config.DNSMode)
ips, err := vip.LookupHost(c.VIP, p.config.DNSMode)
if err != nil {
return err
}
@@ -478,7 +483,7 @@ func (sm *Manager) updateStatus(i *cluster.Instance) error {
}
if !cmp.Equal(currentService.Status.LoadBalancer.Ingress, ingresses) {
currentService.Status.LoadBalancer.Ingress = ingresses
_, err = sm.clientSet.CoreV1().Services(currentService.Namespace).UpdateStatus(context.TODO(), currentService, metav1.UpdateOptions{})
_, err = p.clientSet.CoreV1().Services(currentService.Namespace).UpdateStatus(context.TODO(), currentService, metav1.UpdateOptions{})
if err != nil {
log.Error("updating Service", "namespace", i.ServiceSnapshot.Namespace, "name", i.ServiceSnapshot.Name, "err", err)
return err
@@ -491,5 +496,21 @@ func (sm *Manager) updateStatus(i *cluster.Instance) error {
}
func isUPNPEnabled(s *v1.Service) bool {
return metav1.HasAnnotation(s.ObjectMeta, upnpEnabled) && s.Annotations[upnpEnabled] == "true"
return metav1.HasAnnotation(s.ObjectMeta, kubevip.UpnpEnabled) && s.Annotations[kubevip.UpnpEnabled] == "true"
}
// Refresh UPNP Port Forwards for all Service Instances registered in the processor
func (p *Processor) RefreshUPNPForwards() {
log.Info("Starting UPNP Port Refresher")
for {
time.Sleep(300 * time.Second)
log.Info("[UPNP] Refreshing Instances", "number of instances", len(p.ServiceInstances))
for i := range p.ServiceInstances {
p.upnpMap(context.TODO(), p.ServiceInstances[i])
if err := p.updateStatus(p.ServiceInstances[i]); err != nil {
log.Warn("[UPNP] Error updating service", "ip", p.ServiceInstances[i].ServiceSnapshot.Name, "err", err)
}
}
}
}

View File

@@ -0,0 +1,94 @@
package services
import (
"context"
"fmt"
log "log/slog"
"github.com/kube-vip/kube-vip/pkg/endpoints"
"github.com/kube-vip/kube-vip/pkg/endpoints/providers"
"github.com/kube-vip/kube-vip/pkg/servicecontext"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/watch"
)
func (p *Processor) watchEndpoint(svcCtx *servicecontext.Context, id string, service *v1.Service, provider providers.Provider) error {
log.Info("watching", "provider", provider.GetLabel(), "service_name", service.Name, "namespace", service.Namespace)
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
leaderCtx, cancel := context.WithCancel(svcCtx.Ctx)
defer cancel()
var leaderElectionActive bool
rw, err := provider.CreateRetryWatcher(leaderCtx, p.rwClientSet, service)
if err != nil {
return fmt.Errorf("[%s] error watching endpoints: %w", provider.GetLabel(), err)
}
exitFunction := make(chan struct{})
go func() {
select {
case <-svcCtx.Ctx.Done():
log.Debug("context cancelled", "provider", provider.GetLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-p.shutdownChan:
log.Debug("shutdown called", "provider", provider.GetLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
case <-exitFunction:
log.Debug("function ending", "provider", provider.GetLabel())
// Stop the retry watcher
rw.Stop()
// Cancel the context, which will in turn cancel the leadership
cancel()
return
}
}()
ch := rw.ResultChan()
epProcessor := endpoints.NewEndpointProcessor(p.config, provider, p.bgpServer, &p.ServiceInstances)
var lastKnownGoodEndpoint string
for event := range ch {
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
restart, err := epProcessor.AddOrModify(svcCtx, event, &lastKnownGoodEndpoint, service, id, &leaderElectionActive, p.StartServicesLeaderElection, &leaderCtx, &cancel)
if restart {
continue
} else if err != nil {
return fmt.Errorf("[%s] error while processing add/modify event: %w", provider.GetLabel(), err)
}
case watch.Deleted:
if err := epProcessor.Delete(service, id); err != nil {
return fmt.Errorf("[%s] error while processing delete event: %w", provider.GetLabel(), err)
}
// Close the goroutine that will end the retry watcher, then exit the endpoint watcher function
close(exitFunction)
log.Info("stopping watching", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil
case watch.Error:
errObject := apierrors.FromObject(event.Object)
statusErr, _ := errObject.(*apierrors.StatusError)
log.Error("watch error", "provider", provider.GetLabel(), "err", statusErr)
}
}
close(exitFunction)
log.Info("stopping watching", "provider", provider.GetLabel(), "service name", service.Name, "namespace", service.Namespace)
return nil //nolint:govet
}

View File

@@ -0,0 +1,183 @@
package services
import (
"context"
"fmt"
log "log/slog"
"github.com/davecgh/go-spew/spew"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/trafficmirror"
"github.com/prometheus/client_golang/prometheus"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
)
// This function handles the watching of a services endpoints and updates a load balancers endpoint configurations accordingly
func (p *Processor) ServicesWatcher(ctx context.Context, serviceFunc func(context.Context, *v1.Service) error) error {
// first start port mirroring if enabled
if err := p.startTrafficMirroringIfEnabled(); err != nil {
return err
}
defer func() {
// clean up traffic mirror related config
err := p.stopTrafficMirroringIfEnabled()
if err != nil {
log.Error("Stopping traffic mirroring", "err", err)
}
}()
if p.config.ServiceNamespace == "" {
// v1.NamespaceAll is actually "", but we'll stay with the const in case things change upstream
p.config.ServiceNamespace = v1.NamespaceAll
log.Info("(svcs) starting services watcher for all namespaces")
} else {
log.Info("(svcs) starting services watcher", "namespace", p.config.ServiceNamespace)
}
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return p.rwClientSet.CoreV1().Services(p.config.ServiceNamespace).Watch(ctx, metav1.ListOptions{})
},
})
if err != nil {
return fmt.Errorf("error creating services watcher: %s", err.Error())
}
exitFunction := make(chan struct{})
go func() {
select {
case <-p.shutdownChan:
log.Debug("(svcs) shutdown called")
// Stop the retry watcher
rw.Stop()
return
case <-exitFunction:
log.Debug("(svcs) function ending")
// Stop the retry watcher
rw.Stop()
return
}
}()
ch := rw.ResultChan()
// Used for tracking an active endpoint / pod
for event := range ch {
p.CountServiceWatchEvent.With(prometheus.Labels{"type": string(event.Type)}).Add(1)
// We need to inspect the event and get ResourceVersion out of it
switch event.Type {
case watch.Added, watch.Modified:
restart, err := p.AddOrModify(ctx, event, serviceFunc)
if restart {
break
}
if err != nil {
return fmt.Errorf("add/modify service error: %w", err)
}
case watch.Deleted:
restart, err := p.Delete(event)
if restart {
break
}
if err != nil {
return fmt.Errorf("delete service error: %w", err)
}
case watch.Bookmark:
// Un-used
case watch.Error:
log.Error("Error attempting to watch Kubernetes services")
// This round trip allows us to handle unstructured status
errObject := apierrors.FromObject(event.Object)
statusErr, ok := errObject.(*apierrors.StatusError)
if !ok {
log.Error(spew.Sprintf("Received an error which is not *metav1.Status but %#+v", event.Object))
}
status := statusErr.ErrStatus
log.Error("services", "err", status)
default:
}
}
close(exitFunction)
log.Warn("Stopping watching services for type: LoadBalancer in all namespaces")
return nil
}
func lbClassFilterLegacy(svc *v1.Service, config *kubevip.Config) bool {
if svc == nil {
log.Info("(svcs) service is nil, ignoring")
return true
}
if svc.Spec.LoadBalancerClass != nil {
// if this isn't nil then it has been configured, check if it the kube-vip loadBalancer class
if *svc.Spec.LoadBalancerClass != config.LoadBalancerClassName {
log.Info("(svcs) specified the wrong loadBalancer class", "service name", svc.Name, "lbClass", *svc.Spec.LoadBalancerClass)
return true
}
} else if config.LoadBalancerClassOnly {
// if kube-vip is configured to only recognize services with kube-vip's lb class, then ignore the services without any lb class
log.Info("(svcs) kube-vip configured to only recognize services with kube-vip's lb class but the service didn't specify any loadBalancer class, ignoring", "service name", svc.Name)
return true
}
return false
}
func lbClassFilter(svc *v1.Service, config *kubevip.Config) bool {
if svc == nil {
log.Info("(svcs) service is nil, ignoring")
return true
}
if svc.Spec.LoadBalancerClass == nil && config.LoadBalancerClassName != "" {
log.Info("(svcs) no loadBalancer class, ignoring", "service name", svc.Name, "expected lbClass", config.LoadBalancerClassName)
return true
}
if svc.Spec.LoadBalancerClass == nil && config.LoadBalancerClassName == "" {
return false
}
if *svc.Spec.LoadBalancerClass != config.LoadBalancerClassName {
log.Info("(svcs) specified wrong loadBalancer class, ignoring", "service name", svc.Name, "wrong lbClass", *svc.Spec.LoadBalancerClass, "expected lbClass", config.LoadBalancerClassName)
return true
}
return false
}
func (p *Processor) serviceInterface() string {
svcIf := p.config.Interface
if p.config.ServicesInterface != "" {
svcIf = p.config.ServicesInterface
}
return svcIf
}
func (p *Processor) startTrafficMirroringIfEnabled() error {
if p.config.MirrorDestInterface != "" {
svcIf := p.serviceInterface()
log.Info("mirroring traffic", "src", svcIf, "dest", p.config.MirrorDestInterface)
if err := trafficmirror.MirrorTrafficFromNIC(svcIf, p.config.MirrorDestInterface); err != nil {
return err
}
} else {
log.Debug("skip starting traffic mirroring since it's not enabled.")
}
return nil
}
func (p *Processor) stopTrafficMirroringIfEnabled() error {
if p.config.MirrorDestInterface != "" {
svcIf := p.serviceInterface()
log.Info("clean up qdisc config", "interface", svcIf)
if err := trafficmirror.CleanupQDSICFromNIC(svcIf); err != nil {
return err
}
} else {
log.Debug("skip stopping traffic mirroring since it's not enabled.")
}
return nil
}

View File

@@ -2,8 +2,8 @@ package vip
import (
"fmt"
"math"
"net"
"os"
"slices"
"strconv"
"strings"
@@ -36,6 +36,7 @@ type Network interface {
DeleteIP() (bool, error)
DeleteRoute() error
UpdateRoutes() (bool, error)
IsSet() (bool, error)
IP() string
CIDR() string
IPisLinkLocal() bool
@@ -62,6 +63,7 @@ type network struct {
link *networkinterface.Link
ports []v1.ServicePort
serviceName string
enableSecurity bool
ignoreSecurity bool
dnsName string
@@ -81,7 +83,8 @@ type network struct {
// NewConfig will attempt to provide an interface to the kernel network configuration
func NewConfig(address string, iface string, loGlobalScope bool, subnet string, isDDNS bool, tableID int, tableType int,
routingProtocol int, dnsMode, forwardMethod, iptablesBackend string, ipvsEnabled bool, intfMgr *networkinterface.Manager) ([]Network, error) {
routingProtocol int, dnsMode, forwardMethod, iptablesBackend string,
ipvsEnabled, enableSecurity bool, intfMgr *networkinterface.Manager) ([]Network, error) {
networks := []Network{}
link, err := netlink.LinkByName(iface)
@@ -117,6 +120,12 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
return networks, errors.Wrapf(err, "could not parse address '%s'", address)
}
// set address as deprecated so it isn't used as source address according to RFC 3484
result.address.PreferedLft = 0
// Also set ValidLft so the netlink library actually sets them
result.address.ValidLft = math.MaxInt
if iface == "lo" && !loGlobalScope {
// set host scope on loopback, otherwise global scope will be used by default
result.address.Scope = unix.RT_SCOPE_HOST
@@ -139,6 +148,8 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
iptablesBackend: iptablesBackend,
isDDNS: isDDNS,
dnsName: address,
ipvsEnabled: ipvsEnabled,
enableSecurity: enableSecurity,
}
networks = append(networks, result)
@@ -157,6 +168,8 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
iptablesBackend: iptablesBackend,
isDDNS: isDDNS,
dnsName: address,
ipvsEnabled: ipvsEnabled,
enableSecurity: enableSecurity,
}
// we're able to resolve store this as the initial IP
@@ -167,6 +180,9 @@ func NewConfig(address string, iface string, loGlobalScope bool, subnet string,
// set ValidLft so that the VIP expires if the DNS entry is updated, otherwise it'll be refreshed by the DNS prober
result.address.ValidLft = defaultValidLft
// set address as deprecated so it isn't used as source address according to RFC 3484
result.address.PreferedLft = 0
networks = append(networks, result)
}
@@ -298,7 +314,7 @@ func (configurator *network) AddIP(precheck bool) (bool, error) {
exists := false
var err error
if precheck {
if exists, err = configurator.isSet(); err != nil {
if exists, err = configurator.IsSet(); err != nil {
return false, errors.Wrap(err, "could not check if address exists")
}
}
@@ -319,7 +335,7 @@ func (configurator *network) AddIP(precheck bool) (bool, error) {
}
func (configurator *network) configureIPTables() error {
if os.Getenv("enable_service_security") == "true" && !configurator.ignoreSecurity {
if configurator.enableSecurity && !configurator.ignoreSecurity {
if err := configurator.addIptablesRulesToLimitTrafficPorts(); err != nil {
return errors.Wrap(err, "could not add iptables rules to limit traffic ports")
}
@@ -468,7 +484,7 @@ func (configurator *network) DeleteIP() (bool, error) {
configurator.link.Lock.Lock()
defer configurator.link.Lock.Unlock()
result, err := configurator.isSet()
result, err := configurator.IsSet()
if err != nil {
return false, errors.Wrap(err, "ip check in DeleteIP failed")
}
@@ -479,9 +495,10 @@ func (configurator *network) DeleteIP() (bool, error) {
}
if err = netlink.AddrDel(configurator.link.Intf, configurator.address); err != nil {
return false, fmt.Errorf("could not delete IP %q from interface %q: %w", configurator.address.IPNet.String(), configurator.link.Intf.Attrs().Name, err)
return false, errors.Wrap(err, "could not delete ip")
}
if os.Getenv("enable_service_security") == "true" && !configurator.ignoreSecurity {
if configurator.enableSecurity && !configurator.ignoreSecurity {
if err := configurator.removeIptablesRuleToLimitTrafficPorts(); err != nil {
return true, errors.Wrap(err, "could not remove iptables rules to limit traffic ports")
}
@@ -588,7 +605,7 @@ func addressHasDADFAILEDFlag(address netlink.Addr) bool {
}
// isSet - Check to see if VIP is set
func (configurator *network) isSet() (result bool, err error) {
func (configurator *network) IsSet() (result bool, err error) {
var addresses []netlink.Addr
if configurator.address == nil {
@@ -633,7 +650,13 @@ func (configurator *network) SetIP(ip string) error {
}
if configurator.address != nil && configurator.IsDNS() {
addr.ValidLft = defaultValidLft
} else {
addr.ValidLft = math.MaxInt
}
// set address as deprecated so it isn't used as source address according to RFC 3484
addr.PreferedLft = 0
configurator.address = addr
return nil
}

View File

@@ -135,9 +135,7 @@ func sendARP(iface *net.Interface, m *arpMessage) error {
Halen: m.hardwareAddressLength,
}
target := ethernetBroadcast
for i := 0; i < len(target); i++ { //nolint
ll.Addr[i] = target[i]
}
copy(ll.Addr[:], target)
b, err := m.bytes()
if err != nil {

View File

@@ -0,0 +1,3 @@
[global.config]
as = {{ .AS }}
router-id = "1.1.1.1"

683
testing/e2e/e2e_bgp_test.go Normal file
View File

@@ -0,0 +1,683 @@
//go:build e2e
// +build e2e
package e2e_test
import (
"context"
"fmt"
"io"
"net"
"os"
"os/exec"
"path/filepath"
"slices"
"strconv"
"strings"
"text/template"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
kindconfigv1alpha4 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
"sigs.k8s.io/kind/pkg/log"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/kube-vip/kube-vip/testing/e2e"
"github.com/kube-vip/kube-vip/testing/services/pkg/deployment"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/client"
api "github.com/osrg/gobgp/v3/api"
)
const (
goBGPAS uint32 = 65500
kubevipAS uint32 = 65501
goBGPPort uint32 = 50051
defaultFixedNexthopv6 = "fc00:1000:1000:1000::100"
defaultFixedNexthopv4 = "172.18.0.100"
)
var _ = Describe("kube-vip BGP mode", Ordered, func() {
if Mode == ModeBGP {
var (
logger log.Logger
imagePath string
k8sImagePath string
configPath string
kubeVIPBGPManifestTemplate *template.Template
goBGPConfigTemplate *template.Template
tempDirPath string
v129 bool
localIPv4 string
localIPv6 string
curDir string
networkInterface string
bgpKill chan any
)
BeforeAll(func() {
klog.SetOutput(GinkgoWriter)
logger = e2e.TestLogger{}
imagePath = os.Getenv("E2E_IMAGE_PATH") // Path to kube-vip image
configPath = os.Getenv("CONFIG_PATH") // path to the api server config
k8sImagePath = os.Getenv("K8S_IMAGE_PATH") // path to the kubernetes image (version for kind)
if configPath == "" {
configPath = "/etc/kubernetes/admin.conf"
}
if networkInterface = os.Getenv("NETWORK_INTERFACE"); networkInterface == "" {
networkInterface = "br-"
}
_, v129 = os.LookupEnv("V129")
var err error
curDir, err = os.Getwd()
Expect(err).NotTo(HaveOccurred())
templateBGPPath := filepath.Join(curDir, "kube-vip-bgp.yaml.tmpl")
kubeVIPBGPManifestTemplate, err = template.New("kube-vip-bgp.yaml.tmpl").ParseFiles(templateBGPPath)
Expect(err).NotTo(HaveOccurred())
tempDirPath, err = os.MkdirTemp("", "kube-vip-test")
Expect(err).NotTo(HaveOccurred())
v4addr, _, err := deployment.GetLocalIPv4(networkInterface)
Expect(err).ToNot(HaveOccurred())
localIPv4 = v4addr.String()
v6addr, _, err := deployment.GetLocalIPv6(networkInterface)
Expect(err).ToNot(HaveOccurred())
localIPv6 = v6addr.String()
goBGPConfig := &e2e.BGPPeerValues{
AS: goBGPAS,
}
bgpKill = make(chan any)
goBGPConfigPath := filepath.Join(filepath.Join(curDir, "bgp"), "config.toml.tmpl")
goBGPConfigTemplate, err = template.New("config.toml.tmpl").ParseFiles(goBGPConfigPath)
Expect(err).ToNot(HaveOccurred())
goBGPConfigPath = filepath.Join(tempDirPath, "config.toml")
f, err := os.OpenFile(goBGPConfigPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
Expect(err).ToNot(HaveOccurred())
defer f.Close()
err = goBGPConfigTemplate.Execute(f, goBGPConfig)
Expect(err).ToNot(HaveOccurred())
go startGoBGP(goBGPConfigPath, bgpKill)
})
AfterAll(func() {
close(bgpKill)
})
Describe("kube-vip IPv4 services BGP mode functionality", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
nodesNumber = 1
)
BeforeAll(func() {
setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath,
k8sImagePath, e2e.IPv4Family, e2e.IPv4Family, []string{e2e.IPv4Family}, &client, &gobgpPeers, v129,
kubeVIPBGPManifestTemplate, &gobgpClient, logger, nodesNumber, "", "bgp-ipv4")
})
AfterAll(func() {
for _, p := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: p.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv4 routes for services",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, "")
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, "")
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip IPv6 services BGP mode functionality", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
nodesNumber = 1
)
BeforeAll(func() {
setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath,
k8sImagePath, e2e.IPv6Family, e2e.IPv6Family, []string{e2e.IPv6Family}, &client, &gobgpPeers, v129,
kubeVIPBGPManifestTemplate, &gobgpClient, logger, nodesNumber, "", "bgp-ipv4")
})
AfterAll(func() {
for _, p := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: p.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv6 routes for services",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, "")
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, "")
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv6 over IPv4 - fixed nexthop", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
nodesNumber = 1
)
BeforeAll(func() {
setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
e2e.DualstackFamily, e2e.IPv4Family, []string{e2e.IPv4Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
logger, nodesNumber, "fixed", "mpbgp-ipv4")
})
AfterAll(func() {
for _, p := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: p.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv6 routes over IPv4 session",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, defaultFixedNexthopv6)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, defaultFixedNexthopv6)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv4 over IPv6 - fixed nexthop", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
nodesNumber = 1
)
BeforeAll(func() {
setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
e2e.DualstackFamilyIPv6, e2e.IPv6Family, []string{e2e.IPv6Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
logger, nodesNumber, "fixed", "mpbgp-ipv6")
})
AfterAll(func() {
for _, n := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: n.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv4 routes over IPv6 session",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, defaultFixedNexthopv4)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, defaultFixedNexthopv4)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv6 over IPv4 - auto_sourceif nexthop", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
containerIP string
nodesNumber = 1
)
BeforeAll(func() {
_, containerIP = setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
e2e.DualstackFamily, e2e.IPv4Family, []string{e2e.IPv4Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
logger, nodesNumber, "auto_sourceif", "mpbgp-if-ipv4")
})
AfterAll(func() {
for _, p := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: p.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv6 routes over IPv4 session",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, containerIP)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv6Family, api.Family_AFI_IP6, []corev1.IPFamily{corev1.IPv6Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, containerIP)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
Describe("kube-vip DualStack services BGP mode functionality with MP-BGP IPv4 over IPv6 - fixed nexthop", Ordered, func() {
var (
cpVIP string
clusterName string
client kubernetes.Interface
manifestValues *e2e.KubevipManifestValues
gobgpClient api.GobgpApiClient
gobgpPeers []*e2e.BGPPeerValues
containerIP string
nodesNumber = 1
)
BeforeAll(func() {
containerIP, _ = setupEnv(&tempDirPath, &cpVIP, &clusterName, manifestValues, localIPv4, localIPv6, imagePath, configPath, k8sImagePath,
e2e.DualstackFamilyIPv6, e2e.IPv6Family, []string{e2e.IPv6Family}, &client, &gobgpPeers, v129, kubeVIPBGPManifestTemplate, &gobgpClient,
logger, nodesNumber, "auto_sourceif", "mpbgp-if-ipv6")
})
AfterAll(func() {
for _, n := range gobgpPeers {
_, err := gobgpClient.DeletePeer(context.TODO(), &api.DeletePeerRequest{
Address: n.IP,
})
Expect(err).ToNot(HaveOccurred())
}
cleanupCluster(clusterName, tempDirPath, ConfigMtx, logger)
})
DescribeTable("advertise IPv4 routes over IPv6 session",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 1, gobgpClient, containerIP)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
DescribeTable("only stops advertising route if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
testBGP(offset, e2e.IPv4Family, api.Family_AFI_IP, []corev1.IPFamily{corev1.IPv4Protocol}, svcName, trafficPolicy, client, 2, gobgpClient, containerIP)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
Entry("with external traffic policy - local", "test-svc-local", SOffset.Get(), corev1.ServiceExternalTrafficPolicyLocal),
)
})
}
})
func testBGP(offset uint, lbFamily string, afiFamily api.Family_Afi, svcFamily []corev1.IPFamily, svcName string,
trafficPolicy corev1.ServiceExternalTrafficPolicy, client kubernetes.Interface, numberOfServices int, gobgpClient api.GobgpApiClient, expectedNexthop string) {
lbAddress := e2e.GenerateVIP(lbFamily, offset)
routeCheckFamily := &api.Family{
Afi: afiFamily,
Safi: api.Family_SAFI_UNICAST,
}
testServiceBGP(svcName, lbAddress, trafficPolicy, client, svcFamily, numberOfServices, gobgpClient, routeCheckFamily, expectedNexthop)
}
func setupEnv(tempDirPath, cpVIP, clusterName *string, manifestValues *e2e.KubevipManifestValues,
localIPv4, localIPv6, imagePath, configPath, k8sImagePath, clusterAddrFamily, bgpClientAddrFamily string, peerAddrFamily []string, client *kubernetes.Interface,
gobgpPeers *[]*e2e.BGPPeerValues, v129 bool, kubeVIPBGPManifestTemplate *template.Template, gobgpClient *api.GobgpApiClient,
logger log.Logger, nodesNumber int, mpbgpnexthop, clusterNameSuffix string) (string, string) {
var err error
*tempDirPath, err = os.MkdirTemp("", "kube-vip-test")
Expect(err).ToNot(HaveOccurred())
*cpVIP = e2e.GenerateVIP(clusterAddrFamily, SOffset.Get())
var clusterIPFamily kindconfigv1alpha4.ClusterIPFamily
var podSubnet, serviceSubnet string
switch clusterAddrFamily {
case e2e.IPv6Family:
clusterIPFamily = kindconfigv1alpha4.IPv6Family
case e2e.DualstackFamily:
clusterIPFamily = kindconfigv1alpha4.DualStackFamily
case e2e.DualstackFamilyIPv6:
clusterIPFamily = kindconfigv1alpha4.DualStackFamily
podSubnet = "fd00:10:244::/56,10.244.0.0/16"
serviceSubnet = "fd00:10:96::/112,10.96.0.0/16"
default:
clusterIPFamily = kindconfigv1alpha4.IPv4Family
}
networking := &kindconfigv1alpha4.Networking{
IPFamily: clusterIPFamily,
}
if podSubnet != "" && serviceSubnet != "" {
networking.PodSubnet = podSubnet
networking.ServiceSubnet = serviceSubnet
}
kvPeers := []*e2e.BGPPeerValues{}
if slices.Contains(peerAddrFamily, e2e.IPv4Family) {
kvPeers = append(kvPeers, &e2e.BGPPeerValues{
IP: localIPv4,
AS: goBGPAS,
IPFamily: e2e.IPv4Family,
})
}
if slices.Contains(peerAddrFamily, e2e.IPv6Family) {
kvPeers = append(kvPeers, &e2e.BGPPeerValues{
IP: localIPv6,
AS: goBGPAS,
IPFamily: e2e.IPv6Family,
})
}
kvPeersStr := []string{}
for _, p := range kvPeers {
kvPeersStr = append(kvPeersStr, p.String())
}
manifestValues = &e2e.KubevipManifestValues{
ControlPlaneVIP: *cpVIP,
ImagePath: imagePath,
ConfigPath: configPath,
ControlPlaneEnable: "false",
SvcEnable: "true",
SvcElectionEnable: "false",
BGPAS: kubevipAS,
BGPPeers: strings.Join(kvPeersStr, ","),
MPBGPNexthop: mpbgpnexthop,
MPBGPNexthopIPv4: defaultFixedNexthopv4,
MPBGPNexthopIPv6: defaultFixedNexthopv6,
}
By(manifestValues.BGPPeers)
*clusterName, *client = prepareCluster(*tempDirPath, clusterNameSuffix, k8sImagePath, v129, kubeVIPBGPManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
container := fmt.Sprintf("%s-control-plane", *clusterName)
containerIPv4, containerIPv6, err := GetContainerIPs(container)
Expect(err).ToNot(HaveOccurred())
if slices.Contains(peerAddrFamily, e2e.IPv4Family) {
*gobgpPeers = append(*gobgpPeers, &e2e.BGPPeerValues{
IP: containerIPv4,
AS: kubevipAS,
IPFamily: e2e.IPv4Family,
})
}
if slices.Contains(peerAddrFamily, e2e.IPv6Family) {
*gobgpPeers = append(*gobgpPeers, &e2e.BGPPeerValues{
IP: containerIPv6,
AS: kubevipAS,
IPFamily: e2e.IPv6Family,
})
}
if bgpClientAddrFamily == e2e.IPv6Family {
*gobgpClient, err = newGoBGPClient(localIPv6, goBGPPort)
} else {
*gobgpClient, err = newGoBGPClient(localIPv4, goBGPPort)
}
Expect(err).ToNot(HaveOccurred())
for _, p := range *gobgpPeers {
if slices.Contains(peerAddrFamily, p.IPFamily) {
peerCtx := context.TODO()
Eventually(peerCtx, func() error {
_, err = (*gobgpClient).AddPeer(context.TODO(), &api.AddPeerRequest{
Peer: &api.Peer{
Conf: &api.PeerConf{
NeighborAddress: p.IP,
PeerAsn: uint32(p.AS),
},
AfiSafis: []*api.AfiSafi{
{
Config: &api.AfiSafiConfig{
Enabled: true,
Family: &api.Family{
Afi: api.Family_AFI_IP6,
Safi: api.Family_SAFI_UNICAST,
},
},
},
{
Config: &api.AfiSafiConfig{
Enabled: true,
Family: &api.Family{
Afi: api.Family_AFI_IP,
Safi: api.Family_SAFI_UNICAST,
},
},
},
},
},
})
return err
}, "120s", "100ms").Should(Succeed())
}
}
return containerIPv4, containerIPv6
}
func testServiceBGP(svcName, lbAddress string, trafficPolicy corev1.ServiceExternalTrafficPolicy,
client kubernetes.Interface, serviceAddrFamily []corev1.IPFamily, numberOfServices int,
gobgpClient api.GobgpApiClient, gobgpFamily *api.Family, expectedNexthop string) {
lbAddresses := vip.Split(lbAddress)
services := []string{}
for i := range numberOfServices {
services = append(services, fmt.Sprintf("%s-%d", svcName, i))
}
for _, svc := range services {
createTestService(svc, dsNamespace, dsName, lbAddress,
client, corev1.IPFamilyPolicyPreferDualStack, serviceAddrFamily, trafficPolicy)
}
for _, addr := range lbAddresses {
paths := checkGoBGPPaths(context.Background(), gobgpClient, gobgpFamily, []*api.TableLookupPrefix{{Prefix: addr}}, 1)
Expect(strings.Contains(paths[0].Prefix, lbAddress)).To(BeTrue())
if expectedNexthop != "" {
Expect(strings.Contains(paths[0].String(), fmt.Sprintf("next_hop:\"%s\"", expectedNexthop)) || strings.Contains(paths[0].String(), fmt.Sprintf("next_hops:\"%s\"", expectedNexthop))).To(BeTrue())
}
}
for i := range numberOfServices {
err := client.CoreV1().Services(dsNamespace).Delete(context.TODO(), services[i], metav1.DeleteOptions{})
Expect(err).ToNot(HaveOccurred())
if i < numberOfServices-1 {
for _, addr := range lbAddresses {
paths := checkGoBGPPaths(context.Background(), gobgpClient, gobgpFamily, []*api.TableLookupPrefix{{Prefix: addr}}, 1)
Expect(strings.Contains(paths[0].Prefix, lbAddress)).To(BeTrue())
if expectedNexthop != "" {
Expect(strings.Contains(paths[0].String(), fmt.Sprintf("next_hop:\"%s\"", expectedNexthop)) || strings.Contains(paths[0].String(), fmt.Sprintf("next_hops:\"%s\"", expectedNexthop))).To(BeTrue())
}
}
}
}
for _, addr := range lbAddresses {
checkGoBGPPaths(context.Background(), gobgpClient, gobgpFamily, []*api.TableLookupPrefix{{Prefix: addr}}, 0)
}
}
func GetContainerIPs(containerName string) (string, string, error) {
cli, err := client.NewClientWithOpts(client.FromEnv)
if err != nil {
panic(err)
}
containers, err := cli.ContainerList(context.Background(), container.ListOptions{})
if err != nil {
return "", "", fmt.Errorf("failed to list containers: %w", err)
}
for _, c := range containers {
for _, n := range c.Names {
if n[1:] == containerName {
fmt.Println(n)
for _, n := range c.NetworkSettings.Networks {
return n.IPAddress, n.GlobalIPv6Address, nil
}
}
}
}
return "", "", nil
}
func newGoBGPClient(address string, port uint32) (api.GobgpApiClient, error) {
grpcOpts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
target := net.JoinHostPort(address, strconv.Itoa(int(port)))
conn, err := grpc.NewClient(target, grpcOpts...)
if err != nil {
return nil, fmt.Errorf("failed to connect to GoBGP server %q: %w", target, err)
}
return api.NewGobgpApiClient(conn), nil
}
func checkGoBGPPaths(ctx context.Context, client api.GobgpApiClient, family *api.Family, prefixes []*api.TableLookupPrefix, expectedPaths int) []*api.Destination {
var paths []*api.Destination
Eventually(func() error {
var err error
paths, err = getGoBGPPaths(ctx, client, family, prefixes)
if err != nil {
return err
}
if len(paths) != expectedPaths {
return fmt.Errorf("expected %d paths, but found %d", expectedPaths, len(paths))
}
return nil
}, "120s").ShouldNot(HaveOccurred())
return paths
}
func getGoBGPPaths(ctx context.Context, client api.GobgpApiClient, family *api.Family, prefixes []*api.TableLookupPrefix) ([]*api.Destination, error) {
pathCtx, cancel := context.WithTimeout(ctx, time.Second*5)
defer cancel()
stream, err := client.ListPath(pathCtx, &api.ListPathRequest{
TableType: api.TableType_GLOBAL,
Family: family,
Name: "",
Prefixes: prefixes,
SortType: api.ListPathRequest_PREFIX,
})
if err != nil {
return nil, err
}
rib := make([]*api.Destination, 0)
for {
r, err := stream.Recv()
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
rib = append(rib, r.Destination)
}
return rib, nil
}
func startGoBGP(config string, kill chan any) {
By("starting GoBGP server")
cmd := exec.Command("../../bin/gobgpd", "-f", config)
go cmd.Run()
<-kill
By("stopping GoBGP server")
err := cmd.Process.Kill()
Expect(err).ToNot(HaveOccurred())
}

View File

@@ -6,6 +6,7 @@ package e2e_test
import (
"context"
"fmt"
"net"
"os"
"path/filepath"
"strconv"
@@ -24,6 +25,7 @@ import (
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/kube-vip/kube-vip/testing/e2e"
"github.com/kube-vip/kube-vip/testing/services/pkg/deployment"
)
var _ = Describe("kube-vip routing table mode", func() {
@@ -84,7 +86,7 @@ var _ = Describe("kube-vip routing table mode", func() {
SvcElectionEnable: "false",
}
clusterName, _ = prepareCluster(tempDirPath, "rt-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber)
clusterName, _ = prepareCluster(tempDirPath, "rt-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
@@ -134,7 +136,7 @@ var _ = Describe("kube-vip routing table mode", func() {
SvcElectionEnable: "false",
}
clusterName, _ = prepareCluster(tempDirPath, "rt-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber)
clusterName, _ = prepareCluster(tempDirPath, "rt-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
@@ -183,7 +185,20 @@ var _ = Describe("kube-vip routing table mode", func() {
SvcElectionEnable: "false",
}
clusterName, _ = prepareCluster(tempDirPath, "rt-ds-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber)
networkInterface := ""
if networkInterface = os.Getenv("NETWORK_INTERFACE"); networkInterface == "" {
networkInterface = "br-"
}
localIPv6, localIPv6Net, err := deployment.GetLocalIPv6(networkInterface)
Expect(err).ToNot(HaveOccurred())
addSAN := &san{
ip: localIPv6,
ipnet: localIPv6Net,
}
clusterName, _ = prepareCluster(tempDirPath, "rt-ds-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, addSAN)
})
AfterAll(func() {
@@ -238,7 +253,20 @@ var _ = Describe("kube-vip routing table mode", func() {
SvcElectionEnable: "false",
}
clusterName, _ = prepareCluster(tempDirPath, "rt-ds-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber)
networkInterface := ""
if networkInterface = os.Getenv("NETWORK_INTERFACE"); networkInterface == "" {
networkInterface = "br-"
}
localIPv4, localIPv4Net, err := deployment.GetLocalIPv4(networkInterface)
Expect(err).ToNot(HaveOccurred())
addSAN := &san{
ip: localIPv4,
ipnet: localIPv4Net,
}
clusterName, _ = prepareCluster(tempDirPath, "rt-ds-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, addSAN)
})
AfterAll(func() {
@@ -301,7 +329,7 @@ var _ = Describe("kube-vip routing table mode", func() {
ipFamily = []corev1.IPFamily{corev1.IPv4Protocol}
clusterName, client = prepareCluster(tempDirPath, "rt-svc-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber)
clusterName, client = prepareCluster(tempDirPath, "rt-svc-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
@@ -361,7 +389,7 @@ var _ = Describe("kube-vip routing table mode", func() {
ipFamily = []corev1.IPFamily{corev1.IPv6Protocol}
clusterName, client = prepareCluster(tempDirPath, "rt-svc-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber)
clusterName, client = prepareCluster(tempDirPath, "rt-svc-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
@@ -422,7 +450,7 @@ var _ = Describe("kube-vip routing table mode", func() {
ipFamily = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}
clusterName, client = prepareCluster(tempDirPath, "rt-ds-svc-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber)
clusterName, client = prepareCluster(tempDirPath, "rt-ds-svc-ipv4", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
@@ -485,7 +513,7 @@ var _ = Describe("kube-vip routing table mode", func() {
ipFamily = []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}
clusterName, client = prepareCluster(tempDirPath, "rt-ds-svc-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber)
clusterName, client = prepareCluster(tempDirPath, "rt-ds-svc-ipv6", k8sImagePath, v129, kubeVIPRoutingTableManifestTemplate, logger, manifestValues, networking, nodesNumber, nil)
})
AfterAll(func() {
@@ -548,3 +576,8 @@ func testServiceRT(svcName, lbAddress, leaseName, leaseNamespace, clusterName st
}
}
}
type san struct {
ip *net.IP
ipnet *net.IPNet
}

View File

@@ -18,6 +18,7 @@ const (
ModeEnv = "TEST_MODE"
ModeARP = "arp"
ModeRT = "rt"
ModeBGP = "bgp"
)
var (
@@ -30,7 +31,7 @@ func TestE2E(t *testing.T) {
mode := os.Getenv(ModeEnv)
if mode == "" {
Mode = ModeARP
} else if mode != ModeARP && mode != ModeRT {
} else if mode != ModeARP && mode != ModeRT && mode != ModeBGP {
log.Fatal("invalid", "mode", mode)
os.Exit(1)
} else {

View File

@@ -34,7 +34,7 @@ import (
"github.com/onsi/gomega/format"
"github.com/onsi/gomega/gexec"
kvcluster "github.com/kube-vip/kube-vip/pkg/cluster"
"github.com/kube-vip/kube-vip/pkg/kubevip"
"github.com/kube-vip/kube-vip/pkg/vip"
"github.com/kube-vip/kube-vip/testing/e2e"
)
@@ -44,6 +44,11 @@ const (
dsNamespace = "default"
)
const testJSON = `
- op: add
path: "/apiServer/certSANs/-"
value: `
var _ = Describe("kube-vip ARP/NDP broadcast neighbor", func() {
if Mode == ModeARP {
var (
@@ -105,7 +110,7 @@ var _ = Describe("kube-vip ARP/NDP broadcast neighbor", func() {
EnableEndpointslices: "false",
}
clusterName, _ = prepareCluster(tempDirPath, "ipv4", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 3)
clusterName, _ = prepareCluster(tempDirPath, "ipv4", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 3, nil)
})
AfterAll(func() {
@@ -140,7 +145,7 @@ var _ = Describe("kube-vip ARP/NDP broadcast neighbor", func() {
EnableEndpointslices: "false",
}
clusterName, client = prepareCluster(tempDirPath, "svc-ipv4", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 1)
clusterName, client = prepareCluster(tempDirPath, "svc-ipv4", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 1, nil)
})
AfterAll(func() {
@@ -159,7 +164,6 @@ var _ = Describe("kube-vip ARP/NDP broadcast neighbor", func() {
DescribeTable("only removes VIP address if it was referenced by multiple services and all of them were deleted",
func(svcName string, offset uint, trafficPolicy corev1.ServiceExternalTrafficPolicy) {
lbAddress := e2e.GenerateVIP(e2e.IPv4Family, offset)
testService(svcName, lbAddress, "plndr-svcs-lock", "kube-system", trafficPolicy, client, false, []corev1.IPFamily{corev1.IPv4Protocol}, 2)
},
Entry("with external traffic policy - cluster", "test-svc-cluster", SOffset.Get(), corev1.ServiceExternalTrafficPolicyCluster),
@@ -190,7 +194,7 @@ var _ = Describe("kube-vip ARP/NDP broadcast neighbor", func() {
EnableEndpointslices: "false",
}
clusterName, client = prepareCluster(tempDirPath, "svc-el-ipv4", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 1)
clusterName, client = prepareCluster(tempDirPath, "svc-el-ipv4", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 1, nil)
})
AfterAll(func() {
@@ -229,7 +233,7 @@ var _ = Describe("kube-vip ARP/NDP broadcast neighbor", func() {
EnableEndpointslices: "false",
}
clusterName, _ = prepareCluster(tempDirPath, "ipv6", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 3)
clusterName, _ = prepareCluster(tempDirPath, "ipv6", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 3, nil)
})
AfterAll(func() {
@@ -264,7 +268,7 @@ var _ = Describe("kube-vip ARP/NDP broadcast neighbor", func() {
EnableEndpointslices: "false",
}
clusterName, client = prepareCluster(tempDirPath, "svc-ipv6", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 1)
clusterName, client = prepareCluster(tempDirPath, "svc-ipv6", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 1, nil)
})
AfterAll(func() {
@@ -313,7 +317,7 @@ var _ = Describe("kube-vip ARP/NDP broadcast neighbor", func() {
EnableEndpointslices: "false",
}
clusterName, client = prepareCluster(tempDirPath, "svc-el-ipv6", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 1)
clusterName, client = prepareCluster(tempDirPath, "svc-el-ipv6", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 1, nil)
})
AfterAll(func() {
@@ -352,7 +356,7 @@ var _ = Describe("kube-vip ARP/NDP broadcast neighbor", func() {
EnableEndpointslices: "true",
}
clusterName, _ = prepareCluster(tempDirPath, "ds-ipv4", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 3)
clusterName, _ = prepareCluster(tempDirPath, "ds-ipv4", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 3, nil)
})
AfterAll(func() {
@@ -387,7 +391,7 @@ var _ = Describe("kube-vip ARP/NDP broadcast neighbor", func() {
EnableEndpointslices: "true",
}
clusterName, client = prepareCluster(tempDirPath, "ds-svc-ipv4", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 1)
clusterName, client = prepareCluster(tempDirPath, "ds-svc-ipv4", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 1, nil)
})
AfterAll(func() {
@@ -436,7 +440,7 @@ var _ = Describe("kube-vip ARP/NDP broadcast neighbor", func() {
EnableEndpointslices: "true",
}
clusterName, client = prepareCluster(tempDirPath, "ds-svc-el-ipv4", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 1)
clusterName, client = prepareCluster(tempDirPath, "ds-svc-el-ipv4", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 1, nil)
})
AfterAll(func() {
@@ -477,7 +481,7 @@ var _ = Describe("kube-vip ARP/NDP broadcast neighbor", func() {
EnableEndpointslices: "true",
}
clusterName, _ = prepareCluster(tempDirPath, "ds-ipv6", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 3)
clusterName, _ = prepareCluster(tempDirPath, "ds-ipv6", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 3, nil)
})
AfterAll(func() {
@@ -514,7 +518,7 @@ var _ = Describe("kube-vip ARP/NDP broadcast neighbor", func() {
EnableEndpointslices: "true",
}
clusterName, client = prepareCluster(tempDirPath, "ds-svc-ipv6", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 1)
clusterName, client = prepareCluster(tempDirPath, "ds-svc-ipv6", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 1, nil)
})
AfterAll(func() {
@@ -565,7 +569,7 @@ var _ = Describe("kube-vip ARP/NDP broadcast neighbor", func() {
EnableEndpointslices: "true",
}
clusterName, client = prepareCluster(tempDirPath, "ds-svc-el-ipv6", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 1)
clusterName, client = prepareCluster(tempDirPath, "ds-svc-el-ipv6", k8sImagePath, v129, kubeVIPManifestTemplate, logger, manifestValues, networking, 1, nil)
})
AfterAll(func() {
@@ -604,7 +608,7 @@ var _ = Describe("kube-vip ARP/NDP broadcast neighbor", func() {
EnableEndpointslices: "false",
}
clusterName, _ = prepareCluster(tempDirPath, "ipv4-hostname", k8sImagePath, v129, kubeVIPHostnameManifestTemplate, logger, manifestValues, networking, 3)
clusterName, _ = prepareCluster(tempDirPath, "ipv4-hostname", k8sImagePath, v129, kubeVIPHostnameManifestTemplate, logger, manifestValues, networking, 3, nil)
})
AfterAll(func() {
@@ -792,7 +796,7 @@ func createTestDS(name, namespace string, client kubernetes.Interface) {
func createTestService(name, namespace, target, lbAddress string, client kubernetes.Interface, ipfPolicy corev1.IPFamilyPolicy, ipFamiles []corev1.IPFamily, externalPolicy corev1.ServiceExternalTrafficPolicy) {
svcAnnotations := make(map[string]string)
svcAnnotations[kvcluster.LoadbalancerIPAnnotation] = lbAddress
svcAnnotations[kubevip.LoadbalancerIPAnnotation] = lbAddress
labels := make(map[string]string)
labels["app"] = target
@@ -835,7 +839,8 @@ func checkIPAddressByLease(name, namespace, lbAddress string, expected bool, cli
func prepareCluster(tempDirPath, clusterNameSuffix, k8sImagePath string,
v129 bool, kubeVIPManifestTemplate *template.Template, logger log.Logger,
manifestValues *e2e.KubevipManifestValues, networking *kindconfigv1alpha4.Networking, nodesNum int) (string, kubernetes.Interface) {
manifestValues *e2e.KubevipManifestValues, networking *kindconfigv1alpha4.Networking, nodesNum int,
addSAN *san) (string, kubernetes.Interface) {
manifestPath := filepath.Join(tempDirPath, fmt.Sprintf("kube-vip-%s.yaml", clusterNameSuffix))
@@ -849,6 +854,24 @@ func prepareCluster(tempDirPath, clusterNameSuffix, k8sImagePath string,
Nodes: []kindconfigv1alpha4.Node{},
}
kubeadmPatches := []kindconfigv1alpha4.PatchJSON6902{}
if addSAN != nil {
for i := 0; i < 64; i++ {
(*addSAN.ip)[len(*addSAN.ip)-1]++
if addSAN.ipnet.Contains(*addSAN.ip) {
kubeadmPatches = append(kubeadmPatches, kindconfigv1alpha4.PatchJSON6902{
Group: "kubeadm.k8s.io",
Version: "v1beta3",
Kind: "ClusterConfiguration",
Patch: testJSON + addSAN.ip.String(),
})
}
}
}
clusterConfig.KubeadmConfigPatchesJSON6902 = kubeadmPatches
for range nodesNum {
nodeConfig := kindconfigv1alpha4.Node{
Role: kindconfigv1alpha4.ControlPlaneRole,

View File

@@ -28,8 +28,10 @@ import (
)
const (
IPv4Family = "IPv4"
IPv6Family = "IPv6"
IPv4Family = "IPv4"
IPv6Family = "IPv6"
DualstackFamily = "DualStack"
DualstackFamilyIPv6 = "DualStackIPv6"
)
func EnsureKindNetwork() {
@@ -69,31 +71,37 @@ func EnsureKindNetwork() {
}
func GenerateVIP(family string, offset uint) string {
if family == DualstackFamily || family == DualstackFamilyIPv6 {
return fmt.Sprintf("%s,%s", GenerateVIP(IPv4Family, offset), GenerateVIP(IPv6Family, offset))
}
cidrs := getKindNetworkSubnetCIDRs()
for _, cidr := range cidrs {
ip, ipNet, parseErr := net.ParseCIDR(cidr)
Expect(parseErr).NotTo(HaveOccurred())
if cidr != "" {
ip, ipNet, parseErr := net.ParseCIDR(cidr)
Expect(parseErr).NotTo(HaveOccurred())
if ip.To4() == nil && family == IPv6Family {
lowerMask := binary.BigEndian.Uint64(ipNet.Mask[8:])
lowerStart := binary.BigEndian.Uint64(ipNet.IP[8:])
lowerEnd := (lowerStart & lowerMask) | (^lowerMask)
if ip.To4() == nil && family == IPv6Family {
lowerMask := binary.BigEndian.Uint64(ipNet.Mask[8:])
lowerStart := binary.BigEndian.Uint64(ipNet.IP[8:])
lowerEnd := (lowerStart & lowerMask) | (^lowerMask)
chosenVIP := make([]byte, 16)
// Copy upper half into chosenVIP
copy(chosenVIP, ipNet.IP[0:8])
// Copy lower half into chosenVIP
binary.BigEndian.PutUint64(chosenVIP[8:], lowerEnd-uint64(offset))
return net.IP(chosenVIP).String()
} else if ip.To4() != nil && family == IPv4Family {
mask := binary.BigEndian.Uint32(ipNet.Mask)
start := binary.BigEndian.Uint32(ipNet.IP)
end := (start & mask) | (^mask)
chosenVIP := make([]byte, 16)
// Copy upper half into chosenVIP
copy(chosenVIP, ipNet.IP[0:8])
// Copy lower half into chosenVIP
binary.BigEndian.PutUint64(chosenVIP[8:], lowerEnd-uint64(offset))
return net.IP(chosenVIP).String()
} else if ip.To4() != nil && family == IPv4Family {
mask := binary.BigEndian.Uint32(ipNet.Mask)
start := binary.BigEndian.Uint32(ipNet.IP)
end := (start & mask) | (^mask)
chosenVIP := make([]byte, 4)
binary.BigEndian.PutUint32(chosenVIP, end-uint32(offset))
return net.IP(chosenVIP).String()
chosenVIP := make([]byte, 4)
binary.BigEndian.PutUint32(chosenVIP, end-uint32(offset))
return net.IP(chosenVIP).String()
}
}
}

View File

@@ -0,0 +1,82 @@
---
apiVersion: v1
kind: Pod
metadata:
name: kube-vip
namespace: kube-system
spec:
containers:
- name: kube-vip
args:
- manager
- --prometheusHTTPServer
- ""
env:
- name: vip_loglevel
value: "-4"
- name: lb_class_legacy_handling
value: "false"
- name: lb_class_name
- name: prometheus_server
value: :2112
- name: disable_service_updates
value: "false"
- name: vip_arp
value: "false"
- name: port
value: "6443"
- name: vip_cidr
value: "32"
- name: vip_leaderelection
value: "false"
- name: vip_interface
value: lo
- name: bgp_enable
value: "true"
- name: bgp_hold_time
value: "3"
- name: bgp_keepalive_interval
value: "1"
- name: bgp_sourceif
value: "eth0"
- name: bgp_routerid
value: "2.2.2.2"
- name: bgp_as
value: "{{ .BGPAS }}"
- name: bgp_peers
value: "{{ .BGPPeers }}"
- name: mpbgp_nexthop
value: "{{ .MPBGPNexthop }}"
- name: mpbgp_ipv4
value: "{{ .MPBGPNexthopIPv4 }}"
- name: mpbgp_ipv6
value: "{{ .MPBGPNexthopIPv6 }}"
- name: address
value: "{{ .ControlPlaneVIP }}"
- name: cp_enable
value: "{{ .ControlPlaneEnable }}"
- name: svc_enable
value: "{{ .SvcEnable }}"
- name: svc_election
value: "{{ .SvcElectionEnable }}"
- name: enable_endpointslices
value: "{{ .EnableEndpointslices }}"
image: "{{ .ImagePath }}"
imagePullPolicy: Never
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
volumes:
- hostPath:
path: "{{ .ConfigPath }}"
name: kubeconfig

View File

@@ -3,6 +3,11 @@
package e2e
import (
"fmt"
"net"
)
type KubevipManifestValues struct {
ControlPlaneVIP string
ImagePath string
@@ -11,4 +16,34 @@ type KubevipManifestValues struct {
SvcElectionEnable string
EnableEndpointslices string
ControlPlaneEnable string
BGPAS uint32
BGPPeers string
MPBGPNexthop string
MPBGPNexthopIPv4 string
MPBGPNexthopIPv6 string
}
type BGPPeerValues struct {
IP string
AS uint32
MPBGP string
IPFamily string
}
func (pv *BGPPeerValues) String() string {
tmpIP := pv.IP
ip := net.ParseIP(tmpIP)
if ip == nil {
return ""
}
if ip.To4() == nil {
tmpIP = fmt.Sprintf("[%s]", tmpIP)
}
if pv.MPBGP != "" {
return fmt.Sprintf("%s:%d::false/mpbgp_nexthop=%s", tmpIP, pv.AS, pv.MPBGP)
}
return fmt.Sprintf("%s:%d::false", tmpIP, pv.AS)
}

View File

@@ -38,6 +38,7 @@ func main() {
flag.BoolVar(&t.LeaderActive, "leaderActive", false, "Perform a test on the active leader")
flag.BoolVar(&t.LocalDeploy, "localDeploy", false, "Perform a test on the active leader")
flag.BoolVar(&t.Egress, "egress", false, "Perform an egress test")
flag.BoolVar(&t.EgressInternal, "egressInternal", false, "Perform an egress test, using the internal functionality")
flag.BoolVar(&t.EgressIPv6, "egressIPv6", false, "Perform an egress test")
flag.BoolVar(&t.DualStack, "dualStack", false, "Perform an dual stack test")
flag.BoolVar(&t.RetainCluster, "retain", false, "Retain the cluster")

View File

@@ -18,13 +18,14 @@ import (
// service defines the settings for a new service
type Service struct {
name string
egress bool // enable egress
egressIPv6 bool // egress should be IPv6
policyLocal bool // set the policy to local pods
testHTTP bool
testDualstack bool // test dualstack loadbalancer services
timeout int // how long to wait for the service to be created
name string
egress bool // enable egress
egressInternal bool // enable Internal egress
egressIPv6 bool // egress should be IPv6
policyLocal bool // set the policy to local pods
testHTTP bool
testDualstack bool // test dualstack loadbalancer services
timeout int // how long to wait for the service to be created
}
type Deployment struct {
@@ -238,6 +239,11 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
"kube-vip.io/egress": "true",
}
}
if s.egressInternal {
svc.Annotations["kube-vip.io/egress-internal"] = "true"
}
if s.egressIPv6 {
svc.Annotations["kube-vip.io/egress-ipv6"] = "true"
}
@@ -276,7 +282,7 @@ func (s *Service) CreateService(ctx context.Context, clientset *kubernetes.Clien
slog.Fatal(err)
}
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return clientset.CoreV1().Services(v1.NamespaceDefault).Watch(ctx, metav1.ListOptions{})
},

View File

@@ -62,6 +62,7 @@ func (config *TestConfig) StartServiceTest(ctx context.Context, clientset *kuber
errs = append(errs, err)
}
}
if config.LocalDeploy {
// Failover tests
err = config.LocalDeployment(ctx, clientset)
@@ -72,8 +73,17 @@ func (config *TestConfig) StartServiceTest(ctx context.Context, clientset *kuber
}
if config.Egress {
// Failover tests
err = config.EgressDeployment(ctx, clientset)
// Egress test
err = config.EgressDeployment(ctx, clientset, false)
if err != nil {
slog.Error(err)
errs = append(errs, err)
}
}
if config.Egress && config.EgressInternal {
// Egress test
err = config.EgressDeployment(ctx, clientset, true)
if err != nil {
slog.Error(err)
errs = append(errs, err)
@@ -81,8 +91,17 @@ func (config *TestConfig) StartServiceTest(ctx context.Context, clientset *kuber
}
if config.EgressIPv6 {
// Failover tests
err = config.Egressv6Deployment(ctx, clientset)
// Egress v6 tests
err = config.Egressv6Deployment(ctx, clientset, false)
if err != nil {
slog.Error(err)
errs = append(errs, err)
}
}
if config.EgressIPv6 && config.EgressInternal {
// Egress v6 tests
err = config.Egressv6Deployment(ctx, clientset, true)
if err != nil {
slog.Error(err)
errs = append(errs, err)
@@ -90,7 +109,7 @@ func (config *TestConfig) StartServiceTest(ctx context.Context, clientset *kuber
}
if config.DualStack {
// Failover tests
// Dualstack tests
err = config.DualStackDeployment(ctx, clientset)
if err != nil {
slog.Error(err)
@@ -165,7 +184,7 @@ func leaderFailover(ctx context.Context, name, leaderNode *string, clientset *ku
slog.Infof("👀 service [%s] for updates", *name)
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return clientset.CoreV1().Services(v1.NamespaceDefault).Watch(ctx, metav1.ListOptions{})
},
@@ -248,7 +267,7 @@ func podFailover(ctx context.Context, name, leaderNode *string, clientset *kuber
slog.Infof("👀 service [%s] for updates", *name)
// Use a restartable watcher, as this should help in the event of etcd or timeout issues
rw, err := watchtools.NewRetryWatcher("1", &cache.ListWatch{
rw, err := watchtools.NewRetryWatcherWithContext(ctx, "1", &cache.ListWatch{
WatchFunc: func(_ metav1.ListOptions) (watch.Interface, error) {
return clientset.CoreV1().Services(v1.NamespaceDefault).Watch(ctx, metav1.ListOptions{})
},
@@ -376,32 +395,32 @@ func handleRequest(conn net.Conn) {
conn.Close()
}
func GetLocalIP(ifName string, family int) (string, error) {
func GetLocalIP(ifName string, family int) (*net.IP, *net.IPNet, error) {
links, err := netlink.LinkList()
if err != nil {
return "", fmt.Errorf("netlink: failed to list links: %w", err)
return nil, nil, fmt.Errorf("netlink: failed to list links: %w", err)
}
for _, link := range links {
if strings.Contains(link.Attrs().Name, ifName) {
ip, _, err := getNetwork(link, family)
ip, ipnet, err := getNetwork(link, family)
if err != nil {
return "", fmt.Errorf("failed to get IPv4 address: %w", err)
return nil, nil, fmt.Errorf("failed to get IPv4 address: %w", err)
}
if ip == nil {
return "", fmt.Errorf("failed to find IPv4 address on the interface %q", ifName)
return nil, nil, fmt.Errorf("failed to find IPv4 address on the interface %q", ifName)
}
return ip.String(), nil
return ip, ipnet, nil
}
}
return "", nil
return nil, nil, nil
}
func GetLocalIPv4(ifName string) (string, error) {
func GetLocalIPv4(ifName string) (*net.IP, *net.IPNet, error) {
return GetLocalIP(ifName, netlink.FAMILY_V4)
}
func GetLocalIPv6(ifName string) (string, error) {
func GetLocalIPv6(ifName string) (*net.IP, *net.IPNet, error) {
return GetLocalIP(ifName, netlink.FAMILY_V6)
}

View File

@@ -32,6 +32,7 @@ type TestConfig struct {
LocalDeploy bool
DualStack bool
Egress bool
EgressInternal bool
EgressIPv6 bool
RetainCluster bool
SkipHostnameChange bool
@@ -307,8 +308,8 @@ func (config *TestConfig) LocalDeployment(ctx context.Context, clientset *kubern
return nil
}
func (config *TestConfig) EgressDeployment(ctx context.Context, clientset *kubernetes.Clientset) error {
// pod Failover tests
func (config *TestConfig) EgressDeployment(ctx context.Context, clientset *kubernetes.Clientset, internal bool) error {
// egress test
var err error
defer func() error {
@@ -324,7 +325,7 @@ func (config *TestConfig) EgressDeployment(ctx context.Context, clientset *kuber
return nil
}() //nolint
slog.Infof("🧪 ---> egress IP re-write (local policy) <---")
slog.Infof("🧪 ---> egress IP re-write (local policy, internal: %t) <---", internal)
var egress string
var found bool
timeout := 30
@@ -337,10 +338,11 @@ func (config *TestConfig) EgressDeployment(ctx context.Context, clientset *kuber
}
// Find this machines IP address
deploy.address, err = GetLocalIPv4(config.DockerNIC)
addr, _, err := GetLocalIPv4(config.DockerNIC)
if err != nil {
return fmt.Errorf("unable to detect local IP address: %w", err)
}
deploy.address = addr.String()
if deploy.address == "" {
return fmt.Errorf("unable to detect local IP address")
}
@@ -359,6 +361,10 @@ func (config *TestConfig) EgressDeployment(ctx context.Context, clientset *kuber
timeout: 30,
}
if internal {
svc.egressInternal = true
}
_, lbAddresses, err := svc.CreateService(ctx, clientset)
if err != nil {
return err
@@ -381,8 +387,8 @@ func (config *TestConfig) EgressDeployment(ctx context.Context, clientset *kuber
return nil
}
func (config *TestConfig) Egressv6Deployment(ctx context.Context, clientset *kubernetes.Clientset) error {
// pod Failover tests
func (config *TestConfig) Egressv6Deployment(ctx context.Context, clientset *kubernetes.Clientset, internal bool) error {
// egress v6 test
var err error
defer func() error {
@@ -398,7 +404,7 @@ func (config *TestConfig) Egressv6Deployment(ctx context.Context, clientset *kub
return nil
}() //nolint
slog.Infof("🧪 ---> egress IP re-write IPv6 (local policy) <---")
slog.Infof("🧪 ---> egress IP re-write IPv6 (local policy, internal: %t) <---", internal)
var egress string
var found bool
timeout := 30
@@ -411,10 +417,11 @@ func (config *TestConfig) Egressv6Deployment(ctx context.Context, clientset *kub
}
// Find this machines IP address
deploy.address, err = GetLocalIPv6(config.DockerNIC)
addr, _, err := GetLocalIPv6(config.DockerNIC)
if err != nil {
return fmt.Errorf("unable to detect local IP address: %w", err)
}
deploy.address = addr.String()
if deploy.address == "" {
return fmt.Errorf("unable to detect local IP address")
}
@@ -434,6 +441,10 @@ func (config *TestConfig) Egressv6Deployment(ctx context.Context, clientset *kub
testDualstack: true,
}
if internal {
svc.egressInternal = true
}
_, lbAddresses, err := svc.CreateService(ctx, clientset)
if err != nil {
return err